Detect and use physical disks for Swift storage

This change let Swift detect and use physical disk for storage. The
old named volume for storage isn't really useful for any serious setup.
Also updated swift-guide.rst accordingly.

Change-Id: I4f577b7b69d8bcd8b3961500946241c65a16db22
Partially-Implements: blueprint swift-physical-disk

ansible/roles/swift/defaults/main.yml

@@ -37,5 +37,7 @@ swift_keystone_user: "swift"
 swift_admin_tenant_name: "admin"
 
 swift_devices_mount_point: "/srv/node"
+swift_devices_match_mode: "strict"
+swift_devices_name: "KOLLA_SWIFT_DATA"
 
 openstack_swift_auth: "{'auth_url':'{{ openstack_auth_v2.auth_url }}','username':'{{ openstack_auth_v2.username }}','password':'{{ openstack_auth_v2.password }}','project_name':'{{ openstack_auth_v2.project_name }}'}"

ansible/roles/swift/tasks/start.yml

@@ -1,4 +1,27 @@
 ---
+- name: Looking up disks for Swift
+  command: docker exec -t kolla_ansible /usr/bin/ansible localhost
+    -m find_disks
+    -a "name={{ swift_devices_name }}
+        match_mode={{ swift_devices_match_mode }}"
+  register: swift_disk_lookup
+  changed_when: "{{ swift_disk_lookup.stdout.find('localhost | SUCCESS => ') != -1 and
+                    (swift_disk_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  failed_when: swift_disk_lookup.stdout.split()[2] != 'SUCCESS'
+
+- name: Reading data from variable
+  set_fact:
+    swift_disks: "{{ (swift_disk_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).disks|from_json }}"
+
+- name: Mounting Swift disks
+  mount:
+    src: "UUID={{ item.fs_uuid }}"
+    fstype: xfs
+    opts: "noatime,nodiratime,nobarrier,logbufs=8"
+    state: mounted
+    name: "{{ swift_devices_mount_point }}/{{ item['fs_label'] }}"
+  with_items: swift_disks
+
 - name: Starting swift-rsyncd container
   kolla_docker:
     action: "start_container"
@@ -7,10 +30,8 @@
     name: "swift_rsyncd"
     volumes:
       - "{{ node_config_directory }}/swift-rsyncd/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname not in groups['swift-proxy-server']
 
 - name: Starting swift-proxy-server container
@@ -22,10 +43,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-proxy-server/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-proxy-server']
 
 - name: Starting swift-account-server container
@@ -37,10 +56,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-account-server/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-account-server']
 
 - name: Starting swift-account-auditor container
@@ -52,10 +69,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-account-auditor/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-account-server']
 
 - name: Starting swift-account-replicator container
@@ -67,10 +82,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-account-replicator/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-account-server']
 
 - name: Starting swift-account-reaper container
@@ -82,10 +95,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-account-reaper/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-account-server']
 
 - name: Starting swift-container-server container
@@ -97,10 +108,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-container-server/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-container-server']
 
 - name: Starting swift-container-auditor container
@@ -112,10 +121,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-container-auditor/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-container-server']
 
 - name: Starting swift-container-replicator container
@@ -127,10 +134,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-container-replicator/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-container-server']
 
 - name: Starting swift-container-updater container
@@ -142,10 +147,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-container-updater/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-container-server']
 
 - name: Starting swift-object-server container
@@ -157,10 +160,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-object-server/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-object-server']
 
 - name: Starting swift-object-auditor container
@@ -172,10 +173,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-object-auditor/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-object-server']
 
 - name: Starting swift-object-replicator container
@@ -187,10 +186,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-object-replicator/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-object-server']
 
 - name: Starting swift-object-updater container
@@ -202,10 +199,8 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-object-updater/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-object-server']
 
 - name: Starting swift-object-expirer container
@@ -217,8 +212,6 @@
     volumes:
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-object-expirer/:{{ container_config_directory }}/:ro"
+      - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}"
       - "/var/lib/kolla/dev/log:/dev/log"
-      - "swift_sdb1:/srv/node/sdb1"
-      - "swift_sdb2:/srv/node/sdb2"
-      - "swift_sdb3:/srv/node/sdb3"
   when: inventory_hostname in groups['swift-object-server']

doc/swift-guide.rst

@@ -7,19 +7,51 @@ Kolla can deploy a full working Swift setup in either a AIO or multi node setup.
 
 Prerequisites
 -------------
-Before running Swift we need to generate "rings", which are binary compressed files that at a high
+Before running Swift we need to generate "rings", which are binary compressed
-level let the various Swift services know where data is in the cluster. We hope to automate this
+files that at a high level let the various Swift services know where data is in
-process in a future release.
+the cluster. We hope to automate this process in a future release.
 
-Swift also expects block devices to be available and partitioned on the host, which Swift uses in
+disks with partition table (recommended)
-combination with the rings to store data. Swift demos commonly just use directories created under
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-/srv/node to simulate these devices. In order to ease "out of the box" testing of Kolla, we offer a
-similar setup with a data container. *Note*, data containers are very inefficient for this purpose.
-In production setups operators will want to provision disks according to the Swift operator guide,
-which can then be added the rings and used in Kolla.
 
-For an AIO setup, the following commands can be used, locally, to generate rings containing the data
+Swift also expects block devices to be available for storage. To prepare a disk
-container directories:
+for use as Swift storage device, a special partition name and filesystem label
+need to be added.  So that Kolla can detect those disks and mount for services.
+
+Follow the example below to add 3 disks for an AIO demo setup.
+
+::
+
+    # <WARNING ALL DATA ON DISK will be LOST!>
+    index=0
+    for d in sdc sdd sde; do
+        parted /dev/${d} -s -- mklabel gpt mkpart KOLLA_SWIFT_DATA 1 -1
+        sudo mkfs.xfs -f -L d${index} /dev/${d}1
+        (( index++ ))
+    done
+
+disks without partition table
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Kolla also supports unpartitioned disk (filesystem on /dev/sdc instead of
+/dev/sdc1) detection purely based on filesystem label. This is generally not a
+recommended practice but can be helpful for Kolla to take over Swift deployment
+already using disk like this.
+
+Given hard disks with labels swd1, swd2, swd3, use the following settings in
+ansible/roles/swift/defaults/main.yml
+
+::
+
+    swift_devices_match_mode: "prefix"
+    swift_devices_name: "swd"
+
+rings
+~~~~~
+
+Run following commands locally to generate Rings for AIO demo setup. The
+commands work with "disks with partition table" example listed above. Please
+modify accordingly if your setup is different.
 
 ::
 
@@ -33,11 +65,11 @@ container directories:
     kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base \
     swift-ring-builder /etc/kolla/config/swift/object.builder create 10 3 1
 
-  for partition in sdb1 sdb2 sdb3; do
+  for i in {0..2}; do
     docker run \
       -v /etc/kolla/config/swift/:/etc/kolla/config/swift/ \
       kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base swift-ring-builder \
-      /etc/kolla/config/swift/object.builder add z1-${KOLLA_INTERNAL_ADDRESS}:6000/${partition} 1;
+      /etc/kolla/config/swift/object.builder add r1z1-${KOLLA_INTERNAL_ADDRESS}:6000/d${i} 1;
   done
 
   # Account ring
@@ -46,11 +78,11 @@ container directories:
     kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base \
     swift-ring-builder /etc/kolla/config/swift/account.builder create 10 3 1
 
-  for partition in sdb1 sdb2 sdb3; do
+  for i in {0..2}; do
     docker run \
       -v /etc/kolla/config/swift/:/etc/kolla/config/swift/ \
       kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base swift-ring-builder \
-      /etc/kolla/config/swift/account.builder add z1-${KOLLA_INTERNAL_ADDRESS}:6001/${partition} 1;
+      /etc/kolla/config/swift/account.builder add r1z1-${KOLLA_INTERNAL_ADDRESS}:6001/d${i} 1;
   done
 
   # Container ring
@@ -59,11 +91,11 @@ container directories:
     kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base \
     swift-ring-builder /etc/kolla/config/swift/container.builder create 10 3 1
 
-  for partition in sdb1 sdb2 sdb3; do
+  for i in {0..2}; do
     docker run \
       -v /etc/kolla/config/swift/:/etc/kolla/config/swift/ \
       kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base swift-ring-builder \
-      /etc/kolla/config/swift/container.builder add z1-${KOLLA_INTERNAL_ADDRESS}:6002/${partition} 1;
+      /etc/kolla/config/swift/container.builder add r1z1-${KOLLA_INTERNAL_ADDRESS}:6002/d${i} 1;
   done
 
   for ring in object account container; do
@@ -81,6 +113,12 @@ http://docs.openstack.org/kilo/install-guide/install/apt/content/swift-initial-r
 
 Deploying
 ---------
+Enable Swift in /etc/kolla/globals.yml:
+
+::
+
+    enable_swift : "yes"
+
 Once the rings are in place, deploying Swift is the same as any other Kolla Ansible service. Below
 is the minimal command to bring up Swift AIO, and it's dependencies: