From a0fc2bd644ab24b482975c3741872384c754ab03 Mon Sep 17 00:00:00 2001 From: Qiu Yu Date: Mon, 25 Jan 2016 23:52:53 -0700 Subject: [PATCH] Detect and use physical disks for Swift storage This change let Swift detect and use physical disk for storage. The old named volume for storage isn't really useful for any serious setup. Also updated swift-guide.rst accordingly. Change-Id: I4f577b7b69d8bcd8b3961500946241c65a16db22 Partially-Implements: blueprint swift-physical-disk --- ansible/roles/swift/defaults/main.yml | 2 + ansible/roles/swift/tasks/start.yml | 83 ++++++++++++--------------- doc/swift-guide.rst | 72 +++++++++++++++++------ 3 files changed, 95 insertions(+), 62 deletions(-) diff --git a/ansible/roles/swift/defaults/main.yml b/ansible/roles/swift/defaults/main.yml index b969843cb9..27f7e05ce1 100644 --- a/ansible/roles/swift/defaults/main.yml +++ b/ansible/roles/swift/defaults/main.yml @@ -37,5 +37,7 @@ swift_keystone_user: "swift" swift_admin_tenant_name: "admin" swift_devices_mount_point: "/srv/node" +swift_devices_match_mode: "strict" +swift_devices_name: "KOLLA_SWIFT_DATA" openstack_swift_auth: "{'auth_url':'{{ openstack_auth_v2.auth_url }}','username':'{{ openstack_auth_v2.username }}','password':'{{ openstack_auth_v2.password }}','project_name':'{{ openstack_auth_v2.project_name }}'}" diff --git a/ansible/roles/swift/tasks/start.yml b/ansible/roles/swift/tasks/start.yml index 6caf6abff3..c5bb8ba545 100644 --- a/ansible/roles/swift/tasks/start.yml +++ b/ansible/roles/swift/tasks/start.yml @@ -1,4 +1,27 @@ --- +- name: Looking up disks for Swift + command: docker exec -t kolla_ansible /usr/bin/ansible localhost + -m find_disks + -a "name={{ swift_devices_name }} + match_mode={{ swift_devices_match_mode }}" + register: swift_disk_lookup + changed_when: "{{ swift_disk_lookup.stdout.find('localhost | SUCCESS => ') != -1 and + (swift_disk_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}" + failed_when: swift_disk_lookup.stdout.split()[2] != 'SUCCESS' + +- name: Reading data from variable + set_fact: + swift_disks: "{{ (swift_disk_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).disks|from_json }}" + +- name: Mounting Swift disks + mount: + src: "UUID={{ item.fs_uuid }}" + fstype: xfs + opts: "noatime,nodiratime,nobarrier,logbufs=8" + state: mounted + name: "{{ swift_devices_mount_point }}/{{ item['fs_label'] }}" + with_items: swift_disks + - name: Starting swift-rsyncd container kolla_docker: action: "start_container" @@ -7,10 +30,8 @@ name: "swift_rsyncd" volumes: - "{{ node_config_directory }}/swift-rsyncd/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname not in groups['swift-proxy-server'] - name: Starting swift-proxy-server container @@ -22,10 +43,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-proxy-server/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-proxy-server'] - name: Starting swift-account-server container @@ -37,10 +56,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-account-server/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-account-server'] - name: Starting swift-account-auditor container @@ -52,10 +69,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-account-auditor/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-account-server'] - name: Starting swift-account-replicator container @@ -67,10 +82,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-account-replicator/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-account-server'] - name: Starting swift-account-reaper container @@ -82,10 +95,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-account-reaper/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-account-server'] - name: Starting swift-container-server container @@ -97,10 +108,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-container-server/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-container-server'] - name: Starting swift-container-auditor container @@ -112,10 +121,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-container-auditor/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-container-server'] - name: Starting swift-container-replicator container @@ -127,10 +134,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-container-replicator/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-container-server'] - name: Starting swift-container-updater container @@ -142,10 +147,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-container-updater/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-container-server'] - name: Starting swift-object-server container @@ -157,10 +160,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-object-server/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-object-server'] - name: Starting swift-object-auditor container @@ -172,10 +173,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-object-auditor/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-object-server'] - name: Starting swift-object-replicator container @@ -187,10 +186,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-object-replicator/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-object-server'] - name: Starting swift-object-updater container @@ -202,10 +199,8 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-object-updater/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-object-server'] - name: Starting swift-object-expirer container @@ -217,8 +212,6 @@ volumes: - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" - "{{ node_config_directory }}/swift-object-expirer/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}" - "/var/lib/kolla/dev/log:/dev/log" - - "swift_sdb1:/srv/node/sdb1" - - "swift_sdb2:/srv/node/sdb2" - - "swift_sdb3:/srv/node/sdb3" when: inventory_hostname in groups['swift-object-server'] diff --git a/doc/swift-guide.rst b/doc/swift-guide.rst index bf50c770bd..c159b05363 100644 --- a/doc/swift-guide.rst +++ b/doc/swift-guide.rst @@ -7,19 +7,51 @@ Kolla can deploy a full working Swift setup in either a AIO or multi node setup. Prerequisites ------------- -Before running Swift we need to generate "rings", which are binary compressed files that at a high -level let the various Swift services know where data is in the cluster. We hope to automate this -process in a future release. +Before running Swift we need to generate "rings", which are binary compressed +files that at a high level let the various Swift services know where data is in +the cluster. We hope to automate this process in a future release. -Swift also expects block devices to be available and partitioned on the host, which Swift uses in -combination with the rings to store data. Swift demos commonly just use directories created under -/srv/node to simulate these devices. In order to ease "out of the box" testing of Kolla, we offer a -similar setup with a data container. *Note*, data containers are very inefficient for this purpose. -In production setups operators will want to provision disks according to the Swift operator guide, -which can then be added the rings and used in Kolla. +disks with partition table (recommended) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -For an AIO setup, the following commands can be used, locally, to generate rings containing the data -container directories: +Swift also expects block devices to be available for storage. To prepare a disk +for use as Swift storage device, a special partition name and filesystem label +need to be added. So that Kolla can detect those disks and mount for services. + +Follow the example below to add 3 disks for an AIO demo setup. + +:: + + # + index=0 + for d in sdc sdd sde; do + parted /dev/${d} -s -- mklabel gpt mkpart KOLLA_SWIFT_DATA 1 -1 + sudo mkfs.xfs -f -L d${index} /dev/${d}1 + (( index++ )) + done + +disks without partition table +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Kolla also supports unpartitioned disk (filesystem on /dev/sdc instead of +/dev/sdc1) detection purely based on filesystem label. This is generally not a +recommended practice but can be helpful for Kolla to take over Swift deployment +already using disk like this. + +Given hard disks with labels swd1, swd2, swd3, use the following settings in +ansible/roles/swift/defaults/main.yml + +:: + + swift_devices_match_mode: "prefix" + swift_devices_name: "swd" + +rings +~~~~~ + +Run following commands locally to generate Rings for AIO demo setup. The +commands work with "disks with partition table" example listed above. Please +modify accordingly if your setup is different. :: @@ -33,11 +65,11 @@ container directories: kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base \ swift-ring-builder /etc/kolla/config/swift/object.builder create 10 3 1 - for partition in sdb1 sdb2 sdb3; do + for i in {0..2}; do docker run \ -v /etc/kolla/config/swift/:/etc/kolla/config/swift/ \ kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base swift-ring-builder \ - /etc/kolla/config/swift/object.builder add z1-${KOLLA_INTERNAL_ADDRESS}:6000/${partition} 1; + /etc/kolla/config/swift/object.builder add r1z1-${KOLLA_INTERNAL_ADDRESS}:6000/d${i} 1; done # Account ring @@ -46,11 +78,11 @@ container directories: kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base \ swift-ring-builder /etc/kolla/config/swift/account.builder create 10 3 1 - for partition in sdb1 sdb2 sdb3; do + for i in {0..2}; do docker run \ -v /etc/kolla/config/swift/:/etc/kolla/config/swift/ \ kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base swift-ring-builder \ - /etc/kolla/config/swift/account.builder add z1-${KOLLA_INTERNAL_ADDRESS}:6001/${partition} 1; + /etc/kolla/config/swift/account.builder add r1z1-${KOLLA_INTERNAL_ADDRESS}:6001/d${i} 1; done # Container ring @@ -59,11 +91,11 @@ container directories: kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base \ swift-ring-builder /etc/kolla/config/swift/container.builder create 10 3 1 - for partition in sdb1 sdb2 sdb3; do + for i in {0..2}; do docker run \ -v /etc/kolla/config/swift/:/etc/kolla/config/swift/ \ kollaglue/${KOLLA_BASE_DISTRO}-${KOLLA_INSTALL_TYPE}-swift-base swift-ring-builder \ - /etc/kolla/config/swift/container.builder add z1-${KOLLA_INTERNAL_ADDRESS}:6002/${partition} 1; + /etc/kolla/config/swift/container.builder add r1z1-${KOLLA_INTERNAL_ADDRESS}:6002/d${i} 1; done for ring in object account container; do @@ -81,6 +113,12 @@ http://docs.openstack.org/kilo/install-guide/install/apt/content/swift-initial-r Deploying --------- +Enable Swift in /etc/kolla/globals.yml: + +:: + + enable_swift : "yes" + Once the rings are in place, deploying Swift is the same as any other Kolla Ansible service. Below is the minimal command to bring up Swift AIO, and it's dependencies: