From a781c64319b17dfa4503a7d260d9ba764d0c69b4 Mon Sep 17 00:00:00 2001 From: Scott Solkhon Date: Fri, 8 Mar 2019 15:33:12 +0000 Subject: [PATCH] Support separate Swift storage networks Adds support to seperate Swift access and replication traffic from other storage traffic. In a deployment where both Ceph and Swift have been deployed, this changes adds functionalality to support optional seperation of storage network traffic. This adds two new network interfaces 'swift_storage_interface' and 'swift_replication_interface' which maintain backwards compatibility. The Swift access network interface is configured via 'swift_storage_interface', which defaults to 'storage_interface'. The Swift replication network interface is configured via 'swift_replication_interface', which defaults to 'swift_storage_interface'. If a separate replication network is used, Kolla Ansible now deploys separate replication servers for the accounts, containers and objects, that listen on this network. In this case, these services handle only replication traffic, and the original account-, container- and object- servers only handle storage user requests. Change-Id: Ib39e081574e030126f2d08f51de89641ddb0d42e --- ansible/group_vars/all.yml | 2 + ansible/roles/swift/defaults/main.yml | 3 ++ ansible/roles/swift/tasks/config.yml | 15 ++++++ ansible/roles/swift/tasks/legacy_upgrade.yml | 3 ++ ansible/roles/swift/tasks/precheck.yml | 41 ++++++++++++-- ansible/roles/swift/tasks/reconfigure.yml | 15 ++++++ ansible/roles/swift/tasks/start.yml | 54 +++++++++++++++++++ ansible/roles/swift/templates/account.conf.j2 | 6 ++- .../roles/swift/templates/container.conf.j2 | 6 ++- ansible/roles/swift/templates/object.conf.j2 | 6 ++- ansible/roles/swift/templates/rsyncd.conf.j2 | 2 +- .../swift-account-replication-server.json.j2 | 30 +++++++++++ ...swift-container-replication-server.json.j2 | 30 +++++++++++ .../swift-object-replication-server.json.j2 | 36 +++++++++++++ .../admin/production-architecture-guide.rst | 10 +++- doc/source/reference/storage/swift-guide.rst | 32 +++++++++++ etc/kolla/globals.yml | 2 + ...-replication-network-40ecd13e4339f299.yaml | 22 ++++++++ 18 files changed, 306 insertions(+), 9 deletions(-) create mode 100644 ansible/roles/swift/templates/swift-account-replication-server.json.j2 create mode 100644 ansible/roles/swift/templates/swift-container-replication-server.json.j2 create mode 100644 ansible/roles/swift/templates/swift-object-replication-server.json.j2 create mode 100644 releasenotes/notes/swift-replication-network-40ecd13e4339f299.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 7b124a214f..1cda7afc72 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -167,6 +167,8 @@ kolla_external_vip_interface: "{{ network_interface }}" api_interface: "{{ network_interface }}" storage_interface: "{{ network_interface }}" cluster_interface: "{{ network_interface }}" +swift_storage_interface: "{{ storage_interface }}" +swift_replication_interface: "{{ swift_storage_interface }}" migration_interface: "{{ network_interface }}" tunnel_interface: "{{ network_interface }}" octavia_network_interface: "{{ api_interface }}" diff --git a/ansible/roles/swift/defaults/main.yml b/ansible/roles/swift/defaults/main.yml index d1d7d31386..c79532c54d 100644 --- a/ansible/roles/swift/defaults/main.yml +++ b/ansible/roles/swift/defaults/main.yml @@ -66,6 +66,9 @@ swift_devices_match_mode: "strict" swift_devices_name: "KOLLA_SWIFT_DATA" swift_delay_auth_decision: "False" +# Boolean, true if there is a dedicated replication network. +swift_has_replication_network: "{{ swift_storage_interface != swift_replication_interface }}" + openstack_swift_auth: "{{ openstack_auth }}" syslog_server: "{{ api_interface_address }}" diff --git a/ansible/roles/swift/tasks/config.yml b/ansible/roles/swift/tasks/config.yml index 3d72639d7d..a2dbae52e3 100644 --- a/ansible/roles/swift/tasks/config.yml +++ b/ansible/roles/swift/tasks/config.yml @@ -11,14 +11,17 @@ - "swift" - "swift-account-auditor" - "swift-account-reaper" + - "swift-account-replication-server" - "swift-account-replicator" - "swift-account-server" - "swift-container-auditor" + - "swift-container-replication-server" - "swift-container-replicator" - "swift-container-server" - "swift-container-updater" - "swift-object-auditor" - "swift-object-expirer" + - "swift-object-replication-server" - "swift-object-replicator" - "swift-object-server" - "swift-object-updater" @@ -34,14 +37,17 @@ with_items: - "swift-account-auditor" - "swift-account-reaper" + - "swift-account-replication-server" - "swift-account-replicator" - "swift-account-server" - "swift-container-auditor" + - "swift-container-replication-server" - "swift-container-replicator" - "swift-container-server" - "swift-container-updater" - "swift-object-auditor" - "swift-object-expirer" + - "swift-object-replication-server" - "swift-object-replicator" - "swift-object-server" - "swift-object-updater" @@ -64,14 +70,17 @@ with_items: - "account-auditor" - "account-reaper" + - "account-replication-server" - "account-replicator" - "account-server" - "container-auditor" + - "container-replication-server" - "container-replicator" - "container-server" - "container-updater" - "object-auditor" - "object-expirer" + - "object-replication-server" - "object-replicator" - "object-server" - "object-updater" @@ -93,6 +102,7 @@ with_items: - "account-auditor" - "account-reaper" + - "account-replication-server" - "account-replicator" - "account-server" @@ -111,6 +121,7 @@ become: true with_items: - "container-auditor" + - "container-replication-server" - "container-replicator" - "container-server" - "container-updater" @@ -131,6 +142,7 @@ with_items: - "object-auditor" - "object-expirer" + - "object-replication-server" - "object-replicator" - "object-server" - "object-updater" @@ -184,14 +196,17 @@ with_items: - "swift-account-auditor" - "swift-account-reaper" + - "swift-account-replication-server" - "swift-account-replicator" - "swift-account-server" - "swift-container-auditor" + - "swift-container-replication-server" - "swift-container-replicator" - "swift-container-server" - "swift-container-updater" - "swift-object-auditor" - "swift-object-expirer" + - "swift-object-replication-server" - "swift-object-replicator" - "swift-object-server" - "swift-object-updater" diff --git a/ansible/roles/swift/tasks/legacy_upgrade.yml b/ansible/roles/swift/tasks/legacy_upgrade.yml index aa69d3bd11..289eed0d62 100644 --- a/ansible/roles/swift/tasks/legacy_upgrade.yml +++ b/ansible/roles/swift/tasks/legacy_upgrade.yml @@ -13,14 +13,17 @@ - "swift_rsyncd" - "swift_account_server" - "swift_account_auditor" + - "swift_account_replication_server" - "swift_account_replicator" - "swift_account_reaper" - "swift_container_server" - "swift_container_auditor" + - "swift_container_replication_server" - "swift_container_replicator" - "swift_container_updater" - "swift_object_server" - "swift_object_auditor" + - "swift_object_replication_server" - "swift_object_replicator" - "swift_object_updater" - "swift_object_expirer" diff --git a/ansible/roles/swift/tasks/precheck.yml b/ansible/roles/swift/tasks/precheck.yml index dc53cabf3c..97e927166d 100644 --- a/ansible/roles/swift/tasks/precheck.yml +++ b/ansible/roles/swift/tasks/precheck.yml @@ -10,7 +10,7 @@ - name: Checking free port for Swift Account Server wait_for: - host: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" + host: "{{ hostvars[inventory_hostname]['ansible_' + swift_storage_interface]['ipv4']['address'] }}" port: "{{ swift_account_server_port }}" connect_timeout: 1 timeout: 1 @@ -21,7 +21,7 @@ - name: Checking free port for Swift Container Server wait_for: - host: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" + host: "{{ hostvars[inventory_hostname]['ansible_' + swift_storage_interface]['ipv4']['address'] }}" port: "{{ swift_container_server_port }}" connect_timeout: 1 timeout: 1 @@ -32,7 +32,40 @@ - name: Checking free port for Swift Object Server wait_for: - host: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" + host: "{{ hostvars[inventory_hostname]['ansible_' + swift_storage_interface]['ipv4']['address'] }}" + port: "{{ swift_object_server_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - container_facts['swift_object_server'] is not defined + - inventory_hostname in groups['swift-object-server'] + +- name: Checking free port for Swift Account Replication Server + wait_for: + host: "{{ hostvars[inventory_hostname]['ansible_' + swift_replication_interface]['ipv4']['address'] }}" + port: "{{ swift_account_server_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - container_facts['swift_account_server'] is not defined + - inventory_hostname in groups['swift-account-server'] + +- name: Checking free port for Swift Container Replication Server + wait_for: + host: "{{ hostvars[inventory_hostname]['ansible_' + swift_replication_interface]['ipv4']['address'] }}" + port: "{{ swift_container_server_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - container_facts['swift_container_server'] is not defined + - inventory_hostname in groups['swift-container-server'] + +- name: Checking free port for Swift Object Replication Server + wait_for: + host: "{{ hostvars[inventory_hostname]['ansible_' + swift_replication_interface]['ipv4']['address'] }}" port: "{{ swift_object_server_port }}" connect_timeout: 1 timeout: 1 @@ -43,7 +76,7 @@ - name: Checking free port for Rsync wait_for: - host: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" + host: "{{ hostvars[inventory_hostname]['ansible_' + swift_replication_interface]['ipv4']['address'] }}" port: "873" connect_timeout: 1 timeout: 1 diff --git a/ansible/roles/swift/tasks/reconfigure.yml b/ansible/roles/swift/tasks/reconfigure.yml index 590320b0fe..5b5ef0aece 100644 --- a/ansible/roles/swift/tasks/reconfigure.yml +++ b/ansible/roles/swift/tasks/reconfigure.yml @@ -10,16 +10,19 @@ with_items: - { name: swift_account_server, group: swift-account-server } - { name: swift_account_auditor, group: swift-account-server } + - { name: swift_account_replication_server, group: swift-account-server } - { name: swift_account_replicator, group: swift-account-server } - { name: swift_account_reaper, group: swift-account-server } - { name: swift_rsyncd, group: swift-account-server } - { name: swift_container_server, group: swift-container-server } - { name: swift_container_auditor, group: swift-container-server } + - { name: swift_container_replication_server, group: swift-container-server } - { name: swift_container_replicator, group: swift-container-server } - { name: swift_container_updater, group: swift-container-server } - { name: swift_rsyncd, group: swift-container-server } - { name: swift_object_server, group: swift-object-server } - { name: swift_object_auditor, group: swift-object-server } + - { name: swift_object_replication_server, group: swift-object-server } - { name: swift_object_replicator, group: swift-object-server } - { name: swift_object_updater, group: swift-object-server } - { name: swift_object_expirer, group: swift-object-server } @@ -37,16 +40,19 @@ with_items: - { name: swift_account_server, group: swift-account-server } - { name: swift_account_auditor, group: swift-account-server } + - { name: swift_account_replication_server, group: swift-account-server } - { name: swift_account_replicator, group: swift-account-server } - { name: swift_account_reaper, group: swift-account-server } - { name: swift_rsyncd, group: swift-account-server } - { name: swift_container_server, group: swift-container-server } - { name: swift_container_auditor, group: swift-container-server } + - { name: swift_container_replication_server, group: swift-container-server } - { name: swift_container_replicator, group: swift-container-server } - { name: swift_container_updater, group: swift-container-server } - { name: swift_rsyncd, group: swift-container-server } - { name: swift_object_server, group: swift-object-server } - { name: swift_object_auditor, group: swift-object-server } + - { name: swift_object_replication_server, group: swift-object-server } - { name: swift_object_replicator, group: swift-object-server } - { name: swift_object_updater, group: swift-object-server } - { name: swift_object_expirer, group: swift-object-server } @@ -66,16 +72,19 @@ with_items: - { name: swift_account_server, group: swift-account-server } - { name: swift_account_auditor, group: swift-account-server } + - { name: swift_account_replication_server, group: swift-account-server } - { name: swift_account_replicator, group: swift-account-server } - { name: swift_account_reaper, group: swift-account-server } - { name: swift_rsyncd, group: swift-account-server } - { name: swift_container_server, group: swift-container-server } - { name: swift_container_auditor, group: swift-container-server } + - { name: swift_container_replication_server, group: swift-container-server } - { name: swift_container_replicator, group: swift-container-server } - { name: swift_container_updater, group: swift-container-server } - { name: swift_rsyncd, group: swift-container-server } - { name: swift_object_server, group: swift-object-server } - { name: swift_object_auditor, group: swift-object-server } + - { name: swift_object_replication_server, group: swift-object-server } - { name: swift_object_replicator, group: swift-object-server } - { name: swift_object_updater, group: swift-object-server } - { name: swift_object_expirer, group: swift-object-server } @@ -95,16 +104,19 @@ with_together: - [{ name: swift_account_server, group: swift-account-server }, { name: swift_account_auditor, group: swift-account-server }, + { name: swift_account_replication_server, group: swift-account-server }, { name: swift_account_replicator, group: swift-account-server }, { name: swift_account_reaper, group: swift-account-server }, { name: swift_rsyncd, group: swift-account-server }, { name: swift_container_server, group: swift-container-server }, { name: swift_container_auditor, group: swift-container-server }, + { name: swift_container_replication_server, group: swift-container-server }, { name: swift_container_replicator, group: swift-container-server }, { name: swift_container_updater, group: swift-container-server }, { name: swift_rsyncd, group: swift-container-server }, { name: swift_object_server, group: swift-object-server }, { name: swift_object_auditor, group: swift-object-server }, + { name: swift_object_replication_server, group: swift-object-server }, { name: swift_object_replicator, group: swift-object-server }, { name: swift_object_updater, group: swift-object-server }, { name: swift_object_expirer, group: swift-object-server }, @@ -131,16 +143,19 @@ with_together: - [{ name: swift_account_server, group: swift-account-server }, { name: swift_account_auditor, group: swift-account-server }, + { name: swift_account_replication_server, group: swift-account-server }, { name: swift_account_replicator, group: swift-account-server }, { name: swift_account_reaper, group: swift-account-server }, { name: swift_rsyncd, group: swift-account-server }, { name: swift_container_server, group: swift-container-server }, { name: swift_container_auditor, group: swift-container-server }, + { name: swift_container_replication_server, group: swift-container-server }, { name: swift_container_replicator, group: swift-container-server }, { name: swift_container_updater, group: swift-container-server }, { name: swift_rsyncd, group: swift-container-server }, { name: swift_object_server, group: swift-object-server }, { name: swift_object_auditor, group: swift-object-server }, + { name: swift_object_replication_server, group: swift-object-server }, { name: swift_object_replicator, group: swift-object-server }, { name: swift_object_updater, group: swift-object-server }, { name: swift_object_expirer, group: swift-object-server }, diff --git a/ansible/roles/swift/tasks/start.yml b/ansible/roles/swift/tasks/start.yml index 368d0bd554..55ffc33620 100644 --- a/ansible/roles/swift/tasks/start.yml +++ b/ansible/roles/swift/tasks/start.yml @@ -75,6 +75,24 @@ - "/etc/localtime:/etc/localtime:ro" when: inventory_hostname in groups['swift-account-server'] +- name: Starting swift-account-replication-server container + become: true + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + environment: + RSYNC_CONNECT_PROG: "nc %H {{ swift_rsync_port }}" + image: "{{ swift_account_image_full }}" + name: "swift_account_replication_server" + volumes: + - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" + - "{{ node_config_directory }}/swift-account-replication-server/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}:shared" + - "/etc/localtime:/etc/localtime:ro" + when: + - inventory_hostname in groups['swift-account-server'] + - swift_has_replication_network + - name: Starting swift-account-replicator container become: true kolla_docker: @@ -133,6 +151,24 @@ - "/etc/localtime:/etc/localtime:ro" when: inventory_hostname in groups['swift-container-server'] +- name: Starting swift-container-replication-server container + become: true + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + environment: + RSYNC_CONNECT_PROG: "nc %H {{ swift_rsync_port }}" + image: "{{ swift_container_image_full }}" + name: "swift_container_replication_server" + volumes: + - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" + - "{{ node_config_directory }}/swift-container-replication-server/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}:shared" + - "/etc/localtime:/etc/localtime:ro" + when: + - inventory_hostname in groups['swift-container-server'] + - swift_has_replication_network + - name: Starting swift-container-replicator container become: true kolla_docker: @@ -191,6 +227,24 @@ - "/etc/localtime:/etc/localtime:ro" when: inventory_hostname in groups['swift-object-server'] +- name: Starting swift-object-replication-server container + become: true + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + environment: + RSYNC_CONNECT_PROG: "nc %H {{ swift_rsync_port }}" + image: "{{ swift_object_image_full }}" + name: "swift_object_replication_server" + volumes: + - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro" + - "{{ node_config_directory }}/swift-object-replication-server/:{{ container_config_directory }}/:ro" + - "{{ swift_devices_mount_point }}:{{ swift_devices_mount_point }}:shared" + - "/etc/localtime:/etc/localtime:ro" + when: + - inventory_hostname in groups['swift-object-server'] + - swift_has_replication_network + - name: Starting swift-object-replicator container become: true kolla_docker: diff --git a/ansible/roles/swift/templates/account.conf.j2 b/ansible/roles/swift/templates/account.conf.j2 index 36b2d7367d..e0a6a522f1 100644 --- a/ansible/roles/swift/templates/account.conf.j2 +++ b/ansible/roles/swift/templates/account.conf.j2 @@ -1,5 +1,6 @@ +{% set interface = swift_replication_interface if 'replicat' in service_name else swift_storage_interface %} [DEFAULT] -bind_ip = {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }} +bind_ip = {{ hostvars[inventory_hostname]['ansible_' + interface]['ipv4']['address'] }} bind_port = {{ swift_account_server_port }} devices = {{ swift_devices_mount_point }} mount_check = false @@ -15,6 +16,9 @@ pipeline = account-server [app:account-server] use = egg:swift#account +{% if swift_has_replication_network %} +replication_server = {{ service_name == 'swift-account-replication-server' }} +{% endif %} {% if service_name == 'swift-account-auditor' %} [account-auditor] diff --git a/ansible/roles/swift/templates/container.conf.j2 b/ansible/roles/swift/templates/container.conf.j2 index a12537580e..f68464a9e7 100644 --- a/ansible/roles/swift/templates/container.conf.j2 +++ b/ansible/roles/swift/templates/container.conf.j2 @@ -1,5 +1,6 @@ +{% set interface = swift_replication_interface if 'replicat' in service_name else swift_storage_interface %} [DEFAULT] -bind_ip = {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }} +bind_ip = {{ hostvars[inventory_hostname]['ansible_' + interface]['ipv4']['address'] }} bind_port = {{ swift_container_server_port }} devices = {{ swift_devices_mount_point }} mount_check = false @@ -16,6 +17,9 @@ pipeline = container-server [app:container-server] use = egg:swift#container allow_versions = True +{% if swift_has_replication_network %} +replication_server = {{ service_name == 'swift-container-replication-server' }} +{% endif %} {% if service_name == 'swift-container-auditor' %} [container-auditor] diff --git a/ansible/roles/swift/templates/object.conf.j2 b/ansible/roles/swift/templates/object.conf.j2 index 204a9a0bb9..63c81b0639 100644 --- a/ansible/roles/swift/templates/object.conf.j2 +++ b/ansible/roles/swift/templates/object.conf.j2 @@ -1,5 +1,6 @@ +{% set interface = swift_replication_interface if 'replicat' in service_name else swift_storage_interface %} [DEFAULT] -bind_ip = {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }} +bind_ip = {{ hostvars[inventory_hostname]['ansible_' + interface]['ipv4']['address'] }} bind_port = {{ swift_object_server_port }} devices = {{ swift_devices_mount_point }} mount_check = false @@ -20,6 +21,9 @@ pipeline = object-server [app:object-server] use = egg:swift#object +{% if swift_has_replication_network %} +replication_server = {{ service_name == 'swift-object-replication-server' }} +{% endif %} {% if service_name == 'swift-object-auditor' %} [object-auditor] diff --git a/ansible/roles/swift/templates/rsyncd.conf.j2 b/ansible/roles/swift/templates/rsyncd.conf.j2 index 2797f9761a..3831415f57 100644 --- a/ansible/roles/swift/templates/rsyncd.conf.j2 +++ b/ansible/roles/swift/templates/rsyncd.conf.j2 @@ -1,4 +1,4 @@ -address = {{ api_interface_address }} +address = {{ hostvars[inventory_hostname]['ansible_' + swift_replication_interface]['ipv4']['address'] }} {% if inventory_hostname in groups['swift-account-server'] %} [account] diff --git a/ansible/roles/swift/templates/swift-account-replication-server.json.j2 b/ansible/roles/swift/templates/swift-account-replication-server.json.j2 new file mode 100644 index 0000000000..1c9c50b44a --- /dev/null +++ b/ansible/roles/swift/templates/swift-account-replication-server.json.j2 @@ -0,0 +1,30 @@ +{ + "command": "swift-account-server /etc/swift/account-server.conf --verbose", + "config_files": [ + { + "source": "/var/lib/kolla/swift/account.ring.gz", + "dest": "/etc/swift/account.ring.gz", + "owner": "swift", + "perm": "0640" + }, + { + "source": "{{ container_config_directory }}/swift.conf", + "dest": "/etc/swift/swift.conf", + "owner": "swift", + "perm": "0640" + }, + { + "source": "{{ container_config_directory }}/account-replication-server.conf", + "dest": "/etc/swift/account-server.conf", + "owner": "swift", + "perm": "0640" + }, + { + "source": "{{ container_config_directory }}/policy.json", + "dest": "/etc/swift/policy.json", + "owner": "swift", + "perm": "0600", + "optional": true + } + ] +} diff --git a/ansible/roles/swift/templates/swift-container-replication-server.json.j2 b/ansible/roles/swift/templates/swift-container-replication-server.json.j2 new file mode 100644 index 0000000000..02c202cab5 --- /dev/null +++ b/ansible/roles/swift/templates/swift-container-replication-server.json.j2 @@ -0,0 +1,30 @@ +{ + "command": "swift-container-server /etc/swift/container-server.conf --verbose", + "config_files": [ + { + "source": "/var/lib/kolla/swift/container.ring.gz", + "dest": "/etc/swift/container.ring.gz", + "owner": "swift", + "perm": "0640" + }, + { + "source": "{{ container_config_directory }}/swift.conf", + "dest": "/etc/swift/swift.conf", + "owner": "swift", + "perm": "0640" + }, + { + "source": "{{ container_config_directory }}/container-replication-server.conf", + "dest": "/etc/swift/container-server.conf", + "owner": "swift", + "perm": "0640" + }, + { + "source": "{{ container_config_directory }}/policy.json", + "dest": "/etc/swift/policy.json", + "owner": "swift", + "perm": "0600", + "optional": true + } + ] +} diff --git a/ansible/roles/swift/templates/swift-object-replication-server.json.j2 b/ansible/roles/swift/templates/swift-object-replication-server.json.j2 new file mode 100644 index 0000000000..c63005e70e --- /dev/null +++ b/ansible/roles/swift/templates/swift-object-replication-server.json.j2 @@ -0,0 +1,36 @@ +{ + "command": "swift-object-server /etc/swift/object-server.conf --verbose", + "config_files": [ + { + "source": "/var/lib/kolla/swift/container.ring.gz", + "dest": "/etc/swift/container.ring.gz", + "owner": "swift", + "perm": "0640" + }, + { + "source": "/var/lib/kolla/swift/object.ring.gz", + "dest": "/etc/swift/object.ring.gz", + "owner": "swift", + "perm": "0640" + }, + { + "source": "{{ container_config_directory }}/swift.conf", + "dest": "/etc/swift/swift.conf", + "owner": "swift", + "perm": "0640" + }, + { + "source": "{{ container_config_directory }}/object-replication-server.conf", + "dest": "/etc/swift/object-server.conf", + "owner": "swift", + "perm": "0640" + }, + { + "source": "{{ container_config_directory }}/policy.json", + "dest": "/etc/swift/policy.json", + "owner": "swift", + "perm": "0600", + "optional": true + } + ] +} diff --git a/doc/source/admin/production-architecture-guide.rst b/doc/source/admin/production-architecture-guide.rst index 47614e9e07..c7f8d9f97a 100644 --- a/doc/source/admin/production-architecture-guide.rst +++ b/doc/source/admin/production-architecture-guide.rst @@ -56,7 +56,7 @@ In Kolla operators should configure following network interfaces: * ``storage_interface`` - This is the interface that is used by virtual machines to communicate to Ceph. This can be heavily utilized so it's - recommended to put this network on 10Gig networking. Defaults to + recommended to use a high speed network fabric. Defaults to ``network_interface``. * ``cluster_interface`` - This is another interface used by Ceph. It's used for @@ -64,6 +64,14 @@ In Kolla operators should configure following network interfaces: bottleneck it can affect data consistency and performance of whole cluster. Defaults to ``network_interface``. +* ``swift_storage_interface`` - This interface is used by Swift for storage + access traffic. This can be heavily utilized so it's recommended to use + a high speed network fabric. Defaults to ``storage_interface``. + +* ``swift_replication_interface`` - This interface is used by Swift for storage + replication traffic. This can be heavily utilized so it's recommended to use + a high speed network fabric. Defaults to ``swift_storage_interface``. + * ``tunnel_interface`` - This interface is used by Neutron for vm-to-vm traffic over tunneled networks (like VxLan). Defaults to ``network_interface``. diff --git a/doc/source/reference/storage/swift-guide.rst b/doc/source/reference/storage/swift-guide.rst index 29004d16f1..3b53783b75 100644 --- a/doc/source/reference/storage/swift-guide.rst +++ b/doc/source/reference/storage/swift-guide.rst @@ -10,6 +10,24 @@ Overview Kolla can deploy a full working Swift setup in either a **all-in-one** or **multinode** setup. +Networking +~~~~~~~~~~ + +The following networks are used by Swift: + +External API network (``kolla_external_vip_interface``) + This network is used by users to access the Swift public API. +Internal API network (``api_interface``) + This network is used by users to access the Swift internal API. It is also + used by HAProxy to access the Swift proxy servers. +Swift Storage network (``swift_storage_interface``) + This network is used by the Swift proxy server to access the account, + container and object servers. Defaults to ``storage_interface``. +Swift replication network (``swift_replication_network``) + This network is used for Swift storage replication traffic. + This is optional as the default configuration uses + the ``swift_storage_interface`` for replication traffic. + Disks with a partition table (recommended) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -75,6 +93,11 @@ generate rings for a demo setup. The commands work with **disks with partition table** example listed above. Please modify accordingly if your setup is different. +If using a separate replication network it is necessary to add the replication +network IP addresses to the rings. See the `Swift documentation +`__ +for details on how to do that. + Prepare for Rings generating ---------------------------- @@ -162,6 +185,15 @@ To generate Swift container ring, run the following commands: done done +.. end + +Rebalance +--------- + +To rebalance the ring files: + +.. code-block:: console + for ring in object account container; do docker run \ --rm \ diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 0b95913d3c..92ac059847 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -90,6 +90,8 @@ kolla_internal_vip_address: "10.10.10.254" #api_interface: "{{ network_interface }}" #storage_interface: "{{ network_interface }}" #cluster_interface: "{{ network_interface }}" +#swift_storage_interface: "{{ storage_interface }}" +#swift_replication_interface: "{{ swift_storage_interface }}" #tunnel_interface: "{{ network_interface }}" #dns_interface: "{{ network_interface }}" #octavia_network_interface: "{{ api_interface }}" diff --git a/releasenotes/notes/swift-replication-network-40ecd13e4339f299.yaml b/releasenotes/notes/swift-replication-network-40ecd13e4339f299.yaml new file mode 100644 index 0000000000..e7bcbb3669 --- /dev/null +++ b/releasenotes/notes/swift-replication-network-40ecd13e4339f299.yaml @@ -0,0 +1,22 @@ +--- +features: + - | + Adds support to seperate Swift access and replication traffic from other + storage traffic. + + In a deployment where both Ceph and Swift have been deployed, + this changes adds functionalality to support optional seperation + of storage network traffic. This adds two new network interfaces + 'swift_storage_interface' and 'swift_replication_interface' which maintain + backwards compatibility. + + The Swift access network interface is configured via 'swift_storage_interface', + which defaults to 'storage_interface'. The Swift replication network + interface is configured via 'swift_replication_interface', which + defaults to 'swift_storage_interface'. + + If a separate replication network is used, Kolla Ansible now deploys separate + replication servers for the accounts, containers and objects, that listen on + this network. In this case, these services handle only replication traffic, and + the original account-, container- and object- servers only handle storage + user requests.