Ceph RadosGW integration

This adds a new scenario for Ceph Rados GW integration:
- It adds the RGW into haproxy to the default swift port if
swift port isn't deployed already
- It adds tempest swift API testing on the rados gw in the
check scenario
- It adds ceph rgw in default inventories.

Change-Id: I5f6ff3fa05a4a8019bf5b695b02184d9f065bc2e
Co-Authored-By: Jean-Philippe Evrard <jean-philippe@evrard.me>
Co-Authored-By: Maxime Guyot <maxime.guyot@elits.com>

doc/source/user/ceph/full-deploy.rst

@@ -35,9 +35,9 @@ integration in two ways:
   in ``user_variables.yml``
 * deploying a ceph cluster by using the roles maintained by the
   `Ceph-Ansible`_ project. Deployers can enable the ``ceph-install``
-  playbook by adding hosts to the ``ceph-mon_hosts`` and ``ceph-osd_hosts``
-  groups in ``openstack_user_config.yml``, and then configuring
-  `Ceph-Ansible specific vars
+  playbook by adding hosts to the ``ceph-mon_hosts``, ``ceph-osd_hosts`` and
+  ``ceph-rgw_hosts`` groups in ``openstack_user_config.yml``, and then
+  configuring `Ceph-Ansible specific vars
   <https://github.com/ceph/ceph-ansible/blob/master/group_vars/all.yml.sample>`_
   in the OpenStack-Ansible ``user_variables.yml`` file.
 

etc/openstack_deploy/conf.d/ceph.yml.aio

@@ -7,3 +7,8 @@ ceph-mon_hosts:
 ceph-osd_hosts:
   aio1:
     ip: 172.29.236.100
+
+# The nodes that the Ceph RadosGW object gateways will be running on
+ceph-rgw_hosts:
+  aio1:
+    ip: 172.29.236.100

etc/openstack_deploy/user_secrets.yml

@@ -160,3 +160,6 @@ molteniron_container_mysql_password:
 tacker_rabbitmq_password:
 tacker_service_password:
 tacker_container_mysql_password:
+
+## Ceph RadosGW Keystone password
+radosgw_admin_password:

inventory/env.d/ceph.yml

@@ -20,6 +20,9 @@ component_skel:
   ceph-osd:
     belongs_to:
       - ceph_all
+  ceph-rgw:
+    belongs_to:
+      - ceph_all
 
 container_skel:
   ceph-mon_container:
@@ -34,6 +37,11 @@ container_skel:
       - ceph-osd
     properties:
       is_metal: true
+  ceph-rgw_container:
+    belongs_to:
+      - ceph-rgw_containers
+    contains:
+      - ceph-rgw
 
 physical_skel:
   ceph-mon_containers:
@@ -48,3 +56,9 @@ physical_skel:
   ceph-osd_hosts:
     belongs_to:
       - hosts
+  ceph-rgw_containers:
+    belongs_to:
+      - all_containers
+  ceph-rgw_hosts:
+    belongs_to:
+      - hosts

inventory/group_vars/all/ceph.yml

@@ -19,6 +19,7 @@ ceph_client_package_state: "{{ package_state }}"
 mon_group_name: ceph-mon
 mgr_group_name: "{{ mon_group_name }}"
 osd_group_name: ceph-osd
+rgw_group_name: ceph-rgw
 ceph_origin: "{{ (ansible_pkg_mgr == 'zypper') | ternary('distro', 'repository') }}"
 ceph_repository: community
 # The _stable_release var is used by both the OSA ceph_client role and the

inventory/group_vars/ceph_all.yml

@@ -22,3 +22,45 @@ ceph_container_bind_mounts:
 # needed.
 nfs_file_gw: False
 nfs_obj_gw: False
+
+# radosgw
+
+radosgw_service_name: "radosgw"
+radosgw_service_type: "object-store"
+radosgw_service_description: "Object Storage Service"
+radosgw_service_region: "{{ service_region }}"
+radosgw_admin_user: radosgw
+radosgw_admin_tenant: service
+
+radosgw_service_port: "{{ (groups['swift_proxy'] is defined and groups['swift_proxy'] | length > 0) | ternary(7980,8080) }}"
+radosgw_address: "{{ container_address }}"
+radosgw_service_proto: http
+radosgw_service_publicuri_proto: "{{ openstack_service_publicuri_proto | default(radosgw_service_proto) }}"
+radosgw_service_adminuri_proto: "{{ openstack_service_adminuri_proto | default(radosgw_service_proto) }}"
+radosgw_service_internaluri_proto: "{{ openstack_service_internaluri_proto | default(radosgw_service_proto) }}"
+radosgw_service_publicuri: "{{ radosgw_service_publicuri_proto }}://{{ external_lb_vip_address }}:{{ radosgw_service_port }}"
+radosgw_service_publicurl: "{{ radosgw_service_publicuri }}/swift/v1"
+radosgw_service_adminuri: "{{ radosgw_service_adminuri_proto }}://{{ internal_lb_vip_address }}:{{ radosgw_service_port }}"
+radosgw_service_adminurl: "{{ radosgw_service_adminuri }}/swift/v1"
+radosgw_service_internaluri: "{{ radosgw_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ radosgw_service_port }}"
+radosgw_service_internalurl: "{{ radosgw_service_internaluri }}/swift/v1"
+
+# This is the endpoint hostname that will be configured in the Keystone catalog for object storage
+radosgw_dns_name: "{{ ansible_host }}"
+# To extend ceph_conf_overrides while keeping this configuration, create a
+# dict of config_template style overrides for ceph.conf in a var, ie.
+# ceph_conf_overrides_custom, then use:
+# ceph_conf_overrides: "{{ ceph_conf_overrides_rgw | combine(ceph_conf_overrides_custom, recursive=True) }}"
+ceph_conf_overrides: "{{ ceph_conf_overrides_rgw }}"
+ceph_conf_overrides_rgw:
+  "client.rgw.{{ hostvars[inventory_hostname]['ansible_hostname'] }}":
+    # OpenStack integration with Keystone
+    rgw_keystone_url: "{{ keystone_service_adminuri }}"
+    rgw_keystone_api_version: 3
+    rgw_keystone_admin_user: "{{ radosgw_admin_user }}"
+    rgw_keystone_admin_password: "{{ radosgw_admin_password }}"
+    rgw_keystone_admin_tenant: "{{ radosgw_admin_tenant }}"
+    rgw_keystone_admin_domain: default
+    rgw_keystone_accepted_roles: 'Member, _member_, admin, swiftoperator'
+    rgw_s3_auth_use_keystone: true
+    rgw_enable_apis: swift

inventory/group_vars/haproxy_all/haproxy.yml

@@ -353,3 +353,15 @@ haproxy_default_services:
       haproxy_timeout_server: 5000s
       haproxy_whitelist_networks: "{{ haproxy_opendaylight_whitelist_networks }}"
       haproxy_service_enabled: "{{ neutron_plugin_type == 'ml2.opendaylight' }}"
+  - service:
+      haproxy_service_name: ceph-rgw
+      haproxy_backend_nodes: "{{ groups['ceph-rgw'] | default([]) }}"
+      haproxy_ssl: "{{ haproxy_ssl }}"
+      haproxy_balance_alg: source
+      haproxy_port: "{{ hostvars[(groups['ceph-rgw'] | default(['localhost']))[0] | default('localhost')]['radosgw_service_port'] | default(7980) }}"
+      haproxy_balance_type: http
+      haproxy_backend_options:
+        - httpchk HEAD /
+      haproxy_backend_httpcheck_options:
+        - expect status 405
+      haproxy_service_enabled: "{{ groups['ceph-rgw'] is defined and groups['ceph-rgw'] | length > 0 }}"

inventory/group_vars/utility_all.yml

@@ -52,7 +52,7 @@ tempest_service_available_heat: "{{ groups['heat_all'] is defined and groups['he
 tempest_service_available_horizon: "{{ groups['horizon_all'] is defined and groups['horizon_all'] | length > 0 }}"
 tempest_service_available_neutron: "{{ groups['neutron_all'] is defined and groups['neutron_all'] | length > 0 }}"
 tempest_service_available_nova: "{{ groups['nova_all'] is defined and groups['nova_all'] | length > 0 }}"
-tempest_service_available_swift: "{{ groups['swift_all'] is defined and groups['swift_all'] | length > 0 }}"
+tempest_service_available_swift: "{{ (groups['swift_all'] is defined and groups['swift_all'] | length > 0) or (groups['ceph-rgw'] is defined and groups['ceph-rgw'] | length > 0) }}"
 
 tempest_log_dir: /var/log/utility
 tempest_venv_tag: "{{ openstack_release }}"

playbooks/ceph-rgw-install.yml

@@ -0,0 +1,175 @@
+---
+# Copyright 2017, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install ceph radosgw
+  hosts: ceph-rgw
+  user: root
+  pre_tasks:
+    - name: Ensure RGW service
+      keystone:
+        command: "ensure_service"
+        endpoint: "{{ keystone_service_adminurl }}"
+        login_user: "{{ keystone_admin_user_name }}"
+        login_password: "{{ keystone_auth_admin_password }}"
+        login_project_name: "{{ keystone_admin_tenant_name }}"
+        service_name: "{{ radosgw_service_name }}"
+        service_type: "{{ radosgw_service_type }}"
+        description: "{{ radosgw_service_description }}"
+        insecure: "{{ keystone_service_adminuri_insecure }}"
+      delegate_to: "{{ groups['utility_all'][0] }}"
+      run_once: true
+      when: radosgw_keystone | bool
+      register: add_service
+      until: add_service|success
+      retries: 5
+      delay: 2
+      tags:
+        - ceph-rgw-setup
+        - rgw-service-add
+    - name: Ensure RGW user
+      keystone:
+        command: "ensure_user"
+        endpoint: "{{ keystone_service_adminurl }}"
+        login_user: "{{ keystone_admin_user_name }}"
+        login_password: "{{ keystone_auth_admin_password }}"
+        login_project_name: "{{ keystone_admin_tenant_name }}"
+        user_name: "{{ radosgw_admin_user }}"
+        tenant_name: "{{ radosgw_admin_tenant }}"
+        role_name: "{{ radosgw_role_name | default('service') }}"
+        password: "{{ radosgw_admin_password }}"
+        insecure: "{{ keystone_service_adminuri_insecure }}"
+      delegate_to: "{{ groups['utility_all'][0] }}"
+      run_once: true
+      when: radosgw_keystone | bool
+      register: add_service
+      until: add_service|success
+      retries: 5
+      delay: 10
+      tags:
+        - ceph-rgw-setup
+        - rgw-service-add
+    - name: Ensure RGW user to admin role
+      keystone:
+        command: "ensure_user_role"
+        endpoint: "{{ keystone_service_adminurl }}"
+        login_user: "{{ keystone_admin_user_name }}"
+        login_password: "{{ keystone_auth_admin_password }}"
+        login_project_name: "{{ keystone_admin_tenant_name }}"
+        user_name: "{{ radosgw_admin_user }}"
+        tenant_name: "{{ radosgw_admin_tenant }}"
+        role_name: "{{ radosgw_role_name | default('admin') }}"
+        insecure: "{{ keystone_service_adminuri_insecure }}"
+      delegate_to: "{{ groups['utility_all'][0] }}"
+      run_once: true
+      register: add_admin_role
+      when: radosgw_keystone | bool
+      until: add_admin_role|success
+      retries: 5
+      delay: 10
+      tags:
+        - ceph-rgw-setup
+        - rgw-service-add
+    - name: Ensure swiftoperator role
+      keystone:
+        command: "ensure_role"
+        endpoint: "{{ keystone_service_adminurl }}"
+        login_user: "{{ keystone_admin_user_name }}"
+        login_password: "{{ keystone_auth_admin_password }}"
+        login_project_name: "{{ keystone_admin_tenant_name }}"
+        role_name: "swiftoperator"
+        insecure: "{{ keystone_service_adminuri_insecure }}"
+      delegate_to: "{{ groups['utility_all'][0] }}"
+      register: add_swiftoperator_role
+      until: add_swiftoperator_role|success
+      retries: 5
+      delay: 10
+      tags:
+        - ceph-rgw-setup
+        - rgw-service-add
+    - name: Ensure RGW endpoint
+      keystone:
+        command: "ensure_endpoint"
+        endpoint: "{{ keystone_service_adminurl }}"
+        login_user: "{{ keystone_admin_user_name }}"
+        login_password: "{{ keystone_auth_admin_password }}"
+        login_project_name: "{{ keystone_admin_tenant_name }}"
+        region_name: "{{ radosgw_service_region }}"
+        service_name: "{{ radosgw_service_name }}"
+        service_type: "{{ radosgw_service_type }}"
+        insecure: "{{ keystone_service_adminuri_insecure }}"
+        endpoint_list:
+          - url: "{{ radosgw_service_publicurl }}"
+            interface: "public"
+          - url: "{{ radosgw_service_adminurl }}"
+            interface: "admin"
+          - url: "{{ radosgw_service_internalurl }}"
+            interface: "internal"
+      delegate_to: "{{ groups['utility_all'][0] }}"
+      run_once: true
+      register: add_service
+      when: radosgw_keystone | bool
+      until: add_service|success
+      retries: 5
+      delay: 10
+      tags:
+        - ceph-rgw-setup
+        - rgw-service-add
+    - include: common-tasks/os-log-dir-setup.yml
+      vars:
+        log_dirs:
+          - src: "/openstack/log/{{ inventory_hostname }}-ceph"
+            dest: "/var/log/ceph"
+    - include: common-tasks/os-lxc-container-setup.yml
+    - name: Gather ceph-mon facts
+      action: setup
+      delegate_to: "{{ item }}"
+      delegate_facts: yes
+      with_items: "{{ groups[mon_group_name] }}"
+      when:
+        - inventory_hostname == ansible_play_hosts[0]
+      tags:
+        - ceph-mon-facts
+  roles:
+    - role: ceph-defaults
+      tags:
+        - skip_ansible_lint
+    - role: ceph-common
+      tags:
+        - skip_ansible_lint
+    - role: ceph-config
+      tags:
+        - skip_ansible_lint
+    - role: ceph-rgw
+      tags:
+        - skip_ansible_lint
+    - role: "rsyslog_client"
+      rsyslog_client_log_rotate_file: ceph_log_rotate
+      rsyslog_client_log_dir: "/var/log/ceph"
+      rsyslog_client_config_name: "99-ceph-rsyslog-client.conf"
+      tags:
+        - "ceph-rsyslog-client"
+        - "rsyslog-client"
+    - role: "system_crontab_coordination"
+      tags:
+        - "system-crontab-coordination"
+  vars:
+    is_metal: "{{ properties.is_metal|default(false) }}"
+    radosgw_keystone: yes
+    # TODO(logan): Remove the following line once
+    # upstream issue https://github.com/ceph/ceph-ansible/issues/2111 is fixed
+    radosgw_ssl: no
+    radosgw_civetweb_port: "{{ radosgw_service_port }}"
+  tags:
+    - ceph-rgw

playbooks/setup-openstack.yml

@@ -40,6 +40,11 @@
 - include: os-molteniron-install.yml
 - include: os-octavia-install.yml
 - include: os-tacker-install.yml
+
+# This is not an OpenStack service, but integrates with Keystone and must be
+# deployed afterward.
+- include: ceph-rgw-install.yml
+
 - include: os-tempest-install.yml
   when: (tempest_install | default(False)) | bool or (tempest_run | default(False)) | bool
 

tests/roles/bootstrap-host/templates/user_variables_ceph.yml.j2

@@ -19,7 +19,7 @@ monitor_interface: eth1 # Management network in the AIO
 public_network: "{{ (mgmt_range ~ '.0/' ~ netmask) | ipaddr('net') }}"
 journal_size: 100
 osd_scenario: collocated
-pool_default_pg_num: 32
+pool_default_pg_num: 16
 openstack_config: true # Ceph ansible automatically creates pools & keys
 cinder_ceph_client: cinder
 cinder_default_volume_type: RBD

tests/test_inventory.py

@@ -161,6 +161,11 @@ class TestAnsibleInventoryFormatConstraints(unittest.TestCase):
         'ceph-osd_container',
         'ceph-osd_hosts',
         'ceph-osd',
+        'ceph-rgw_all',
+        'ceph-rgw_containers',
+        'ceph-rgw_container',
+        'ceph-rgw_hosts',
+        'ceph-rgw',
         'cinder_all',
         'cinder_api',
         'cinder_api_container',