Merge "Add support for Ceph RadosGW integration"

ansible/group_vars/all.yml

@@ -285,6 +285,10 @@ barbican_api_listen_port: "{{ barbican_api_port }}"
 
 blazar_api_port: "1234"
 
+ceph_rgw_internal_fqdn: "{{ kolla_internal_fqdn }}"
+ceph_rgw_external_fqdn: "{{ kolla_external_fqdn }}"
+ceph_rgw_port: "6780"
+
 cinder_internal_fqdn: "{{ kolla_internal_fqdn }}"
 cinder_external_fqdn: "{{ kolla_external_fqdn }}"
 cinder_api_port: "8776"
@@ -601,6 +605,8 @@ enable_ceilometer: "no"
 enable_ceilometer_ipmi: "no"
 enable_cells: "no"
 enable_central_logging: "no"
+enable_ceph_rgw: "no"
+enable_ceph_rgw_loadbalancer: "{{ enable_ceph_rgw | bool }}"
 enable_chrony: "no"
 enable_cinder: "no"
 enable_cinder_backup: "yes"

ansible/roles/ceph-rgw/defaults/main.yml

@@ -0,0 +1,92 @@
+---
+project_name: "ceph-rgw"
+
+ceph_rgw_services:
+  # NOTE(mgoddard): There is no container deployment, this is used for load
+  # balancer configuration.
+  ceph-rgw:
+    group: "all"
+    enabled: "{{ enable_ceph_rgw | bool }}"
+    haproxy:
+      radosgw:
+        enabled: "{{ enable_ceph_rgw_loadbalancer | bool }}"
+        mode: "http"
+        external: false
+        port: "{{ ceph_rgw_port }}"
+        custom_member_list: "{{ ceph_rgw_haproxy_members }}"
+      radosgw_external:
+        enabled: "{{ enable_ceph_rgw_loadbalancer | bool }}"
+        mode: "http"
+        external: true
+        port: "{{ ceph_rgw_port }}"
+        custom_member_list: "{{ ceph_rgw_haproxy_members }}"
+
+####################
+# Load balancer
+####################
+
+# List of Ceph hosts to use as HAProxy backends. Each item should contain
+# 'host' and 'port'` keys. The 'ip' and 'port' keys are optional. If 'ip' is
+# not specified, the 'host' values should be resolvable from the host running
+# HAProxy. If the ``port`` is not specified, the default HTTP (80) or HTTPS
+# (443) port will be used.
+ceph_rgw_hosts: []
+ceph_rgw_haproxy_members: >-
+  {%- set members = [] -%}
+  {%- for host in ceph_rgw_hosts -%}
+  {%- set port = (":" ~ host.port) if host.port is defined else "" -%}
+  {%- set member = "server " ~ host.host ~ " " ~ host.ip | default(host.host) ~ port ~ " " ~ ceph_rgw_haproxy_healthcheck -%}
+  {%- set _ = members.append(member) -%}
+  {%- endfor -%}
+  {{ members }}
+ceph_rgw_haproxy_healthcheck: "check inter 2000 rise 2 fall 5"
+
+
+####################
+# OpenStack
+####################
+
+# Whether to register Ceph RadosGW swift-compatible endpoints in Keystone.
+enable_ceph_rgw_keystone: "{{ enable_ceph_rgw | bool }}"
+
+# Enable/disable ceph-rgw compatibility with OpenStack Swift.
+# This should match the configuration used by Ceph RadosGW.
+ceph_rgw_swift_compatibility: false
+
+# Enable/disable including the account (project) in the endpoint URL. This
+# allows for cross-project and public object access.
+# This should match the 'rgw_swift_account_in_url' config option used by Ceph
+# RadosGW.
+ceph_rgw_swift_account_in_url: false
+
+ceph_rgw_endpoint_path: "{{ '/' if ceph_rgw_swift_compatibility | bool else '/swift/' }}v1{% if ceph_rgw_swift_account_in_url | bool %}/AUTH_%(project_id)s{% endif %}"
+
+ceph_rgw_admin_endpoint: "{{ admin_protocol }}://{{ ceph_rgw_internal_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}"
+ceph_rgw_internal_endpoint: "{{ internal_protocol }}://{{ ceph_rgw_internal_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}"
+ceph_rgw_public_endpoint: "{{ public_protocol }}://{{ ceph_rgw_external_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}"
+
+ceph_rgw_keystone_user: "ceph_rgw"
+
+openstack_ceph_rgw_auth: "{{ openstack_auth }}"
+
+
+####################
+# Keystone
+####################
+ceph_rgw_ks_services:
+  - name: "swift"
+    type: "object-store"
+    description: "Openstack Object Storage"
+    endpoints:
+      - {'interface': 'admin', 'url': '{{ ceph_rgw_admin_endpoint }}'}
+      - {'interface': 'internal', 'url': '{{ ceph_rgw_internal_endpoint }}'}
+      - {'interface': 'public', 'url': '{{ ceph_rgw_public_endpoint }}'}
+
+ceph_rgw_ks_users:
+  - project: "service"
+    user: "{{ ceph_rgw_keystone_user }}"
+    password: "{{ ceph_rgw_keystone_password }}"
+    role: "admin"
+
+ceph_rgw_ks_roles:
+  - "ResellerAdmin"

ansible/roles/ceph-rgw/tasks/check.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/config.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/deploy-containers.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/deploy.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: register.yml

ansible/roles/ceph-rgw/tasks/loadbalancer.yml

@@ -0,0 +1,7 @@
+---
+- name: "Configure haproxy for {{ project_name }}"
+  import_role:
+    role: haproxy-config
+  vars:
+    project_services: "{{ ceph_rgw_services }}"
+  tags: always

ansible/roles/ceph-rgw/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include_tasks: "{{ kolla_action }}.yml"

ansible/roles/ceph-rgw/tasks/precheck.yml

@@ -0,0 +1,10 @@
+---
+- name: Fail if load balancer members not set
+  fail:
+    msg: >-
+      Ceph RadosGW load balancer configuration is enabled
+      (enable_ceph_rgw_loadbalancer) but no HAProxy members are configured.
+      Have you set ceph_rgw_hosts?
+  when:
+    - enable_ceph_rgw_loadbalancer | bool
+    - ceph_rgw_haproxy_members | length == 0

ansible/roles/ceph-rgw/tasks/pull.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/reconfigure.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: deploy.yml

ansible/roles/ceph-rgw/tasks/register.yml

@@ -0,0 +1,9 @@
+---
+- import_role:
+    name: service-ks-register
+  vars:
+    service_ks_register_auth: "{{ openstack_ceph_rgw_auth }}"
+    service_ks_register_services: "{{ ceph_rgw_ks_services }}"
+    service_ks_register_users: "{{ ceph_rgw_ks_users }}"
+    service_ks_register_roles: "{{ ceph_rgw_ks_roles }}"
+  when: enable_ceph_rgw_keystone | bool

ansible/roles/ceph-rgw/tasks/stop.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/upgrade.yml

@@ -0,0 +1 @@
+---

ansible/roles/loadbalancer/tasks/precheck.yml

@@ -208,6 +208,20 @@
     - haproxy_stat.find('blazar_api') == -1
     - haproxy_vip_prechecks
 
+- name: Checking free port for Ceph RadosGW HAProxy
+  wait_for:
+    host: "{{ kolla_internal_vip_address }}"
+    port: "{{ ceph_rgw_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - enable_ceph_rgw | bool
+    - enable_ceph_rgw_loadbalancer | bool
+    - inventory_hostname in groups['loadbalancer']
+    - haproxy_stat.find('radosgw') == -1
+    - haproxy_vip_prechecks
+
 - name: Checking free port for Cinder API HAProxy
   wait_for:
     host: "{{ kolla_internal_vip_address }}"

ansible/site.yml

@@ -22,6 +22,7 @@
         - enable_barbican_{{ enable_barbican | bool }}
         - enable_blazar_{{ enable_blazar | bool }}
         - enable_ceilometer_{{ enable_ceilometer | bool }}
+        - enable_ceph_rgw_{{ enable_ceph_rgw | bool }}
         - enable_chrony_{{ enable_chrony | bool }}
         - enable_cinder_{{ enable_cinder | bool }}
         - enable_cloudkitty_{{ enable_cloudkitty | bool }}
@@ -143,6 +144,11 @@
             tasks_from: loadbalancer
           tags: blazar
           when: enable_blazar | bool
+        - include_role:
+            name: ceph-rgw
+            tasks_from: loadbalancer
+          tags: ceph-rgw
+          when: enable_ceph_rgw | bool
         - include_role:
             name: cinder
             tasks_from: loadbalancer
@@ -603,6 +609,19 @@
         tags: swift,
         when: enable_swift | bool }
 
+- name: Apply role ceph-rgw
+  gather_facts: false
+  hosts:
+    # NOTE(mgoddard): This is only used to register Keystone services, and
+    # can run on any host running kolla-toolbox.
+    - kolla-toolbox
+    - '&enable_ceph_rgw_True'
+  serial: '{{ kolla_serial|default("0") }}'
+  roles:
+    - { role: ceph-rgw,
+        tags: ceph-rgw,
+        when: enable_ceph_rgw | bool }
+
 - name: Apply role glance
   gather_facts: false
   hosts:

doc/source/reference/storage/external-ceph-guide.rst

@@ -211,3 +211,74 @@ type ``default_share_type``, please see :doc:`Manila in Kolla <manila-guide>`.
 
 For more details on the CephFS Native driver, please see
 :manila-doc:`CephFS Native driver <admin/cephfs_driver.html>`.
+
+RadosGW
+-------
+
+As of the Xena 13.0.0 release, Kolla Ansible supports integration with Ceph
+RadosGW. This includes:
+
+* Registration of Swift-compatible endpoints in Keystone
+* Load balancing across RadosGW API servers using HAProxy
+
+See the `Ceph documentation
+<https://docs.ceph.com/en/latest/radosgw/keystone/>`__ for further information,
+including changes that must be applied to the Ceph cluster configuration.
+
+Enable Ceph RadosGW integration:
+
+.. code-block:: yaml
+
+   enable_ceph_rgw: true
+
+Keystone integration
+====================
+
+A Keystone user and endpoints are registered by default, however this may be
+avoided by setting ``enable_ceph_rgw_keystone`` to ``false``. If registration
+is enabled, the username is defined via ``ceph_rgw_keystone_user``, and this
+defaults to ``ceph_rgw``. The hostnames used by the endpoints default to
+``ceph_rgw_external_fqdn`` and ``ceph_rgw_internal_fqdn`` for the public and
+internal endpoints respectively. These default to ``kolla_external_fqdn`` and
+``kolla_internal_fqdn`` respectively. The port used by the endpoints is defined
+via ``ceph_rgw_port``, and defaults to 6780.
+
+By default RadosGW supports both Swift and S3 API, and it is not completely
+compatible with Swift API. The option ``ceph_rgw_swift_compatibility`` can
+enable/disable complete RadosGW compatibility with Swift API.  This should
+match the configuration used by Ceph RadosGW. After changing the value, run
+the ``kolla-ansible deploy`` command to enable.
+
+By default, the RadosGW endpoint URL does not include the project (account) ID.
+This prevents cross-project and public object access. This can be resolved by
+setting ``ceph_rgw_swift_account_in_url`` to ``true``. This should match the
+``rgw_swift_account_in_url`` configuration option in Ceph RadosGW.
+
+Load balancing
+==============
+
+.. warning::
+
+   Users of Ceph RadosGW can generate very high volumes of traffic. It is
+   advisable to use a separate load balancer for RadosGW for anything other
+   than small or lightly utilised RadosGW deployments, however this is
+   currently out of scope for Kolla Ansible.
+
+Load balancing is enabled by default, however this may be avoided by setting
+``enable_ceph_rgw_loadbalancer`` to ``false``. If using load balancing, the
+RadosGW hosts and ports must be configured. Each item should contain
+``host`` and ``port`` keys. The ``ip`` and ``port`` keys are optional. If
+``ip`` is not specified, the ``host`` values should be resolvable from the host
+running HAProxy. If the ``port`` is not specified, the default HTTP (80) or
+HTTPS (443) port will be used. For example:
+
+.. code-block:: yaml
+
+   ceph_rgw_hosts:
+     - host: rgw-host-1
+     - host: rgw-host-2
+       ip: 10.0.0.42
+       port: 8080
+
+The HAProxy frontend port is defined via ``ceph_rgw_port``, and defaults to
+6780.

etc/kolla/globals.yml

@@ -296,6 +296,8 @@
 #enable_ceilometer_ipmi: "no"
 #enable_cells: "no"
 #enable_central_logging: "no"
+#enable_ceph_rgw: "no"
+#enable_ceph_rgw_loadbalancer: "{{ enable_ceph_rgw | bool }}"
 #enable_chrony: "no"
 #enable_cinder: "no"
 #enable_cinder_backup: "yes"

etc/kolla/passwords.yml

@@ -246,3 +246,8 @@ prometheus_alertmanager_password:
 # OpenStack identity federation
 ###############################
 keystone_federation_openid_crypto_password:
+
+####################
+# Ceph RadosGW options
+####################
+ceph_rgw_keystone_password:

releasenotes/notes/ceph-rgw-062e0544a004f7b1.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Adds support for integration with Ceph RadosGW.

tests/templates/globals-default.j2

@@ -132,6 +132,14 @@ nova_backend_ceph: "yes"
 # TODO(yoctozepto): Remove this in the Xena cycle.
 # cephadm doesn't support chrony in a container (checks for chrony.service)
 enable_chrony: "no"
+
+enable_ceph_rgw: {{ not is_upgrade or previous_release != 'wallaby' }}
+ceph_rgw_hosts:
+{% for host in hostvars %}
+  - host: {{ host }}
+    ip: {{ hostvars[host]['ansible_host'] }}
+    port: 6780
+{% endfor %}
 {% endif %}
 
 {% if tls_enabled %}