diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index a06b2f41da..07887764f9 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -285,6 +285,10 @@ barbican_api_listen_port: "{{ barbican_api_port }}" blazar_api_port: "1234" +ceph_rgw_internal_fqdn: "{{ kolla_internal_fqdn }}" +ceph_rgw_external_fqdn: "{{ kolla_external_fqdn }}" +ceph_rgw_port: "6780" + cinder_internal_fqdn: "{{ kolla_internal_fqdn }}" cinder_external_fqdn: "{{ kolla_external_fqdn }}" cinder_api_port: "8776" @@ -601,6 +605,8 @@ enable_ceilometer: "no" enable_ceilometer_ipmi: "no" enable_cells: "no" enable_central_logging: "no" +enable_ceph_rgw: "no" +enable_ceph_rgw_loadbalancer: "{{ enable_ceph_rgw | bool }}" enable_chrony: "no" enable_cinder: "no" enable_cinder_backup: "yes" diff --git a/ansible/roles/ceph-rgw/defaults/main.yml b/ansible/roles/ceph-rgw/defaults/main.yml new file mode 100644 index 0000000000..07345434d8 --- /dev/null +++ b/ansible/roles/ceph-rgw/defaults/main.yml @@ -0,0 +1,92 @@ +--- +project_name: "ceph-rgw" + +ceph_rgw_services: + # NOTE(mgoddard): There is no container deployment, this is used for load + # balancer configuration. + ceph-rgw: + group: "all" + enabled: "{{ enable_ceph_rgw | bool }}" + haproxy: + radosgw: + enabled: "{{ enable_ceph_rgw_loadbalancer | bool }}" + mode: "http" + external: false + port: "{{ ceph_rgw_port }}" + custom_member_list: "{{ ceph_rgw_haproxy_members }}" + radosgw_external: + enabled: "{{ enable_ceph_rgw_loadbalancer | bool }}" + mode: "http" + external: true + port: "{{ ceph_rgw_port }}" + custom_member_list: "{{ ceph_rgw_haproxy_members }}" + +#################### +# Load balancer +#################### + +# List of Ceph hosts to use as HAProxy backends. Each item should contain +# 'host' and 'port'` keys. The 'ip' and 'port' keys are optional. If 'ip' is +# not specified, the 'host' values should be resolvable from the host running +# HAProxy. If the ``port`` is not specified, the default HTTP (80) or HTTPS +# (443) port will be used. +ceph_rgw_hosts: [] +ceph_rgw_haproxy_members: >- + {%- set members = [] -%} + {%- for host in ceph_rgw_hosts -%} + {%- set port = (":" ~ host.port) if host.port is defined else "" -%} + {%- set member = "server " ~ host.host ~ " " ~ host.ip | default(host.host) ~ port ~ " " ~ ceph_rgw_haproxy_healthcheck -%} + {%- set _ = members.append(member) -%} + {%- endfor -%} + {{ members }} +ceph_rgw_haproxy_healthcheck: "check inter 2000 rise 2 fall 5" + + +#################### +# OpenStack +#################### + +# Whether to register Ceph RadosGW swift-compatible endpoints in Keystone. +enable_ceph_rgw_keystone: "{{ enable_ceph_rgw | bool }}" + +# Enable/disable ceph-rgw compatibility with OpenStack Swift. +# This should match the configuration used by Ceph RadosGW. +ceph_rgw_swift_compatibility: false + +# Enable/disable including the account (project) in the endpoint URL. This +# allows for cross-project and public object access. +# This should match the 'rgw_swift_account_in_url' config option used by Ceph +# RadosGW. +ceph_rgw_swift_account_in_url: false + +ceph_rgw_endpoint_path: "{{ '/' if ceph_rgw_swift_compatibility | bool else '/swift/' }}v1{% if ceph_rgw_swift_account_in_url | bool %}/AUTH_%(project_id)s{% endif %}" + +ceph_rgw_admin_endpoint: "{{ admin_protocol }}://{{ ceph_rgw_internal_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}" +ceph_rgw_internal_endpoint: "{{ internal_protocol }}://{{ ceph_rgw_internal_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}" +ceph_rgw_public_endpoint: "{{ public_protocol }}://{{ ceph_rgw_external_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}" + +ceph_rgw_keystone_user: "ceph_rgw" + +openstack_ceph_rgw_auth: "{{ openstack_auth }}" + + +#################### +# Keystone +#################### +ceph_rgw_ks_services: + - name: "swift" + type: "object-store" + description: "Openstack Object Storage" + endpoints: + - {'interface': 'admin', 'url': '{{ ceph_rgw_admin_endpoint }}'} + - {'interface': 'internal', 'url': '{{ ceph_rgw_internal_endpoint }}'} + - {'interface': 'public', 'url': '{{ ceph_rgw_public_endpoint }}'} + +ceph_rgw_ks_users: + - project: "service" + user: "{{ ceph_rgw_keystone_user }}" + password: "{{ ceph_rgw_keystone_password }}" + role: "admin" + +ceph_rgw_ks_roles: + - "ResellerAdmin" diff --git a/ansible/roles/ceph-rgw/tasks/check.yml b/ansible/roles/ceph-rgw/tasks/check.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/check.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ceph-rgw/tasks/config.yml b/ansible/roles/ceph-rgw/tasks/config.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/config.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ceph-rgw/tasks/deploy-containers.yml b/ansible/roles/ceph-rgw/tasks/deploy-containers.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/deploy-containers.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ceph-rgw/tasks/deploy.yml b/ansible/roles/ceph-rgw/tasks/deploy.yml new file mode 100644 index 0000000000..40daddd63b --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/deploy.yml @@ -0,0 +1,2 @@ +--- +- import_tasks: register.yml diff --git a/ansible/roles/ceph-rgw/tasks/loadbalancer.yml b/ansible/roles/ceph-rgw/tasks/loadbalancer.yml new file mode 100644 index 0000000000..d29f3e56d1 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/loadbalancer.yml @@ -0,0 +1,7 @@ +--- +- name: "Configure haproxy for {{ project_name }}" + import_role: + role: haproxy-config + vars: + project_services: "{{ ceph_rgw_services }}" + tags: always diff --git a/ansible/roles/ceph-rgw/tasks/main.yml b/ansible/roles/ceph-rgw/tasks/main.yml new file mode 100644 index 0000000000..bc5d1e6257 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include_tasks: "{{ kolla_action }}.yml" diff --git a/ansible/roles/ceph-rgw/tasks/precheck.yml b/ansible/roles/ceph-rgw/tasks/precheck.yml new file mode 100644 index 0000000000..5430f4837a --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/precheck.yml @@ -0,0 +1,10 @@ +--- +- name: Fail if load balancer members not set + fail: + msg: >- + Ceph RadosGW load balancer configuration is enabled + (enable_ceph_rgw_loadbalancer) but no HAProxy members are configured. + Have you set ceph_rgw_hosts? + when: + - enable_ceph_rgw_loadbalancer | bool + - ceph_rgw_haproxy_members | length == 0 diff --git a/ansible/roles/ceph-rgw/tasks/pull.yml b/ansible/roles/ceph-rgw/tasks/pull.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/pull.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ceph-rgw/tasks/reconfigure.yml b/ansible/roles/ceph-rgw/tasks/reconfigure.yml new file mode 100644 index 0000000000..5b10a7e111 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/reconfigure.yml @@ -0,0 +1,2 @@ +--- +- import_tasks: deploy.yml diff --git a/ansible/roles/ceph-rgw/tasks/register.yml b/ansible/roles/ceph-rgw/tasks/register.yml new file mode 100644 index 0000000000..c33683163c --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/register.yml @@ -0,0 +1,9 @@ +--- +- import_role: + name: service-ks-register + vars: + service_ks_register_auth: "{{ openstack_ceph_rgw_auth }}" + service_ks_register_services: "{{ ceph_rgw_ks_services }}" + service_ks_register_users: "{{ ceph_rgw_ks_users }}" + service_ks_register_roles: "{{ ceph_rgw_ks_roles }}" + when: enable_ceph_rgw_keystone | bool diff --git a/ansible/roles/ceph-rgw/tasks/stop.yml b/ansible/roles/ceph-rgw/tasks/stop.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/stop.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ceph-rgw/tasks/upgrade.yml b/ansible/roles/ceph-rgw/tasks/upgrade.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/upgrade.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/loadbalancer/tasks/precheck.yml b/ansible/roles/loadbalancer/tasks/precheck.yml index 10d13d45e1..6f5b2742f1 100644 --- a/ansible/roles/loadbalancer/tasks/precheck.yml +++ b/ansible/roles/loadbalancer/tasks/precheck.yml @@ -208,6 +208,20 @@ - haproxy_stat.find('blazar_api') == -1 - haproxy_vip_prechecks +- name: Checking free port for Ceph RadosGW HAProxy + wait_for: + host: "{{ kolla_internal_vip_address }}" + port: "{{ ceph_rgw_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - enable_ceph_rgw | bool + - enable_ceph_rgw_loadbalancer | bool + - inventory_hostname in groups['loadbalancer'] + - haproxy_stat.find('radosgw') == -1 + - haproxy_vip_prechecks + - name: Checking free port for Cinder API HAProxy wait_for: host: "{{ kolla_internal_vip_address }}" diff --git a/ansible/site.yml b/ansible/site.yml index 832f3912a7..ca177a6d77 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -22,6 +22,7 @@ - enable_barbican_{{ enable_barbican | bool }} - enable_blazar_{{ enable_blazar | bool }} - enable_ceilometer_{{ enable_ceilometer | bool }} + - enable_ceph_rgw_{{ enable_ceph_rgw | bool }} - enable_chrony_{{ enable_chrony | bool }} - enable_cinder_{{ enable_cinder | bool }} - enable_cloudkitty_{{ enable_cloudkitty | bool }} @@ -143,6 +144,11 @@ tasks_from: loadbalancer tags: blazar when: enable_blazar | bool + - include_role: + name: ceph-rgw + tasks_from: loadbalancer + tags: ceph-rgw + when: enable_ceph_rgw | bool - include_role: name: cinder tasks_from: loadbalancer @@ -603,6 +609,19 @@ tags: swift, when: enable_swift | bool } +- name: Apply role ceph-rgw + gather_facts: false + hosts: + # NOTE(mgoddard): This is only used to register Keystone services, and + # can run on any host running kolla-toolbox. + - kolla-toolbox + - '&enable_ceph_rgw_True' + serial: '{{ kolla_serial|default("0") }}' + roles: + - { role: ceph-rgw, + tags: ceph-rgw, + when: enable_ceph_rgw | bool } + - name: Apply role glance gather_facts: false hosts: diff --git a/doc/source/reference/storage/external-ceph-guide.rst b/doc/source/reference/storage/external-ceph-guide.rst index 87e085856e..304a2ed060 100644 --- a/doc/source/reference/storage/external-ceph-guide.rst +++ b/doc/source/reference/storage/external-ceph-guide.rst @@ -211,3 +211,74 @@ type ``default_share_type``, please see :doc:`Manila in Kolla `. For more details on the CephFS Native driver, please see :manila-doc:`CephFS Native driver `. + +RadosGW +------- + +As of the Xena 13.0.0 release, Kolla Ansible supports integration with Ceph +RadosGW. This includes: + +* Registration of Swift-compatible endpoints in Keystone +* Load balancing across RadosGW API servers using HAProxy + +See the `Ceph documentation +`__ for further information, +including changes that must be applied to the Ceph cluster configuration. + +Enable Ceph RadosGW integration: + +.. code-block:: yaml + + enable_ceph_rgw: true + +Keystone integration +==================== + +A Keystone user and endpoints are registered by default, however this may be +avoided by setting ``enable_ceph_rgw_keystone`` to ``false``. If registration +is enabled, the username is defined via ``ceph_rgw_keystone_user``, and this +defaults to ``ceph_rgw``. The hostnames used by the endpoints default to +``ceph_rgw_external_fqdn`` and ``ceph_rgw_internal_fqdn`` for the public and +internal endpoints respectively. These default to ``kolla_external_fqdn`` and +``kolla_internal_fqdn`` respectively. The port used by the endpoints is defined +via ``ceph_rgw_port``, and defaults to 6780. + +By default RadosGW supports both Swift and S3 API, and it is not completely +compatible with Swift API. The option ``ceph_rgw_swift_compatibility`` can +enable/disable complete RadosGW compatibility with Swift API. This should +match the configuration used by Ceph RadosGW. After changing the value, run +the ``kolla-ansible deploy`` command to enable. + +By default, the RadosGW endpoint URL does not include the project (account) ID. +This prevents cross-project and public object access. This can be resolved by +setting ``ceph_rgw_swift_account_in_url`` to ``true``. This should match the +``rgw_swift_account_in_url`` configuration option in Ceph RadosGW. + +Load balancing +============== + +.. warning:: + + Users of Ceph RadosGW can generate very high volumes of traffic. It is + advisable to use a separate load balancer for RadosGW for anything other + than small or lightly utilised RadosGW deployments, however this is + currently out of scope for Kolla Ansible. + +Load balancing is enabled by default, however this may be avoided by setting +``enable_ceph_rgw_loadbalancer`` to ``false``. If using load balancing, the +RadosGW hosts and ports must be configured. Each item should contain +``host`` and ``port`` keys. The ``ip`` and ``port`` keys are optional. If +``ip`` is not specified, the ``host`` values should be resolvable from the host +running HAProxy. If the ``port`` is not specified, the default HTTP (80) or +HTTPS (443) port will be used. For example: + +.. code-block:: yaml + + ceph_rgw_hosts: + - host: rgw-host-1 + - host: rgw-host-2 + ip: 10.0.0.42 + port: 8080 + +The HAProxy frontend port is defined via ``ceph_rgw_port``, and defaults to +6780. diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 7c38d7ee0b..001576e9b8 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -296,6 +296,8 @@ #enable_ceilometer_ipmi: "no" #enable_cells: "no" #enable_central_logging: "no" +#enable_ceph_rgw: "no" +#enable_ceph_rgw_loadbalancer: "{{ enable_ceph_rgw | bool }}" #enable_chrony: "no" #enable_cinder: "no" #enable_cinder_backup: "yes" diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml index 684d313531..9b0dd99257 100644 --- a/etc/kolla/passwords.yml +++ b/etc/kolla/passwords.yml @@ -246,3 +246,8 @@ prometheus_alertmanager_password: # OpenStack identity federation ############################### keystone_federation_openid_crypto_password: + +#################### +# Ceph RadosGW options +#################### +ceph_rgw_keystone_password: diff --git a/releasenotes/notes/ceph-rgw-062e0544a004f7b1.yaml b/releasenotes/notes/ceph-rgw-062e0544a004f7b1.yaml new file mode 100644 index 0000000000..b17de1205a --- /dev/null +++ b/releasenotes/notes/ceph-rgw-062e0544a004f7b1.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds support for integration with Ceph RadosGW. diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index 87fa60051b..3aa8f2793e 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -132,6 +132,14 @@ nova_backend_ceph: "yes" # TODO(yoctozepto): Remove this in the Xena cycle. # cephadm doesn't support chrony in a container (checks for chrony.service) enable_chrony: "no" + +enable_ceph_rgw: {{ not is_upgrade or previous_release != 'wallaby' }} +ceph_rgw_hosts: +{% for host in hostvars %} + - host: {{ host }} + ip: {{ hostvars[host]['ansible_host'] }} + port: 6780 +{% endfor %} {% endif %} {% if tls_enabled %}