From ccfa2a6c16b0bab44816fba679df042591a1396a Mon Sep 17 00:00:00 2001 From: Jan Gutter Date: Thu, 3 Aug 2023 23:20:59 +0100 Subject: [PATCH] etcd: update to v3.4 * Updates etcd to v3.4 * Updated the config to use v3.4's logging mechanism * Deprecated etcd CA parameters aren't used, so we are not affected by their removal. * Note that we are not currently guarding against skip-version updates for etcd. Notable non-voting jobs exercising some of this: * kolla-ansible-ubuntu-upgrade-cephadm (cinder->tooz->etcd3gw->etcd) * kolla-ansible-ubuntu-zun (see https://review.opendev.org/c/openstack/openstack-ansible/+/883194 ) Depends-On: https://review.opendev.org/c/openstack/kolla/+/890464 Change-Id: I086e7bbc7db64421445731a533265e7056fbdb43 --- ansible/roles/cinder/templates/cinder.conf.j2 | 4 +++- ansible/roles/etcd/defaults/main.yml | 3 ++- ansible/roles/etcd/handlers/main.yml | 13 +++++++++++++ ansible/roles/etcd/tasks/restart_services.yml | 16 ++++++++++++++++ .../ironic/templates/ironic-inspector.conf.j2 | 4 +++- .../roles/masakari/templates/masakari.conf.j2 | 4 +++- .../notes/etcd-3.4-8ced41674f25674f.yaml | 13 +++++++++++++ 7 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/etcd-3.4-8ced41674f25674f.yaml diff --git a/ansible/roles/cinder/templates/cinder.conf.j2 b/ansible/roles/cinder/templates/cinder.conf.j2 index f4b72fa43b..e89e2008e0 100644 --- a/ansible/roles/cinder/templates/cinder.conf.j2 +++ b/ansible/roles/cinder/templates/cinder.conf.j2 @@ -259,5 +259,7 @@ backend_url = {{ redis_connection_string }} # NOTE(yoctozepto): we must use etcd3gw (aka etcd3+http) due to issues with alternative (etcd3) and eventlet (as used by cinder) # see https://bugs.launchpad.net/kolla-ansible/+bug/1854932 # and https://review.opendev.org/466098 for details -backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} +# NOTE(jan.gutter): etcd v3.4 removed the default `v3alpha` api_version. Until +# tooz defaults to a newer version, we should explicitly specify `v3` +backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}?api_version=v3{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} {% endif %} diff --git a/ansible/roles/etcd/defaults/main.yml b/ansible/roles/etcd/defaults/main.yml index c19bd2f328..02b3d80d9f 100644 --- a/ansible/roles/etcd/defaults/main.yml +++ b/ansible/roles/etcd/defaults/main.yml @@ -32,7 +32,8 @@ etcd_base_environment: ETCD_ADVERTISE_CLIENT_URLS: "{{ etcd_client_internal_endpoint }}" ETCD_LISTEN_CLIENT_URLS: "{{ etcd_client_internal_endpoint }}" ETCD_LISTEN_PEER_URLS: "{{ etcd_peer_internal_endpoint }}" - ETCD_OUT_FILE: "/var/log/kolla/etcd/etcd.log" + ETCD_LOGGER: "zap" + ETCD_LOG_OUTPUTS: "stderr,/var/log/kolla/etcd/etcd.log" KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" ETCD_CERT_FILE: "{% if etcd_enable_tls | bool %}/etc/etcd/certs/etcd-cert.pem{% endif %}" ETCD_KEY_FILE: "{% if etcd_enable_tls | bool %}/etc/etcd/certs/etcd-key.pem{% endif %}" diff --git a/ansible/roles/etcd/handlers/main.yml b/ansible/roles/etcd/handlers/main.yml index 5813fcfaf5..408b97b6e5 100644 --- a/ansible/roles/etcd/handlers/main.yml +++ b/ansible/roles/etcd/handlers/main.yml @@ -25,12 +25,25 @@ listen: - Bootstrap etcd services +# When upgrading an etcd cluster we have to do it one by one +- name: Upgrade etcd non-leaders + include_tasks: 'restart_services.yml' + when: + - inventory_hostname not in (groups.etcd_is_leader_True | default([])) + - kolla_action == "upgrade" + listen: + - Restart etcd container + loop: "{{ groups.etcd }}" + +# When there is no upgrade we can restart 25% of the services without +# losing quorum. - name: Rolling restart of etcd non-leaders include_tasks: 'restart_services.yml' when: - inventory_hostname not in (groups.etcd_is_leader_True | default([])) - groups.etcd.index(inventory_hostname) % 4 == item - kolla_action != "config" + - kolla_action != "upgrade" listen: - Restart etcd container - Bootstrap etcd services diff --git a/ansible/roles/etcd/tasks/restart_services.yml b/ansible/roles/etcd/tasks/restart_services.yml index b3c4de5264..622e1b467e 100644 --- a/ansible/roles/etcd/tasks/restart_services.yml +++ b/ansible/roles/etcd/tasks/restart_services.yml @@ -23,3 +23,19 @@ until: check_etcd_client_port is success retries: 10 delay: 6 + +- name: Wait for etcd endpoints to be healthy + become: true + vars: + service_name: "etcd" + service: "{{ etcd_services[service_name] }}" + command: >- + {{ kolla_container_engine }} exec {{ service.container_name }} + etcdctl endpoint health + changed_when: false + register: result + until: + - result is success + - ((result.stdout | from_json | first)['health'] | default(False) | bool) + retries: 10 + delay: 6 diff --git a/ansible/roles/ironic/templates/ironic-inspector.conf.j2 b/ansible/roles/ironic/templates/ironic-inspector.conf.j2 index d0fe3a3bf0..8781919efe 100644 --- a/ansible/roles/ironic/templates/ironic-inspector.conf.j2 +++ b/ansible/roles/ironic/templates/ironic-inspector.conf.j2 @@ -86,5 +86,7 @@ backend_url = {{ redis_connection_string }} # NOTE(yoctozepto): we must use etcd3gw (aka etcd3+http) due to issues with alternative (etcd3) and eventlet (as used by cinder) # see https://bugs.launchpad.net/kolla-ansible/+bug/1854932 # and https://review.opendev.org/466098 for details -backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} +# NOTE(jan.gutter): etcd v3.4 removed the default `v3alpha` api_version. Until +# tooz defaults to a newer version, we should explicitly specify `v3` +backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}?api_version=v3{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} {% endif %} diff --git a/ansible/roles/masakari/templates/masakari.conf.j2 b/ansible/roles/masakari/templates/masakari.conf.j2 index 081688aed1..7f4b8f3fba 100644 --- a/ansible/roles/masakari/templates/masakari.conf.j2 +++ b/ansible/roles/masakari/templates/masakari.conf.j2 @@ -76,6 +76,8 @@ connection = mysql+pymysql://{{ masakari_database_user }}:{{ masakari_database_p {% if masakari_coordination_backend == 'redis' %} backend_url = {{ redis_connection_string }} {% elif masakari_coordination_backend == 'etcd' %} -backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} +# NOTE(jan.gutter): etcd v3.4 removed the default `v3alpha` api_version. Until +# tooz defaults to a newer version, we should explicitly specify `v3` +backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}?api_version=v3{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} {% endif %} {% endif %} diff --git a/releasenotes/notes/etcd-3.4-8ced41674f25674f.yaml b/releasenotes/notes/etcd-3.4-8ced41674f25674f.yaml new file mode 100644 index 0000000000..b3ded9c1d4 --- /dev/null +++ b/releasenotes/notes/etcd-3.4-8ced41674f25674f.yaml @@ -0,0 +1,13 @@ +--- +upgrade: + - | + `etcd` has been upgraded to version `3.4` in this release. Operators are + highly encouraged to read the `upgrade notes `__ + for impacts on `etcd` clients. Upgrades are only supported from + `etcd v3.3`: Skip version upgrades are not supported. Please ensure that + adequate backups are taken before running the upgrade to guard against + dataloss. + - | + `etcd` version `3.4` drops support for the `v3alpha` endpoint. Internal + kolla-ansible endpoints have been updated, but operators are strongly + encouraged to audit any customizations or external users of `etcd`.