Implement rolling upgrades for glance

Using the pattern established in [1], install glance services in a way
that allows for minimal interruptions during upgrades. Upgrades are
serialized such that only one glance host is unavailable at a time.

[1] Change I0bdb51ce0d8b3b9a145d29ef6808e1fe595924e2

Depends-On: I4a4d8f23f05a16d25878340ede019be7ccdb7196
Depends-On: I743089b9a4f13ce974cbeb9cc91b67b013f2f323
Change-Id: I1c70563c7b5bb3e5666821e9288d723b26d9903e

group_vars/glance_all.yml

@@ -13,6 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# MariaDB details for the glance service
+glance_galera_user: glance
+glance_galera_database: glance
+glance_galera_address: "{{ galera_address }}"
+
 glance_service_region: "{{ service_region }}"
 glance_service_in_ldap: "{{ service_ldap_backend_enabled }}"
 

playbooks/common-playbooks/glance.yml

@@ -0,0 +1,98 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install glance services
+  hosts: "{{ glance_hosts }}"
+  serial: "{{ glance_serial }}"
+  gather_facts: "{{ gather_facts | default(True) }}"
+  user: root
+  environment: "{{ deployment_environment_variables | default({}) }}"
+  tags:
+    - glance
+  pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - include: ../common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: glance_api-back
+        haproxy_state: disabled
+      when:
+        - "'glance_api' in group_names"
+        - "groups['glance_api'] | length > 1"
+
+    - name: Configure container (non-nfs)
+      static: no
+      include: ../common-tasks/os-lxc-container-setup.yml
+      vars:
+        list_of_bind_mounts: "{{ glance_container_bind_mounts }}"
+      when:
+        - glance_default_store == "file"
+        - (glance_nfs_client is not defined) or (glance_nfs_client | length == 0)
+
+    - name: Configure container (nfs)
+      include: ../common-tasks/os-lxc-container-setup.yml
+      static: no
+      when: (glance_default_store != "file") or (glance_nfs_client is defined)
+
+    - name: Configure log directories (on metal)
+      include: ../common-tasks/os-log-dir-setup.yml
+      vars:
+        log_dirs:
+          - src: "/openstack/log/{{ inventory_hostname }}-glance"
+            dest: "/var/log/glance"
+
+    - name: Configure package proxy cache
+      include: ../common-tasks/package-cache-proxy.yml
+
+  roles:
+    - role: "os_glance"
+
+    - role: "ceph_client"
+      openstack_service_system_user: "{{ glance_system_user_name }}"
+      openstack_service_venv_bin: "{{ glance_bin }}"
+      when:
+        - "'glance_api' in group_names"
+        - "{{ 'rbd' in [glance_default_store | default('none')] + glance_additional_stores | default([]) }}"
+      tags:
+        - ceph
+
+    - role: "openstack_openrc"
+      tags:
+        - openrc
+
+    - role: "rsyslog_client"
+      rsyslog_client_log_rotate_file: glance_log_rotate
+      rsyslog_client_log_dir: "/var/log/glance"
+      rsyslog_client_config_name: "99-glance-rsyslog-client.conf"
+      tags:
+        - rsyslog
+
+    - role: "system_crontab_coordination"
+      tags:
+        - crontab
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - include: ../common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: glance_api-back
+        haproxy_state: enabled
+      when:
+        - "'glance_api' in group_names"
+        - "groups['glance_api'] | length > 1"

playbooks/os-glance-install.yml

@@ -13,34 +13,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Install glance server
+- name: Prepare MQ/DB services
   hosts: glance_all
   gather_facts: "{{ gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
-  pre_tasks:
+  environment: "{{ deployment_environment_variables | default({}) }}"
-    - include: common-tasks/os-lxc-container-setup.yml
+  tags:
-      static: no
+    - glance
-      vars:
+  tasks:
-        list_of_bind_mounts: "{{ glance_container_bind_mounts }}"
+
-      when:
+    - name: Configure rabbitmq vhost/user
-        - glance_default_store == "file"
+      include: common-tasks/rabbitmq-vhost-user.yml
-        - (glance_nfs_client is not defined) or (glance_nfs_client | length == 0)
-    - include: common-tasks/os-lxc-container-setup.yml
-      static: no
-      when: (glance_default_store != "file") or (glance_nfs_client is defined)
-    - include: common-tasks/rabbitmq-vhost-user.yml
-      static: no
       vars:
         user: "{{ glance_rabbitmq_userid }}"
         password: "{{ glance_rabbitmq_password }}"
         vhost: "{{ glance_rabbitmq_vhost }}"
         _rabbitmq_host_group: "{{ glance_rabbitmq_host_group }}"
       when:
-        - inventory_hostname == groups['glance_all'][0]
         - groups[glance_rabbitmq_host_group] | length > 0
-    - include: common-tasks/rabbitmq-vhost-user.yml
+      run_once: yes
-      static: no
+
+    - name: Configure rabbitmq vhost/user (telemetry)
+      include: common-tasks/rabbitmq-vhost-user.yml
       vars:
         user: "{{ glance_rabbitmq_telemetry_userid }}"
         password: "{{ glance_rabbitmq_telemetry_password }}"
@@ -48,51 +42,118 @@
         _rabbitmq_host_group: "{{ glance_rabbitmq_telemetry_host_group }}"
       when:
         - glance_ceilometer_enabled | bool
-        - inventory_hostname == groups['glance_all'][0]
         - groups[glance_rabbitmq_telemetry_host_group] is defined
         - groups[glance_rabbitmq_telemetry_host_group] | length > 0
         - groups[glance_rabbitmq_telemetry_host_group] != groups[glance_rabbitmq_host_group]
-    - include: common-tasks/os-log-dir-setup.yml
+      run_once: yes
-      vars:
+
-        log_dirs:
+    - name: Configure MySQL user
-          - src: "/openstack/log/{{ inventory_hostname }}-glance"
+      include: common-tasks/mysql-db-user.yml
-            dest: "/var/log/glance"
-    - include: common-tasks/mysql-db-user.yml
-      static: no
       vars:
         user_name: "{{ glance_galera_user }}"
         password: "{{ glance_container_mysql_password }}"
         login_host: "{{ glance_galera_address }}"
         db_name: "{{ glance_galera_database }}"
-      when: inventory_hostname == groups['glance_all'][0]
+      run_once: yes
-    - include: common-tasks/package-cache-proxy.yml
+
-  roles:
+
-    - role: "os_glance"
+
-    - role: "ceph_client"
+- name: Install glance API services
-      openstack_service_system_user: "{{ glance_system_user_name }}"
+  include: common-playbooks/glance.yml
-      openstack_service_venv_bin: "{{ glance_bin }}"
-      when:
-        - inventory_hostname in groups['glance_api']
-        - "{{ 'rbd' in [glance_default_store | default('none')] + glance_additional_stores | default([]) }}"
-      tags:
-        - ceph
-    - role: "openstack_openrc"
-      tags:
-        - openrc
-    - role: "rsyslog_client"
-      rsyslog_client_log_rotate_file: glance_log_rotate
-      rsyslog_client_log_dir: "/var/log/glance"
-      rsyslog_client_config_name: "99-glance-rsyslog-client.conf"
-      tags:
-        - rsyslog
-    - role: "system_crontab_coordination"
-      tags:
-        - crontab
   vars:
-    is_metal: "{{ properties.is_metal|default(false) }}"
+    glance_hosts: "glance_api"
-    glance_galera_user: glance
+    glance_serial: "{{ glance_api_serial | default(['1', '100%']) }}"
-    glance_galera_database: glance
+
-    glance_galera_address: "{{ galera_address }}"
+
+
+- name: Install glance registry services
+  include: common-playbooks/glance.yml
+  vars:
+    glance_hosts: "glance_registry:!glance_api"
+    glance_serial: "{{ glance_registry_serial | default(['1', '100%']) }}"
+
+
+
+# These facts are set against the deployment host to ensure that
+# they are fast to access. This is done in preference to setting
+# them against each target as the hostvars extraction will take
+# a long time if executed against a large inventory.
+- name: Refresh local facts after all software changes are made
+  hosts: glance_all
+  max_fail_percentage: 20
+  user: root
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
     - glance
+  tasks:
+    - name: refresh local facts
+      setup:
+        filter: ansible_local
+        gather_subset: "!all"
+
+    # This variable contains the values of the local fact set for the glance
+    # venv tag for all hosts in the 'glance_all' host group.
+    - name: Gather software version list
+      set_fact:
+        glance_all_software_versions: "{{ (groups['glance_all'] | map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'glance', 'venv_tag'])) | list }}"
+      delegate_to: localhost
+      run_once: yes
+
+    # This variable outputs a boolean value which is True when
+    # glance_all_software_versions contains a list of defined
+    # values. If they are not defined, it means that not all
+    # hosts have their software deployed yet.
+    - name: Set software deployed fact
+      set_fact:
+        glance_all_software_deployed: "{{ (glance_all_software_versions | select('defined')) | list == glance_all_software_versions }}"
+      delegate_to: localhost
+      run_once: yes
+
+    # This variable outputs a boolean when all the values in
+    # glance_all_software_versions are the same and the software
+    # has been deployed to all hosts in the group.
+    - name: Set software updated fact
+      set_fact:
+        glance_all_software_updated: "{{ ((glance_all_software_versions | unique) | length == 1) and (glance_all_software_deployed | bool) }}"
+      delegate_to: localhost
+      run_once: yes
+
+
+- name: Restart glance API to ensure new RPC object version is used
+  hosts: glance_api
+  gather_facts: no
+  serial: "{{ glance_api_serial | default(['1','100%']) }}"
+  user: root
+  environment: "{{ deployment_environment_variables | default({}) }}"
+  tags:
+    - glance
+  tasks:
+    # In order to ensure that the service restart does not
+    # cause an unexpected outage, we drain the load balancer
+    # back end for this container.
+    - include: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: glance_api-back
+        haproxy_state: disabled
+      when:
+        - "glance_all_software_updated | bool"
+        - "ansible_local['openstack_ansible']['glance']['need_service_restart'] | bool"
+        - "groups['glance_api'] | length > 1"
+
+    - name: Execute glance service restart
+      include: common-tasks/restart-service.yml
+      vars:
+        service_name: "glance-api"
+        service_action: "restarted"
+      when:
+        - "glance_all_software_updated | bool"
+        - "ansible_local['openstack_ansible']['glance']['need_service_restart'] | bool"
+
+    # Now that service restart is done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - include: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: glance_api-back
+        haproxy_state: enabled
+      when: "groups['glance_api'] | length > 1"