From ce4230fab19c3507db76ac71b0c6071b17eafe13 Mon Sep 17 00:00:00 2001 From: Michal Jastrzebski Date: Mon, 7 Dec 2015 15:34:22 -0600 Subject: [PATCH] Playbook for nova upgrade This creates tree and playbook for nova upgrade. Also other service upgrades will follow standard setup here. Change-Id: Ic31759efaee4986eb87b9ff0968f13189d130d48 Partially-Implements: blueprint upgrade-kolla Implements: blueprint upgrade-nova --- ansible/library/kolla_docker.py | 35 ++++- ansible/roles/nova/tasks/bootstrap.yml | 48 +----- .../roles/nova/tasks/bootstrap_service.yml | 48 ++++++ ansible/roles/nova/tasks/reload.yml | 33 +++++ ansible/roles/nova/tasks/start.yml | 137 +----------------- ansible/roles/nova/tasks/start_compute.yml | 65 +++++++++ ansible/roles/nova/tasks/start_conductors.yml | 11 ++ .../roles/nova/tasks/start_controllers.yml | 61 ++++++++ ansible/roles/nova/tasks/upgrade.yml | 38 +++++ 9 files changed, 287 insertions(+), 189 deletions(-) create mode 100644 ansible/roles/nova/tasks/bootstrap_service.yml create mode 100644 ansible/roles/nova/tasks/reload.yml create mode 100644 ansible/roles/nova/tasks/start_compute.yml create mode 100644 ansible/roles/nova/tasks/start_conductors.yml create mode 100644 ansible/roles/nova/tasks/start_controllers.yml diff --git a/ansible/library/kolla_docker.py b/ansible/library/kolla_docker.py index f095c4db8e..3b1e9bd612 100644 --- a/ansible/library/kolla_docker.py +++ b/ansible/library/kolla_docker.py @@ -33,11 +33,13 @@ options: required: True type: str choices: + - compare_image - create_volume - pull_image - remove_container - remove_volume - start_container + - stop_container api_version: description: - The version of the api for docker-py to use when contacting docker @@ -230,12 +232,14 @@ class DockerWorker(object): if find_name in cont['Names']: return cont - def check_container_differs(self): + def get_container_info(self): container = self.check_container() if not container: - return True - container_info = self.dc.inspect_container(self.params.get('name')) + return None + return self.dc.inspect_container(self.params.get('name')) + def check_container_differs(self): + container_info = self.get_container_info() return ( self.compare_image(container_info) or self.compare_labels(container_info) or @@ -261,7 +265,10 @@ class DockerWorker(object): if new_privileged != current_privileged: return True - def compare_image(self, container_info): + def compare_image(self, container_info=None): + container_info = container_info or self.get_container_info() + if not container_info: + return True new_image = self.check_image() current_image = container_info['Image'] if new_image['Id'] != current_image: @@ -493,6 +500,13 @@ class DockerWorker(object): if self.params.get('remove_on_exit'): self.remove_container() + def stop_container(self): + name = self.params.get('name') + container = self.check_container() + if not container['Status'].startswith('Exited '): + self.changed = True + self.dc.stop(name) + def create_volume(self): if not self.check_volume(): self.changed = True @@ -517,11 +531,13 @@ class DockerWorker(object): def generate_module(): argument_spec = dict( common_options=dict(required=False, type='dict', default=dict()), - action=dict(requried=True, type='str', choices=['create_volume', + action=dict(requried=True, type='str', choices=['compare_image', + 'create_volume', 'pull_image', 'remove_container', 'remove_volume', - 'start_container']), + 'start_container', + 'stop_container']), api_version=dict(required=False, type='str', default='auto'), auth_email=dict(required=False, type='str'), auth_password=dict(required=False, type='str'), @@ -594,8 +610,11 @@ def main(): try: dw = DockerWorker(module) - getattr(dw, module.params.get('action'))() - module.exit_json(changed=dw.changed) + # TODO(inc0): We keep it bool to have ansible deal with cosistent + # types. If we ever add method that will have to return some + # meaningful data, we need to refactor all methods to return dicts. + result = bool(getattr(dw, module.params.get('action'))()) + module.exit_json(changed=dw.changed, result=result) except Exception as e: module.exit_json(failed=True, changed=True, msg=repr(e)) diff --git a/ansible/roles/nova/tasks/bootstrap.yml b/ansible/roles/nova/tasks/bootstrap.yml index f67d440913..e7014527cc 100644 --- a/ansible/roles/nova/tasks/bootstrap.yml +++ b/ansible/roles/nova/tasks/bootstrap.yml @@ -71,51 +71,5 @@ run_once: True delegate_to: "{{ groups['nova-api'][0] }}" -- name: Running Nova bootstrap container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - detach: False - environment: - KOLLA_BOOTSTRAP: - KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" - image: "{{ nova_api_image_full }}" - labels: - BOOTSTRAP: - name: "bootstrap_nova" - restart_policy: "never" - volumes: "{{ node_config_directory }}/nova-api/:{{ container_config_directory }}/:ro" - run_once: True - delegate_to: "{{ groups['nova-api'][0] }}" +- include: bootstrap_service.yml when: database_created - -- name: Creating nova-compute volume - kolla_docker: - action: "create_volume" - common_options: "{{ docker_common_options }}" - name: "nova_compute" - register: nova_compute_volume - when: - - inventory_hostname in groups['compute'] - - not enable_nova_fake | bool - -- name: Running Nova compute bootstrap container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - detach: False - environment: - KOLLA_BOOTSTRAP: - KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" - image: "{{ nova_compute_image_full }}" - labels: - BOOTSTRAP: - name: "bootstrap_nova_compute" - restart_policy: "never" - volumes: - - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro" - - "nova_compute:/var/lib/nova/" - when: - - inventory_hostname in groups['compute'] - - not enable_nova_fake | bool - - nova_compute_volume.changed diff --git a/ansible/roles/nova/tasks/bootstrap_service.yml b/ansible/roles/nova/tasks/bootstrap_service.yml new file mode 100644 index 0000000000..787b2787b2 --- /dev/null +++ b/ansible/roles/nova/tasks/bootstrap_service.yml @@ -0,0 +1,48 @@ +--- +- name: Running Nova bootstrap container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + detach: False + environment: + KOLLA_BOOTSTRAP: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + image: "{{ nova_api_image_full }}" + labels: + BOOTSTRAP: + name: "bootstrap_nova" + restart_policy: "never" + volumes: "{{ node_config_directory }}/nova-api/:{{ container_config_directory }}/:ro" + run_once: True + delegate_to: "{{ groups['nova-api'][0] }}" + +- name: Creating nova-compute volume + kolla_docker: + action: "create_volume" + common_options: "{{ docker_common_options }}" + name: "nova_compute" + register: nova_compute_volume + when: + - inventory_hostname in groups['compute'] + - not enable_nova_fake | bool + +- name: Running Nova compute bootstrap container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + detach: False + environment: + KOLLA_BOOTSTRAP: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + image: "{{ nova_compute_image_full }}" + labels: + BOOTSTRAP: + name: "bootstrap_nova_compute" + restart_policy: "never" + volumes: + - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro" + - "nova_compute:/var/lib/nova/" + when: + - inventory_hostname in groups['compute'] + - not enable_nova_fake | bool + - nova_compute_volume.changed diff --git a/ansible/roles/nova/tasks/reload.yml b/ansible/roles/nova/tasks/reload.yml new file mode 100644 index 0000000000..25c959e149 --- /dev/null +++ b/ansible/roles/nova/tasks/reload.yml @@ -0,0 +1,33 @@ +--- +# This play calls sighup on every service to refresh upgrade levels +- name: Sighup nova-api + command: docker exec -t nova_api kill -1 1 + when: inventory_hostname in groups['nova-api'] + +- name: Sighup nova-conductor + command: docker exec -t nova_conductor kill -1 1 + when: inventory_hostname in groups['nova-conductor'] + +- name: Sighup nova-consoleauth + command: docker exec -t nova_consoleauth kill -1 1 + when: inventory_hostname in groups['nova-consoleauth'] + +- name: Sighup nova-novncproxy + command: docker exec -t nova_novncproxy kill -1 1 + when: + - inventory_hostname in groups['nova-novncproxy'] + - nova_console == 'novnc' + +- name: Sighup nova-scheduler + command: docker exec -t nova_scheduler kill -1 1 + when: inventory_hostname in groups['nova-scheduler'] + +- name: Sighup nova-spicehtml5proxy + command: docker exec -t nova_spicehtml5proxy kill -1 1 + when: + - inventory_hostname in groups['nova-spicehtml5proxy'] + - nova_console == 'spice' + +- name: Sighup nova-compute + command: docker exec -t nova_compute kill -1 1 + when: inventory_hostname in groups['compute'] diff --git a/ansible/roles/nova/tasks/start.yml b/ansible/roles/nova/tasks/start.yml index 795e2ca5dc..659729b199 100644 --- a/ansible/roles/nova/tasks/start.yml +++ b/ansible/roles/nova/tasks/start.yml @@ -1,137 +1,6 @@ --- -- name: Starting nova-libvirt container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - image: "{{ nova_libvirt_image_full }}" - name: "nova_libvirt" - pid_mode: "host" - privileged: True - volumes: - - "{{ node_config_directory }}/nova-libvirt/:{{ container_config_directory }}/:ro" - - "/lib/modules:/lib/modules:ro" - - "/run/:/run/" - - "/sys/fs/cgroup:/sys/fs/cgroup" - - "nova_compute:/var/lib/nova/" - - "nova_libvirt:/var/lib/libvirt" - when: inventory_hostname in groups['compute'] +- include: start_controllers.yml -- name: Starting nova-api container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - image: "{{ nova_api_image_full }}" - name: "nova_api" - privileged: True - volumes: - - "{{ node_config_directory }}/nova-api/:{{ container_config_directory }}/:ro" - - "/lib/modules:/lib/modules:ro" - - "rsyslog_socket:/var/lib/kolla/rsyslog/" - when: inventory_hostname in groups['nova-api'] +- include: start_conductors.yml -- name: Starting nova-conductor container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - name: "nova_conductor" - image: "{{ nova_conductor_image_full }}" - volumes: - - "{{ node_config_directory }}/nova-conductor/:{{ container_config_directory }}/:ro" - - "rsyslog_socket:/var/lib/kolla/rsyslog/" - when: inventory_hostname in groups['nova-conductor'] - -- name: Starting nova-consoleauth container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - image: "{{ nova_consoleauth_image_full }}" - name: "nova_consoleauth" - volumes: - - "{{ node_config_directory }}/nova-consoleauth/:{{ container_config_directory }}/:ro" - - "rsyslog_socket:/var/lib/kolla/rsyslog/" - when: inventory_hostname in groups['nova-consoleauth'] - -- name: Starting nova-novncproxy container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - image: "{{ nova_novncproxy_image_full }}" - name: "nova_novncproxy" - volumes: - - "{{ node_config_directory }}/nova-novncproxy/:{{ container_config_directory }}/:ro" - - "rsyslog_socket:/var/lib/kolla/rsyslog/" - when: - - inventory_hostname in groups['nova-novncproxy'] - - nova_console == 'novnc' - -- name: Starting nova-scheduler container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - image: "{{ nova_scheduler_image_full }}" - name: "nova_scheduler" - volumes: - - "{{ node_config_directory }}/nova-scheduler/:{{ container_config_directory }}/:ro" - - "rsyslog_socket:/var/lib/kolla/rsyslog/" - when: inventory_hostname in groups['nova-scheduler'] - -- name: Starting nova-spicehtml5proxy container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - image: "{{ nova_spicehtml5proxy_image_full }}" - name: "nova_spicehtml5proxy" - volumes: - - "{{ node_config_directory }}/nova-spicehtml5proxy/:{{ container_config_directory }}/:ro" - - "rsyslog_socket:/var/lib/kolla/rsyslog/" - when: - - inventory_hostname in groups['nova-spicehtml5proxy'] - - nova_console == 'spice' - -- name: Starting nova-compute container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - image: "{{ nova_compute_image_full }}" - name: "nova_compute" - privileged: True - volumes: - - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro" - - "/lib/modules:/lib/modules:ro" - - "/run:/run" - - "nova_compute:/var/lib/nova/" - - "nova_libvirt:/var/lib/libvirt" - - "rsyslog_socket:/var/lib/kolla/rsyslog/" - when: - - inventory_hostname in groups['compute'] - - not enable_nova_fake | bool - -- name: Starting nova-compute-ironic container - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - image: "{{ nova_compute_ironic_image_full }}" - name: "nova_compute_ironic" - volumes: - - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro" - - "rsyslog_socket:/var/lib/kolla/rsyslog/" - when: - - inventory_hostname in groups['nova-compute-ironic'] - - enable_ironic | bool - -- name: Starting nova-compute-fake containers - kolla_docker: - action: "start_container" - common_options: "{{ docker_common_options }}" - image: "{{ nova_compute_image_full }}" - name: "nova_compute_fake_{{ item }}" - privileged: True - volumes: - - "{{ node_config_directory }}/nova-compute-fake-{{ item }}/:{{ container_config_directory }}/:ro" - - "/lib/modules:/lib/modules:ro" - - "/run:/run" - - "rsyslog_socket:/var/lib/kolla/rsyslog/" - with_sequence: start=1 end={{ num_nova_fake_per_node }} - when: - - inventory_hostname in groups['compute'] - - enable_nova_fake | bool +- include: start_compute.yml diff --git a/ansible/roles/nova/tasks/start_compute.yml b/ansible/roles/nova/tasks/start_compute.yml new file mode 100644 index 0000000000..bbb8419cb1 --- /dev/null +++ b/ansible/roles/nova/tasks/start_compute.yml @@ -0,0 +1,65 @@ +--- +- name: Starting nova-libvirt container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + image: "{{ nova_libvirt_image_full }}" + name: "nova_libvirt" + pid_mode: "host" + privileged: True + volumes: + - "{{ node_config_directory }}/nova-libvirt/:{{ container_config_directory }}/:ro" + - "/lib/modules:/lib/modules:ro" + - "/run/:/run/" + - "/sys/fs/cgroup:/sys/fs/cgroup" + - "nova_compute:/var/lib/nova/" + - "nova_libvirt:/var/lib/libvirt" + when: inventory_hostname in groups['compute'] + +- name: Starting nova-compute container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + image: "{{ nova_compute_image_full }}" + name: "nova_compute" + privileged: True + volumes: + - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro" + - "/lib/modules:/lib/modules:ro" + - "/run:/run" + - "nova_compute:/var/lib/nova/" + - "nova_libvirt:/var/lib/libvirt" + - "rsyslog_socket:/var/lib/kolla/rsyslog/" + when: + - inventory_hostname in groups['compute'] + - not enable_nova_fake | bool + +- name: Starting nova-compute-ironic container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + image: "{{ nova_compute_ironic_image_full }}" + name: "nova_compute_ironic" + volumes: + - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro" + - "rsyslog_socket:/var/lib/kolla/rsyslog/" + when: + - inventory_hostname in groups['nova-compute-ironic'] + - enable_ironic | bool + +- name: Starting nova-compute-fake containers + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + image: "{{ nova_compute_image_full }}" + name: "nova_compute_fake_{{ item }}" + privileged: True + volumes: + - "{{ node_config_directory }}/nova-compute-fake-{{ item }}/:{{ container_config_directory }}/:ro" + - "/lib/modules:/lib/modules:ro" + - "/run:/run" + - "rsyslog_socket:/var/lib/kolla/rsyslog/" + with_sequence: start=1 end={{ num_nova_fake_per_node }} + when: + - inventory_hostname in groups['compute'] + - enable_nova_fake | bool diff --git a/ansible/roles/nova/tasks/start_conductors.yml b/ansible/roles/nova/tasks/start_conductors.yml new file mode 100644 index 0000000000..11ebf2ff3c --- /dev/null +++ b/ansible/roles/nova/tasks/start_conductors.yml @@ -0,0 +1,11 @@ +--- +- name: Starting nova-conductor container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + name: "nova_conductor" + image: "{{ nova_conductor_image_full }}" + volumes: + - "{{ node_config_directory }}/nova-conductor/:{{ container_config_directory }}/:ro" + - "rsyslog_socket:/var/lib/kolla/rsyslog/" + when: inventory_hostname in groups['nova-conductor'] diff --git a/ansible/roles/nova/tasks/start_controllers.yml b/ansible/roles/nova/tasks/start_controllers.yml new file mode 100644 index 0000000000..c9b6668004 --- /dev/null +++ b/ansible/roles/nova/tasks/start_controllers.yml @@ -0,0 +1,61 @@ +--- +- name: Starting nova-api container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + image: "{{ nova_api_image_full }}" + name: "nova_api" + privileged: True + volumes: + - "{{ node_config_directory }}/nova-api/:{{ container_config_directory }}/:ro" + - "/lib/modules:/lib/modules:ro" + - "rsyslog_socket:/var/lib/kolla/rsyslog/" + when: inventory_hostname in groups['nova-api'] + +- name: Starting nova-consoleauth container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + image: "{{ nova_consoleauth_image_full }}" + name: "nova_consoleauth" + volumes: + - "{{ node_config_directory }}/nova-consoleauth/:{{ container_config_directory }}/:ro" + - "rsyslog_socket:/var/lib/kolla/rsyslog/" + when: inventory_hostname in groups['nova-consoleauth'] + +- name: Starting nova-novncproxy container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + image: "{{ nova_novncproxy_image_full }}" + name: "nova_novncproxy" + volumes: + - "{{ node_config_directory }}/nova-novncproxy/:{{ container_config_directory }}/:ro" + - "rsyslog_socket:/var/lib/kolla/rsyslog/" + when: + - inventory_hostname in groups['nova-novncproxy'] + - nova_console == 'novnc' + +- name: Starting nova-scheduler container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + image: "{{ nova_scheduler_image_full }}" + name: "nova_scheduler" + volumes: + - "{{ node_config_directory }}/nova-scheduler/:{{ container_config_directory }}/:ro" + - "rsyslog_socket:/var/lib/kolla/rsyslog/" + when: inventory_hostname in groups['nova-scheduler'] + +- name: Starting nova-spicehtml5proxy container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + image: "{{ nova_spicehtml5proxy_image_full }}" + name: "nova_spicehtml5proxy" + volumes: + - "{{ node_config_directory }}/nova-spicehtml5proxy/:{{ container_config_directory }}/:ro" + - "rsyslog_socket:/var/lib/kolla/rsyslog/" + when: + - inventory_hostname in groups['nova-spicehtml5proxy'] + - nova_console == 'spice' diff --git a/ansible/roles/nova/tasks/upgrade.yml b/ansible/roles/nova/tasks/upgrade.yml index ed97d539c0..f8f2f7deec 100644 --- a/ansible/roles/nova/tasks/upgrade.yml +++ b/ansible/roles/nova/tasks/upgrade.yml @@ -1 +1,39 @@ --- +# Create new set of configs on nodes +- include: config.yml + +# TODO(inc0): since nova is creating new database in L->M, we need to call it. +# It should be removed later +- include: bootstrap.yml + +- include: bootstrap_service.yml + +- name: Checking if conductor container needs upgrading + kolla_docker: + action: "compare_image" + common_options: "{{ docker_common_options }}" + name: "nova_conductor" + image: "{{ nova_conductor_image_full }}" + when: inventory_hostname in groups['nova-conductor'] + register: conductor_differs + +# Short downtime here, but from user perspective his call will just timeout or execute later +- name: Stopping all nova_conductor containers + kolla_docker: + action: "stop_container" + common_options: "{{ docker_common_options }}" + name: "nova_conductor" + when: + - inventory_hostname in groups['nova-conductor'] + - conductor_differs['result'] + +- include: start_conductors.yml + +- include: start_controllers.yml + serial: "30%" + +- include: start_compute.yml + serial: "10%" + +- include: reload.yml + serial: "30%"