CI: Refactor a lot

Separate upgrade logic to is_upgrade job var and rename scenarios to match. Rename "ACTION" to "SCENARIO" (as it is a scenario). Separate testing of dashboard (aka Horizon) and increase its timeout to 5 minutes (CentOS 7 slow as always). Separate initialization of core OpenStack. Use gate setup script from ./tests/ Remove useless tox setupenv. Do not deploy Heat when not really necessary. Change-Id: I4fca319ccc3de7188f8b7b44c9c71321e3899467
2019-11-20 14:25:16 +01:00 · 2019-11-20 14:25:16 +01:00 · a2fc684164
commit a2fc684164
parent a1cd432630
12 changed files with 211 additions and 174 deletions
--- a/tests/deploy.sh
+++ b/tests/deploy.sh
@ -22,25 +22,6 @@ function deploy {
    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy &> /tmp/logs/ansible/deploy
    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy &> /tmp/logs/ansible/post-deploy
    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check &> /tmp/logs/ansible/check-deploy
-
-    if [[ ${ACTION} != "mariadb" ]]; then
-        init_runonce
-    fi
-}
-
-function init_runonce {
-    . /etc/kolla/admin-openrc.sh
-    . ~/openstackclient-venv/bin/activate
-
-    # Wait for service ready
-    sleep 15
-
-    if ! openstack image show cirros >/dev/null 2>&1; then
-        echo "Initialising OpenStack resources via init-runonce"
-        tools/init-runonce &> /tmp/logs/ansible/init-runonce
-    else
-        echo "Not running init-runonce - resources exist"
-    fi
 }


--- a/tests/init-core-openstack.sh
+++ b/tests/init-core-openstack.sh
@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+export PYTHONUNBUFFERED=1
+
+
+function init_runonce {
+    . /etc/kolla/admin-openrc.sh
+    . ~/openstackclient-venv/bin/activate
+
+    echo "Initialising OpenStack resources via init-runonce"
+    tools/init-runonce &> /tmp/logs/ansible/init-runonce
+}
+
+
+init_runonce
--- a/tests/run.yml
+++ b/tests/run.yml
@ -8,6 +8,9 @@

    # NOTE(yoctozepto): setting vars as facts for all to have them around in all the plays
    - name: set facts for commonly used variables
+      vars:
+        # NOTE(yoctozepto): needed here to use in other facts too
+        openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb'] }}"
      set_fact:
        kolla_inventory_path: "/etc/kolla/inventory"
        logs_dir: "/tmp/logs"
@ -16,15 +19,16 @@
        infra_dockerhub_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082/"
        need_build_image: false
        build_image_tag: "change_{{ zuul.change | default('none') }}"
-        is_upgrade: "{{ 'upgrade' in scenario }}"
-        is_ceph: "{{ 'ceph' in scenario }}"
+        openstack_core_enabled: "{{ openstack_core_enabled }}"
+        openstack_core_tested: "{{ scenario in ['core', 'ceph', 'cinder-lvm', 'cells'] }}"
+        dashboard_enabled: "{{ openstack_core_enabled }}"

    - name: Prepare disks for Ceph or LVM
      script: "setup_disks.sh {{ disk_type }}"
-      when: scenario == "cinder-lvm" or is_ceph
+      when: scenario in ['cinder-lvm', 'ceph']
      become: true
      vars:
-        disk_type: "{{ ceph_storetype if is_ceph else 'cinder-lvm' }}"
+        disk_type: "{{ ceph_storetype if scenario == 'ceph' else 'cinder-lvm' }}"
        ceph_storetype: "{{ hostvars[inventory_hostname].get('ceph_osd_storetype') }}"

 - hosts: primary
@ -100,15 +104,16 @@
        # nova-compute.conf
        - src: "tests/templates/nova-compute-overrides.j2"
          dest: /etc/kolla/config/nova/nova-compute.conf
-          when: "{{ scenario not in ['bifrost', 'mariadb'] }}"
+          when: "{{ openstack_core_enabled }}"
        # ceph.conf
        - src: "tests/templates/ceph-overrides.j2"
          dest: /etc/kolla/config/ceph.conf
-          when: "{{ is_ceph }}"
+          when: "{{ scenario == 'ceph' }}"
        # bifrost/dib.yml
        - src: "tests/templates/bifrost-dib-overrides.j2"
          dest: /etc/kolla/config/bifrost/dib.yml
          when: "{{ scenario == 'bifrost' }}"
+        # ironic.conf
        - src: "tests/templates/ironic-overrides.j2"
          dest: /etc/kolla/config/ironic.conf
          when: "{{ scenario == 'ironic' }}"
@ -180,7 +185,7 @@
    # job).
    - name: Run setup_gate.sh script
      script:
-        cmd: ../tools/setup_gate.sh
+        cmd: setup_gate.sh
        executable: /bin/bash
        chdir: "{{ kolla_ansible_src_dir }}"
      environment:
@ -189,7 +194,7 @@
        BUILD_IMAGE: "{{ need_build_image }}"
        TAG: "{{ build_image_tag }}"
        KOLLA_SRC_DIR: "{{ ansible_env.HOME }}/src/opendev.org/openstack/kolla"
-        ACTION: "{{ scenario }}"
+        SCENARIO: "{{ scenario }}"

    # At this point we have generated all necessary configuration, and are
    # ready to deploy the control plane services. Control flow now depends on
@ -202,24 +207,39 @@
            cmd: deploy.sh
            executable: /bin/bash
            chdir: "{{ kolla_ansible_src_dir }}"
-          environment:
-            ACTION: "{{ scenario }}"

-        - name: Run test-openstack.sh script
+        # NOTE(yoctozepto): this is nice as the first step after the deployment
+        # because it waits for the services to stabilize well enough so that
+        # the dashboard is able to show the login prompt
+        - name: Run test-dashboard.sh script
          script:
-            cmd: test-openstack.sh
+            cmd: test-dashboard.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          when: dashboard_enabled
+
+        - name: Run init-core-openstack.sh script
+          script:
+            cmd: init-core-openstack.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          when: openstack_core_tested or scenario in ['ironic', 'scenario_nfv', 'zun']
+
+        - name: Run test-core-openstack.sh script
+          script:
+            cmd: test-core-openstack.sh
            executable: /bin/bash
            chdir: "{{ kolla_ansible_src_dir }}"
          environment:
-            ACTION: "{{ scenario }}"
-          when: scenario not in ['ironic', 'mariadb', 'scenario_nfv']
+            SCENARIO: "{{ scenario }}"
+          when: openstack_core_tested

        - name: Run test-zun.sh script
-          shell:
-            cmd: tests/test-zun.sh
+          script:
+            cmd: test-zun.sh
            executable: /bin/bash
            chdir: "{{ kolla_ansible_src_dir }}"
-          when: "{{ 'zun' in scenario }}"
+          when: scenario == 'zun'

        - name: Run test-scenario-nfv.sh script
          script:
@ -311,7 +331,7 @@
            # ceph.conf
            - src: "tests/templates/ceph-overrides.j2"
              dest: /etc/kolla/config/ceph.conf
-              when: "{{ is_ceph }}"
+              when: "{{ scenario == 'ceph' }}"
          when: item.when | default(true)

        - name: upgrade kolla-ansible
@ -346,23 +366,25 @@
            cmd: tests/upgrade.sh
            executable: /bin/bash
            chdir: "{{ kolla_ansible_src_dir }}"
-          environment:
-            ACTION: "{{ scenario }}"

-        - name: Run test-openstack.sh script
+        # NOTE(yoctozepto): this is nice as the first step after the upgrade
+        # because it waits for the services to stabilize well enough so that
+        # the dashboard is able to show the login prompt
+        - name: Run test-dashboard.sh script
          shell:
-            cmd: tests/test-openstack.sh
+            cmd: tests/test-dashboard.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          when: dashboard_enabled
+
+        - name: Run test-core-openstack.sh script
+          shell:
+            cmd: tests/test-core-openstack.sh
            executable: /bin/bash
            chdir: "{{ kolla_ansible_src_dir }}"
          environment:
-            ACTION: "{{ scenario }}"
-
-        - name: Run test-zun.sh script
-          shell:
-            cmd: tests/test-zun.sh
-            executable: /bin/bash
-            chdir: "{{ kolla_ansible_src_dir }}"
-          when: "{{ 'zun' in scenario }}"
+            SCENARIO: "{{ scenario }}"
+          when: openstack_core_tested
      when: is_upgrade

    # Bifrost testing.
@ -390,6 +412,8 @@
 - hosts: all
  any_errors_fatal: true
  tasks:
+    - name: Post-deploy/upgrade sanity checks
+      block:
        - name: Run check-failure.sh script
          shell:
            cmd: tests/check-failure.sh
@ -416,8 +440,6 @@
        cmd: reconfigure.sh
        executable: /bin/bash
        chdir: "{{ kolla_ansible_src_dir }}"
-      environment:
-        ACTION: "{{ scenario }}"
      when:
        - not is_upgrade
        - scenario != "bifrost"
@ -426,6 +448,8 @@
 - hosts: all
  any_errors_fatal: true
  tasks:
+    - name: Post-reconfigure sanity checks
+      block:
        - name: Run check-failure.sh script
          shell:
            cmd: tests/check-failure.sh
@ -443,3 +467,6 @@
            cmd: tests/check-logs.sh
            executable: /bin/bash
            chdir: "{{ kolla_ansible_src_dir }}"
+      when:
+        - not is_upgrade
+        - scenario != "bifrost"
--- a/tests/templates/globals-default.j2
+++ b/tests/templates/globals-default.j2
@ -13,7 +13,11 @@ docker_custom_config:
 # in the CI gate.
 keepalived_virtual_router_id: "{{ 250 | random(1) }}"

-{% if enable_core_openstack | bool %}
+enable_openstack_core: "{{ openstack_core_enabled }}"
+enable_horizon: "{{ dashboard_enabled }}"
+enable_heat: "{{ openstack_core_tested }}"
+
+{% if scenario != 'bifrost' %}
 kolla_internal_vip_address: "{{ kolla_internal_vip_address }}"
 neutron_external_interface: "fake_interface"
 openstack_logging_debug: "True"
@ -45,7 +49,7 @@ openstack_release: "{{ previous_release }}"
 {% endif %}
 {% endif %}

-{% if is_ceph %}
+{% if scenario == "ceph" %}
 enable_ceph: "yes"
 enable_ceph_mds: "yes"
 enable_ceph_rgw: "yes"
@ -86,6 +90,7 @@ enable_neutron_sfc: "yes"
 enable_mistral: "yes"
 enable_redis: "yes"
 enable_barbican: "yes"
+enable_heat: "yes"
 {% endif %}

 {% if scenario == "ironic" %}
@ -106,6 +111,5 @@ enable_chrony: "no"
 enable_fluentd: "no"
 enable_mariadb: "yes"
 enable_memcached: "no"
-enable_openstack_core: "no"
 enable_rabbitmq: "no"
 {% endif %}
--- a/tests/test-core-openstack.sh
+++ b/tests/test-core-openstack.sh
@ -11,7 +11,7 @@ function test_smoke {
    openstack --debug compute service list
    openstack --debug network agent list
    openstack --debug orchestration service list
-    if [[ $ACTION =~ "ceph" ]] || [[ $ACTION == "cinder-lvm" ]]; then
+    if [[ $SCENARIO == "ceph" ]] || [[ $SCENARIO == "cinder-lvm" ]]; then
        openstack --debug volume service list
    fi
 }
@ -28,7 +28,7 @@ function test_instance_boot {
    fi
    echo "SUCCESS: Server creation"

-    if [[ $ACTION =~ "ceph" ]] || [[ $ACTION == "cinder-lvm" ]]; then
+    if [[ $SCENARIO == "ceph" ]] || [[ $SCENARIO == "cinder-lvm" ]]; then
        echo "TESTING: Cinder volume attachment"
        openstack volume create --size 2 test_volume
        attempt=1
@ -75,58 +75,25 @@ function test_instance_boot {
    echo "SUCCESS: Server deletion"
 }

-function check_dashboard {
-    # Query the dashboard, and check that the returned page looks like a login
-    # page.
-    DASHBOARD_URL=${OS_AUTH_URL%:*}
-    output_path=$1
-    if ! curl --include --location --fail $DASHBOARD_URL > $output_path; then
-        return 1
-    fi
-    if ! grep Login $output_path >/dev/null; then
-        return 1
-    fi
-}
-
-function test_dashboard {
-    echo "TESTING: Dashboard"
-    # The dashboard has been known to take some time to become accessible, so
-    # use retries.
-    output_path=$(mktemp)
-    attempt=1
-    while ! check_dashboard $output_path; do
-        echo "Dashboard not accessible yet"
-        attempt=$((attempt+1))
-        if [[ $attempt -eq 10 ]]; then
-            echo "FAILED: Dashboard did not become accessible. Response:"
-            cat $output_path
-            return 1
-        fi
-        sleep 10
-    done
-    echo "SUCCESS: Dashboard"
-}
-
 function test_openstack_logged {
    . /etc/kolla/admin-openrc.sh
    . ~/openstackclient-venv/bin/activate
    test_smoke
    test_instance_boot
-    test_dashboard
 }

 function test_openstack {
    echo "Testing OpenStack"
-    log_file=/tmp/logs/ansible/test-openstack
+    log_file=/tmp/logs/ansible/test-core-openstack
    if [[ -f $log_file ]]; then
        log_file=${log_file}-upgrade
    fi
    test_openstack_logged > $log_file 2>&1
    result=$?
    if [[ $result != 0 ]]; then
-        echo "Testing OpenStack failed. See ansible/test-openstack for details"
+        echo "Testing OpenStack failed. See ansible/test-core-openstack for details"
    else
-        echo "Successfully tested OpenStack. See ansible/test-openstack for details"
+        echo "Successfully tested OpenStack. See ansible/test-core-openstack for details"
    fi
    return $result
 }
--- a/tests/test-dashboard.sh
+++ b/tests/test-dashboard.sh
@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+export PYTHONUNBUFFERED=1
+
+
+function check_dashboard {
+    # Query the dashboard, and check that the returned page looks like a login
+    # page.
+    DASHBOARD_URL=${OS_AUTH_URL%:*}
+    output_path=$1
+    if ! curl --include --location --fail $DASHBOARD_URL > $output_path; then
+        return 1
+    fi
+    if ! grep Login $output_path >/dev/null; then
+        return 1
+    fi
+}
+
+function test_dashboard_logged {
+    . /etc/kolla/admin-openrc.sh
+
+    echo "TESTING: Dashboard"
+    # The dashboard has been known to take some time to become accessible, so
+    # use retries.
+    output_path=$(mktemp)
+    attempt=1
+    while ! check_dashboard $output_path; do
+        echo "Dashboard not accessible yet"
+        attempt=$((attempt+1))
+        if [[ $attempt -eq 30 ]]; then
+            echo "FAILED: Dashboard did not become accessible. Response:"
+            cat $output_path
+            return 1
+        fi
+        sleep 10
+    done
+    echo "SUCCESS: Dashboard"
+}
+
+function test_dashboard {
+    echo "Testing dashboard"
+    log_file=/tmp/logs/ansible/test-dashboard
+    if [[ -f $log_file ]]; then
+        log_file=${log_file}-upgrade
+    fi
+    test_dashboard_logged > $log_file 2>&1
+    result=$?
+    if [[ $result != 0 ]]; then
+        echo "Testing dashboard failed. See ansible/test-dashboard for details"
+    else
+        echo "Successfully tested dashboard. See ansible/test-dashboard for details"
+    fi
+    return $result
+}
+
+
+test_dashboard
--- a/tools/dump_info.sh
+++ b/tools/dump_info.sh
@ -1,26 +0,0 @@
-#!/bin/bash
-
-set -o xtrace
-
-function dump_node_info {
-    # NOTE(SamYaple): function for debugging gate
-    set +o errexit
-    local OLD_PATH="${PATH}"
-    PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    sudo parted -l
-
-    sudo mount
-
-    df -h
-
-    uname -a
-
-    cat /etc/*release*
-
-    cat /proc/meminfo
-
-    PATH="${OLD_PATH}"
-    set -o errexit
-}
-
-(dump_node_info 2>&1) > /tmp/logs/node_info_$(date +%s)
--- a/tools/setup_gate.sh
+++ b/tools/setup_gate.sh
@ -14,47 +14,47 @@ function setup_openstack_clients {
    ~/openstackclient-venv/bin/pip install -U pip
    ~/openstackclient-venv/bin/pip install python-openstackclient
    ~/openstackclient-venv/bin/pip install python-heatclient
-    if [[ $ACTION == zun ]]; then
+    if [[ $SCENARIO == zun ]]; then
        ~/openstackclient-venv/bin/pip install python-zunclient
    fi
-    if [[ $ACTION == ironic ]]; then
+    if [[ $SCENARIO == ironic ]]; then
        ~/openstackclient-venv/bin/pip install python-ironicclient
    fi
-    if [[ $ACTION == masakari ]]; then
+    if [[ $SCENARIO == masakari ]]; then
        ~/openstackclient-venv/bin/pip install python-masakariclient
    fi
 }

 function setup_config {
-    if [[ $ACTION != "bifrost" ]]; then
+    if [[ $SCENARIO != "bifrost" ]]; then
        GATE_IMAGES="cron,fluentd,glance,haproxy,keepalived,keystone,kolla-toolbox,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,horizon,chrony,heat,placement"
    else
        GATE_IMAGES="bifrost"
    fi

-    if [[ $ACTION =~ "ceph" ]]; then
+    if [[ $SCENARIO == "ceph" ]]; then
        GATE_IMAGES+=",ceph,cinder"
    fi

-    if [[ $ACTION == "cinder-lvm" ]]; then
+    if [[ $SCENARIO == "cinder-lvm" ]]; then
        GATE_IMAGES+=",cinder,iscsid,tgtd"
    fi

-    if [[ $ACTION == "zun" ]]; then
+    if [[ $SCENARIO == "zun" ]]; then
        GATE_IMAGES+=",zun,kuryr,etcd"
    fi

-    if [[ $ACTION == "scenario_nfv" ]]; then
+    if [[ $SCENARIO == "scenario_nfv" ]]; then
        GATE_IMAGES+=",tacker,mistral,redis,barbican"
    fi
-    if [[ $ACTION == "ironic" ]]; then
+    if [[ $SCENARIO == "ironic" ]]; then
        GATE_IMAGES+=",dnsmasq,ironic,iscsid"
    fi
-    if [[ $ACTION == "masakari" ]]; then
+    if [[ $SCENARIO == "masakari" ]]; then
        GATE_IMAGES+=",masakari"
    fi

-    if [[ $ACTION == "mariadb" ]]; then
+    if [[ $SCENARIO == "mariadb" ]]; then
        GATE_IMAGES="cron,haproxy,keepalived,kolla-toolbox,mariadb"
    fi

--- a/tox.ini
+++ b/tox.ini
@ -94,12 +94,6 @@ commands =
  sphinx-build -W --keep-going -b latex doc/source doc/build/pdf
  make -C doc/build/pdf

-[testenv:setupenv]
-basepython = python3
-commands =
-  {toxinidir}/tools/setup_gate.sh
-  {toxinidir}/tools/dump_info.sh
-
 [testenv:releasenotes]
 basepython = python3
 deps =
--- a/zuul.d/base.yaml
+++ b/zuul.d/base.yaml
@ -23,8 +23,8 @@
      - ^specs/
      - ^kolla_ansible/tests/
    vars:
-      scenario: aio
-      enable_core_openstack: yes
+      scenario: core
+      is_upgrade: no
      api_network_prefix: "192.0.2."
      api_network_prefix_length: "24"
      api_interface_name: vxlan0
@ -38,7 +38,7 @@
    parent: kolla-ansible-base
    vars:
      previous_release: train
-      scenario: upgrade
+      is_upgrade: yes

 - job:
    name: kolla-ansible-ipv6-base
@ -56,7 +56,6 @@
    vars:
      scenario: bifrost
      install_type: source
-      enable_core_openstack: no

 - job:
    name: kolla-ansible-ironic-base
@ -65,6 +64,7 @@
    files:
      - ^ansible/roles/(ironic|nova)/
      - ^tests/test-ironic.sh
+      - ^tests/test-dashboard.sh
    vars:
      scenario: ironic

@ -75,5 +75,6 @@
    files:
      - ^ansible/roles/(zun|kuryr|etcd)/
      - ^tests/test-zun.sh
+      - ^tests/test-dashboard.sh
    vars:
      scenario: zun
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@ -153,7 +153,7 @@
    vars:
      base_distro: ubuntu
      install_type: source
-      scenario: upgrade-ceph
+      scenario: ceph
    host-vars:
      primary:
        ceph_osd_storetype: filestore
@ -171,7 +171,7 @@
    vars:
      base_distro: centos
      install_type: source
-      scenario: upgrade-ceph
+      scenario: ceph
    host-vars:
      primary:
        ceph_osd_storetype: filestore
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@ -18,25 +18,36 @@
        - kolla-ansible-ubuntu-source-ceph
        - kolla-ansible-centos-source-ceph
        - kolla-ansible-ubuntu-source-cinder-lvm:
-            files: ^ansible\/roles\/(cinder|iscsi)\/.*
+            files:
+              - ^ansible/roles/(cinder|iscsi)/
+              - ^tests/test-core-openstack.sh
+              - ^tests/test-dashboard.sh
        - kolla-ansible-centos-source-cinder-lvm:
-            files: ^ansible\/roles\/(cinder|iscsi)\/.*
+            files:
+              - ^ansible/roles/(cinder|iscsi)/
+              - ^tests/test-core-openstack.sh
+              - ^tests/test-dashboard.sh
        - kolla-ansible-bifrost-centos-source:
-            files: ^ansible\/roles\/bifrost\/.*
+            files:
+              - ^ansible/roles/bifrost/
+              - ^tests/test-bifrost.sh
        - kolla-ansible-centos-source-zun
        - kolla-ansible-ubuntu-source-zun
        - kolla-ansible-centos-source-scenario-nfv:
            files:
              - ^ansible/roles/(barbican|heat|mistral|redis|tacker)/
              - ^tests/test-scenario-nfv.sh
+              - ^tests/test-dashboard.sh
        - kolla-ansible-ubuntu-source-masakari:
            files:
              - ^ansible/roles/masakari/
              - ^tests/test-masakari.sh
+              - ^tests/test-dashboard.sh
        - kolla-ansible-centos-source-masakari:
            files:
              - ^ansible/roles/masakari/
              - ^tests/test-masakari.sh
+              - ^tests/test-dashboard.sh
        - kolla-ansible-centos-source-ironic
        - kolla-ansible-centos-binary-ironic
        - kolla-ansible-ubuntu-source-ironic