From baa9319a75b5da17d948fa2ec488b61632aa7cc4 Mon Sep 17 00:00:00 2001 From: Jeffrey Zhang Date: Sun, 1 Oct 2017 23:21:38 +0800 Subject: [PATCH] Move to zuul v3 in project jobs Partial-Bug: #1720601 Change-Id: Ibc20a6ae8c645ff82f3c14a6286073dffd4cfae2 --- .zuul.yaml | 57 ++++++++++++ .../roles/neutron/templates/neutron.conf.j2 | 3 + tests/ansible_generate_config.yml | 40 --------- tests/ansible_generate_inventory.yml | 26 ------ tests/ansible_get_logs.yml | 88 ------------------- tests/get_logs.sh | 70 +++++++-------- tests/post.yml | 28 ++++++ tests/pre.yml | 20 +++++ tests/run.yml | 83 +++++++++++++++++ tests/templates/globals-default.j2 | 8 +- tests/templates/inventory.j2 | 6 -- tools/setup_Debian.sh | 4 +- tools/setup_gate.sh | 69 +-------------- 13 files changed, 229 insertions(+), 273 deletions(-) create mode 100644 .zuul.yaml delete mode 100644 tests/ansible_generate_config.yml delete mode 100644 tests/ansible_generate_inventory.yml delete mode 100644 tests/ansible_get_logs.yml create mode 100644 tests/post.yml create mode 100644 tests/pre.yml create mode 100644 tests/run.yml delete mode 100644 tests/templates/inventory.j2 diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000000..958368d1ce --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,57 @@ +- project: + name: openstack/kolla-ansible + check: + jobs: + - kolla-ansible-centos-source + - kolla-ansible-ubuntu-source + - kolla-ansible-oraclelinux-source + +- nodeset: + name: kolla-ansible-centos + nodes: + - name: primary + label: centos-7 + +- nodeset: + name: kolla-ansible-xenial + nodes: + - name: primary + label: ubuntu-xenial + +- job: + name: kolla-ansible-base + pre-run: tests/pre + run: tests/run + post-run: tests/post + attempts: 1 + timeout: 10800 + required-projects: + - openstack/kolla + - openstack/requirements + irrelevant-files: + - ^.*\.rst$ + - ^doc/.* + +- job: + name: kolla-ansible-centos-source + parent: kolla-ansible-base + nodeset: kolla-ansible-centos + vars: + base_distro: centos + install_type: source + +- job: + name: kolla-ansible-ubuntu-source + parent: kolla-ansible-base + nodeset: kolla-ansible-xenial + vars: + base_distro: ubuntu + install_type: source + +- job: + name: kolla-ansible-oraclelinux-source + parent: kolla-ansible-base + nodeset: kolla-ansible-centos + vars: + base_distro: oraclelinux + install_type: source diff --git a/ansible/roles/neutron/templates/neutron.conf.j2 b/ansible/roles/neutron/templates/neutron.conf.j2 index f831a63314..2db1c20dc2 100644 --- a/ansible/roles/neutron/templates/neutron.conf.j2 +++ b/ansible/roles/neutron/templates/neutron.conf.j2 @@ -161,3 +161,6 @@ project_name = service project_domain_name = {{ default_project_domain_name }} os_region_name = {{ openstack_region_name }} os_interface = internal + +[privsep] +helper_command=sudo neutron-rootwrap /etc/neutron/rootwrap.conf privsep-helper diff --git a/tests/ansible_generate_config.yml b/tests/ansible_generate_config.yml deleted file mode 100644 index 61a543bcb1..0000000000 --- a/tests/ansible_generate_config.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -- hosts: localhost - connection: local - become: True - tasks: - - name: Get api_interface name - set_fact: api_interface_address="{{ lookup('file', '/etc/nodepool/primary_node_private') }}" - - - shell: "ip a | grep {{ api_interface_address }}" - register: api_interface_name - - - set_fact: api_interface_name="{{ api_interface_name.stdout_lines[0].split(" ")[-1] }}" - - - set_fact: - is_multinode: "{{ lookup('file', '/etc/nodepool/sub_nodes') }}" - - - name: Ensure /etc/kolla dir - file: - path: /etc/kolla - state: "directory" - - - name: Setup globals.yml - template: - src: "templates/globals-default.j2" - dest: "/etc/kolla/globals.yml" - - - name: Copy passwords.yml - copy: - src: "../etc/kolla/passwords.yml" - dest: "/etc/kolla/passwords.yml" - - - name: Ensure /etc/kolla/config directory - file: - path: /etc/kolla/config/nova - state: "directory" - - - name: Setup overrides - template: - src: "templates/nova-compute-overrides.j2" - dest: "/etc/kolla/config/nova/nova-compute.conf" diff --git a/tests/ansible_generate_inventory.yml b/tests/ansible_generate_inventory.yml deleted file mode 100644 index 9d93d9eeff..0000000000 --- a/tests/ansible_generate_inventory.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- hosts: localhost - connection: local - tasks: - - name: Get node addresses - set_fact: - primary_node_address: "{{ lookup('file', '/etc/nodepool/primary_node_private') }}" - sub_node_addresses: "{{ lookup('file', '/etc/nodepool/sub_nodes_private').split('\n') }}" - - - set_fact: node_group="{{ lookup('template', 'templates/inventory.j2')}}" - - - name: Ensure /tmp/kolla exists - file: - path: "/tmp/kolla" - state: "directory" - - - name: Copy default ansible kolla-ansible inventory - copy: - src: ../ansible/inventory/all-in-one - dest: /tmp/kolla/raw_inventory - - - name: Replace localhost with IPs - replace: - path: /tmp/kolla/raw_inventory - regexp: "localhost.*$" - replace: "{{ node_group }}" diff --git a/tests/ansible_get_logs.yml b/tests/ansible_get_logs.yml deleted file mode 100644 index 68d9e2a97e..0000000000 --- a/tests/ansible_get_logs.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- -- hosts: all - tasks: - - name: Check node role - command: "cat /etc/nodepool/role" - register: node_role - - - set_fact: node_role="{{ node_role.stdout }}" - - - name: Ensure /tmp/logs dir - file: - path: "/tmp/logs" - state: "directory" - when: - - node_role == "sub" - - - name: Get /tmp/logs symlink target - command: "readlink -f /tmp/logs" - register: logs_target - - - name: Ensure primary node directories - file: - path: "{{ logs_target.stdout }}/{{ item }}" - state: "directory" - mode: 0777 - when: - - node_role == "primary" - with_items: - - "subnodes" - - "docker_logs" - - "kolla_configs" - - "system_logs" - - "kolla" - - - name: Ensure sub node directories - file: - path: "/tmp/logs/{{ item }}" - state: "directory" - mode: 0777 - when: - - node_role == "sub" - with_items: - - "docker_logs" - - "kolla_configs" - - "system_logs" - - "kolla" - - - name: Run diagnostics script - script: get_logs.sh - register: get_logs_result - failed_when: false - - - name: Print get_logs output - debug: - msg: "{{ get_logs_result.stdout }}" - - - name: Run dump_info script - script: ../tools/dump_info.sh - - - name: Download logs from all subnodes - synchronize: - src: "/tmp/logs" - dest: "{{ logs_target.stdout }}/subnodes/{{ ansible_hostname }}" - mode: "pull" - when: - - node_role == "sub" - - - name: Change permission of all log files - command: "chmod -R 777 {{ logs_target.stdout }}" - - - name: Fail if get_logs has failed - fail: - msg: "{{ get_logs_result.stdout }}" - when: get_logs_result.rc != 0 - - - name: Copy inventory file to logs - copy: - src: "/tmp/kolla/raw_inventory" - dest: "{{ logs_target.stdout }}/ansible/inventory" - when: - - node_role == "primary" - - - name: Copy hosts file to logs - copy: - src: "/etc/hosts" - dest: "{{ logs_target.stdout }}/system_logs/hosts" - when: - - node_role == "primary" diff --git a/tests/get_logs.sh b/tests/get_logs.sh index 06f082b365..81961ad7c4 100644 --- a/tests/get_logs.sh +++ b/tests/get_logs.sh @@ -1,8 +1,25 @@ #!/bin/bash +set +o errexit + +check_failure() { + # All docker container's status are created, restarting, running, removing, + # paused, exited and dead. Containers without running status are treated as + # failure. removing is added in docker 1.13, just ignore it now. + failed_containers=$(docker ps -a --format "{{.Names}}" \ + --filter status=created \ + --filter status=restarting \ + --filter status=paused \ + --filter status=exited \ + --filter status=dead) + + if [[ -n "$failed_containers" ]]; then + exit 1; + fi +} + copy_logs() { - set +o errexit - LOG_DIR=$(readlink -f /tmp/logs) + LOG_DIR=/tmp/logs cp -rnL /var/lib/docker/volumes/kolla_logs/_data/* ${LOG_DIR}/kolla/ cp -rnL /etc/kolla/* ${LOG_DIR}/kolla_configs/ @@ -16,12 +33,12 @@ copy_logs() { cp /var/log/upstart/docker.log ${LOG_DIR}/system_logs/docker.log fi - sudo cp -r /etc/sudoers.d ${LOG_DIR}/system_logs/ - sudo cp /etc/sudoers ${LOG_DIR}/system_logs/sudoers.txt + cp -r /etc/sudoers.d ${LOG_DIR}/system_logs/ + cp /etc/sudoers ${LOG_DIR}/system_logs/sudoers.txt df -h > ${LOG_DIR}/system_logs/df.txt free > ${LOG_DIR}/system_logs/free.txt - sudo parted -l > ${LOG_DIR}/system_logs/parted-l.txt + parted -l > ${LOG_DIR}/system_logs/parted-l.txt mount > ${LOG_DIR}/system_logs/mount.txt env > ${LOG_DIR}/system_logs/env.txt @@ -45,9 +62,9 @@ copy_logs() { docker exec ceph_mon ceph osd tree > ${LOG_DIR}/kolla/ceph/ceph_osd_tree.txt fi - # container logs - mkdir ${LOG_DIR}/kolla/fluentd/ - docker logs fluentd > ${LOG_DIR}/kolla/fluentd/fluentd.txt + for container in $(docker ps -a --format "{{.Names}}"); do + docker logs --tail all ${container} > ${LOG_DIR}/docker_logs/${container}.txt + done # Rename files to .txt; this is so that when displayed via # logs.openstack.org clicking results in the browser shows the @@ -56,40 +73,13 @@ copy_logs() { # Rename all .log files to .txt files for f in $(find ${LOG_DIR}/{system_logs,kolla,docker_logs} -name "*.log"); do - sudo mv $f ${f/.log/.txt} + mv $f ${f/.log/.txt} done - sudo find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.txt' -execdir gzip -f -9 {} \+ - sudo find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.json' -execdir gzip -f -9 {} \+ - - set +o errexit -} - -check_failure() { - # Command failures after this point can be expected - set +o errexit - - docker images - docker ps -a - # All docker container's status are created, restarting, running, removing, - # paused, exited and dead. Containers without running status are treated as - # failure. removing is added in docker 1.13, just ignore it now. - failed_containers=$(docker ps -a --format "{{.Names}}" \ - --filter status=created \ - --filter status=restarting \ - --filter status=paused \ - --filter status=exited \ - --filter status=dead) - - for failed in ${failed_containers}; do - docker logs --tail all ${failed} > /tmp/logs/docker_logs/${failed} - done - - copy_logs - - if [[ -n "$failed_containers" ]]; then - exit 1; - fi + chmod -R 777 ${LOG_DIR} + find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.txt' -execdir gzip -f -9 {} \+ + find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.json' -execdir gzip -f -9 {} \+ } +copy_logs check_failure diff --git a/tests/post.yml b/tests/post.yml new file mode 100644 index 0000000000..97a9cfaaa2 --- /dev/null +++ b/tests/post.yml @@ -0,0 +1,28 @@ +--- +- hosts: all + vars: + logs_dir: "/tmp/logs" + tasks: + - name: Run diagnostics script + script: get_logs.sh + register: get_logs_result + become: true + failed_when: false + + - name: Print get_logs output + debug: + msg: "{{ get_logs_result.stdout }}" + + - name: Download logs to executor + synchronize: + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + src: "{{ logs_dir }}/" + ignore_errors: yes + + - name: Download /etc/hosts file to executor + synchronize: + src: "/etc/hosts" + dest: "{{ zuul.executor.log_root }}/{{inventory_hostname }}/" + mode: pull + ignore_errors: yes diff --git a/tests/pre.yml b/tests/pre.yml new file mode 100644 index 0000000000..dd614b89c9 --- /dev/null +++ b/tests/pre.yml @@ -0,0 +1,20 @@ +- hosts: all + vars: + logs_dir: "/tmp/logs" + tasks: + - name: Ensure /tmp/logs/ dir + file: + path: "{{ logs_dir }}" + state: "directory" + + - name: Ensure node directories + file: + path: "{{ logs_dir }}/{{ item }}" + state: "directory" + mode: 0777 + with_items: + - "docker_logs" + - "kolla_configs" + - "system_logs" + - "kolla" + - "ansible" diff --git a/tests/run.yml b/tests/run.yml new file mode 100644 index 0000000000..25eae0b2e9 --- /dev/null +++ b/tests/run.yml @@ -0,0 +1,83 @@ +--- +- hosts: all + vars: + kolla_inventory_path: "/etc/kolla/inventory" + logs_dir: "/tmp/logs" + tasks: + - name: ensure /etc/kolla exists + file: + path: "/etc/kolla" + state: "directory" + mode: 0777 + become: true + + - set_fact: + nodes: | + {% for host in hostvars %} + {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_become=true ansible_user={{ hostvars[host]['ansible_user'] }} + {% endfor %} + + - name: copy default ansible kolla-ansible inventory + copy: + src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/ansible/inventory/all-in-one" + dest: "{{ kolla_inventory_path }}" + delegate_to: "primary" + + - name: replace localhost with IPs + replace: + path: "{{ kolla_inventory_path }}" + regexp: "localhost.*$" + replace: "{{ nodes }}" + delegate_to: "primary" + + - name: generate global.yml file + vars: + api_interface_name: "{{ hostvars['primary']['ansible_default_ipv4'].alias }}" + api_interface_address: "{{ hostvars['primary']['nodepool']['private_ipv4'] }}" + template: + src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/globals-default.j2" + dest: /etc/kolla/globals.yml + delegate_to: "primary" + + - name: ensure nova conf overrides dir exists + file: + path: "/etc/kolla/config/nova" + state: "directory" + mode: 0777 + become: true + delegate_to: "primary" + + - name: generate config overrides + template: + src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/nova-compute-overrides.j2" + dest: /etc/kolla/config/nova/nova-compute.conf + delegate_to: "primary" + + - name: copy passwords.yml file + copy: + src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/etc/kolla/passwords.yml" + dest: /etc/kolla/passwords.yml + delegate_to: "primary" + + - name: install kolla-ansible requirements + pip: + requirements: "{{ ansible_env.HOME }}/{{ zuul.project.src_dir }}/requirements.txt" + become: true + + - name: generate passwords + shell: "{{ zuul.project.src_dir }}/tools/generate_passwords.py" + delegate_to: "primary" + + - shell: + cmd: | + set -e + set -x + export BASE_DISTRO={{ base_distro }} + export INSTALL_TYPE={{ install_type }} + export NODEPOOL_TARBALLS_MIRROR=http://{{ zuul_site_mirror_fqdn }}:8080/tarballs + + tools/setup_gate.sh + executable: /bin/bash + chdir: "{{ zuul.project.src_dir }}" + delegate_to: "primary" + environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index 89f0432d8c..4404f9875c 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -1,8 +1,8 @@ --- -kolla_base_distro: "{{ base }}" -kolla_install_type: "{{ type }}" +kolla_base_distro: "{{ base_distro }}" +kolla_install_type: "{{ install_type }}" -{% if is_multinode %} +{% if hostvars|length > 2 %} enable_haproxy: "no" kolla_internal_vip_address: "{{ api_interface_address }}" {% else %} @@ -20,7 +20,7 @@ enable_heat: "no" openstack_logging_debug: "True" openstack_service_workers: "1" -{% if 'ceph' in action %} +{% if 'ceph' in zuul.job %} enable_ceph: "yes" enable_cinder: "yes" {% endif %} diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2 deleted file mode 100644 index 0a1f2ecdc6..0000000000 --- a/tests/templates/inventory.j2 +++ /dev/null @@ -1,6 +0,0 @@ -{{ primary_node_address }} ansible_become=true ansible_connection=local -{% for addr in sub_node_addresses %} -{% if addr %} -{{ addr }} ansible_user=jenkins ansible_become=true ansible_ssh_private_key_file=/etc/nodepool/id_rsa -{% endif %} -{% endfor %} diff --git a/tools/setup_Debian.sh b/tools/setup_Debian.sh index c3c965a995..f5417ca3cc 100755 --- a/tools/setup_Debian.sh +++ b/tools/setup_Debian.sh @@ -82,7 +82,7 @@ sudo service docker stop if [[ ${DISTRIB_CODENAME} == "trusty" ]]; then sudo apt-get -y install --no-install-recommends btrfs-tools setup_disk - echo "DOCKER_OPTS=\"-s btrfs --insecure-registry $(cat /etc/nodepool/primary_node_private):4000\"" | sudo tee /etc/default/docker + echo "DOCKER_OPTS=\"-s btrfs --insecure-registry 0.0.0.0/0\"" | sudo tee /etc/default/docker sudo mount --make-shared /run sudo service docker start else @@ -90,7 +90,7 @@ else sudo tee /etc/systemd/system/docker.service.d/kolla.conf << EOF [Service] ExecStart= -ExecStart=/usr/bin/dockerd --storage-driver overlay2 --insecure-registry $(cat /etc/nodepool/primary_node_private):4000 +ExecStart=/usr/bin/dockerd --storage-driver overlay2 --insecure-registry 0.0.0.0/0 MountFlags=shared EOF sudo systemctl daemon-reload diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh index 20db357e90..8e97d6b681 100755 --- a/tools/setup_gate.sh +++ b/tools/setup_gate.sh @@ -6,20 +6,8 @@ set -o errexit # Enable unbuffered output for Ansible in Jenkins. export PYTHONUNBUFFERED=1 -. /etc/nodepool/provider - -NODEPOOL_MIRROR_HOST=${NODEPOOL_MIRROR_HOST:-mirror.$NODEPOOL_REGION.$NODEPOOL_CLOUD.openstack.org} -NODEPOOL_MIRROR_HOST=$(echo $NODEPOOL_MIRROR_HOST|tr '[:upper:]' '[:lower:]') -NODEPOOL_PYPI_MIRROR=${NODEPOOL_PYPI_MIRROR:-http://$NODEPOOL_MIRROR_HOST/pypi/simple} -NODEPOOL_TARBALLS_MIRROR=${NODEPOOL_TARBALLS_MIRROR:-http://$NODEPOOL_MIRROR_HOST:8080/tarballs} - GIT_PROJECT_DIR=$(mktemp -d) -# Just for mandre :) -if [[ ! -f /etc/sudoers.d/jenkins ]]; then - echo "jenkins ALL=(:docker) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/jenkins -fi - function clone_repos { cat > /tmp/clonemap < /tmp/hosts - ansible-playbook tests/ansible_generate_inventory.yml - sudo chown root: /tmp/hosts - sudo chmod 644 /tmp/hosts - sudo mv /tmp/hosts /etc/hosts -} - function setup_ansible { - RAW_INVENTORY=/tmp/kolla/raw_inventory - mkdir /tmp/kolla + RAW_INVENTORY=/etc/kolla/inventory # TODO(SamYaple): Move to virtualenv sudo -H pip install -U "ansible>=2,<2.4" "docker>=2.0.0" "python-openstackclient" "python-neutronclient" "ara" detect_distro - setup_inventory - sudo mkdir /etc/ansible ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))") sudo tee /etc/ansible/ansible.cfg</logs - mkdir logs - - # For ease of access we symlink that logs directory to a known path - ln -s $(pwd)/logs /tmp/logs - mkdir -p /tmp/logs/{ansible,build,kolla,kolla_configs,system_logs} -} - function prepare_images { sudo docker run -d -p 4000:5000 --restart=always -v /opt/kolla_registry/:/var/lib/registry --name registry registry:2 @@ -179,8 +129,6 @@ function prepare_images { fi } - - function sanity_check { # Wait for service ready sleep 15 @@ -203,30 +151,19 @@ function sanity_check { fi } -function get_logs { - ansible-playbook -i ${RAW_INVENTORY} tests/ansible_get_logs.yml > /tmp/logs/ansible/get-logs -} -setup_logging -tools/dump_info.sh clone_repos -setup_workaround_broken_nodepool -setup_ssh setup_ansible setup_config setup_node -ansible-playbook -e type=$INSTALL_TYPE -e base=$BASE_DISTRO -e action=$ACTION tests/ansible_generate_config.yml > /tmp/logs/ansible/generate_config tools/kolla-ansible -i ${RAW_INVENTORY} bootstrap-servers > /tmp/logs/ansible/bootstrap-servers -sudo tools/generate_passwords.py prepare_images if echo $ACTION | grep -q "ceph"; then ansible-playbook -i ${RAW_INVENTORY} tests/ansible_setup_ceph_disks.yml > /tmp/logs/ansible/setup_ceph_disks fi -trap get_logs EXIT - # Create dummy interface for neutron ansible -m shell -i ${RAW_INVENTORY} -a "ip l a fake_interface type dummy" all @@ -252,7 +189,5 @@ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade # run prechecks again tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2 -get_logs - ara generate html /tmp/logs/playbook_reports/ gzip --recursive --best /tmp/logs/playbook_reports/