From 7760676d94a2e44a4f8b9be1eaf3cb337ba779f1 Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Fri, 20 Oct 2017 11:12:47 -0400 Subject: [PATCH] Migrate legacy jobs for feature/zuulv3 branch The nodepool-functional-py35 and nodepool-functional-py35-src jobs are copied directly from the master branch since there is currently a zuulv3 bug preventing us from using those jobs in this branch. Change-Id: If8636dc36dbab145ef976ce3dd38dd3b593ef1b1 Depends-On: I22007434b38129379690f4e469a1981ed7dcb68c --- .zuul.yaml | 44 +++++++ .../nodepool-functional-py35-src/post.yaml | 15 +++ .../nodepool-functional-py35-src/run.yaml | 121 ++++++++++++++++++ playbooks/nodepool-functional-py35/post.yaml | 15 +++ playbooks/nodepool-functional-py35/run.yaml | 85 ++++++++++++ playbooks/nodepool-zuul-functional/post.yaml | 80 ++++++++++++ playbooks/nodepool-zuul-functional/run.yaml | 52 ++++++++ 7 files changed, 412 insertions(+) create mode 100644 playbooks/nodepool-functional-py35-src/post.yaml create mode 100644 playbooks/nodepool-functional-py35-src/run.yaml create mode 100644 playbooks/nodepool-functional-py35/post.yaml create mode 100644 playbooks/nodepool-functional-py35/run.yaml create mode 100644 playbooks/nodepool-zuul-functional/post.yaml create mode 100644 playbooks/nodepool-zuul-functional/run.yaml diff --git a/.zuul.yaml b/.zuul.yaml index 435e98e5a..1ca6ba59c 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,3 +1,36 @@ +- job: + name: nodepool-functional-py35 + parent: legacy-dsvm-base + run: playbooks/nodepool-functional-py35/run + post-run: playbooks/nodepool-functional-py35/post + timeout: 5400 + required-projects: + - openstack-infra/devstack-gate + - openstack-infra/nodepool + +- job: + name: nodepool-functional-py35-src + parent: legacy-dsvm-base + run: playbooks/nodepool-functional-py35-src/run + post-run: playbooks/nodepool-functional-py35-src/post + timeout: 5400 + required-projects: + - openstack-infra/devstack-gate + - openstack-infra/glean + - openstack-infra/nodepool + - openstack-infra/shade + - openstack/diskimage-builder + +- job: + name: nodepool-zuul-functional + parent: legacy-base + run: playbooks/nodepool-zuul-functional/run + post-run: playbooks/nodepool-zuul-functional/post + timeout: 1800 + required-projects: + - openstack-infra/nodepool + - openstack-infra/zuul + - project: name: openstack-infra/nodepool check: @@ -6,3 +39,14 @@ - tox-cover - tox-pep8 - tox-py35 + - nodepool-functional-py35: + voting: false + - nodepool-functional-py35-src: + voting: false + gate: + jobs: + - tox-pep8 + - tox-py35 + experimental: + jobs: + - nodepool-zuul-functional diff --git a/playbooks/nodepool-functional-py35-src/post.yaml b/playbooks/nodepool-functional-py35-src/post.yaml new file mode 100644 index 000000000..e07f5510a --- /dev/null +++ b/playbooks/nodepool-functional-py35-src/post.yaml @@ -0,0 +1,15 @@ +- hosts: primary + tasks: + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/logs/** + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff --git a/playbooks/nodepool-functional-py35-src/run.yaml b/playbooks/nodepool-functional-py35-src/run.yaml new file mode 100644 index 000000000..812532ee1 --- /dev/null +++ b/playbooks/nodepool-functional-py35-src/run.yaml @@ -0,0 +1,121 @@ +- hosts: all + name: Autoconverted job legacy-dsvm-nodepool-py35-src from old job gate-dsvm-nodepool-py35-src-nv + tasks: + + - name: Ensure legacy workspace directory + file: + path: '{{ ansible_user_dir }}/workspace' + state: directory + + - shell: + cmd: | + set -e + set -x + cat > clonemap.yaml << EOF + clonemap: + - name: openstack-infra/devstack-gate + dest: devstack-gate + EOF + /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ + git://git.openstack.org \ + openstack-infra/devstack-gate + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + set -e + set -x + export PYTHONUNBUFFERED=true + # Disable tempest as nodepool is talking to the cloud not tempest. + export DEVSTACK_GATE_TEMPEST=0 + # Use neutron as the public clouds in use are neutron based. + export DEVSTACK_GATE_NEUTRON=1 + # The nodepool process needs sudo rights in order to + # perform dib image builds + export DEVSTACK_GATE_REMOVE_STACK_SUDO=0 + # Disable services we do not need for nodepool + export DEVSTACK_LOCAL_CONFIG="disable_service horizon" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acentral" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acompute" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-evaluator" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-notifier" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-anotification" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-api" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-collector" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service cinder" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-bak" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-sch" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-api" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-vol" + + if [ "-py35" == "-py35" ]; then + export DEVSTACK_GATE_USE_PYTHON3=True + + # swift is not ready for python3 yet + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-account" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-container" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-object" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-proxy" + fi + + export BRANCH_OVERRIDE=default + if [ "$BRANCH_OVERRIDE" != "default" ] ; then + export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE + fi + # Because we are testing a non standard project, add the + # our project repository. This makes zuul do the right + # reference magic for testing changes. + export PROJECTS="openstack-infra/nodepool $PROJECTS" + # note the actual url here is somewhat irrelevant because it + # caches in nodepool, however make it a valid url for + # documentation purposes. + export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin nodepool git://git.openstack.org/openstack-infra/nodepool" + export DEVSTACK_PROJECT_FROM_GIT="diskimage-builder" + export DEVSTACK_PROJECT_FROM_GIT+=",glean" + export DEVSTACK_PROJECT_FROM_GIT+=",shade" + + # Start with all images disabled. + export NODEPOOL_PAUSE_CENTOS_7_DIB=true + export NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB=true + export NODEPOOL_PAUSE_FEDORA_26_DIB=true + export NODEPOOL_PAUSE_OPENSUSE_423_DIB=true + export NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=true + export NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB=true + + if [ "" == "" ] ; then + # dsvm-nodepool-src + export NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=false + elif [ "" == "-debian" ] ; then + # dsvm-nodepool-debian-src + export NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB=false + elif [ "" == "-opensuse" ] ; then + # dsvm-nodepool-opensuse-src + export NODEPOOL_PAUSE_OPENSUSE_423_DIB=false + elif [ "" == "-redhat" ] ; then + # dsvm-nodepool-redhat-src + export NODEPOOL_PAUSE_CENTOS_7_DIB=false + export NODEPOOL_PAUSE_FEDORA_26_DIB=false + elif [ "" == "-ubuntu" ] ; then + # dsvm-nodepool-ubuntu-src + export NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=false + export NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB=false + fi + export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_CENTOS_7_DIB=$NODEPOOL_PAUSE_CENTOS_7_DIB" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB=$NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_FEDORA_26_DIB=$NODEPOOL_PAUSE_FEDORA_26_DIB" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_OPENSUSE_423_DIB=$NODEPOOL_PAUSE_OPENSUSE_423_DIB" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=$NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB=$NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB" + + function post_test_hook { + /opt/stack/new/nodepool/tools/check_devstack_plugin.sh + } + export -f post_test_hook + + cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh + ./safe-devstack-vm-gate-wrap.sh + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/nodepool-functional-py35/post.yaml b/playbooks/nodepool-functional-py35/post.yaml new file mode 100644 index 000000000..e07f5510a --- /dev/null +++ b/playbooks/nodepool-functional-py35/post.yaml @@ -0,0 +1,15 @@ +- hosts: primary + tasks: + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/logs/** + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff --git a/playbooks/nodepool-functional-py35/run.yaml b/playbooks/nodepool-functional-py35/run.yaml new file mode 100644 index 000000000..d62470167 --- /dev/null +++ b/playbooks/nodepool-functional-py35/run.yaml @@ -0,0 +1,85 @@ +- hosts: all + name: Autoconverted job legacy-dsvm-nodepool-py35 from old job gate-dsvm-nodepool-py35-nv + tasks: + + - name: Ensure legacy workspace directory + file: + path: '{{ ansible_user_dir }}/workspace' + state: directory + + - shell: + cmd: | + set -e + set -x + cat > clonemap.yaml << EOF + clonemap: + - name: openstack-infra/devstack-gate + dest: devstack-gate + EOF + /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ + git://git.openstack.org \ + openstack-infra/devstack-gate + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + set -e + set -x + export PYTHONUNBUFFERED=true + # Disable tempest as nodepool is talking to the cloud not tempest. + export DEVSTACK_GATE_TEMPEST=0 + # Use neutron as the public clouds in use are neutron based. + export DEVSTACK_GATE_NEUTRON=1 + # The nodepool process needs sudo rights in order to + # perform dib image builds + export DEVSTACK_GATE_REMOVE_STACK_SUDO=0 + # Disable services we do not need for nodepool + export DEVSTACK_LOCAL_CONFIG="disable_service horizon" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acentral" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acompute" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-evaluator" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-notifier" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-anotification" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-api" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-collector" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service cinder" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-bak" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-sch" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-api" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-vol" + + if [ "-py35" == "-py35" ]; then + export DEVSTACK_GATE_USE_PYTHON3=True + + # swift is not ready for python3 yet + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-account" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-container" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-object" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-proxy" + fi + + export BRANCH_OVERRIDE=default + if [ "$BRANCH_OVERRIDE" != "default" ] ; then + export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE + fi + # Because we are testing a non standard project, add the + # our project repository. This makes zuul do the right + # reference magic for testing changes. + export PROJECTS="openstack-infra/nodepool $PROJECTS" + # note the actual url here is somewhat irrelevant because it + # caches in nodepool, however make it a valid url for + # documentation purposes. + export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin nodepool git://git.openstack.org/openstack-infra/nodepool" + + function post_test_hook { + /opt/stack/new/nodepool/tools/check_devstack_plugin.sh + } + export -f post_test_hook + + cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh + ./safe-devstack-vm-gate-wrap.sh + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/nodepool-zuul-functional/post.yaml b/playbooks/nodepool-zuul-functional/post.yaml new file mode 100644 index 000000000..b495a60ae --- /dev/null +++ b/playbooks/nodepool-zuul-functional/post.yaml @@ -0,0 +1,80 @@ +- hosts: primary + tasks: + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*nose_results.html + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testr_results.html.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.testrepository/tmp* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testrepository.subunit.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}/tox' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.tox/*/log/* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}/logs' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/logs/** + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff --git a/playbooks/nodepool-zuul-functional/run.yaml b/playbooks/nodepool-zuul-functional/run.yaml new file mode 100644 index 000000000..201ad98b3 --- /dev/null +++ b/playbooks/nodepool-zuul-functional/run.yaml @@ -0,0 +1,52 @@ +- hosts: all + name: Autoconverted job legacy-zuul-nodepool from old job gate-zuul-nodepool + tasks: + + - name: Ensure legacy workspace directory + file: + path: '{{ ansible_user_dir }}/workspace' + state: directory + + - shell: + cmd: | + set -e + set -x + CLONEMAP=`mktemp` + function cleanup { + # In cases where zuul-cloner is aborted during a git + # clone operation, git will remove the git work tree in + # its cleanup. The work tree in these jobs is the + # workspace directory, which means that subsequent + # jenkins post-build actions can not run because the + # workspace has been removed. + # To reduce the likelyhood of this having an impact, + # recreate the workspace directory if needed + mkdir -p $WORKSPACE + rm -f $CLONEMAP + } + trap cleanup EXIT + cat > $CLONEMAP << EOF + clonemap: + - name: openstack-infra/zuul + dest: . + EOF + /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \ + git://git.openstack.org openstack-infra/zuul + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: /usr/local/jenkins/slave_scripts/install-distro-packages.sh + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: tools/nodepool-integration-setup.sh + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: /usr/local/jenkins/slave_scripts/run-tox.sh nodepool + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}'