Add upgrade support for Vitrage

1. Devstack plugin now supports upgrade
2. Grenade job is added based on legacy dsvm job

Additional tests for resourses will be added in a following patch.

Task: #22494
Story: #2002676
Change-Id: I93cb85d66cbae9fb19a275a66a9f8dff8e935f3c

.zuul.yaml

@@ -11,14 +11,15 @@
         - vitrage-dsvm-mock-py35:
             branches: ^(?!stable/(newton|ocata|pike)).*$
         - openstack-tox-lower-constraints
+        - grenade-vitrage
     gate:
       jobs:
         - vitrage-dsvm-api-py27
         - vitrage-dsvm-datasources-py27
         - vitrage-dsvm-api-py35
         - vitrage-dsvm-datasources-py35
-
         - openstack-tox-lower-constraints
+        - grenade-vitrage
 - job:
     name: vitrage-dsvm-api-py27
     parent: legacy-dsvm-base
@@ -124,3 +125,23 @@
       - ^.*\.rst$
       - ^doc/.*$
       - ^releasenotes/.*$
+
+- job:
+    name: grenade-vitrage
+    parent: legacy-dsvm-base
+    run: playbooks/legacy/vitrage-grenade/run.yaml
+    post-run: playbooks/legacy/vitrage-grenade/post.yaml
+    timeout: 7800
+    required-projects:
+      - openstack-dev/grenade
+      - openstack-infra/devstack-gate
+      - openstack/vitrage
+      - openstack/python-vitrageclient
+    irrelevant-files:
+      - ^(test-|)requirements.txt$
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^setup.cfg$
+    vars:
+      branch_override: default

devstack/plugin.sh

@@ -14,7 +14,8 @@
 # --------
 GITDIR["python-vitrageclient"]=$DEST/python-vitrageclient
 GITREPO["python-vitrageclient"]=${VITRAGECLIENT_REPO:-${GIT_BASE}/openstack/python-vitrageclient.git}
-GITBRANCH["python-vitrageclient"]=${VITRAGECLIENT_BRANCH:-master}
+GITBRANCH["vitrage"]=${VITRAGE_BRANCH:-$TARGET_BRANCH}
+GITBRANCH["python-vitrageclient"]=${VITRAGECLIENT_BRANCH:-$TARGET_BRANCH}
 
 # Support potential entry-points console scripts in VENV or not
 if [[ ${USE_VENV} = True ]]; then

devstack/upgrade/settings

@@ -0,0 +1,13 @@
+register_project_for_upgrade vitrage
+register_db_to_save vitrage
+
+# Duplicate some settings from devstack. Use old devstack as we install base
+# environment from it.
+VITRAGE_BASE_DEVSTACK_DIR=$TOP_DIR/../../old/vitrage/devstack
+VITRAGE_DIR=$TOP_DIR/../../new/vitrage
+source $VITRAGE_BASE_DEVSTACK_DIR/settings
+
+devstack_localrc base enable_service vitrage-api vitrage-collector vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing
+devstack_localrc target enable_service vitrage-api vitrage-collector vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing
+BASE_RUN_SMOKE=False
+TARGET_RUN_SMOKE=False

devstack/upgrade/shutdhown.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+set -o errexit
+
+source $GRENADE_DIR/grenaderc
+source $GRENADE_DIR/functions
+
+# We need base DevStack functions for this
+source $BASE_DEVSTACK_DIR/functions
+source $BASE_DEVSTACK_DIR/stackrc # needed for status directory
+source $BASE_DEVSTACK_DIR/lib/tls
+source $BASE_DEVSTACK_DIR/lib/apache
+
+VITRAGE_DEVSTACK_DIR=$(dirname $(dirname $0))
+
+set -o xtrace
+
+stop_vitrage
+
+SERVICES_DOWN="vitrage-api vitrage-collector vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing"
+
+# sanity check that services are actually down
+ensure_services_stopped $SERVICES_DOWN

devstack/upgrade/upgrade.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# ``upgrade-vitrage``
+
+echo "*********************************************************************"
+echo "Begin $0"
+echo "*********************************************************************"
+
+# Clean up any resources that may be in use
+cleanup() {
+    set +o errexit
+
+    echo "*********************************************************************"
+    echo "ERROR: Abort $0" >&2
+    echo "*********************************************************************"
+
+    # Kill ourselves to signal any calling process
+    trap 2; kill -2 $$
+}
+
+trap cleanup SIGHUP SIGINT SIGTERM
+
+# Keep track of the grenade directory
+RUN_DIR=$(cd $(dirname "$0") && pwd)
+
+# Source params
+source $GRENADE_DIR/grenaderc
+
+# Import common functions
+source $GRENADE_DIR/functions
+
+# This script exits on an error so that errors don't compound and you see
+# only the first error that occurred.
+set -o errexit
+
+# Upgrade Vitrage
+# ===============
+
+# Locate vitrage devstack plugin, the directory above the
+# grenade plugin.
+VITRAGE_DEVSTACK_DIR=$(dirname $(dirname $0))
+VITRAGE_DIR=$(dirname $(dirname $0))/../../vitrage
+VITRAGE_CONF_DIR=/etc/vitrage
+VITRAGE_CONF=$VITRAGE_CONF_DIR/vitrage.conf
+# Duplicate some setup bits from target DevStack
+source $TARGET_DEVSTACK_DIR/functions
+source $TARGET_DEVSTACK_DIR/stackrc
+source $TARGET_DEVSTACK_DIR/lib/tls
+source $TARGET_DEVSTACK_DIR/lib/stack
+source $TARGET_DEVSTACK_DIR/lib/apache
+source $TARGET_DEVSTACK_DIR/lib/rpc_backend
+
+# Print the commands being run so that we can see the command that triggers
+# an error.  It is also useful for following allowing as the install occurs.
+set -o xtrace
+
+# Save current config files for posterity
+[[ -d $SAVE_DIR/etc.vitrage ]] || cp -pr $VITRAGE_CONF_DIR $SAVE_DIR/etc.vitrage
+
+# Install the target vitrage
+source $VITRAGE_DEVSTACK_DIR/plugin.sh stack install
+
+# calls upgrade-vitrage for specific release
+upgrade_project vitrage $RUN_DIR $BASE_DEVSTACK_BRANCH $TARGET_DEVSTACK_BRANCH
+
+# Simulate init_vitrage()
+VITRAGE_BIN_DIR=$(dirname $(which vitrage-dbsync))
+$VITRAGE_BIN_DIR/vitrage-dbsync --config-file $VITRAGE_CONF || die $LINENO "DB sync error"
+
+# Start Vitrage
+start_vitrage
+
+# Don't succeed unless the services come up
+# Truncating some service names to 11 characters
+ensure_services_started vitrage-api vitrage-collector vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing
+
+set +o xtrace
+echo "*********************************************************************"
+echo "SUCCESS: End $0"
+echo "*********************************************************************"

playbooks/legacy/vitrage-grenade/post.yaml

@@ -0,0 +1,16 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
+

playbooks/legacy/vitrage-grenade/run.yaml

@@ -0,0 +1,58 @@
+- hosts: all
+  name: job for grenade-vitrage
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat << 'EOF' >>"/tmp/dg-local.conf"
+          [[local|localrc]]
+          enable_plugin vitrage git://git.openstack.org/openstack/vitrage
+          # TEMPEST_PLUGINS+=' ../'
+          EOF
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PROJECTS="openstack-dev/grenade $PROJECTS"
+          export PROJECTS="openstack/vitrage $PROJECTS"
+          export PROJECTS="openstack/python-vitrageclient $PROJECTS"
+          export PYTHONUNBUFFERED=true
+          export GRENADE_PLUGINRC="enable_grenade_plugin vitrage https://git.openstack.org/openstack/vitrage"
+          export DEVSTACK_GATE_TEMPEST=1
+          export DEVSTACK_GATE_TEMPEST_NOTESTS=1
+          export DEVSTACK_GATE_GRENADE=pullup
+          if [ "{{ branch_override }}" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'