First version 3rd party CI configuration

Change-Id: I54d05577089a0d7accffe55edae35eca42ed54b5

jenkins-tp/conf/jenkins_job.ini.example

@@ -0,0 +1,9 @@
+[jenkins]
+user=USER
+password=APIKEY
+url=https://jenkins-tp.ng.mirantis.net
+query_plugins_info=False
+
+[job_builder]
+ignore_cache=True
+keep_descriptions=False

jenkins-tp/jobs/build-job-template.yaml

@@ -0,0 +1,18 @@
+#
+# Template used to execute tox tests
+#
+- job-template:
+    name: test-build-{component}
+    builders:
+      - get-from-zuul-container
+      - build-image:
+          component: '{component}'
+          repos: '{repos}'
+    concurrent: true
+    description: |
+      Test build {component} image job
+    node: standard
+    wrappers:
+      - ng-cleanup
+    triggers:
+      - zuul

jenkins-tp/jobs/build-job-with-tox-template.yaml

@@ -0,0 +1,20 @@
+#
+# Template used to execute tox tests
+#
+- job-template:
+    name: test-build-with-tox-{component}
+    builders:
+      - get-from-zuul-container
+      - build-image:
+          component: '{component}'
+          repos: '{repos}'
+      - check-tox:
+          tox-test: '{tox-test}'
+    concurrent: true
+    description: |
+      Test build {component} image job
+    node: standard
+    wrappers:
+      - ng-cleanup
+    triggers:
+      - zuul

jenkins-tp/jobs/builders/fuel-ccp-ext-config-gate.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -xe
+
+# run pep8, py27 and py34 checks
+tox

jenkins-tp/jobs/builders/fuel-ccp-gate.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -xe
+
+virtualenv microenv
+
+source microenv/bin/activate
+
+pip install .
+
+mcp-microservices --images-maintainer mirantis@mirantis.com \
+    --images-namespace microbuild --images-tag latest \
+    --repositories-path containers/nextgen \
+build
+
+deactivate
+

jenkins-tp/jobs/builders/fuel-ccp-installer-gate.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -xe
+
+# for now it alwyas exits with 0
+exit 0

jenkins-tp/jobs/builders/fuel-ccp-mariadb-gate.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -xe
+
+virtualenv mariadbenv
+
+source mariadbenv/bin/activate
+
+pip install fuel-ccp/
+
+mcp-microservices --images-maintainer mirantis@mirantis.com \
+    --images-namespace mariadbbuild --images-tag latest \
+    --repositories-names {repos} \
+    --repositories-path containers/nextgen \
+    build -c mariadb
+
+deactivate
+
+pushd containers/nextgen/ms-mariadb
+tox -e py27
+popd

jenkins-tp/jobs/builders/fuel-ccp-memcached-gate.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -xe
+
+virtualenv memcachedenv
+
+source memcachedenv/bin/activate
+
+pip install fuel-ccp/
+
+mcp-microservices --images-maintainer mirantis@mirantis.com \
+    --images-namespace memcachedbuild --images-tag latest \
+    --repositories-names {repos} \
+    --repositories-path containers/nextgen \
+    build -c memcached
+
+deactivate
+
+pushd containers/nextgen/ms-memcached
+tox -e py27
+popd

jenkins-tp/jobs/builders/fuel-ccp-qa-gate.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -xe
+
+export PATH=/bin:/usr/bin:/sbin:/usr/sbin:${PATH}
+
+function check_venv_exists() {
+  VIRTUAL_ENV=mcp-qa-venv
+
+  if [ -f ${VIRTUAL_ENV}/bin/activate ]; then
+    source ${VIRTUAL_ENV}/bin/activate
+    echo "Python virtual env exist"
+  else
+    rm -rf ${VIRTUAL_ENV}
+    virtualenv  --no-site-packages ${VIRTUAL_ENV}
+    source ${VIRTUAL_ENV}/bin/activate
+  fi
+}
+
+check_venv_exists
+
+pip install -r mcp_tests/requirements.txt
+
+py.test -k mysql_is_running
+
+deactivate

jenkins-tp/jobs/builders/mcp-build-images.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -ex
+
+ls -al
+
+virtualenv mcp
+source mcp/bin/activate
+pip install .
+
+mcp-microservices \
+--images-maintainer mos-microservices@mirantis.com \
+--builder-push \
+--auth-registry \
+--builder-registry registry01-bud.ng.mirantis.net \
+--auth-registry-username $REG_USER \
+--auth-registry-password $REG_PASS \
+--images-namespace nextgen \
+--images-tag latest \
+--repositories-path microservices-repos \
+build
+
+deactivate

jenkins-tp/jobs/builders/update-jenkins-post-job.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -xe
+
+
+echo "This is job for update jenkins jobs after merge"
+
+git clone https://git.openstack.org/$ZUUL_PROJECT .
+
+cat > jenkins_jobs.ini << EOF
+[jenkins]
+user=${JJB_USER}
+password=${JJB_PASS}
+url=https://jenkins-tp.ng.mirantis.net/
+query_plugins_info=False
+[job_builder]
+ignore_cache=True
+recursive=True
+EOF
+
+tox -e fuel-ccp-ci -- jenkins-jobs --flush-cache --conf jenkins_jobs.ini update \
+	                           --delete-old jenkins-tp/jobs
+

jenkins-tp/jobs/fuel-ccp-docs-build-jobs.yaml

@@ -0,0 +1,51 @@
+#
+# Template for job executed manually
+#
+- job-template:
+    name: build-all-docs-manual-job-{repo}
+    description: |
+      Docs build job executed manually
+    node: docs
+    builders:
+      - shell:
+          !include-raw 'builders/ng-build-specs.sh'
+    concurrent: true
+    wrappers:
+      - ng-cleanup
+      - fuel-ccp-ci-docs
+    scm:
+      - fuel-infra:
+          scm-basedir: ''
+          scm-branch: '{branch}'
+          scm-repo: 'nextgen/{repo}'
+#
+# Template for triggered job
+#
+- job-template:
+    name: build-all-docs-triggered-job-{repo}
+    description: |
+      Docs build job triggered
+    node: docs
+    builders:
+      - shell: |
+          git clone https://git.openstack.org/openstack/$ZUUL_PROJECT .
+          git fetch $ZUUL_URL/$ZUUL_PROJECT $ZUUL_REF
+          git checkout FETCH_HEAD
+      - shell:
+          !include-raw 'builders/ng-build-specs.sh'
+    concurrent: true
+    wrappers:
+      - ng-cleanup
+      - fuel-ccp-ci-docs
+    triggers:
+      - zuul
+
+
+- project:
+    name: fuel-ccp-docs-build-jobs
+    jobs:
+       - 'build-all-docs-manual-job-{repo}'
+       - 'build-all-docs-triggered-job-{repo}'
+    repo:
+       - 'fuel-ccp-specs':
+           branch: 'master'

jenkins-tp/jobs/fuel-ccp-gate-jobs.yaml

@@ -0,0 +1,55 @@
+- job:
+    name: fuel-ccp-gate-job
+    description: |
+      Fuel CCP gate job
+    node: standard
+    builders:
+      - get-from-zuul
+      - shell: !include-raw builders/fuel-ccp-gate.sh
+    concurrent: true
+    wrappers:
+      - ng-cleanup
+    triggers:
+      - zuul
+
+- job:
+    name: fuel-ccp-ms-ext-config-gate-job
+    description: |
+      Some gate fuel-ccp-ms-ext-config test job
+    node: standard
+    builders:
+      - get-from-zuul
+      - shell: !include-raw builders/fuel-ccp-ext-config-gate.sh
+    concurrent: true
+    wrappers:
+      - ng-cleanup
+    triggers:
+      - zuul
+
+- job:
+    name: fuel-ccp-installer
+    description: |
+      Fuel CCP Installer gate job
+    node: standard
+    builders:
+      - get-from-zuul
+      - shell: !include-raw builders/fuel-ccp-installer-gate.sh
+    concurrent: true
+    wrappers:
+      - ng-cleanup
+    triggers:
+      - zuul
+
+- job:
+    name: fuel-ccp-mcp-qa
+    description: |
+      Gate mcp-qa test job
+    node: standard
+    builders:
+      - get-from-zuul
+      - shell: !include-raw builders/fuel-ccp-qa-gate.sh
+    concurrent: true
+    wrappers:
+      - ng-cleanup
+    triggers:
+      - zuul

jenkins-tp/jobs/fuel-ccp-post-jobs.yaml

@@ -0,0 +1,15 @@
+- job:
+    name: update-jenkins-jobs
+    description: |
+      Update jenkins jobs after merge
+    node: standard
+    builders:
+      - shell: !include-raw builders/update-jenkins-post-job.sh
+    concurrent: true
+    wrappers:
+      - ng-cleanup
+      - inject-passwords:
+          global: true
+    triggers:
+      - zuul
+

jenkins-tp/jobs/macros.yaml

@@ -0,0 +1,143 @@
+#
+# Configurable repository sources
+#
+- scm:
+    name: mcpinstaller
+    scm:
+      - git:
+         url: 'ssh://nextgen-ci@review.fuel-infra.org:29418/{scm-repo}'
+         branches:
+          - '{scm-branch}'
+         basedir: '{scm-basedir}'
+
+- scm:
+    name: openstack
+    scm:
+      - git:
+         url: 'https://git.openstack.org/openstack/{scm-repo}.git'
+         branches:
+          - '{scm-branch}'
+         basedir: '{scm-basedir}'
+
+- scm:
+    name: fuel-infra
+    scm:
+      - git:
+          url: 'ssh://nextgen-ci@review.fuel-infra.org:29418/{scm-repo}.git'
+          basedir: '{scm-basedir}'
+          skip-tag: true
+          branches:
+            - '{scm-branch}'
+
+
+- scm:
+    name: zuul_repo
+    scm:
+      - git:
+          url: $ZUUL_URL/$ZUUL_PROJECT
+          branches:
+            - $ZUUL_BRANCH
+
+#
+# Repository source with gerrit changes
+#
+- scm:
+    name: zuul_repo_trigger
+    scm:
+      - git:
+          basedir: '{scm-basedir}'
+          branches:
+            - $ZUUL_BRANCH
+          remotes:
+            - gerrit:
+                refspec: $ZUUL_REF
+                url: $ZUUL_URL/$ZUUL_PROJECT
+#
+# Static repositories
+#
+
+
+#
+# SSH credentials
+#
+- wrapper:
+    name: fuel-ccp-ci-jenkins
+    wrappers:
+      - ssh-agent-credentials:
+          users:
+            - '77acc946-f346-4665-8271-c141b6ae9764'
+
+- wrapper:
+    name: fuel-ccp-ci-docs
+    wrappers:
+      - ssh-agent-credentials:
+          users:
+            - '04a7dbe6-f3cd-4cc1-afc5-b8f2bd6cbb38'
+
+- wrapper:
+    name: 'ng-cleanup'
+    wrappers:
+      - workspace-cleanup
+
+#
+# Publishers
+#
+#TODO: check target fuel-ccp-ci
+- publisher:
+    name: logs-public
+    publishers:
+      - ssh:
+          site: 'ci-logs.fuel-infra.org'
+          target: 'fuel-ccp-ci/$JOB_NAME/$BUILD_ID'
+          source: 'logs/*'
+          flatten: true
+
+- publisher:
+    name: logs-private
+    publishers:
+      - archive:
+          allow-empty: true
+          artifacts: 'logs/*'
+          latest-only: false
+
+#
+# Builders
+#
+- builder:
+    name: check-tox
+    builders:
+      - shell: 'tox -v -e {tox-test}'
+
+- builder:
+    name: build-image
+    builders:
+      - shell: |
+          virtualenv {component}env
+          source {component}env/bin/activate
+          pip install fuel-ccp/
+          mcp-microservices --images-maintainer mirantis@mirantis.com \
+              --images-namespace {component}build --images-tag latest \
+              --repositories-names {repos} \
+              --repositories-path containers/nextgen \
+              build -c {component}
+          deactivate
+
+- builder:
+    name: get-from-zuul-container
+    builders:
+      - shell: |
+          git clone https://git.openstack.org/openstack/fuel-ccp
+          mkdir -p containers/$ZUUL_PROJECT
+          cd containers/$ZUUL_PROJECT
+          git clone https://git.openstack.org/$ZUUL_PROJECT .
+          git fetch $ZUUL_URL/$ZUUL_PROJECT $ZUUL_REF
+          git checkout FETCH_HEAD
+
+- builder:
+    name: get-from-zuul
+    builders:
+      - shell: |
+          git clone https://git.openstack.org/$ZUUL_PROJECT .
+          git fetch $ZUUL_URL/$ZUUL_PROJECT $ZUUL_REF
+          git checkout FETCH_HEAD
+          git status

jenkins-tp/jobs/mcp-build-images.yaml

@@ -0,0 +1,21 @@
+- job-template:
+    name: mcp-build-images
+    description: |
+      Build MCP Docker images and store them into Docker registry.
+    node: standard
+    builders:
+      - shell:
+          !include-raw 'builders/mcp-build-images.sh'
+    concurrent: false
+    scm:
+      - openstack:
+          scm-basedir: ''
+          scm-branch: 'master'
+          scm-repo: 'openstack/fuel-ccp'
+    wrappers:
+      - ng-cleanup
+
+- project:
+    name: Build MCP Docker images and store them into Docker registry.
+    jobs:
+       - 'mcp-build-images'

jenkins-tp/jobs/projects.yaml

@@ -0,0 +1,65 @@
+- project:
+    name: fuel-ccp-debian-base
+    jobs:
+      - test-build-{component}:
+          component: debianbase
+          repos: fuel-ccp-debian-base
+
+- project:
+    name: fuel-ccp-glance
+    jobs:
+      - test-build-{component}:
+          component: glance
+          repos: fuel-ccp-debian-base,fuel-ccp-openstack-base,fuel-ccp-glance
+
+- project:
+    name: fuel-ccp-horizon
+    jobs:
+      - test-build-{component}:
+          component: horizon
+          repos: fuel-ccp-debian-base,fuel-ccp-openstack-base,fuel-ccp-horizon
+
+- project:
+    name: fuel-ccp-keystone
+    jobs:
+      - test-build-{component}:
+          component: keystone
+          repos: fuel-ccp-debian-base,fuel-ccp-openstack-base,fuel-ccp-keystone
+
+- project:
+    name: fuel-ccp-memcached
+    jobs:
+      - test-build-{component}:
+          component: memcached
+          repos: fuel-ccp-debian-base,fuel-ccp-memcached
+          tox-test: py27
+
+- project:
+    name: fuel-ccp-mariadb
+    jobs:
+      - test-build-{component}:
+          component: mariadb
+          repos: fuel-ccp-debian-base,fuel-ccp-mariadb
+          tox-test: py27
+
+- project:
+    name: fuel-ccp-neutron
+    jobs:
+      - test-build-{component}:
+          component: neutron
+          repos: fuel-ccp-debian-base,fuel-ccp-openstack-base,fuel-ccp-neutron
+
+- project:
+    name: fuel-ccp-openstack-base
+    jobs:
+      - test-build-{component}:
+          component: openstackbase
+          repos: fuel-ccp-debian-base,fuel-ccp-openstack-base
+
+- project:
+    name: fuel-ccp-rabbitmq
+    jobs:
+      - test-build-{component}:
+          component: rabbitmq
+          repos: fuel-ccp-debian-base,fuel-ccp-rabbitmq
+

jenkins-tp/jobs/tox-job-template.yaml

@@ -0,0 +1,21 @@
+#
+# Template used to execute tox tests
+#
+- job-template:
+    name: verify-tox-{tox-test}
+    builders:
+      - shell: |
+          env | grep ZUUL
+          git clone https://git.openstack.org/$ZUUL_PROJECT .
+          git fetch $ZUUL_URL/$ZUUL_PROJECT $ZUUL_REF
+          git checkout FETCH_HEAD
+      - check-tox:
+          tox-test: '{tox-test}'
+    concurrent: true
+    description: |
+      Tox {tox-test} syntax test
+    node: standard
+    wrappers:
+      - ng-cleanup
+    triggers:
+      - zuul

jenkins-tp/jobs/verify-ci.yaml

@@ -0,0 +1,45 @@
+#
+# Jobs used to verify syntax in CI repository
+#
+
+# Shell
+- job-template:
+    name: verify-bashate-{repo}
+    builders:
+      - get-from-zuul
+      - check-tox:
+          tox-test: bashate
+    concurrent: true
+    description: |
+      Bashate syntax test
+    node: standard
+    wrappers:
+      - ng-cleanup
+
+# JJB
+- job-template:
+    name: verify-jjb-{repo}
+    builders:
+      - get-from-zuul
+      - check-tox:
+          tox-test: nextgen-ci
+    concurrent: true
+    description: |
+      JJB syntax test
+    node: standard
+    wrappers:
+      - ng-cleanup
+
+- project:
+    name: verify-ci-bashate
+    jobs:
+       - 'verify-bashate-{repo}'
+    repo:
+       - 'project-config'
+
+- project:
+    name: verify-ci-jjb
+    jobs:
+       - 'verify-jjb-{repo}'
+    repo:
+       - 'project-config'

zuul-tp/layout.yaml

@@ -0,0 +1,185 @@
+pipelines:
+  - name: check
+    description: Newly uploaded patchsets enter this pipeline to receive an initial +/-1 Verified vote from Jenkins.
+    success-message: Build succeeded (check pipeline).
+    failure-message: Build failed (check pipeline).  For information on how to proceed, see http://docs.openstack.org/infra/manual/developers.html#automated-testing
+    manager: IndependentPipelineManager
+    source: gerrit
+    precedence: normal
+    require:
+      open: True
+      current-patchset: True
+    trigger:
+      gerrit:
+        - event: patchset-created
+        - event: change-restored
+        - event: comment-added
+          comment: (?i)^(Patch Set [0-9]+:)?( [\w\\+-]*)*(\n\n)?\s*(recheck|reverify)
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: 1
+
+  - name: gate
+    description: Changes that have been approved by core developers are enqueued in order in this pipeline, and if they pass tests in Jenkins, will be merged.
+    success-message: Build succeeded (gate pipeline).
+    failure-message: Build failed (gate pipeline).  For information on how to proceed, see http://docs.openstack.org/infra/manual/developers.html#automated-testing
+    manager: DependentPipelineManager
+    source: gerrit
+    precedence: high
+    require:
+      open: True
+      current-patchset: True
+      approval:
+        - verified: [1, 2]
+          username: mirantis-fuel-ccp-ci
+        - workflow: 1
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - workflow: 1
+        - event: comment-added
+          approval:
+            - verified: 1
+          username: mirantis-fuel-ccp-ci
+    start:
+      gerrit:
+        verified: 0
+    success:
+      gerrit:
+        verified: 2
+        submit: true
+    failure:
+      gerrit:
+        verified: -2
+    window-floor: 20
+    window-increase-factor: 2
+
+  - name: post
+    description: This pipeline runs jobs that operate after each change is merged.
+    manager: IndependentPipelineManager
+    source: gerrit
+    precedence: low
+    trigger:
+      gerrit:
+        - event: ref-updated
+          ref: ^(?!refs/).*$
+
+
+
+project-templates:
+  - name: verify-tox
+    check:
+      - 'verify-tox-pep8'
+      - 'verify-tox-py27'
+
+projects:
+  - name: openstack-dev/ci-sandbox
+    check:
+      - verify-tox-py27
+      - verify-tox-pep8
+
+  - name: openstack/fuel-ccp-nextgen-specs
+    check:
+      - verify-tox-pep8
+
+  - name: openstack/fuel-ccp
+    check:
+      - fuel-ccp-gate-job
+    gate:
+      - fuel-ccp-gate-job
+
+  - name: openstack/fuel-ccp-debian-base
+    check:
+      - test-build-debianbase
+    gate:
+      - test-build-debianbase
+
+  - name: openstack/fuel-ccp-ext-config
+    check:
+      - fuel-ccp-ext-config-gate-job
+    gate:
+      - fuel-ccp-ext-config-gate-job
+
+  - name: openstack/fuel-ccp-glance
+    check:
+      - test-build-glance
+    gate:
+      - test-build-glance
+
+  - name: openstack/fuel-ccp-horizon
+    check:
+      - test-build-horizon
+    gate:
+      - test-build-horizon
+
+  - name: openstack/fuel-ccp-keystone
+    check:
+      - test-build-keystone
+    gate:
+      - test-build-keystone
+
+  - name: openstack/fuel-ccp-mariadb
+    check:
+      - test-build-mariadb
+    gate:
+      - test-build-mariadb
+
+  - name: openstack/fuel-ccp-memcached
+    check:
+      - test-build-memcached
+    gate:
+      - test-build-memcached
+
+  - name: openstack/fuel-ccp-neutron
+    check:
+      - test-build-neutron
+    gate:
+      - test-build-neutron
+
+  - name: openstack/fuel-ccp-nova
+    check:
+      - test-build-nova
+    gate:
+      - test-build-nova
+
+  - name: openstack/fuel-ccp-openstack-base
+    check:
+      - test-build-openstackbase
+    gate:
+      - test-build-openstackbase
+
+  - name: openstack/fuel-ccp-rabbitmq
+    check:
+      - test-build-rabbitmq
+
+  - name: openstack/fuel-ccp-installer
+    check:
+      - fuel-ccp-installer
+    gate:
+      - fuel-ccp-installer
+
+  - name: openstack/fuel-ccp-tests
+    check:
+      - fuel-ccp-mcp-qa
+    gate:
+      - fuel-ccp-mcp-qa
+
+  - name: openstack/fuel-ccp-ci-config
+    check:
+      - verify-jjb-project-config
+    gate:
+      - verify-jjb-project-config
+    post:
+      - update-jenkins-jobs
+
+jobs:
+  - name: ^fuel.*$
+    voting: false
+  - name: ^test.*$
+    voting: false
+  - name: ^verify-jjb.*$
+    voting: false