From 0a5838fa0992b265d5fcab7a5539c906d45e67ab Mon Sep 17 00:00:00 2001 From: Denis Egorenko Date: Wed, 3 Jun 2015 18:35:36 +0300 Subject: [PATCH] Rename all plugins to full plugin version Rename short plugin names to full plugin names and add OS version to job name. Also sort by alphabetical functions-* files. Change-Id: I5155613715fd2ced99034807f14f495487d2d8ac --- config/devstack/local.sh | 30 ++--- config/{sahara => }/pip.conf | 0 config/zuul/layout.yaml | 70 +++++------ jenkins_job_builder/diskimage.yaml | 101 +++++---------- jenkins_job_builder/sahara.yaml | 121 +++++++----------- slave-scripts/dib.sh | 112 +++++++---------- slave-scripts/functions-common.sh | 196 ++++++++++++++++------------- slave-scripts/functions-dib.sh | 136 ++++++++------------ slave-scripts/gate-sahara.sh | 75 +++++------ slave-scripts/gate-ui-tests.sh | 4 +- 10 files changed, 370 insertions(+), 475 deletions(-) rename config/{sahara => }/pip.conf (100%) diff --git a/config/devstack/local.sh b/config/devstack/local.sh index c2e4a5b0..4d2c18a7 100755 --- a/config/devstack/local.sh +++ b/config/devstack/local.sh @@ -24,13 +24,14 @@ else USE_NEUTRON=false fi -VANILLA26_IMAGE_PATH=/home/ubuntu/images/sahara-vanilla-2.6.0-ubuntu-14.04.qcow2 -HDP2_IMAGE_PATH=/home/ubuntu/images/centos_6-6_hdp-2.qcow2 -CENTOS_CDH_IMAGE_PATH=/home/ubuntu/images/centos_sahara_cloudera_latest.qcow2 -UBUNTU_CDH_IMAGE_PATH=/home/ubuntu/images/ubuntu_sahara_cloudera_latest.qcow2 -SPARK_IMAGE_PATH=/home/ubuntu/images/sahara_spark_latest.qcow2 -MAPR_IMAGE_PATH=/home/ubuntu/images/ubuntu_mapr_latest.qcow2 -NATIVE_UBUNTU_IMAGE_PATH=/home/ubuntu/images/ubuntu-12.04-server-cloudimg-amd64-disk1.img +VANILLA_2_6_0_IMAGE_PATH=/home/ubuntu/images/vanilla_2.6.0_u14.qcow2 +HDP_2_0_6_IMAGE_PATH=/home/ubuntu/images/hdp_2.0.6_c6.6.qcow2 +CENTOS_CDH_5_3_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.3.0_c6.6.qcow2 +UBUNTU_CDH_5_3_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.3.0_u12.qcow2 +UBUNTU_CDH_5_4_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.4.0_u12.qcow2 +SPARK_1_0_0_IMAGE_PATH=/home/ubuntu/images/spark_1.0.0_u14.qcow2 +MAPR_4_0_2_IMAGE_PATH=/home/ubuntu/images/mapr_4.0.2_u14.qcow2 +UBUNTU_12_04_IMAGE_PATH=/home/ubuntu/images/ubuntu-12.04-server-cloudimg-amd64-disk1.img # setup ci tenant and ci users CI_TENANT_ID=$(keystone tenant-create --name ci --description 'CI tenant' | grep -w id | get_field 2) @@ -69,13 +70,14 @@ nova flavor-create --is-public true m1.small 2 1024 20 1 source $ADMIN_RCFILE ci-user ci # add images for tests -glance image-create --name ubuntu_vanilla_2.6_latest --file $VANILLA26_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.6.0'='True' --property '_sahara_tag_vanilla'='True' --property '_sahara_username'='ubuntu' -glance image-create --name sahara_hdp_2_latest --file $HDP2_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.0.6'='True' --property '_sahara_tag_hdp'='True' --property '_sahara_username'='cloud-user' -glance image-create --name centos_cdh_latest --file $CENTOS_CDH_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="cloud-user" -glance image-create --name ubuntu_cdh_latest --file $UBUNTU_CDH_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu" -glance image-create --name sahara_spark_latest --file $SPARK_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_spark'='True' --property '_sahara_tag_1.0.0'='True' --property '_sahara_username'="ubuntu" -glance image-create --name ubuntu_mapr_latest --file $MAPR_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_4.0.2.mrv2'='True' --property '_sahara_username'="ubuntu" -glance image-create --name ubuntu-test-image --file $NATIVE_UBUNTU_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true +glance image-create --name $(basename -s .qcow2 $VANILLA_2_6_0_IMAGE_PATH) --file $VANILLA_2_6_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.6.0'='True' --property '_sahara_tag_vanilla'='True' --property '_sahara_username'='ubuntu' +glance image-create --name $(basename -s .qcow2 $HDP_2_0_6_IMAGE_PATH) --file $HDP_2_0_6_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.0.6'='True' --property '_sahara_tag_hdp'='True' --property '_sahara_username'='cloud-user' +glance image-create --name $(basename -s .qcow2 $CENTOS_CDH_5_3_0_IMAGE_PATH) --file $CENTOS_CDH_5_3_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="cloud-user" +glance image-create --name $(basename -s .qcow2 $UBUNTU_CDH_5_3_0_IMAGE_PATH) --file $UBUNTU_CDH_5_3_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu" +glance image-create --name $(basename -s .qcow2 $UBUNTU_CDH_5_4_0_IMAGE_PATH) --file $UBUNTU_CDH_5_4_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.4.0'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu" +glance image-create --name $(basename -s .qcow2 $SPARK_1_0_0_IMAGE_PATH) --file $SPARK_1_0_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_spark'='True' --property '_sahara_tag_1.0.0'='True' --property '_sahara_username'="ubuntu" +glance image-create --name $(basename -s .qcow2 $MAPR_4_0_2_IMAGE_PATH) --file $MAPR_4_0_2_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_4.0.2.mrv2'='True' --property '_sahara_username'="ubuntu" +glance image-create --name ubuntu-test-image --file $UBUNTU_12_04_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true glance image-update --name ubuntu-12.04 --property '_sahara_tag_ci'='True' ubuntu-12.04-server-cloudimg-amd64-disk1 glance image-update --name ubuntu-14.04 trusty-server-cloudimg-amd64-disk1 diff --git a/config/sahara/pip.conf b/config/pip.conf similarity index 100% rename from config/sahara/pip.conf rename to config/pip.conf diff --git a/config/zuul/layout.yaml b/config/zuul/layout.yaml index 6fd01918..f6aed1e6 100644 --- a/config/zuul/layout.yaml +++ b/config/zuul/layout.yaml @@ -27,51 +27,51 @@ pipelines: projects: - name: openstack/sahara check: - - gate-sahara-neutron-heat-vanilla_2.6 - - gate-sahara-neutron-direct-spark-aio - - gate-sahara-neutron-direct-transient - - gate-sahara-neutron-heat-transient - - gate-sahara-neutron-heat-mapr - - gate-sahara-nova-direct-cdh_5.3.0_ubuntu-aio - - gate-sahara-nova-direct-cdh_5.3.0_centos-aio - - gate-sahara-nova-direct-cdh_5.4.0-aio - - gate-sahara-nova-heat-hdp_2 + - gate-sahara-neutron-heat-vanilla_2.6.0-u14 + - gate-sahara-neutron-direct-spark_1.0.0-u14-aio + - gate-sahara-neutron-direct-transient-u14 + - gate-sahara-neutron-heat-transient-u14 + - gate-sahara-neutron-heat-mapr_4.0.2.mrv2-u14 + - gate-sahara-nova-direct-cdh_5.3.0-u12-aio + - gate-sahara-nova-direct-cdh_5.3.0-c6.6-aio + - gate-sahara-nova-direct-cdh_5.4.0-u12-aio + - gate-sahara-nova-heat-hdp_2.0.6-c6.6 - gate-sahara-pep8-trunk # - tempest-sahara-tests - name: openstack/python-saharaclient check: - - gate-saharaclient-neutron-heat-vanilla_2.6 - - gate-saharaclient-neutron-direct-spark-aio - - gate-saharaclient-neutron-direct-transient - - gate-saharaclient-neutron-heat-transient - - gate-saharaclient-neutron-heat-mapr - - gate-saharaclient-nova-direct-cdh_5.3.0_ubuntu-aio - - gate-saharaclient-nova-direct-cdh_5.3.0_centos-aio - - gate-saharaclient-nova-direct-cdh_5.4.0-aio - - gate-saharaclient-nova-heat-hdp_2 + - gate-saharaclient-neutron-heat-vanilla_2.6.0-u14 + - gate-saharaclient-neutron-direct-spark_1.0.0-u14-aio + - gate-saharaclient-neutron-direct-transient-u14 + - gate-saharaclient-neutron-heat-transient-u14 + - gate-saharaclient-neutron-heat-mapr_4.0.2.mrv2-u14 + - gate-saharaclient-nova-direct-cdh_5.3.0-u12-aio + - gate-saharaclient-nova-direct-cdh_5.3.0-c6.6-aio + - gate-saharaclient-nova-direct-cdh_5.4.0-u12-aio + - gate-saharaclient-nova-heat-hdp_2.0.6-c6.6 - gate-sahara-pep8-trunk # - tempest-saharaclient-tests - name: openstack/sahara-image-elements check: - - dib-neutron-heat-vanilla_2.6-ubuntu - - dib-neutron-heat-vanilla_2.6-centos - - dib-neutron-heat-mapr - - dib-nova-direct-spark-aio - - dib-nova-heat-hdp_2 - - dib-nova-direct-cdh_5.3.0-ubuntu-aio - - dib-nova-direct-cdh_5.3.0-centos-aio - - dib-nova-direct-cdh_5.4.0-ubuntu-aio + - dib-neutron-heat-vanilla_2.6.0-u14 + - dib-neutron-heat-vanilla_2.6.0-c6.6 + - dib-neutron-heat-mapr_4.0.2.mrv2-u14 + - dib-neutron-direct-spark_1.0.0-u14-aio + - dib-nova-heat-hdp_2.0.6-c6.6 + - dib-nova-direct-cdh_5.3.0-u12-aio + - dib-nova-direct-cdh_5.3.0-c6.6-aio + - dib-nova-direct-cdh_5.4.0-u12-aio post: - - dib-neutron-heat-vanilla_2.6-ubuntu - - dib-neutron-heat-vanilla_2.6-centos - - dib-neutron-heat-mapr - - dib-nova-direct-spark-aio - - dib-nova-heat-hdp_2 - - dib-nova-direct-cdh_5.3.0-ubuntu-aio - - dib-nova-direct-cdh_5.3.0-centos-aio - - dib-nova-direct-cdh_5.4.0-ubuntu-aio + - dib-neutron-heat-vanilla_2.6.0-u14 + - dib-neutron-heat-vanilla_2.6.0-c6.6 + - dib-neutron-heat-mapr_4.0.2.mrv2-u14 + - dib-neutron-direct-spark_1.0.0-u14-aio + - dib-nova-heat-hdp_2.0.6-c6.6 + - dib-nova-direct-cdh_5.3.0-u12-aio + - dib-nova-direct-cdh_5.3.0-c6.6-aio + - dib-nova-direct-cdh_5.4.0-u12-aio - name: stackforge/sahara-ci-config check: @@ -91,7 +91,7 @@ jobs: - all-files-match-any: - ^doc/.*$ - ^.*\.rst$ - - name: ^.*-neutron-heat-mapr$ + - name: ^.*-mapr_4.0.2.mrv2-u14$ branch: master voting: false - name: ^.*-cdh_5.4.0-.*$ diff --git a/jenkins_job_builder/diskimage.yaml b/jenkins_job_builder/diskimage.yaml index 77a910f3..c4102f0b 100644 --- a/jenkins_job_builder/diskimage.yaml +++ b/jenkins_job_builder/diskimage.yaml @@ -1,101 +1,62 @@ -- job: - name: 'dib-nova-heat-hdp_2' - defaults: global - builders: - - gerrit-git-prep - - ci-config-prep - - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh hdp_2" - - properties: - - zeromq-event - - publishers: - - sahara-logs - - console-log - - trigger-cleanup - node: 'trusty-nova' - -- job: - name: 'dib-neutron-heat-mapr' - defaults: global - builders: - - gerrit-git-prep - - ci-config-prep - - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh mapr" - - properties: - - zeromq-event - - publishers: - - sahara-logs - - console-log - - trigger-cleanup - node: 'trusty-neutron' - - job-template: - name: 'dib-nova-direct-{plugin}-{os}-aio' + name: 'dib-{network}-{engine}-{plugin}-{os}-aio' defaults: global builders: - gerrit-git-prep - ci-config-prep - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh {plugin} {os}" - properties: - zeromq-event - publishers: - sahara-logs - console-log - trigger-cleanup - node: 'trusty-nova' - -- job: - name: 'dib-nova-direct-spark-aio' - defaults: global - builders: - - gerrit-git-prep - - ci-config-prep - - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh {plugin}" - - properties: - - zeromq-event - - publishers: - - sahara-logs - - console-log - - trigger-cleanup - node: 'trusty-nova' + node: 'trusty-{network}' - job-template: - name: 'dib-neutron-heat-vanilla_2.6-{os}' + name: 'dib-{network}-{engine}-{plugin}-{os}' defaults: global builders: - gerrit-git-prep - ci-config-prep - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh {plugin} {os}" - properties: - zeromq-event - publishers: - sahara-logs - console-log - trigger-cleanup - node: 'trusty-neutron' + node: 'trusty-{network}' - project: name: sahara-image-elements github-org: openstack - plugin: cdh_5.3.0 + network: neutron + engine: heat + plugin: vanilla_2.6.0 os: - - ubuntu - - centos + - u14 + - c6.6 + jobs: - - 'dib-neutron-heat-vanilla_2.6-{os}' - - 'dib-nova-heat-hdp_2' - - 'dib-neutron-heat-mapr' - - 'dib-nova-direct-spark-aio' - - 'dib-nova-direct-{plugin}-{os}-aio' - - 'dib-nova-direct-{plugin}-{os}-aio': - plugin: cdh_5.4.0 - os: ubuntu + - 'dib-{network}-{engine}-{plugin}-{os}-aio' + - 'dib-{network}-{engine}-{plugin}-{os}-aio': + network: nova + engine: direct + plugin: + - cdh_5.3.0 + - cdh_5.4.0 + os: + - u12 + - c6.6 + - 'dib-{network}-{engine}-{plugin}-{os}-aio': + engine: direct + plugin: spark_1.0.0 + os: u14 + - 'dib-{network}-{engine}-{plugin}-{os}': + plugin: mapr_4.0.2.mrv2 + os: u14 + - 'dib-{network}-{engine}-{plugin}-{os}': + network: nova + plugin: hdp_2.0.6 + os: c6.6 diff --git a/jenkins_job_builder/sahara.yaml b/jenkins_job_builder/sahara.yaml index 6fb849bc..4465c26d 100644 --- a/jenkins_job_builder/sahara.yaml +++ b/jenkins_job_builder/sahara.yaml @@ -1,7 +1,6 @@ -- job: - name: 'tempest-saharaclient-tests' +- job-template: + name: 'tempest-{project_name}-tests' defaults: global - concurrent: false builders: - gerrit-git-prep - ci-config-prep @@ -10,26 +9,7 @@ - zeromq-event - build-blocker: blocking-jobs: - - "tempest-sahara-tests" - publishers: - - sahara-logs - - console-log - - trigger-cleanup - node: trusty-neutron || trusty-nova - -- job: - name: 'tempest-sahara-tests' - defaults: global - concurrent: false - builders: - - gerrit-git-prep - - ci-config-prep - - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/tempest.sh" - properties: - - zeromq-event - - build-blocker: - blocking-jobs: - - "tempest-saharaclient-tests" + - "tempest-.*-tests" publishers: - sahara-logs - console-log @@ -37,84 +17,72 @@ node: trusty-neutron || trusty-nova - job-template: - name: 'gate-saharaclient-neutron-{plugin-neutron}' + name: 'gate-{project_name}-{network}-{engine}-{plugin}-{os}' defaults: global builders: - gerrit-git-prep - ci-config-prep - - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-saharaclient.sh" + - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-{project_name}.sh" properties: - zeromq-event publishers: - sahara-logs - console-log - trigger-cleanup - node: 'trusty-neutron' + node: 'trusty-{network}' - job-template: - name: 'gate-saharaclient-nova-{plugin-nova_network}' + name: 'gate-{project_name}-{network}-{engine}-{plugin}-{os}-aio' defaults: global builders: - gerrit-git-prep - ci-config-prep - - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-saharaclient.sh" + - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-{project_name}.sh" properties: - zeromq-event publishers: - sahara-logs - console-log - trigger-cleanup - node: 'trusty-nova' - -- job-template: - name: 'gate-sahara-neutron-{plugin-neutron}' - defaults: global - builders: - - gerrit-git-prep - - ci-config-prep - - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-sahara.sh" - properties: - - zeromq-event - publishers: - - sahara-logs - - console-log - - trigger-cleanup - node: 'trusty-neutron' - -- job-template: - name: 'gate-sahara-nova-{plugin-nova_network}' - defaults: global - builders: - - gerrit-git-prep - - ci-config-prep - - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-sahara.sh" - properties: - - zeromq-event - publishers: - - sahara-logs - - console-log - - trigger-cleanup - node: 'trusty-nova' + node: 'trusty-{network}' - project: - name: sahara - github-org: openstack - plugin-neutron: - - heat-vanilla_2.6 - - direct-spark-aio - - direct-transient - - heat-transient - - heat-mapr - plugin-nova_network: - - direct-cdh_5.3.0_ubuntu-aio - - direct-cdh_5.3.0_centos-aio - - direct-cdh_5.4.0-aio - - heat-hdp_2 + name: main + project_name: + - sahara + - saharaclient + network: nova + engine: direct + plugin: + - cdh_5.3.0 + - cdh_5.4.0 + os: + - u12 + - c6.6 jobs: - - 'gate-sahara-neutron-{plugin-neutron}' - - 'gate-sahara-nova-{plugin-nova_network}' - - 'gate-saharaclient-neutron-{plugin-neutron}' - - 'gate-saharaclient-nova-{plugin-nova_network}' + - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}-aio' + - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}': + engine: heat + plugin: + - vanilla_2.6.0 + - transient + - mapr_4.0.2 + os: u14 + network: neutron + - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}': + plugin: transient + os: u14 + network: neutron + - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}-aio': + plugin: spark_1.0.0 + os: u14 + network: neutron + - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}': + network: nova + engine: heat + plugin: hdp_2.0.6 + os: c6.6 + - 'tempest-{project_name}-tests' - job: name: gate-sahara-pep8-trunk @@ -125,7 +93,6 @@ \n

This job runs pep8 check using trunk version of hacking" disabled: false node: trusty-neutron || trusty-nova - builders: - gerrit-git-prep - ci-config-prep diff --git a/slave-scripts/dib.sh b/slave-scripts/dib.sh index ea45f74c..6c57dae6 100755 --- a/slave-scripts/dib.sh +++ b/slave-scripts/dib.sh @@ -10,20 +10,17 @@ CLUSTER_HASH=${CLUSTER_HASH:-$RANDOM} cluster_name="$HOST-$ZUUL_CHANGE-$CLUSTER_HASH" SAHARA_PATH="/tmp/sahara" -sahara_conf_path="$SAHARA_PATH/etc/sahara/sahara.conf" +sahara_conf_file="$SAHARA_PATH/etc/sahara/sahara.conf" sahara_templates_path=$SAHARA_PATH/etc/scenario/sahara-ci engine=$(echo $JOB_NAME | awk -F '-' '{ print $3 }') -job_type="$1" -image_type=${2:-ubuntu} -# Image names -vanilla_two_six_image=$HOST-sahara-vanilla-${image_type}-${ZUUL_CHANGE}-hadoop_2.6 -hdp_two_image=$HOST-sahara-hdp-centos-${ZUUL_CHANGE}-hadoop_2 -spark_image=$HOST-sahara-spark-ubuntu-${ZUUL_CHANGE} -cdh_image=$HOST-${image_type}-cdh-${ZUUL_CHANGE} -cdh_5_4_0_image=$HOST-${image_type}-cdh_5.4.0-${ZUUL_CHANGE} -mapr_402mrv2_image=$HOST-${image_type}-mapr-${ZUUL_CHANGE} +plugin="$1" +os="$2" +image_name=${HOST}_${plugin}_${os}_${ZUUL_CHANGE} +eval ${plugin//./_}_image=$image_name +mode="aio" +sahara_plugin=$(echo $plugin | awk -F '_' '{ print $1 } ') # Clone Sahara git clone https://review.openstack.org/openstack/sahara $SAHARA_PATH -b $ZUUL_BRANCH @@ -31,79 +28,66 @@ git clone https://review.openstack.org/openstack/sahara $SAHARA_PATH -b $ZUUL_BR # make verbose the scripts execution of disk-image-create export DIB_DEBUG_TRACE=1 -case $job_type in - vanilla_2.6) - if [ "${image_type}" == 'centos' ]; then - username='cloud-user' - else - username='ubuntu' - fi +if [ "${os}" == 'c6.6' ]; then + username="cloud-user" + os_type="centos" +else + username="ubuntu" + os_type="ubuntu" +fi - env ${image_type}_vanilla_hadoop_2_6_image_name=${vanilla_two_six_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p vanilla -i $image_type -v 2.6 - check_error_code $? ${vanilla_two_six_image}.qcow2 - upload_image "vanilla-2.6" "${username}" ${vanilla_two_six_image} - DISTRIBUTE_MODE=True - tests_config_file="$sahara_templates_path/vanilla-2.6.0.yaml" - insert_scenario_value $tests_config_file vanilla_two_six_image +case $plugin in + vanilla_2.6.0) + env ${os_type}_vanilla_hadoop_2_6_image_name=${vanilla_2_6_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p vanilla -i $os_type -v 2.6 + check_error_code $? ${vanilla_2_6_0_image}.qcow2 + upload_image "${plugin}" "${username}" ${vanilla_2_6_0_image} + mode=distribute + scenario_conf_file="$sahara_templates_path/vanilla-2.6.0.yaml" ;; - spark) - env ubuntu_spark_image_name=${spark_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p spark - check_error_code $? ${spark_image}.qcow2 - upload_image "spark" "ubuntu" ${spark_image} - tests_config_file="$sahara_templates_path/spark-1.0.0.yaml" - insert_scenario_value $tests_config_file spark_image - insert_config_value $sahara_conf_path DEFAULT plugins spark + spark_1.0.0) + env ubuntu_spark_image_name=${spark_1_0_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p spark + check_error_code $? ${spark_1_0_0_image}.qcow2 + upload_image "${plugin}" "${username}" ${spark_1_0_0_image} + scenario_conf_file="$sahara_templates_path/spark-1.0.0.yaml" ;; - hdp_2) - env centos_hdp_hadoop_2_image_name=${hdp_two_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p hdp -v 2 - check_error_code $? ${hdp_two_image}.qcow2 - upload_image "hdp2" "cloud-user" ${hdp_two_image} - DISTRIBUTE_MODE=True - tests_config_file="$sahara_templates_path/hdp-2.0.6.yaml" - insert_scenario_value $tests_config_file hdp_two_image + hdp_2.0.6) + env centos_hdp_hadoop_2_image_name=${hdp_2_0_6_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p hdp -v 2 + check_error_code $? ${hdp_2_0_6_image}.qcow2 + upload_image "${plugin}" "${username}" ${hdp_2_0_6_image} + mode=distribute + scenario_conf_file="$sahara_templates_path/hdp-2.0.6.yaml" ;; cdh_5.3.0) - if [ "${image_type}" == 'centos' ]; then - username='cloud-user' - else - username='ubuntu' - fi - env cloudera_5_3_${image_type}_image_name=${cdh_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i $image_type -v 5.3 - check_error_code $? ${cdh_image}.qcow2 - upload_image "cdh_5.3.0" ${username} ${cdh_image} - tests_config_file="$sahara_templates_path/cdh-5.3.0.yaml" - insert_config_value $sahara_conf_path DEFAULT plugins cdh - insert_scenario_value $tests_config_file cdh_image + env cloudera_5_3_${os_type}_image_name=${cdh_5_3_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i $os_type -v 5.3 + check_error_code $? ${cdh_5_3_0_image}.qcow2 + upload_image "${plugin}" "${username}" ${cdh_5_3_0_image} + scenario_conf_file="$sahara_templates_path/cdh-5.3.0.yaml" ;; cdh_5.4.0) env cloudera_5_4_ubuntu_image_name=${cdh_5_4_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i ubuntu -v 5.4 check_error_code $? ${cdh_5_4_0_image}.qcow2 - upload_image "cdh_5.4.0" "ubuntu" ${cdh_5_4_0_image} - tests_config_file="$sahara_templates_path/cdh-5.4.0.yaml" - insert_config_value $sahara_conf_path DEFAULT plugins cdh - insert_scenario_value $tests_config_file cdh_5_4_0_image + upload_image "${plugin}" "${username}" ${cdh_5_4_0_image} + scenario_conf_file="$sahara_templates_path/cdh-5.4.0.yaml" ;; - mapr) - env mapr_ubuntu_image_name=${mapr_402mrv2_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p mapr -i ubuntu - check_error_code $? ${mapr_402mrv2_image}.qcow2 - upload_image "mapr" "ubuntu" ${mapr_402mrv2_image} - DISTRIBUTE_MODE=True - tests_config_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml" - insert_config_value $sahara_conf_path DEFAULT plugins mapr - insert_scenario_value $tests_config_file mapr_402mrv2_image + mapr_4.0.2.mrv2) + env mapr_ubuntu_image_name=${mapr_4_0_2_mrv2_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p mapr -i ubuntu + check_error_code $? ${mapr_4_0_2_mrv2_image}.qcow2 + upload_image "${plugin}" "${username}" ${mapr_4_0_2_mrv2_image} + mode=distribute + scenario_conf_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml" ;; esac cd $SAHARA_PATH sudo pip install . --no-cache-dir enable_pypi -write_sahara_main_conf "$sahara_conf_path" "$engine" -write_tests_conf "$tests_config_file" "$cluster_name" -start_sahara "$sahara_conf_path" && run_tests "$tests_config_file" +write_sahara_main_conf "$sahara_conf_file" "$engine" "$sahara_plugin" +write_tests_conf "$scenario_conf_file" "$cluster_name" "$image_name" +start_sahara "$sahara_conf_file" "$mode" && run_tests "$scenario_conf_file" print_python_env -cleanup_image "$job_type" "$image_type" +cleanup_image "$plugin" "$os" diff --git a/slave-scripts/functions-common.sh b/slave-scripts/functions-common.sh index 97b93931..4c99358c 100755 --- a/slave-scripts/functions-common.sh +++ b/slave-scripts/functions-common.sh @@ -1,15 +1,9 @@ #!/bin/bash -xe -sahara_configs_path=$WORKSPACE/sahara-ci-config/config/sahara -ci_flavor_id=\'20\' -medium_flavor_id=\'3\' -large_flavor_id=\'4\' - -enable_pypi() { - mkdir -p ~/.pip - export PIP_USE_MIRRORS=True - cp $sahara_configs_path/pip.conf ~/.pip/pip.conf -} +configs_path=$WORKSPACE/sahara-ci-config/config +eval ci_flavor_id="\'20\'" +eval medium_flavor_id="\'3\'" +eval large_flavor_id="\'4\'" conf_has_option() { local file=$1 @@ -21,6 +15,19 @@ conf_has_option() { [ -n "$line" ] } +enable_pypi() { + mkdir -p ~/.pip + export PIP_USE_MIRRORS=True + cp $configs_path/pip.conf ~/.pip/pip.conf +} + +failure() { + local reason=$1 + print_python_env + echo "$reason" + exit 1 +} + insert_config_value() { local file=$1 local section=$2 @@ -46,30 +53,21 @@ $option = $value } insert_scenario_value() { - local config=$1 - local value=$2 - sed -i "s/%${value}%/${!value}/g" $config -} + local file=$1 + local main_key=$2 + local stop_key=$3 + local sub_key=$4 + local value=$5 + local old_value=$6 -write_sahara_main_conf() { - local conf_path=$1 - local engine=$2 - insert_config_value $conf_path DEFAULT infrastructure_engine $engine - insert_config_value $conf_path DEFAULT api_workers 4 - insert_config_value $conf_path DEFAULT use_identity_api_v3 true - insert_config_value $conf_path DEFAULT use_neutron $USE_NEUTRON - insert_config_value $conf_path DEFAULT min_transient_cluster_active_time 30 - insert_config_value $conf_path DEFAULT node_domain ci - insert_config_value $conf_path database connection mysql://sahara-citest:sahara-citest@localhost/sahara?charset=utf8 - insert_config_value $conf_path keystone_authtoken auth_uri http://$OPENSTACK_HOST:5000/v2.0/ - insert_config_value $conf_path keystone_authtoken identity_uri http://$OPENSTACK_HOST:35357/ - insert_config_value $conf_path keystone_authtoken admin_user $OS_USERNAME - insert_config_value $conf_path keystone_authtoken admin_password $OS_PASSWORD - insert_config_value $conf_path keystone_authtoken admin_tenant_name $OS_TENANT_NAME + [[ -z $main_key || -z $sub_key ]] && return - echo "----------- sahara.conf -----------" - cat $conf_path - echo "--------------- end ---------------" + if ! scenario_has_option "$file" "$main_key" "$stop_key" "$sub_key"; then + echo "No such keys: $main_key -> $sub_key in scenario $file file. Skip setting value $value" + else + local sep=$(echo -ne "\x01") + sed -i -e '/'${main_key}':/,/'${stop_key}'/ s'${sep}'\([ \t]'${sub_key}':[ \t]\).*'${old_value}'.*$'${sep}'\1'${value}${sep} $file + fi } print_python_env() { @@ -78,58 +76,6 @@ print_python_env() { pip freeze > $WORKSPACE/logs/python-system-env.txt } -failure() { - local reason=$1 - print_python_env - echo "$reason" - exit 1 -} - -start_sahara() { - local conf_path=$1 - local conf_dir=$(dirname $1) - mkdir $WORKSPACE/logs - sahara-db-manage --config-file $conf_path upgrade head || failure "Command 'sahara-db-manage' failed" - if [ "$DISTRIBUTE_MODE" == "True" ]; then - screen -dmS sahara-api /bin/bash -c "PYTHONUNBUFFERED=1 sahara-api --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-api.txt" - sleep 2 - screen -dmS sahara-engine_1 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-1.txt" - screen -dmS sahara-engine_2 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-2.txt" - else - screen -dmS sahara-all /bin/bash -c "PYTHONUNBUFFERED=1 sahara-all --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log.txt" - fi - - api_responding_timeout=30 - if ! timeout ${api_responding_timeout} sh -c "while ! curl -s http://127.0.0.1:8386/v1.1/ 2>/dev/null | grep -q 'Authentication required' ; do sleep 1; done"; then - failure "Sahara API failed to respond within ${api_responding_timeout} seconds" - fi -} - -write_tests_conf() { - local test_conf=$1 - local cluster_name=$2 - local addr=$(ifconfig eth0| awk -F ' *|:' '/inet addr/{print $4}') - if [ "$USE_NEUTRON" == "true" ]; then - NETWORK="neutron" - else - NETWORK="nova-network" - fi - local test_scenario_credentials=$(dirname $1)/credentials.yaml - insert_scenario_value $test_scenario_credentials OS_USERNAME - insert_scenario_value $test_scenario_credentials OS_PASSWORD - insert_scenario_value $test_scenario_credentials OS_TENANT_NAME - insert_scenario_value $test_scenario_credentials OPENSTACK_HOST - insert_scenario_value $test_scenario_credentials NETWORK - insert_scenario_value $test_conf cluster_name - insert_scenario_value $test_conf ci_flavor_id - insert_scenario_value $test_conf medium_flavor_id - insert_scenario_value $test_conf large_flavor_id - - echo "----------- tests config -----------" - cat $test_conf - echo "---------------- end ---------------" -} - run_tests() { local config=$1 local concurrency=${2:-"1"} @@ -143,3 +89,85 @@ run_tests() { STATUS=$(grep "\ -\ Failed" tox.log | awk '{print $3}') if [ "$STATUS" != "0" ]; then failure "Integration tests have failed"; fi } + +scenario_has_option() { + local file=$1 + local main_key=$2 + local stop_key=$3 + local sub_key=$4 + local line + + line=$(sed -ne "/$main_key\:/,/$stop_key/ { /[ \t]$sub_key:/ p; }" "$file") + [ -n "$line" ] +} + +start_sahara() { + local conf_path=$1 + local conf_dir=$(dirname $1) + local mode=$2 + mkdir $WORKSPACE/logs + sahara-db-manage --config-file $conf_path upgrade head || failure "Command 'sahara-db-manage' failed" + if [ "$mode" == "distribute" ]; then + screen -dmS sahara-api /bin/bash -c "PYTHONUNBUFFERED=1 sahara-api --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-api.txt" + sleep 2 + screen -dmS sahara-engine_1 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-1.txt" + screen -dmS sahara-engine_2 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-2.txt" + else + screen -dmS sahara-all /bin/bash -c "PYTHONUNBUFFERED=1 sahara-all --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log.txt" + fi + + api_responding_timeout=30 + if ! timeout ${api_responding_timeout} sh -c "while ! curl -s http://127.0.0.1:8386/v1.1/ 2>/dev/null | grep -q 'Authentication required' ; do sleep 1; done"; then + failure "Sahara API failed to respond within ${api_responding_timeout} seconds" + fi +} + +write_sahara_main_conf() { + local conf_path=$1 + local engine=$2 + local plugin=$3 + insert_config_value $conf_path DEFAULT infrastructure_engine $engine + insert_config_value $conf_path DEFAULT api_workers 4 + insert_config_value $conf_path DEFAULT use_identity_api_v3 true + insert_config_value $conf_path DEFAULT use_neutron $USE_NEUTRON + insert_config_value $conf_path DEFAULT min_transient_cluster_active_time 30 + insert_config_value $conf_path DEFAULT node_domain ci + insert_config_value $conf_path DEFAULT plugins $plugin + insert_config_value $conf_path database connection mysql://sahara-citest:sahara-citest@localhost/sahara?charset=utf8 + insert_config_value $conf_path keystone_authtoken auth_uri http://$OPENSTACK_HOST:5000/v2.0/ + insert_config_value $conf_path keystone_authtoken identity_uri http://$OPENSTACK_HOST:35357/ + insert_config_value $conf_path keystone_authtoken admin_user $OS_USERNAME + insert_config_value $conf_path keystone_authtoken admin_password $OS_PASSWORD + insert_config_value $conf_path keystone_authtoken admin_tenant_name $OS_TENANT_NAME + + echo "----------- sahara.conf -----------" + cat $conf_path + echo "--------------- end ---------------" +} + +write_tests_conf() { + local test_conf=$1 + local cluster_name=$2 + local image_name=$3 + local os_auth_url="http://$OPENSTACK_HOST:5000/v2.0/" + if [ "$USE_NEUTRON" == "true" ]; then + NETWORK="neutron" + else + NETWORK="nova-network" + fi + local test_scenario_credentials=$(dirname $1)/credentials.yaml + insert_scenario_value $test_scenario_credentials credentials "" os_username $OS_USERNAME + insert_scenario_value $test_scenario_credentials credentials "" os_password $OS_PASSWORD + insert_scenario_value $test_scenario_credentials credentials "" os_tenant $OS_TENANT_NAME + insert_scenario_value $test_scenario_credentials credentials "" os_auth_url $os_auth_url + insert_scenario_value $test_scenario_credentials network "" "type" $NETWORK + insert_scenario_value $test_conf clusters node_group_templates image $image_name + insert_scenario_value $test_conf cluster "" name $cluster_name + insert_scenario_value $test_conf node_group_templates "\\$" flavor_id $ci_flavor_id ci_flavor_id + insert_scenario_value $test_conf node_group_templates "\\$" flavor_id $medium_flavor_id medium_flavor_id + insert_scenario_value $test_conf node_group_templates "\\$" flavor_id $large_flavor_id large_flavor_id + + echo "----------- tests config -----------" + cat $test_conf + echo "---------------- end ---------------" +} diff --git a/slave-scripts/functions-dib.sh b/slave-scripts/functions-dib.sh index 8cc7fc66..7c43e180 100755 --- a/slave-scripts/functions-dib.sh +++ b/slave-scripts/functions-dib.sh @@ -2,55 +2,6 @@ CUR_IMAGE=none -register_new_image() { - local image_name=$1 - local image_properties=$2 - glance image-create --name $1 --file $1.qcow2 --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' $image_properties -} - -delete_image() { - # "|| true" here, to avoid error code producing in case of missing image - glance image-delete $1 || true -} - -upload_image() { - local plugin=$1 - local username=$2 - local image=$3 - delete_image "$image" - - case "$plugin" in - vanilla-2.4) - image_properties="--property _sahara_tag_2.4.1=True --property _sahara_tag_vanilla=True --property _sahara_username=${username}" - ;; - vanilla-2.6) - image_properties="--property _sahara_tag_2.6.0=True --property _sahara_tag_vanilla=True --property _sahara_username=${username}" - ;; - hdp2) - image_properties="--property _sahara_tag_2.0.6=True --property _sahara_tag_hdp=True --property _sahara_username=${username}" - ;; - cdh_5.3.0) - image_properties="--property _sahara_tag_5.3.0=True --property _sahara_tag_5=True --property _sahara_tag_cdh=True --property _sahara_username=${username}" - ;; - cdh_5.4.0) - image_properties="--property _sahara_tag_5.4.0=True --property _sahara_tag_cdh=True --property _sahara_username=${username}" - ;; - spark) - image_properties="--property _sahara_tag_spark=True --property _sahara_tag_1.0.0=True --property _sahara_username=${username}" - ;; - mapr) - image_properties="--property _sahara_tag_mapr=True --property _sahara_tag_4.0.2.mrv2=True --property _sahara_username=${username}" - ;; - esac - register_new_image "$image" "$image_properties" - CUR_IMAGE="$image" -} - -rename_image() { - # 1 - source image, 2 - target image - glance image-update $1 --name $2 -} - check_error_code() { if [ "$1" != "0" -o ! -f "$2" ]; then echo "$2 image isn't build" @@ -58,6 +9,22 @@ check_error_code() { fi } +cleanup_image() { + local plugin=$1 + local os=$2 + if [ "$ZUUL_PIPELINE" == "check" -o "$ZUUL_BRANCH" != "master" ]; then + delete_image "$CUR_IMAGE" + else + delete_image ${plugin}_${os} + rename_image "$CUR_IMAGE" ${plugin}_${os} + fi +} + +delete_image() { + # "|| true" here, to avoid error code producing in case of missing image + glance image-delete $1 || true +} + failure() { local reason=$1 echo "$reason" @@ -66,37 +33,42 @@ failure() { exit 1 } -cleanup_image() { - local job_type=$1 - local os=$2 - if [ "$ZUUL_PIPELINE" == "check" -o "$ZUUL_BRANCH" != "master" ]; then - delete_image "$CUR_IMAGE" - else - case $job_type in - vanilla_2.6) - delete_image ${os}_vanilla_2.6_latest - rename_image "$CUR_IMAGE" ${os}_vanilla_2.6_latest - ;; - hdp_2) - delete_image sahara_hdp_2_latest - rename_image "$CUR_IMAGE" sahara_hdp_2_latest - ;; - cdh_5.3.0) - delete_image ${os}_cdh_latest - rename_image "$CUR_IMAGE" ${os}_cdh_latest - ;; - cdh_5.4.0) - delete_image ${os}_cdh_5.4.0_latest - rename_image "$CUR_IMAGE" ${os}_cdh_5.4.0_latest - ;; - spark) - delete_image sahara_spark_latest - rename_image "$CUR_IMAGE" sahara_spark_latest - ;; - mapr) - delete_image ubuntu_mapr_latest - rename_image "$CUR_IMAGE" ubuntu_mapr_latest - ;; - esac - fi +register_new_image() { + local image_name=$1 + local image_properties=$2 + glance image-create --name $image_name --file $image_name.qcow2 --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' $image_properties +} + +rename_image() { + # 1 - source image, 2 - target image + glance image-update $1 --name $2 +} + +upload_image() { + local plugin=$1 + local username=$2 + local image=$3 + delete_image "$image" + case "$plugin" in + vanilla_2.6.0) + image_properties="--property _sahara_tag_2.6.0=True --property _sahara_tag_vanilla=True --property _sahara_username=${username}" + ;; + hdp_2.0.6) + image_properties="--property _sahara_tag_2.0.6=True --property _sahara_tag_hdp=True --property _sahara_username=${username}" + ;; + cdh_5.3.0) + image_properties="--property _sahara_tag_5.3.0=True --property _sahara_tag_5=True --property _sahara_tag_cdh=True --property _sahara_username=${username}" + ;; + cdh_5.4.0) + image_properties="--property _sahara_tag_5.4.0=True --property _sahara_tag_cdh=True --property _sahara_username=${username}" + ;; + spark_1.0.0) + image_properties="--property _sahara_tag_spark=True --property _sahara_tag_1.0.0=True --property _sahara_username=${username}" + ;; + mapr_4.0.2.mrv2) + image_properties="--property _sahara_tag_mapr=True --property _sahara_tag_4.0.2.mrv2=True --property _sahara_username=${username}" + ;; + esac + register_new_image "$image" "$image_properties" + CUR_IMAGE="$image" } diff --git a/slave-scripts/gate-sahara.sh b/slave-scripts/gate-sahara.sh index 452fbd6e..4558ad12 100755 --- a/slave-scripts/gate-sahara.sh +++ b/slave-scripts/gate-sahara.sh @@ -9,69 +9,50 @@ CLUSTER_HASH=${CLUSTER_HASH:-$RANDOM} cluster_name="$HOST-$ZUUL_CHANGE-$CLUSTER_HASH" SAHARA_PATH=${1:-$WORKSPACE} -sahara_conf_path=$SAHARA_PATH/etc/sahara/sahara.conf +sahara_conf_file=$SAHARA_PATH/etc/sahara/sahara.conf sahara_templates_path=$SAHARA_PATH/etc/scenario/sahara-ci -job_type=$(echo $JOB_NAME | awk -F '-' '{ print $5 }') engine_type=$(echo $JOB_NAME | awk -F '-' '{ print $4 }') +plugin=$(echo $JOB_NAME | awk -F '-' '{ print $5 }') +os=$(echo $JOB_NAME | awk -F '-' '{ print $6 }') +image_name=${plugin}_${os} +mode="aio" +sahara_plugin=$(echo $plugin | awk -F '_' '{ print $1 } ') -# Image names -hdp_two_image=sahara_hdp_2_latest -vanilla_two_six_image=ubuntu_vanilla_2.6_latest -spark_image=sahara_spark_latest -cdh_5_3_0_centos_image=centos_cdh_latest -cdh_5_3_0_ubuntu_image=ubuntu_cdh_latest -cdh_5_4_0_image=ubuntu_cdh_5.4.0_latest -mapr_402mrv2_image=ubuntu_mapr_latest - -case $job_type in - hdp_2) - DISTRIBUTE_MODE=True - tests_config_file="$sahara_templates_path/hdp-2.0.6.yaml" - insert_scenario_value $tests_config_file hdp_two_image +case $plugin in + hdp_2.0.6) + mode=distribute + scenario_conf_file="$sahara_templates_path/hdp-2.0.6.yaml" ;; - vanilla_2.6) - DISTRIBUTE_MODE=True - tests_config_file="$sahara_templates_path/vanilla-2.6.0.yaml" - insert_scenario_value $tests_config_file vanilla_two_six_image + vanilla_2.6.0) + mode=distribute + scenario_conf_file="$sahara_templates_path/vanilla-2.6.0.yaml" ;; transient) + # transient is using image with latest vanilla version + image_name=vanilla_2.6.0_u14 concurrency=3 - DISTRIBUTE_MODE=True - tests_config_file="$sahara_templates_path/transient.yaml" - insert_scenario_value $tests_config_file vanilla_two_six_image + mode=distribute + scenario_conf_file="$sahara_templates_path/transient.yaml" ;; - cdh_5.3.0*) - insert_config_value $sahara_conf_path DEFAULT plugins cdh - if [[ "$job_type" =~ centos ]]; then - cdh_image=$cdh_5_3_0_centos_image - else - cdh_image=$cdh_5_3_0_ubuntu_image - fi - tests_config_file="$sahara_templates_path/cdh-5.3.0.yaml" - insert_scenario_value $tests_config_file cdh_image + cdh_5.3.0) + scenario_conf_file="$sahara_templates_path/cdh-5.3.0.yaml" ;; cdh_5.4.0) - insert_config_value $sahara_conf_path DEFAULT plugins cdh - tests_config_file="$sahara_templates_path/cdh-5.4.0.yaml" - insert_scenario_value $tests_config_file cdh_5_4_0_image + scenario_conf_file="$sahara_templates_path/cdh-5.4.0.yaml" ;; - spark) - insert_config_value $sahara_conf_path DEFAULT plugins spark - tests_config_file="$sahara_templates_path/spark-1.0.0.yaml" - insert_scenario_value $tests_config_file spark_image + spark_1.0.0) + scenario_conf_file="$sahara_templates_path/spark-1.0.0.yaml" ;; - mapr) - insert_config_value $sahara_conf_path DEFAULT plugins mapr - DISTRIBUTE_MODE=True - tests_config_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml" - insert_scenario_value $tests_config_file mapr_402mrv2_image + mapr_4.0.2.mrv2) + mode=distribute + scenario_conf_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml" ;; esac sudo pip install . --no-cache-dir enable_pypi -write_sahara_main_conf "$sahara_conf_path" "$engine_type" -write_tests_conf "$tests_config_file" "$cluster_name" -start_sahara "$sahara_conf_path" && run_tests "$tests_config_file" "$concurrency" +write_sahara_main_conf "$sahara_conf_file" "$engine_type" "$sahara_plugin" +write_tests_conf "$scenario_conf_file" "$cluster_name" "$image_name" +start_sahara "$sahara_conf_file" "$mode" && run_tests "$scenario_conf_file" "$concurrency" print_python_env diff --git a/slave-scripts/gate-ui-tests.sh b/slave-scripts/gate-ui-tests.sh index 8d36f9de..4167441e 100755 --- a/slave-scripts/gate-ui-tests.sh +++ b/slave-scripts/gate-ui-tests.sh @@ -49,12 +49,12 @@ floating_ip_pool = 'public' keystone_url = 'http://$OPENSTACK_HOST:5000/v2.0' await_element = 120 image_name_for_register = '$TEST_IMAGE' -image_name_for_edit = "sahara-itests-ci-vanilla-image" +image_name_for_edit = 'sahara-itests-ci-vanilla-image' security_groups = default [vanilla] skip_plugin_tests = False skip_edp_test = False -base_image = "sahara-itests-ci-vanilla-image" +base_image = 'sahara-itests-ci-vanilla-image' [hdp] skip_plugin_tests = False hadoop_version = '1.3.2'