From 3277ef2998c12fdaed7e3969c37ef8b189768e8f Mon Sep 17 00:00:00 2001 From: Evgeny Sikachev Date: Mon, 5 Sep 2016 18:33:40 +0300 Subject: [PATCH] Add mapr 520 on sahara-ci Added scripts and jobs for sahara and sahara-image-elements, devstack scripts Change-Id: I37425ae30e7f82f7ad5ea9d2394ad3aea10e1162 --- config/devstack/local.sh | 2 ++ config/devstack/pre_unstack.sh | 2 ++ config/zuul/layout.yaml | 14 +++++++++++++- jenkins_job_builder/diskimage.yaml | 1 + jenkins_job_builder/sahara.yaml | 2 ++ slave-scripts/dib.sh | 12 +++++++++++- slave-scripts/functions-dib.sh | 7 +++++-- slave-scripts/gate-sahara.sh | 5 +++++ 8 files changed, 41 insertions(+), 4 deletions(-) diff --git a/config/devstack/local.sh b/config/devstack/local.sh index a41ab914..6699b429 100755 --- a/config/devstack/local.sh +++ b/config/devstack/local.sh @@ -37,6 +37,7 @@ UBUNTU_CDH_5_7_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.7.0_u14.qcow2 SPARK_1_3_1_IMAGE_PATH=/home/ubuntu/images/spark_1.3.1_u14.qcow2 SPARK_1_6_0_IMAGE_PATH=/home/ubuntu/images/spark_1.6.0_u14.qcow2 MAPR_5_1_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.1.0.mrv2_u14.qcow2 +MAPR_5_2_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.2.0.mrv2_u14.qcow2 STORM_1_0_1_IMAGE_PATH=/home/ubuntu/images/storm_1.0.1_u14.qcow2 export OS_CLOUD='devstack-admin' @@ -94,6 +95,7 @@ openstack image create $(basename -s .qcow2 $UBUNTU_CDH_5_7_0_IMAGE_PATH) --file openstack image create $(basename -s .qcow2 $SPARK_1_3_1_IMAGE_PATH) --file $SPARK_1_3_1_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_spark'='True' --property '_sahara_tag_1.3.1'='True' --property '_sahara_username'="ubuntu" openstack image create $(basename -s .qcow2 $SPARK_1_6_0_IMAGE_PATH) --file $SPARK_1_6_0_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_spark'='True' --property '_sahara_tag_1.6.0'='True' --property '_sahara_username'="ubuntu" openstack image create $(basename -s .qcow2 $MAPR_5_1_0_MRV2_IMAGE_PATH) --file $MAPR_5_1_0_MRV2_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_5.1.0.mrv2'='True' --property '_sahara_username'="ubuntu" +openstack image create $(basename -s .qcow2 $MAPR_5_2_0_MRV2_IMAGE_PATH) --file $MAPR_5_2_0_MRV2_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_5.2.0.mrv2'='True' --property '_sahara_username'="ubuntu" openstack image create $(basename -s .qcow2 $STORM_1_0_1_IMAGE_PATH) --file $STORM_1_0_1_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_storm'='True' --property '_sahara_tag_1.0.1'='True' --property '_sahara_username'="ubuntu" openstack image set --name ubuntu-14.04 trusty-server-cloudimg-amd64-disk1 diff --git a/config/devstack/pre_unstack.sh b/config/devstack/pre_unstack.sh index 7d441dcd..68101cd3 100755 --- a/config/devstack/pre_unstack.sh +++ b/config/devstack/pre_unstack.sh @@ -29,6 +29,7 @@ UBUNTU_CDH_5_7_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.7.0_u14.qcow2 SPARK_1_3_1_IMAGE_PATH=/home/ubuntu/images/spark_1.3.1_u14.qcow2 SPARK_1_6_0_IMAGE_PATH=/home/ubuntu/images/spark_1.6.0_u14.qcow2 MAPR_5_1_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.1.0.mrv2_u14.qcow2 +MAPR_5_2_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.2.0.mrv2_u14.qcow2 STORM_1_0_1_IMAGE_PATH=/home/ubuntu/images/storm_1.0.1_u14.qcow2 @@ -45,6 +46,7 @@ openstack image save --file $UBUNTU_CDH_5_7_0_IMAGE_PATH $(get_id $UBUNTU_CDH_5_ openstack image save --file $SPARK_1_3_1_IMAGE_PATH $(get_id $SPARK_1_3_1_IMAGE_PATH) openstack image save --file $SPARK_1_6_0_IMAGE_PATH $(get_id $SPARK_1_6_0_IMAGE_PATH) openstack image save --file $MAPR_5_1_0_MRV2_IMAGE_PATH $(get_id $MAPR_5_1_0_MRV2_IMAGE_PATH) +openstack image save --file $MAPR_5_2_0_MRV2_IMAGE_PATH $(get_id $MAPR_5_2_0_MRV2_IMAGE_PATH) openstack image save --file $STORM_1_0_1_IMAGE_PATH $(get_id $STORM_1_0_1_IMAGE_PATH) bash $TOP_DIR/unstack.sh diff --git a/config/zuul/layout.yaml b/config/zuul/layout.yaml index e9153294..3c762d2b 100644 --- a/config/zuul/layout.yaml +++ b/config/zuul/layout.yaml @@ -73,6 +73,7 @@ projects: - gate-sahara-spark_1.3.1-u14-aio - gate-sahara-spark_1.6.0-u14-aio - gate-sahara-mapr_5.1.0.mrv2-u14 + - gate-sahara-mapr_5.2.0.mrv2-u14 - gate-sahara-cdh_5.4.0-c6.6-aio - gate-sahara-cdh_5.5.0-u14-aio - gate-sahara-cdh_5.5.0-c7-aio @@ -90,6 +91,7 @@ projects: - gate-sahara_tests-spark_1.3.1-u14-aio - gate-sahara_tests-spark_1.6.0-u14-aio - gate-sahara_tests-mapr_5.1.0.mrv2-u14 + - gate-sahara_tests-mapr_5.2.0.mrv2-u14 - gate-sahara_tests-cdh_5.5.0-u14-aio - gate-sahara_tests-cdh_5.5.0-c7-aio - gate-sahara_tests-ambari_2.3-c7 @@ -127,6 +129,7 @@ projects: - dib-vanilla_2.7.1-u14 - dib-vanilla_2.7.1-c7 - dib-mapr_5.1.0.mrv2-u14 + - dib-mapr_5.2.0.mrv2-u14 - dib-spark_1.6.0-u14-aio - dib-cdh_5.5.0-u14-aio - dib-cdh_5.5.0-c7-aio @@ -138,6 +141,7 @@ projects: - dib-vanilla_2.7.1-u14 - dib-vanilla_2.7.1-c7 - dib-mapr_5.1.0.mrv2-u14 + - dib-mapr_5.2.0.mrv2-u14 - dib-spark_1.6.0-u14-aio - dib-cdh_5.5.0-u14-aio - dib-cdh_5.5.0-c7-aio @@ -145,7 +149,7 @@ projects: - dib-storm_1.0.1-u14-aio - dib-ambari_2.2-c7 - dib-ambari_2.2-u14 - + - name: openstack/sahara-ci-config check: - sahara-ci-syntax-check @@ -173,6 +177,14 @@ jobs: - ^sahara_tests/.*.py$ - ^diskimage-create/.*$ - ^elements/.*$ + - name: ^(?!nightly).*-mapr_5.2.*$ + branch: master + files: + - ^sahara_tests/scenario/defaults/mapr-5.2.*$ + - ^sahara/.*$ + - ^sahara_tests/.*.py$ + - ^diskimage-create/.*$ + - ^elements/.*$ - name: ^(?!nightly|weekly).*-cdh_5.4.0-.*$ branch: stable/liberty files: diff --git a/jenkins_job_builder/diskimage.yaml b/jenkins_job_builder/diskimage.yaml index 82d5ab06..48bef0d6 100644 --- a/jenkins_job_builder/diskimage.yaml +++ b/jenkins_job_builder/diskimage.yaml @@ -56,6 +56,7 @@ - 'dib-{plugin}-{os}': plugin: - mapr_5.1.0.mrv2 + - mapr_5.2.0.mrv2 os: u14 - 'dib-{plugin}-{os}': plugin: diff --git a/jenkins_job_builder/sahara.yaml b/jenkins_job_builder/sahara.yaml index 82a2f8da..00e604f4 100644 --- a/jenkins_job_builder/sahara.yaml +++ b/jenkins_job_builder/sahara.yaml @@ -61,6 +61,7 @@ plugin: - vanilla_2.7.1 - mapr_5.1.0.mrv2 + - mapr_5.2.0.mrv2 os: u14 - '{pipeline}-{project_name}-{plugin}-{os}-aio': plugin: @@ -110,6 +111,7 @@ plugin: - vanilla_2.7.1 - mapr_5.1.0.mrv2 + - mapr_5.2.0.mrv2 os: u14 - '{pipeline}-{project_name}-{plugin}-{os}-aio': plugin: diff --git a/slave-scripts/dib.sh b/slave-scripts/dib.sh index d498c62f..ca1196bb 100755 --- a/slave-scripts/dib.sh +++ b/slave-scripts/dib.sh @@ -110,13 +110,23 @@ case $plugin in ;; mapr_5.1.0.mrv2) - env mapr_ubuntu_image_name=${mapr_5_1_0_mrv2_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p mapr -i ubuntu + env mapr_ubuntu_image_name=${mapr_5_1_0_mrv2_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p mapr -i ubuntu -r 5.1.0 check_error_code $? ${mapr_5_1_0_mrv2_image}.qcow2 upload_image "${plugin}" "${username}" ${mapr_5_1_0_mrv2_image} mode=distribute scenario_conf_file="$sahara_templates_path/mapr-5.1.0.mrv2.yaml.mako" template_image_prefix="mapr_510mrv2" ;; + + mapr_5.2.0.mrv2) + env mapr_ubuntu_image_name=${mapr_5_2_0_mrv2_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p mapr -i ubuntu -r 5.2.0 + check_error_code $? ${mapr_5_2_0_mrv2_image}.qcow2 + upload_image "${plugin}" "${username}" ${mapr_5_2_0_mrv2_image} + mode=distribute + scenario_conf_file="$sahara_templates_path/mapr-5.2.0.mrv2.yaml.mako" + template_image_prefix="mapr_520mrv2" + ;; + storm_1.0.1) env ubuntu_storm_image_name=${storm_1_0_1_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p storm -t 1.0.1 check_error_code $? ${storm_1_0_1_image}.qcow2 diff --git a/slave-scripts/functions-dib.sh b/slave-scripts/functions-dib.sh index 890ff724..023af7ca 100755 --- a/slave-scripts/functions-dib.sh +++ b/slave-scripts/functions-dib.sh @@ -74,12 +74,15 @@ upload_image() { spark_1.3.1) image_properties="--property _sahara_tag_spark=True --property _sahara_tag_1.3.1=True --property _sahara_username=${username}" ;; - spark_1.6.0) + spark_1.6.0) image_properties="--property _sahara_tag_spark=True --property _sahara_tag_1.6.0=True --property _sahara_username=${username}" ;; - mapr_5.1.0.mrv2) + mapr_5.1.0.mrv2) image_properties="--property _sahara_tag_mapr=True --property _sahara_tag_5.1.0.mrv2=True --property _sahara_username=${username}" ;; + mapr_5.2.0.mrv2) + image_properties="--property _sahara_tag_mapr=True --property _sahara_tag_5.2.0.mrv2=True --property _sahara_username=${username}" + ;; storm_1.0.1) image_properties="--property _sahara_tag_storm=True --property _sahara_tag_1.0.1=True --property _sahara_username=${username}" ;; diff --git a/slave-scripts/gate-sahara.sh b/slave-scripts/gate-sahara.sh index ffa71cb2..f948e54d 100755 --- a/slave-scripts/gate-sahara.sh +++ b/slave-scripts/gate-sahara.sh @@ -86,6 +86,11 @@ case $plugin in scenario_conf_file="$sahara_templates_path/mapr-5.1.0.mrv2.yaml.mako" template_image_prefix="mapr_510mrv2" ;; + mapr_5.2.0.mrv2) + mode=distribute + scenario_conf_file="$sahara_templates_path/mapr-5.2.0.mrv2.yaml.mako" + template_image_prefix="mapr_520mrv2" + ;; storm_1.0.1) scenario_conf_file="$sahara_templates_path/storm-1.0.1.yaml.mako" template_image_prefix="storm_1_0"