x/sahara-ci-config
Code Issues Proposed changes

Rename all plugins to full plugin version

Browse Source 
Rename short plugin names to full plugin names and add
OS version to job name.

Also sort by alphabetical functions-* files.

Change-Id: I5155613715fd2ced99034807f14f495487d2d8ac
This commit is contained in:
Denis Egorenko 2015-06-03 18:35:36 +03:00
parent e43b33f033
commit 0a5838fa09
10 changed files with 370 additions and 475 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
config/devstack/local.sh
View File

@ -24,13 +24,14 @@ else
USE_NEUTRON=false USE_NEUTRON=false
fi fi
VANILLA26_IMAGE_PATH=/home/ubuntu/images/sahara-vanilla-2.6.0-ubuntu-14.04.qcow2 VANILLA_2_6_0_IMAGE_PATH=/home/ubuntu/images/vanilla_2.6.0_u14.qcow2
HDP2_IMAGE_PATH=/home/ubuntu/images/centos_6-6_hdp-2.qcow2 HDP_2_0_6_IMAGE_PATH=/home/ubuntu/images/hdp_2.0.6_c6.6.qcow2
CENTOS_CDH_IMAGE_PATH=/home/ubuntu/images/centos_sahara_cloudera_latest.qcow2 CENTOS_CDH_5_3_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.3.0_c6.6.qcow2
UBUNTU_CDH_IMAGE_PATH=/home/ubuntu/images/ubuntu_sahara_cloudera_latest.qcow2 UBUNTU_CDH_5_3_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.3.0_u12.qcow2
SPARK_IMAGE_PATH=/home/ubuntu/images/sahara_spark_latest.qcow2 UBUNTU_CDH_5_4_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.4.0_u12.qcow2
MAPR_IMAGE_PATH=/home/ubuntu/images/ubuntu_mapr_latest.qcow2 SPARK_1_0_0_IMAGE_PATH=/home/ubuntu/images/spark_1.0.0_u14.qcow2
NATIVE_UBUNTU_IMAGE_PATH=/home/ubuntu/images/ubuntu-12.04-server-cloudimg-amd64-disk1.img MAPR_4_0_2_IMAGE_PATH=/home/ubuntu/images/mapr_4.0.2_u14.qcow2
UBUNTU_12_04_IMAGE_PATH=/home/ubuntu/images/ubuntu-12.04-server-cloudimg-amd64-disk1.img
# setup ci tenant and ci users # setup ci tenant and ci users
CI_TENANT_ID=$(keystone tenant-create --name ci --description 'CI tenant' | grep -w id | get_field 2) CI_TENANT_ID=$(keystone tenant-create --name ci --description 'CI tenant' | grep -w id | get_field 2)
@ -69,13 +70,14 @@ nova flavor-create --is-public true m1.small 2 1024 20 1
source $ADMIN_RCFILE ci-user ci source $ADMIN_RCFILE ci-user ci
# add images for tests # add images for tests
glance image-create --name ubuntu_vanilla_2.6_latest --file $VANILLA26_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.6.0'='True' --property '_sahara_tag_vanilla'='True' --property '_sahara_username'='ubuntu' glance image-create --name $(basename -s .qcow2 $VANILLA_2_6_0_IMAGE_PATH) --file $VANILLA_2_6_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.6.0'='True' --property '_sahara_tag_vanilla'='True' --property '_sahara_username'='ubuntu'
glance image-create --name sahara_hdp_2_latest --file $HDP2_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.0.6'='True' --property '_sahara_tag_hdp'='True' --property '_sahara_username'='cloud-user' glance image-create --name $(basename -s .qcow2 $HDP_2_0_6_IMAGE_PATH) --file $HDP_2_0_6_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.0.6'='True' --property '_sahara_tag_hdp'='True' --property '_sahara_username'='cloud-user'
glance image-create --name centos_cdh_latest --file $CENTOS_CDH_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="cloud-user" glance image-create --name $(basename -s .qcow2 $CENTOS_CDH_5_3_0_IMAGE_PATH) --file $CENTOS_CDH_5_3_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="cloud-user"
glance image-create --name ubuntu_cdh_latest --file $UBUNTU_CDH_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu" glance image-create --name $(basename -s .qcow2 $UBUNTU_CDH_5_3_0_IMAGE_PATH) --file $UBUNTU_CDH_5_3_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu"
glance image-create --name sahara_spark_latest --file $SPARK_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_spark'='True' --property '_sahara_tag_1.0.0'='True' --property '_sahara_username'="ubuntu" glance image-create --name $(basename -s .qcow2 $UBUNTU_CDH_5_4_0_IMAGE_PATH) --file $UBUNTU_CDH_5_4_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.4.0'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu"
glance image-create --name ubuntu_mapr_latest --file $MAPR_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_4.0.2.mrv2'='True' --property '_sahara_username'="ubuntu" glance image-create --name $(basename -s .qcow2 $SPARK_1_0_0_IMAGE_PATH) --file $SPARK_1_0_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_spark'='True' --property '_sahara_tag_1.0.0'='True' --property '_sahara_username'="ubuntu"
glance image-create --name ubuntu-test-image --file $NATIVE_UBUNTU_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true glance image-create --name $(basename -s .qcow2 $MAPR_4_0_2_IMAGE_PATH) --file $MAPR_4_0_2_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_4.0.2.mrv2'='True' --property '_sahara_username'="ubuntu"
glance image-create --name ubuntu-test-image --file $UBUNTU_12_04_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true
glance image-update --name ubuntu-12.04 --property '_sahara_tag_ci'='True' ubuntu-12.04-server-cloudimg-amd64-disk1 glance image-update --name ubuntu-12.04 --property '_sahara_tag_ci'='True' ubuntu-12.04-server-cloudimg-amd64-disk1
glance image-update --name ubuntu-14.04 trusty-server-cloudimg-amd64-disk1 glance image-update --name ubuntu-14.04 trusty-server-cloudimg-amd64-disk1

0
config/sahara/pip.conf → config/pip.conf
View File

70
config/zuul/layout.yaml
View File

@ -27,51 +27,51 @@ pipelines:
projects: projects:
- name: openstack/sahara - name: openstack/sahara
check: check:
- gate-sahara-neutron-heat-vanilla_2.6 - gate-sahara-neutron-heat-vanilla_2.6.0-u14
- gate-sahara-neutron-direct-spark-aio - gate-sahara-neutron-direct-spark_1.0.0-u14-aio
- gate-sahara-neutron-direct-transient - gate-sahara-neutron-direct-transient-u14
- gate-sahara-neutron-heat-transient - gate-sahara-neutron-heat-transient-u14
- gate-sahara-neutron-heat-mapr - gate-sahara-neutron-heat-mapr_4.0.2.mrv2-u14
- gate-sahara-nova-direct-cdh_5.3.0_ubuntu-aio - gate-sahara-nova-direct-cdh_5.3.0-u12-aio
- gate-sahara-nova-direct-cdh_5.3.0_centos-aio - gate-sahara-nova-direct-cdh_5.3.0-c6.6-aio
- gate-sahara-nova-direct-cdh_5.4.0-aio - gate-sahara-nova-direct-cdh_5.4.0-u12-aio
- gate-sahara-nova-heat-hdp_2 - gate-sahara-nova-heat-hdp_2.0.6-c6.6
- gate-sahara-pep8-trunk - gate-sahara-pep8-trunk
# - tempest-sahara-tests # - tempest-sahara-tests
- name: openstack/python-saharaclient - name: openstack/python-saharaclient
check: check:
- gate-saharaclient-neutron-heat-vanilla_2.6 - gate-saharaclient-neutron-heat-vanilla_2.6.0-u14
- gate-saharaclient-neutron-direct-spark-aio - gate-saharaclient-neutron-direct-spark_1.0.0-u14-aio
- gate-saharaclient-neutron-direct-transient - gate-saharaclient-neutron-direct-transient-u14
- gate-saharaclient-neutron-heat-transient - gate-saharaclient-neutron-heat-transient-u14
- gate-saharaclient-neutron-heat-mapr - gate-saharaclient-neutron-heat-mapr_4.0.2.mrv2-u14
- gate-saharaclient-nova-direct-cdh_5.3.0_ubuntu-aio - gate-saharaclient-nova-direct-cdh_5.3.0-u12-aio
- gate-saharaclient-nova-direct-cdh_5.3.0_centos-aio - gate-saharaclient-nova-direct-cdh_5.3.0-c6.6-aio
- gate-saharaclient-nova-direct-cdh_5.4.0-aio - gate-saharaclient-nova-direct-cdh_5.4.0-u12-aio
- gate-saharaclient-nova-heat-hdp_2 - gate-saharaclient-nova-heat-hdp_2.0.6-c6.6
- gate-sahara-pep8-trunk - gate-sahara-pep8-trunk
# - tempest-saharaclient-tests # - tempest-saharaclient-tests
- name: openstack/sahara-image-elements - name: openstack/sahara-image-elements
check: check:
- dib-neutron-heat-vanilla_2.6-ubuntu - dib-neutron-heat-vanilla_2.6.0-u14
- dib-neutron-heat-vanilla_2.6-centos - dib-neutron-heat-vanilla_2.6.0-c6.6
- dib-neutron-heat-mapr - dib-neutron-heat-mapr_4.0.2.mrv2-u14
- dib-nova-direct-spark-aio - dib-neutron-direct-spark_1.0.0-u14-aio
- dib-nova-heat-hdp_2 - dib-nova-heat-hdp_2.0.6-c6.6
- dib-nova-direct-cdh_5.3.0-ubuntu-aio - dib-nova-direct-cdh_5.3.0-u12-aio
- dib-nova-direct-cdh_5.3.0-centos-aio - dib-nova-direct-cdh_5.3.0-c6.6-aio
- dib-nova-direct-cdh_5.4.0-ubuntu-aio - dib-nova-direct-cdh_5.4.0-u12-aio
post: post:
- dib-neutron-heat-vanilla_2.6-ubuntu - dib-neutron-heat-vanilla_2.6.0-u14
- dib-neutron-heat-vanilla_2.6-centos - dib-neutron-heat-vanilla_2.6.0-c6.6
- dib-neutron-heat-mapr - dib-neutron-heat-mapr_4.0.2.mrv2-u14
- dib-nova-direct-spark-aio - dib-neutron-direct-spark_1.0.0-u14-aio
- dib-nova-heat-hdp_2 - dib-nova-heat-hdp_2.0.6-c6.6
- dib-nova-direct-cdh_5.3.0-ubuntu-aio - dib-nova-direct-cdh_5.3.0-u12-aio
- dib-nova-direct-cdh_5.3.0-centos-aio - dib-nova-direct-cdh_5.3.0-c6.6-aio
- dib-nova-direct-cdh_5.4.0-ubuntu-aio - dib-nova-direct-cdh_5.4.0-u12-aio
- name: stackforge/sahara-ci-config - name: stackforge/sahara-ci-config
check: check:
@ -91,7 +91,7 @@ jobs:
- all-files-match-any: - all-files-match-any:
- ^doc/.*$ - ^doc/.*$
- ^.*\.rst$ - ^.*\.rst$
- name: ^.*-neutron-heat-mapr$ - name: ^.*-mapr_4.0.2.mrv2-u14$
branch: master branch: master
voting: false voting: false
- name: ^.*-cdh_5.4.0-.*$ - name: ^.*-cdh_5.4.0-.*$

101
jenkins_job_builder/diskimage.yaml
View File

@ -1,101 +1,62 @@
- job:
name: 'dib-nova-heat-hdp_2'
defaults: global
builders:
- gerrit-git-prep
- ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh hdp_2"
properties:
- zeromq-event
publishers:
- sahara-logs
- console-log
- trigger-cleanup
node: 'trusty-nova'
- job:
name: 'dib-neutron-heat-mapr'
defaults: global
builders:
- gerrit-git-prep
- ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh mapr"
properties:
- zeromq-event
publishers:
- sahara-logs
- console-log
- trigger-cleanup
node: 'trusty-neutron'
- job-template: - job-template:
name: 'dib-nova-direct-{plugin}-{os}-aio' name: 'dib-{network}-{engine}-{plugin}-{os}-aio'
defaults: global defaults: global
builders: builders:
- gerrit-git-prep - gerrit-git-prep
- ci-config-prep - ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh {plugin} {os}" - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh {plugin} {os}"
properties: properties:
- zeromq-event - zeromq-event
publishers: publishers:
- sahara-logs - sahara-logs
- console-log - console-log
- trigger-cleanup - trigger-cleanup
node: 'trusty-nova' node: 'trusty-{network}'
- job:
name: 'dib-nova-direct-spark-aio'
defaults: global
builders:
- gerrit-git-prep
- ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh {plugin}"
properties:
- zeromq-event
publishers:
- sahara-logs
- console-log
- trigger-cleanup
node: 'trusty-nova'
- job-template: - job-template:
name: 'dib-neutron-heat-vanilla_2.6-{os}' name: 'dib-{network}-{engine}-{plugin}-{os}'
defaults: global defaults: global
builders: builders:
- gerrit-git-prep - gerrit-git-prep
- ci-config-prep - ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh {plugin} {os}" - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh {plugin} {os}"
properties: properties:
- zeromq-event - zeromq-event
publishers: publishers:
- sahara-logs - sahara-logs
- console-log - console-log
- trigger-cleanup - trigger-cleanup
node: 'trusty-neutron' node: 'trusty-{network}'
- project: - project:
name: sahara-image-elements name: sahara-image-elements
github-org: openstack github-org: openstack
plugin: cdh_5.3.0 network: neutron
engine: heat
plugin: vanilla_2.6.0
os: os:
- ubuntu - u14
- centos - c6.6
jobs: jobs:
- 'dib-neutron-heat-vanilla_2.6-{os}' - 'dib-{network}-{engine}-{plugin}-{os}-aio'
- 'dib-nova-heat-hdp_2' - 'dib-{network}-{engine}-{plugin}-{os}-aio':
- 'dib-neutron-heat-mapr' network: nova
- 'dib-nova-direct-spark-aio' engine: direct
- 'dib-nova-direct-{plugin}-{os}-aio' plugin:
- 'dib-nova-direct-{plugin}-{os}-aio': - cdh_5.3.0
plugin: cdh_5.4.0 - cdh_5.4.0
os: ubuntu os:
- u12
- c6.6
- 'dib-{network}-{engine}-{plugin}-{os}-aio':
engine: direct
plugin: spark_1.0.0
os: u14
- 'dib-{network}-{engine}-{plugin}-{os}':
plugin: mapr_4.0.2.mrv2
os: u14
- 'dib-{network}-{engine}-{plugin}-{os}':
network: nova
plugin: hdp_2.0.6
os: c6.6

121
jenkins_job_builder/sahara.yaml
View File

@ -1,7 +1,6 @@
- job: - job-template:
name: 'tempest-saharaclient-tests' name: 'tempest-{project_name}-tests'
defaults: global defaults: global
concurrent: false
builders: builders:
- gerrit-git-prep - gerrit-git-prep
- ci-config-prep - ci-config-prep
@ -10,26 +9,7 @@
- zeromq-event - zeromq-event
- build-blocker: - build-blocker:
blocking-jobs: blocking-jobs:
- "tempest-sahara-tests" - "tempest-.*-tests"
publishers:
- sahara-logs
- console-log
- trigger-cleanup
node: trusty-neutron || trusty-nova
- job:
name: 'tempest-sahara-tests'
defaults: global
concurrent: false
builders:
- gerrit-git-prep
- ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/tempest.sh"
properties:
- zeromq-event
- build-blocker:
blocking-jobs:
- "tempest-saharaclient-tests"
publishers: publishers:
- sahara-logs - sahara-logs
- console-log - console-log
@ -37,84 +17,72 @@
node: trusty-neutron || trusty-nova node: trusty-neutron || trusty-nova
- job-template: - job-template:
name: 'gate-saharaclient-neutron-{plugin-neutron}' name: 'gate-{project_name}-{network}-{engine}-{plugin}-{os}'
defaults: global defaults: global
builders: builders:
- gerrit-git-prep - gerrit-git-prep
- ci-config-prep - ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-saharaclient.sh" - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-{project_name}.sh"
properties: properties:
- zeromq-event - zeromq-event
publishers: publishers:
- sahara-logs - sahara-logs
- console-log - console-log
- trigger-cleanup - trigger-cleanup
node: 'trusty-neutron' node: 'trusty-{network}'
- job-template: - job-template:
name: 'gate-saharaclient-nova-{plugin-nova_network}' name: 'gate-{project_name}-{network}-{engine}-{plugin}-{os}-aio'
defaults: global defaults: global
builders: builders:
- gerrit-git-prep - gerrit-git-prep
- ci-config-prep - ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-saharaclient.sh" - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-{project_name}.sh"
properties: properties:
- zeromq-event - zeromq-event
publishers: publishers:
- sahara-logs - sahara-logs
- console-log - console-log
- trigger-cleanup - trigger-cleanup
node: 'trusty-nova' node: 'trusty-{network}'
- job-template:
name: 'gate-sahara-neutron-{plugin-neutron}'
defaults: global
builders:
- gerrit-git-prep
- ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-sahara.sh"
properties:
- zeromq-event
publishers:
- sahara-logs
- console-log
- trigger-cleanup
node: 'trusty-neutron'
- job-template:
name: 'gate-sahara-nova-{plugin-nova_network}'
defaults: global
builders:
- gerrit-git-prep
- ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-sahara.sh"
properties:
- zeromq-event
publishers:
- sahara-logs
- console-log
- trigger-cleanup
node: 'trusty-nova'
- project: - project:
name: sahara name: main
github-org: openstack project_name:
plugin-neutron: - sahara
- heat-vanilla_2.6 - saharaclient
- direct-spark-aio network: nova
- direct-transient engine: direct
- heat-transient plugin:
- heat-mapr - cdh_5.3.0
plugin-nova_network: - cdh_5.4.0
- direct-cdh_5.3.0_ubuntu-aio os:
- direct-cdh_5.3.0_centos-aio - u12
- direct-cdh_5.4.0-aio - c6.6
- heat-hdp_2
jobs: jobs:
- 'gate-sahara-neutron-{plugin-neutron}' - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}-aio'
- 'gate-sahara-nova-{plugin-nova_network}' - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}':
- 'gate-saharaclient-neutron-{plugin-neutron}' engine: heat
- 'gate-saharaclient-nova-{plugin-nova_network}' plugin:
- vanilla_2.6.0
- transient
- mapr_4.0.2
os: u14
network: neutron
- 'gate-{project_name}-{network}-{engine}-{plugin}-{os}':
plugin: transient
os: u14
network: neutron
- 'gate-{project_name}-{network}-{engine}-{plugin}-{os}-aio':
plugin: spark_1.0.0
os: u14
network: neutron
- 'gate-{project_name}-{network}-{engine}-{plugin}-{os}':
network: nova
engine: heat
plugin: hdp_2.0.6
os: c6.6
- 'tempest-{project_name}-tests'
- job: - job:
name: gate-sahara-pep8-trunk name: gate-sahara-pep8-trunk
@ -125,7 +93,6 @@
\n<p>This job runs pep8 check using trunk version of hacking" \n<p>This job runs pep8 check using trunk version of hacking"
disabled: false disabled: false
node: trusty-neutron || trusty-nova node: trusty-neutron || trusty-nova
builders: builders:
- gerrit-git-prep - gerrit-git-prep
- ci-config-prep - ci-config-prep

112
slave-scripts/dib.sh
View File

@ -10,20 +10,17 @@ CLUSTER_HASH=${CLUSTER_HASH:-$RANDOM}
cluster_name="$HOST-$ZUUL_CHANGE-$CLUSTER_HASH" cluster_name="$HOST-$ZUUL_CHANGE-$CLUSTER_HASH"
SAHARA_PATH="/tmp/sahara" SAHARA_PATH="/tmp/sahara"
sahara_conf_path="$SAHARA_PATH/etc/sahara/sahara.conf" sahara_conf_file="$SAHARA_PATH/etc/sahara/sahara.conf"
sahara_templates_path=$SAHARA_PATH/etc/scenario/sahara-ci sahara_templates_path=$SAHARA_PATH/etc/scenario/sahara-ci
engine=$(echo $JOB_NAME | awk -F '-' '{ print $3 }') engine=$(echo $JOB_NAME | awk -F '-' '{ print $3 }')
job_type="$1"
image_type=${2:-ubuntu}
# Image names plugin="$1"
vanilla_two_six_image=$HOST-sahara-vanilla-${image_type}-${ZUUL_CHANGE}-hadoop_2.6 os="$2"
hdp_two_image=$HOST-sahara-hdp-centos-${ZUUL_CHANGE}-hadoop_2 image_name=${HOST}_${plugin}_${os}_${ZUUL_CHANGE}
spark_image=$HOST-sahara-spark-ubuntu-${ZUUL_CHANGE} eval ${plugin//./_}_image=$image_name
cdh_image=$HOST-${image_type}-cdh-${ZUUL_CHANGE} mode="aio"
cdh_5_4_0_image=$HOST-${image_type}-cdh_5.4.0-${ZUUL_CHANGE} sahara_plugin=$(echo $plugin | awk -F '_' '{ print $1 } ')
mapr_402mrv2_image=$HOST-${image_type}-mapr-${ZUUL_CHANGE}
# Clone Sahara # Clone Sahara
git clone https://review.openstack.org/openstack/sahara $SAHARA_PATH -b $ZUUL_BRANCH git clone https://review.openstack.org/openstack/sahara $SAHARA_PATH -b $ZUUL_BRANCH
@ -31,79 +28,66 @@ git clone https://review.openstack.org/openstack/sahara $SAHARA_PATH -b $ZUUL_BR
# make verbose the scripts execution of disk-image-create # make verbose the scripts execution of disk-image-create
export DIB_DEBUG_TRACE=1 export DIB_DEBUG_TRACE=1
case $job_type in if [ "${os}" == 'c6.6' ]; then
vanilla_2.6) username="cloud-user"
if [ "${image_type}" == 'centos' ]; then os_type="centos"
username='cloud-user' else
else username="ubuntu"
username='ubuntu' os_type="ubuntu"
fi fi
env ${image_type}_vanilla_hadoop_2_6_image_name=${vanilla_two_six_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p vanilla -i $image_type -v 2.6 case $plugin in
check_error_code $? ${vanilla_two_six_image}.qcow2 vanilla_2.6.0)
upload_image "vanilla-2.6" "${username}" ${vanilla_two_six_image} env ${os_type}_vanilla_hadoop_2_6_image_name=${vanilla_2_6_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p vanilla -i $os_type -v 2.6
DISTRIBUTE_MODE=True check_error_code $? ${vanilla_2_6_0_image}.qcow2
tests_config_file="$sahara_templates_path/vanilla-2.6.0.yaml" upload_image "${plugin}" "${username}" ${vanilla_2_6_0_image}
insert_scenario_value $tests_config_file vanilla_two_six_image mode=distribute
scenario_conf_file="$sahara_templates_path/vanilla-2.6.0.yaml"
;; ;;
spark) spark_1.0.0)
env ubuntu_spark_image_name=${spark_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p spark env ubuntu_spark_image_name=${spark_1_0_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p spark
check_error_code $? ${spark_image}.qcow2 check_error_code $? ${spark_1_0_0_image}.qcow2
upload_image "spark" "ubuntu" ${spark_image} upload_image "${plugin}" "${username}" ${spark_1_0_0_image}
tests_config_file="$sahara_templates_path/spark-1.0.0.yaml" scenario_conf_file="$sahara_templates_path/spark-1.0.0.yaml"
insert_scenario_value $tests_config_file spark_image
insert_config_value $sahara_conf_path DEFAULT plugins spark
;; ;;
hdp_2) hdp_2.0.6)
env centos_hdp_hadoop_2_image_name=${hdp_two_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p hdp -v 2 env centos_hdp_hadoop_2_image_name=${hdp_2_0_6_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p hdp -v 2
check_error_code $? ${hdp_two_image}.qcow2 check_error_code $? ${hdp_2_0_6_image}.qcow2
upload_image "hdp2" "cloud-user" ${hdp_two_image} upload_image "${plugin}" "${username}" ${hdp_2_0_6_image}
DISTRIBUTE_MODE=True mode=distribute
tests_config_file="$sahara_templates_path/hdp-2.0.6.yaml" scenario_conf_file="$sahara_templates_path/hdp-2.0.6.yaml"
insert_scenario_value $tests_config_file hdp_two_image
;; ;;
cdh_5.3.0) cdh_5.3.0)
if [ "${image_type}" == 'centos' ]; then env cloudera_5_3_${os_type}_image_name=${cdh_5_3_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i $os_type -v 5.3
username='cloud-user' check_error_code $? ${cdh_5_3_0_image}.qcow2
else upload_image "${plugin}" "${username}" ${cdh_5_3_0_image}
username='ubuntu' scenario_conf_file="$sahara_templates_path/cdh-5.3.0.yaml"
fi
env cloudera_5_3_${image_type}_image_name=${cdh_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i $image_type -v 5.3
check_error_code $? ${cdh_image}.qcow2
upload_image "cdh_5.3.0" ${username} ${cdh_image}
tests_config_file="$sahara_templates_path/cdh-5.3.0.yaml"
insert_config_value $sahara_conf_path DEFAULT plugins cdh
insert_scenario_value $tests_config_file cdh_image
;; ;;
cdh_5.4.0) cdh_5.4.0)
env cloudera_5_4_ubuntu_image_name=${cdh_5_4_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i ubuntu -v 5.4 env cloudera_5_4_ubuntu_image_name=${cdh_5_4_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i ubuntu -v 5.4
check_error_code $? ${cdh_5_4_0_image}.qcow2 check_error_code $? ${cdh_5_4_0_image}.qcow2
upload_image "cdh_5.4.0" "ubuntu" ${cdh_5_4_0_image} upload_image "${plugin}" "${username}" ${cdh_5_4_0_image}
tests_config_file="$sahara_templates_path/cdh-5.4.0.yaml" scenario_conf_file="$sahara_templates_path/cdh-5.4.0.yaml"
insert_config_value $sahara_conf_path DEFAULT plugins cdh
insert_scenario_value $tests_config_file cdh_5_4_0_image
;; ;;
mapr) mapr_4.0.2.mrv2)
env mapr_ubuntu_image_name=${mapr_402mrv2_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p mapr -i ubuntu env mapr_ubuntu_image_name=${mapr_4_0_2_mrv2_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p mapr -i ubuntu
check_error_code $? ${mapr_402mrv2_image}.qcow2 check_error_code $? ${mapr_4_0_2_mrv2_image}.qcow2
upload_image "mapr" "ubuntu" ${mapr_402mrv2_image} upload_image "${plugin}" "${username}" ${mapr_4_0_2_mrv2_image}
DISTRIBUTE_MODE=True mode=distribute
tests_config_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml" scenario_conf_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml"
insert_config_value $sahara_conf_path DEFAULT plugins mapr
insert_scenario_value $tests_config_file mapr_402mrv2_image
;; ;;
esac esac
cd $SAHARA_PATH cd $SAHARA_PATH
sudo pip install . --no-cache-dir sudo pip install . --no-cache-dir
enable_pypi enable_pypi
write_sahara_main_conf "$sahara_conf_path" "$engine" write_sahara_main_conf "$sahara_conf_file" "$engine" "$sahara_plugin"
write_tests_conf "$tests_config_file" "$cluster_name" write_tests_conf "$scenario_conf_file" "$cluster_name" "$image_name"
start_sahara "$sahara_conf_path" && run_tests "$tests_config_file" start_sahara "$sahara_conf_file" "$mode" && run_tests "$scenario_conf_file"
print_python_env print_python_env
cleanup_image "$job_type" "$image_type" cleanup_image "$plugin" "$os"

196
slave-scripts/functions-common.sh
View File

@ -1,15 +1,9 @@
#!/bin/bash -xe #!/bin/bash -xe
sahara_configs_path=$WORKSPACE/sahara-ci-config/config/sahara configs_path=$WORKSPACE/sahara-ci-config/config
ci_flavor_id=\'20\' eval ci_flavor_id="\'20\'"
medium_flavor_id=\'3\' eval medium_flavor_id="\'3\'"
large_flavor_id=\'4\' eval large_flavor_id="\'4\'"
enable_pypi() {
mkdir -p ~/.pip
export PIP_USE_MIRRORS=True
cp $sahara_configs_path/pip.conf ~/.pip/pip.conf
}
conf_has_option() { conf_has_option() {
local file=$1 local file=$1
@ -21,6 +15,19 @@ conf_has_option() {
[ -n "$line" ] [ -n "$line" ]
} }
enable_pypi() {
mkdir -p ~/.pip
export PIP_USE_MIRRORS=True
cp $configs_path/pip.conf ~/.pip/pip.conf
}
failure() {
local reason=$1
print_python_env
echo "$reason"
exit 1
}
insert_config_value() { insert_config_value() {
local file=$1 local file=$1
local section=$2 local section=$2
@ -46,30 +53,21 @@ $option = $value
} }
insert_scenario_value() { insert_scenario_value() {
local config=$1 local file=$1
local value=$2 local main_key=$2
sed -i "s/%${value}%/${!value}/g" $config local stop_key=$3
} local sub_key=$4
local value=$5
local old_value=$6
write_sahara_main_conf() { [[ -z $main_key || -z $sub_key ]] && return
local conf_path=$1
local engine=$2
insert_config_value $conf_path DEFAULT infrastructure_engine $engine
insert_config_value $conf_path DEFAULT api_workers 4
insert_config_value $conf_path DEFAULT use_identity_api_v3 true
insert_config_value $conf_path DEFAULT use_neutron $USE_NEUTRON
insert_config_value $conf_path DEFAULT min_transient_cluster_active_time 30
insert_config_value $conf_path DEFAULT node_domain ci
insert_config_value $conf_path database connection mysql://sahara-citest:sahara-citest@localhost/sahara?charset=utf8
insert_config_value $conf_path keystone_authtoken auth_uri http://$OPENSTACK_HOST:5000/v2.0/
insert_config_value $conf_path keystone_authtoken identity_uri http://$OPENSTACK_HOST:35357/
insert_config_value $conf_path keystone_authtoken admin_user $OS_USERNAME
insert_config_value $conf_path keystone_authtoken admin_password $OS_PASSWORD
insert_config_value $conf_path keystone_authtoken admin_tenant_name $OS_TENANT_NAME
echo "----------- sahara.conf -----------" if ! scenario_has_option "$file" "$main_key" "$stop_key" "$sub_key"; then
cat $conf_path echo "No such keys: $main_key -> $sub_key in scenario $file file. Skip setting value $value"
echo "--------------- end ---------------" else
local sep=$(echo -ne "\x01")
sed -i -e '/'${main_key}':/,/'${stop_key}'/ s'${sep}'\([ \t]'${sub_key}':[ \t]\).*'${old_value}'.*$'${sep}'\1'${value}${sep} $file
fi
} }
print_python_env() { print_python_env() {
@ -78,58 +76,6 @@ print_python_env() {
pip freeze > $WORKSPACE/logs/python-system-env.txt pip freeze > $WORKSPACE/logs/python-system-env.txt
} }
failure() {
local reason=$1
print_python_env
echo "$reason"
exit 1
}
start_sahara() {
local conf_path=$1
local conf_dir=$(dirname $1)
mkdir $WORKSPACE/logs
sahara-db-manage --config-file $conf_path upgrade head || failure "Command 'sahara-db-manage' failed"
if [ "$DISTRIBUTE_MODE" == "True" ]; then
screen -dmS sahara-api /bin/bash -c "PYTHONUNBUFFERED=1 sahara-api --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-api.txt"
sleep 2
screen -dmS sahara-engine_1 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-1.txt"
screen -dmS sahara-engine_2 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-2.txt"
else
screen -dmS sahara-all /bin/bash -c "PYTHONUNBUFFERED=1 sahara-all --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log.txt"
fi
api_responding_timeout=30
if ! timeout ${api_responding_timeout} sh -c "while ! curl -s http://127.0.0.1:8386/v1.1/ 2>/dev/null | grep -q 'Authentication required' ; do sleep 1; done"; then
failure "Sahara API failed to respond within ${api_responding_timeout} seconds"
fi
}
write_tests_conf() {
local test_conf=$1
local cluster_name=$2
local addr=$(ifconfig eth0| awk -F ' *|:' '/inet addr/{print $4}')
if [ "$USE_NEUTRON" == "true" ]; then
NETWORK="neutron"
else
NETWORK="nova-network"
fi
local test_scenario_credentials=$(dirname $1)/credentials.yaml
insert_scenario_value $test_scenario_credentials OS_USERNAME
insert_scenario_value $test_scenario_credentials OS_PASSWORD
insert_scenario_value $test_scenario_credentials OS_TENANT_NAME
insert_scenario_value $test_scenario_credentials OPENSTACK_HOST
insert_scenario_value $test_scenario_credentials NETWORK
insert_scenario_value $test_conf cluster_name
insert_scenario_value $test_conf ci_flavor_id
insert_scenario_value $test_conf medium_flavor_id
insert_scenario_value $test_conf large_flavor_id
echo "----------- tests config -----------"
cat $test_conf
echo "---------------- end ---------------"
}
run_tests() { run_tests() {
local config=$1 local config=$1
local concurrency=${2:-"1"} local concurrency=${2:-"1"}
@ -143,3 +89,85 @@ run_tests() {
STATUS=$(grep "\ -\ Failed" tox.log | awk '{print $3}') STATUS=$(grep "\ -\ Failed" tox.log | awk '{print $3}')
if [ "$STATUS" != "0" ]; then failure "Integration tests have failed"; fi if [ "$STATUS" != "0" ]; then failure "Integration tests have failed"; fi
} }
scenario_has_option() {
local file=$1
local main_key=$2
local stop_key=$3
local sub_key=$4
local line
line=$(sed -ne "/$main_key\:/,/$stop_key/ { /[ \t]$sub_key:/ p; }" "$file")
[ -n "$line" ]
}
start_sahara() {
local conf_path=$1
local conf_dir=$(dirname $1)
local mode=$2
mkdir $WORKSPACE/logs
sahara-db-manage --config-file $conf_path upgrade head || failure "Command 'sahara-db-manage' failed"
if [ "$mode" == "distribute" ]; then
screen -dmS sahara-api /bin/bash -c "PYTHONUNBUFFERED=1 sahara-api --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-api.txt"
sleep 2
screen -dmS sahara-engine_1 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-1.txt"
screen -dmS sahara-engine_2 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-2.txt"
else
screen -dmS sahara-all /bin/bash -c "PYTHONUNBUFFERED=1 sahara-all --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log.txt"
fi
api_responding_timeout=30
if ! timeout ${api_responding_timeout} sh -c "while ! curl -s http://127.0.0.1:8386/v1.1/ 2>/dev/null | grep -q 'Authentication required' ; do sleep 1; done"; then
failure "Sahara API failed to respond within ${api_responding_timeout} seconds"
fi
}
write_sahara_main_conf() {
local conf_path=$1
local engine=$2
local plugin=$3
insert_config_value $conf_path DEFAULT infrastructure_engine $engine
insert_config_value $conf_path DEFAULT api_workers 4
insert_config_value $conf_path DEFAULT use_identity_api_v3 true
insert_config_value $conf_path DEFAULT use_neutron $USE_NEUTRON
insert_config_value $conf_path DEFAULT min_transient_cluster_active_time 30
insert_config_value $conf_path DEFAULT node_domain ci
insert_config_value $conf_path DEFAULT plugins $plugin
insert_config_value $conf_path database connection mysql://sahara-citest:sahara-citest@localhost/sahara?charset=utf8
insert_config_value $conf_path keystone_authtoken auth_uri http://$OPENSTACK_HOST:5000/v2.0/
insert_config_value $conf_path keystone_authtoken identity_uri http://$OPENSTACK_HOST:35357/
insert_config_value $conf_path keystone_authtoken admin_user $OS_USERNAME
insert_config_value $conf_path keystone_authtoken admin_password $OS_PASSWORD
insert_config_value $conf_path keystone_authtoken admin_tenant_name $OS_TENANT_NAME
echo "----------- sahara.conf -----------"
cat $conf_path
echo "--------------- end ---------------"
}
write_tests_conf() {
local test_conf=$1
local cluster_name=$2
local image_name=$3
local os_auth_url="http://$OPENSTACK_HOST:5000/v2.0/"
if [ "$USE_NEUTRON" == "true" ]; then
NETWORK="neutron"
else
NETWORK="nova-network"
fi
local test_scenario_credentials=$(dirname $1)/credentials.yaml
insert_scenario_value $test_scenario_credentials credentials "" os_username $OS_USERNAME
insert_scenario_value $test_scenario_credentials credentials "" os_password $OS_PASSWORD
insert_scenario_value $test_scenario_credentials credentials "" os_tenant $OS_TENANT_NAME
insert_scenario_value $test_scenario_credentials credentials "" os_auth_url $os_auth_url
insert_scenario_value $test_scenario_credentials network "" "type" $NETWORK
insert_scenario_value $test_conf clusters node_group_templates image $image_name
insert_scenario_value $test_conf cluster "" name $cluster_name
insert_scenario_value $test_conf node_group_templates "\\$" flavor_id $ci_flavor_id ci_flavor_id
insert_scenario_value $test_conf node_group_templates "\\$" flavor_id $medium_flavor_id medium_flavor_id
insert_scenario_value $test_conf node_group_templates "\\$" flavor_id $large_flavor_id large_flavor_id
echo "----------- tests config -----------"
cat $test_conf
echo "---------------- end ---------------"
}

118
slave-scripts/functions-dib.sh
View File

@ -2,55 +2,6 @@
CUR_IMAGE=none CUR_IMAGE=none
register_new_image() {
local image_name=$1
local image_properties=$2
glance image-create --name $1 --file $1.qcow2 --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' $image_properties
}
delete_image() {
# "|| true" here, to avoid error code producing in case of missing image
glance image-delete $1 || true
}
upload_image() {
local plugin=$1
local username=$2
local image=$3
delete_image "$image"
case "$plugin" in
vanilla-2.4)
image_properties="--property _sahara_tag_2.4.1=True --property _sahara_tag_vanilla=True --property _sahara_username=${username}"
;;
vanilla-2.6)
image_properties="--property _sahara_tag_2.6.0=True --property _sahara_tag_vanilla=True --property _sahara_username=${username}"
;;
hdp2)
image_properties="--property _sahara_tag_2.0.6=True --property _sahara_tag_hdp=True --property _sahara_username=${username}"
;;
cdh_5.3.0)
image_properties="--property _sahara_tag_5.3.0=True --property _sahara_tag_5=True --property _sahara_tag_cdh=True --property _sahara_username=${username}"
;;
cdh_5.4.0)
image_properties="--property _sahara_tag_5.4.0=True --property _sahara_tag_cdh=True --property _sahara_username=${username}"
;;
spark)
image_properties="--property _sahara_tag_spark=True --property _sahara_tag_1.0.0=True --property _sahara_username=${username}"
;;
mapr)
image_properties="--property _sahara_tag_mapr=True --property _sahara_tag_4.0.2.mrv2=True --property _sahara_username=${username}"
;;
esac
register_new_image "$image" "$image_properties"
CUR_IMAGE="$image"
}
rename_image() {
# 1 - source image, 2 - target image
glance image-update $1 --name $2
}
check_error_code() { check_error_code() {
if [ "$1" != "0" -o ! -f "$2" ]; then if [ "$1" != "0" -o ! -f "$2" ]; then
echo "$2 image isn't build" echo "$2 image isn't build"
@ -58,6 +9,22 @@ check_error_code() {
fi fi
} }
cleanup_image() {
local plugin=$1
local os=$2
if [ "$ZUUL_PIPELINE" == "check" -o "$ZUUL_BRANCH" != "master" ]; then
delete_image "$CUR_IMAGE"
else
delete_image ${plugin}_${os}
rename_image "$CUR_IMAGE" ${plugin}_${os}
fi
}
delete_image() {
# "|| true" here, to avoid error code producing in case of missing image
glance image-delete $1 || true
}
failure() { failure() {
local reason=$1 local reason=$1
echo "$reason" echo "$reason"
@ -66,37 +33,42 @@ failure() {
exit 1 exit 1
} }
cleanup_image() { register_new_image() {
local job_type=$1 local image_name=$1
local os=$2 local image_properties=$2
if [ "$ZUUL_PIPELINE" == "check" -o "$ZUUL_BRANCH" != "master" ]; then glance image-create --name $image_name --file $image_name.qcow2 --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' $image_properties
delete_image "$CUR_IMAGE" }
else
case $job_type in rename_image() {
vanilla_2.6) # 1 - source image, 2 - target image
delete_image ${os}_vanilla_2.6_latest glance image-update $1 --name $2
rename_image "$CUR_IMAGE" ${os}_vanilla_2.6_latest }
upload_image() {
local plugin=$1
local username=$2
local image=$3
delete_image "$image"
case "$plugin" in
vanilla_2.6.0)
image_properties="--property _sahara_tag_2.6.0=True --property _sahara_tag_vanilla=True --property _sahara_username=${username}"
;; ;;
hdp_2) hdp_2.0.6)
delete_image sahara_hdp_2_latest image_properties="--property _sahara_tag_2.0.6=True --property _sahara_tag_hdp=True --property _sahara_username=${username}"
rename_image "$CUR_IMAGE" sahara_hdp_2_latest
;; ;;
cdh_5.3.0) cdh_5.3.0)
delete_image ${os}_cdh_latest image_properties="--property _sahara_tag_5.3.0=True --property _sahara_tag_5=True --property _sahara_tag_cdh=True --property _sahara_username=${username}"
rename_image "$CUR_IMAGE" ${os}_cdh_latest
;; ;;
cdh_5.4.0) cdh_5.4.0)
delete_image ${os}_cdh_5.4.0_latest image_properties="--property _sahara_tag_5.4.0=True --property _sahara_tag_cdh=True --property _sahara_username=${username}"
rename_image "$CUR_IMAGE" ${os}_cdh_5.4.0_latest
;; ;;
spark) spark_1.0.0)
delete_image sahara_spark_latest image_properties="--property _sahara_tag_spark=True --property _sahara_tag_1.0.0=True --property _sahara_username=${username}"
rename_image "$CUR_IMAGE" sahara_spark_latest
;; ;;
mapr) mapr_4.0.2.mrv2)
delete_image ubuntu_mapr_latest image_properties="--property _sahara_tag_mapr=True --property _sahara_tag_4.0.2.mrv2=True --property _sahara_username=${username}"
rename_image "$CUR_IMAGE" ubuntu_mapr_latest
;; ;;
esac esac
fi register_new_image "$image" "$image_properties"
CUR_IMAGE="$image"
} }

75
slave-scripts/gate-sahara.sh
View File

@ -9,69 +9,50 @@ CLUSTER_HASH=${CLUSTER_HASH:-$RANDOM}
cluster_name="$HOST-$ZUUL_CHANGE-$CLUSTER_HASH" cluster_name="$HOST-$ZUUL_CHANGE-$CLUSTER_HASH"
SAHARA_PATH=${1:-$WORKSPACE} SAHARA_PATH=${1:-$WORKSPACE}
sahara_conf_path=$SAHARA_PATH/etc/sahara/sahara.conf sahara_conf_file=$SAHARA_PATH/etc/sahara/sahara.conf
sahara_templates_path=$SAHARA_PATH/etc/scenario/sahara-ci sahara_templates_path=$SAHARA_PATH/etc/scenario/sahara-ci
job_type=$(echo $JOB_NAME | awk -F '-' '{ print $5 }')
engine_type=$(echo $JOB_NAME | awk -F '-' '{ print $4 }') engine_type=$(echo $JOB_NAME | awk -F '-' '{ print $4 }')
plugin=$(echo $JOB_NAME | awk -F '-' '{ print $5 }')
os=$(echo $JOB_NAME | awk -F '-' '{ print $6 }')
image_name=${plugin}_${os}
mode="aio"
sahara_plugin=$(echo $plugin | awk -F '_' '{ print $1 } ')
# Image names case $plugin in
hdp_two_image=sahara_hdp_2_latest hdp_2.0.6)
vanilla_two_six_image=ubuntu_vanilla_2.6_latest mode=distribute
spark_image=sahara_spark_latest scenario_conf_file="$sahara_templates_path/hdp-2.0.6.yaml"
cdh_5_3_0_centos_image=centos_cdh_latest
cdh_5_3_0_ubuntu_image=ubuntu_cdh_latest
cdh_5_4_0_image=ubuntu_cdh_5.4.0_latest
mapr_402mrv2_image=ubuntu_mapr_latest
case $job_type in
hdp_2)
DISTRIBUTE_MODE=True
tests_config_file="$sahara_templates_path/hdp-2.0.6.yaml"
insert_scenario_value $tests_config_file hdp_two_image
;; ;;
vanilla_2.6) vanilla_2.6.0)
DISTRIBUTE_MODE=True mode=distribute
tests_config_file="$sahara_templates_path/vanilla-2.6.0.yaml" scenario_conf_file="$sahara_templates_path/vanilla-2.6.0.yaml"
insert_scenario_value $tests_config_file vanilla_two_six_image
;; ;;
transient) transient)
# transient is using image with latest vanilla version
image_name=vanilla_2.6.0_u14
concurrency=3 concurrency=3
DISTRIBUTE_MODE=True mode=distribute
tests_config_file="$sahara_templates_path/transient.yaml" scenario_conf_file="$sahara_templates_path/transient.yaml"
insert_scenario_value $tests_config_file vanilla_two_six_image
;; ;;
cdh_5.3.0*) cdh_5.3.0)
insert_config_value $sahara_conf_path DEFAULT plugins cdh scenario_conf_file="$sahara_templates_path/cdh-5.3.0.yaml"
if [[ "$job_type" =~ centos ]]; then
cdh_image=$cdh_5_3_0_centos_image
else
cdh_image=$cdh_5_3_0_ubuntu_image
fi
tests_config_file="$sahara_templates_path/cdh-5.3.0.yaml"
insert_scenario_value $tests_config_file cdh_image
;; ;;
cdh_5.4.0) cdh_5.4.0)
insert_config_value $sahara_conf_path DEFAULT plugins cdh scenario_conf_file="$sahara_templates_path/cdh-5.4.0.yaml"
tests_config_file="$sahara_templates_path/cdh-5.4.0.yaml"
insert_scenario_value $tests_config_file cdh_5_4_0_image
;; ;;
spark) spark_1.0.0)
insert_config_value $sahara_conf_path DEFAULT plugins spark scenario_conf_file="$sahara_templates_path/spark-1.0.0.yaml"
tests_config_file="$sahara_templates_path/spark-1.0.0.yaml"
insert_scenario_value $tests_config_file spark_image
;; ;;
mapr) mapr_4.0.2.mrv2)
insert_config_value $sahara_conf_path DEFAULT plugins mapr mode=distribute
DISTRIBUTE_MODE=True scenario_conf_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml"
tests_config_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml"
insert_scenario_value $tests_config_file mapr_402mrv2_image
;; ;;
esac esac
sudo pip install . --no-cache-dir sudo pip install . --no-cache-dir
enable_pypi enable_pypi
write_sahara_main_conf "$sahara_conf_path" "$engine_type" write_sahara_main_conf "$sahara_conf_file" "$engine_type" "$sahara_plugin"
write_tests_conf "$tests_config_file" "$cluster_name" write_tests_conf "$scenario_conf_file" "$cluster_name" "$image_name"
start_sahara "$sahara_conf_path" && run_tests "$tests_config_file" "$concurrency" start_sahara "$sahara_conf_file" "$mode" && run_tests "$scenario_conf_file" "$concurrency"
print_python_env print_python_env

4
slave-scripts/gate-ui-tests.sh
View File

@ -49,12 +49,12 @@ floating_ip_pool = 'public'
keystone_url = 'http://$OPENSTACK_HOST:5000/v2.0' keystone_url = 'http://$OPENSTACK_HOST:5000/v2.0'
await_element = 120 await_element = 120
image_name_for_register = '$TEST_IMAGE' image_name_for_register = '$TEST_IMAGE'
image_name_for_edit = "sahara-itests-ci-vanilla-image" image_name_for_edit = 'sahara-itests-ci-vanilla-image'
security_groups = default security_groups = default
[vanilla] [vanilla]
skip_plugin_tests = False skip_plugin_tests = False
skip_edp_test = False skip_edp_test = False
base_image = "sahara-itests-ci-vanilla-image" base_image = 'sahara-itests-ci-vanilla-image'
[hdp] [hdp]
skip_plugin_tests = False skip_plugin_tests = False
hadoop_version = '1.3.2' hadoop_version = '1.3.2'