Merge "Gate: Modularise gate for development use"

.zuul.yaml

@@ -96,8 +96,11 @@
 
 - job:
     name: openstack-helm-infra
-    pre-run: tools/gate/playbooks/zuul-pre.yaml
+    pre-run:
-    run: tools/gate/playbooks/zuul-run.yaml
+      - tools/gate/playbooks/osh-infra-deploy-docker.yaml
+      - tools/gate/playbooks/osh-infra-build.yaml
+      - tools/gate/playbooks/osh-infra-deploy-k8s.yaml
+    run: tools/gate/playbooks/osh-infra-deploy-charts.yaml
 
 - job:
     name: openstack-helm-infra-ubuntu

Makefile

@@ -21,6 +21,8 @@ TASK := build
 EXCLUDES := helm-toolkit doc tests tools logs
 CHARTS := helm-toolkit $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.)))
 
+.PHONY: $(EXCLUDES) $(CHARTS)
+
 all: $(CHARTS)
 
 $(CHARTS):
@@ -50,4 +52,8 @@ clean:
 pull-all-images:
 	@./tools/pull-images.sh
 
-.PHONY: $(EXCLUDES) $(CHARTS)
+dev-deploy:
+	@./tools/gate/devel/start.sh $(filter-out $@,$(MAKECMDGOALS))
+
+%:
+	@:

tools/gate/chart-deploys/default.yaml

@@ -0,0 +1,61 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+chart_groups:
+  - name: docker_registry
+    timeout: 600
+    charts:
+      - docker_registry_nfs_provisioner
+      - docker_registry_redis
+      - docker_registry
+
+charts:
+  docker_registry_nfs_provisioner:
+    chart_name: nfs-provisioner
+    release: docker-registry-nfs-provisioner
+    namespace: docker-registry
+    upgrade:
+      pre:
+        delete:
+          - name: docker-bootstrap
+            type: job
+            labels:
+              application: docker
+              component: bootstrap
+    values:
+      labels:
+        node_selector_key: openstack-helm-node-class
+        node_selector_value: primary
+      storageclass:
+        name: openstack-helm-bootstrap
+
+  docker_registry_redis:
+    chart_name: redis
+    release: docker-registry-redis
+    namespace: docker-registry
+    values:
+      labels:
+        node_selector_key: openstack-helm-node-class
+        node_selector_value: primary
+
+  docker_registry:
+    chart_name: registry
+    release: docker-registry
+    namespace: docker-registry
+    values:
+      labels:
+        node_selector_key: openstack-helm-node-class
+        node_selector_value: primary
+      volume:
+        class_name: openstack-helm-bootstrap

tools/gate/devel/start.sh

@@ -16,7 +16,10 @@
 
 set -ex
 : ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../.."}
-export MODE=${1:-"local"}
+export DEPLOY=${1:-"full"}
+export MODE=${2:-"local"}
+export INVENTORY=${WORK_DIR}/tools/gate/devel/${MODE}-inventory.yaml
+export VARS=${WORK_DIR}/tools/gate/devel/${MODE}-vars.yaml
 
 function ansible_install {
   cd /tmp
@@ -28,7 +31,8 @@ function ansible_install {
       python-pip \
       libssl-dev \
       python-dev \
-      build-essential
+      build-essential \
+      jq
   elif [ "x$ID" == "xcentos" ]; then
     sudo yum install -y \
       epel-release
@@ -36,22 +40,41 @@ function ansible_install {
       python-pip \
       python-devel \
       redhat-rpm-config \
-      gcc
+      gcc \
+      curl
+    sudo curl -L -o /usr/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
+    sudo chmod +x /usr/bin/jq
   elif [ "x$ID" == "xfedora" ]; then
     sudo dnf install -y \
       python-devel \
       redhat-rpm-config \
-      gcc
+      gcc \
+      jq
   fi
 
   sudo -H pip install --no-cache-dir --upgrade pip
   sudo -H pip install --no-cache-dir --upgrade setuptools
   sudo -H pip install --no-cache-dir --upgrade pyopenssl
-  sudo -H pip install --no-cache-dir ansible
+  sudo -H pip install --no-cache-dir \
-  sudo -H pip install --no-cache-dir ara
+    ansible \
-  sudo -H pip install --no-cache-dir yq
+    ara \
+    yq
 }
-ansible_install
+
+if [ "x${DEPLOY}" == "xsetup-host" ]; then
+  ansible_install
+  PLAYBOOKS="osh-infra-docker"
+elif [ "x${DEPLOY}" == "xk8s" ]; then
+  PLAYBOOKS="osh-infra-build osh-infra-deploy-k8s"
+elif [ "x${DEPLOY}" == "xcharts" ]; then
+  PLAYBOOKS="osh-infra-deploy-charts"
+elif [ "x${DEPLOY}" == "xfull" ]; then
+  ansible_install
+  PLAYBOOKS="osh-infra-docker osh-infra-build osh-infra-deploy-k8s osh-infra-deploy-charts"
+else
+  echo "Unknown Deploy Option Selected"
+  exit 1
+fi
 
 cd ${WORK_DIR}
 export ANSIBLE_CALLBACK_PLUGINS="$(python -c 'import os,ara; print(os.path.dirname(ara.__file__))')/plugins/callbacks"
@@ -68,7 +91,9 @@ function dump_logs () {
 }
 trap 'dump_logs "$?"' ERR
 
-INVENTORY=${WORK_DIR}/tools/gate/devel/${MODE}-inventory.yaml
+for PLAYBOOK in ${PLAYBOOKS}; do
-VARS=${WORK_DIR}/tools/gate/devel/${MODE}-vars.yaml
+  ansible-playbook ${WORK_DIR}/tools/gate/playbooks/${PLAYBOOK}.yaml \
-ansible-playbook ${WORK_DIR}/tools/gate/playbooks/zuul-pre.yaml -i ${INVENTORY} --extra-vars=@${VARS} --extra-vars "work_dir=${WORK_DIR}"
+    -i ${INVENTORY} \
-ansible-playbook ${WORK_DIR}/tools/gate/playbooks/zuul-run.yaml -i ${INVENTORY} --extra-vars=@${VARS} --extra-vars "work_dir=${WORK_DIR}"
+    --extra-vars=@${VARS} \
+    --extra-vars "work_dir=${WORK_DIR}"
+done

tools/gate/playbooks/osh-infra-build.yaml

@@ -12,18 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- hosts: all
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  gather_facts: False
-  become: yes
-  roles:
-    - deploy-python
-  tags:
-    - deploy-python
-
 - hosts: primary
   vars_files:
     - vars.yaml
@@ -35,24 +23,6 @@
   tags:
     - build-helm-packages
 
-- hosts: all
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  gather_facts: True
-  become: yes
-  roles:
-    - setup-firewall
-    - deploy-python-pip
-    - deploy-docker
-    - deploy-yq
-  tags:
-    - setup-firewall
-    - deploy-python-pip
-    - deploy-docker
-    - deploy-yq
-
 - hosts: all
   vars_files:
     - vars.yaml

tools/gate/playbooks/osh-infra-deploy-charts.yaml

@@ -0,0 +1,35 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: True
+  roles:
+    - build-helm-packages
+  tags:
+    - build-helm-packages
+
+- hosts: primary
+  vars_files:
+    - vars.yaml
+    - ../chart-deploys/default.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  roles:
+    - deploy-helm-packages
+  tags:
+    - deploy-helm-packages

tools/gate/playbooks/osh-infra-deploy-docker.yaml

@@ -0,0 +1,43 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: False
+  become: yes
+  roles:
+    - deploy-python
+  tags:
+    - deploy-python
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: True
+  become: yes
+  roles:
+    - setup-firewall
+    - deploy-python-pip
+    - deploy-docker
+    - deploy-yq
+  tags:
+    - setup-firewall
+    - deploy-python-pip
+    - deploy-docker
+    - deploy-yq

tools/gate/playbooks/osh-infra-deploy-k8s.yaml

@@ -12,15 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- hosts: all
+- hosts: primary
   vars_files:
     - vars.yaml
   vars:
     work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: True
   roles:
-    - clean-host
+    - build-helm-packages
   tags:
-    - clean-host
+    - build-helm-packages
 
 - hosts: primary
   vars_files:
@@ -30,7 +31,7 @@
   roles:
     - deploy-kubeadm-aio-master
   tags:
-    - deploy-kubeadm-aio-master
+    - deploy-kube-master
 
 - hosts: nodes
   vars_files:
@@ -40,14 +41,4 @@
   roles:
     - deploy-kubeadm-aio-node
   tags:
-    - deploy-kubeadm-aio-node
+    - deploy-kube-nodes
-
-- hosts: primary
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  roles:
-    - deploy-helm-packages
-  tags:
-    - deploy-helm-packages

tools/gate/playbooks/osh-infra-docker.yaml

@@ -0,0 +1,40 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: False
+  become: yes
+  roles:
+    - deploy-python
+  tags:
+    - python
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: True
+  become: yes
+  roles:
+    - setup-firewall
+    - deploy-python-pip
+    - deploy-docker
+    - deploy-yq
+  tags:
+    - docker

tools/gate/playbooks/vars.yaml

@@ -52,51 +52,3 @@ nodes:
         value: enabled
       - name: ceph-rgw
         value: enabled
-
-chart_groups:
-  - name: docker_registry
-    timeout: 600
-    charts:
-      - docker_registry_nfs_provisioner
-      - docker_registry_redis
-      - docker_registry
-
-charts:
-  docker_registry_nfs_provisioner:
-    chart_name: nfs-provisioner
-    release: docker-registry-nfs-provisioner
-    namespace: docker-registry
-    upgrade:
-      pre:
-        delete:
-          - name: docker-bootstrap
-            type: job
-            labels:
-              application: docker
-              component: bootstrap
-    values:
-      labels:
-        node_selector_key: openstack-helm-node-class
-        node_selector_value: primary
-      storageclass:
-        name: openstack-helm-bootstrap
-
-  docker_registry_redis:
-    chart_name: redis
-    release: docker-registry-redis
-    namespace: docker-registry
-    values:
-      labels:
-        node_selector_key: openstack-helm-node-class
-        node_selector_value: primary
-
-  docker_registry:
-    chart_name: registry
-    release: docker-registry
-    namespace: docker-registry
-    values:
-      labels:
-        node_selector_key: openstack-helm-node-class
-        node_selector_value: primary
-      volume:
-        class_name: openstack-helm-bootstrap

tools/image-repo-overides.sh

@@ -15,8 +15,7 @@
 # limitations under the License.
 
 KUBE_VERSION=$(yq -r '.version.kubernetes' ./tools/gate/playbooks/vars.yaml)
-KUBE_IMAGES="gcr.io/google_containers/hyperkube-amd64:${KUBE_VERSION}
+KUBE_IMAGES="gcr.io/google_containers/kube-apiserver-amd64:${KUBE_VERSION}
-gcr.io/google_containers/kube-apiserver-amd64:${KUBE_VERSION}
 gcr.io/google_containers/kube-controller-manager-amd64:${KUBE_VERSION}
 gcr.io/google_containers/kube-proxy-amd64:${KUBE_VERSION}
 gcr.io/google_containers/kube-scheduler-amd64:${KUBE_VERSION}