From 3427ee5ab5957ec2a8490e7c1fb559055cfab465 Mon Sep 17 00:00:00 2001 From: Varagini Karthik Date: Sun, 11 Aug 2019 18:58:26 -0500 Subject: [PATCH] Zuul gates setup for Utility Containers Added zuul gate related changes for porthole repo and added zuul gates for ceph and calicoctl utility containers. Change-Id: Ifa24b69a4463de9d25e0958e8f2233eec26aeb6a --- tools/gate/playbooks/zuul-linter.yaml | 20 ++++ tools/helm_install.sh | 44 +++++++++ tools/helm_tk.sh | 76 ++++++++++++++++ tools/image_tags.py | 126 ++++++++++++++++++++++++++ zuul.d/calicoctl-utility.yaml | 35 +++++++ zuul.d/ceph-utility.yaml | 35 +++++++ 6 files changed, 336 insertions(+) create mode 100644 tools/gate/playbooks/zuul-linter.yaml create mode 100644 tools/helm_install.sh create mode 100644 tools/helm_tk.sh create mode 100644 tools/image_tags.py create mode 100644 zuul.d/calicoctl-utility.yaml create mode 100644 zuul.d/ceph-utility.yaml diff --git a/tools/gate/playbooks/zuul-linter.yaml b/tools/gate/playbooks/zuul-linter.yaml new file mode 100644 index 00000000..b86a1279 --- /dev/null +++ b/tools/gate/playbooks/zuul-linter.yaml @@ -0,0 +1,20 @@ +# Copyright 2019 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + tasks: + - name: Execute a Whitespace Linter check + command: find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -not -name "*.png" -type f -exec egrep -l " +$" {} \; + register: result + failed_when: result.stdout != "" \ No newline at end of file diff --git a/tools/helm_install.sh b/tools/helm_install.sh new file mode 100644 index 00000000..9c294ad6 --- /dev/null +++ b/tools/helm_install.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -x + +HELM=$1 +HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://storage.googleapis.com/kubernetes-helm/helm-v2.14.0-linux-amd64.tar.gz"} + + +function install_helm_binary { + if [[ -z "${HELM}" ]] + then + echo "No Helm binary target location." + exit -1 + fi + + if [[ -w "$(dirname ${HELM})" ]] + then + TMP_DIR=${BUILD_DIR:-$(mktemp -d)} + curl -o "${TMP_DIR}/helm.tar.gz" "${HELM_ARTIFACT_URL}" + pushd ${TMP_DIR} + tar -xvzf helm.tar.gz + cp "linux-amd64/helm" "helm" + popd + else + echo "Cannot write to ${HELM}" + exit -1 + fi +} + +install_helm_binary diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh new file mode 100644 index 00000000..c65df800 --- /dev/null +++ b/tools/helm_tk.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Script to setup helm-toolkit and helm dep up the shipyard chart +# +HELM=$1 +HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"} +HTK_PATH=${HTK_PATH:-""} +HTK_STABLE_COMMIT=${HTK_COMMIT:-"200b5e902b3a176fbfbe669b6a10a254c9b50f5d"} +DEP_UP_LIST=${DEP_UP_LIST:-"porthole"} +BUILD_DIR=${BUILD_DIR:-$(mktemp -d)} + +if [[ ! -z $(echo $http_proxy) ]] +then + export no_proxy=$no_proxy,127.0.0.1 +fi + +set -x + +# Use ./helm as we expect this to be run in a already +# configured build directory + +function helm_serve { + if [[ -d "$HOME/.helm" ]]; then + echo ".helm directory found" + else + ${HELM} init --client-only --skip-refresh + fi + if [[ -z $(curl --noproxy '*' -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; then + "${HELM}" serve & > /dev/null + while [[ -z $(curl --noproxy '*' -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; do + sleep 1 + echo "Waiting for Helm Repository" + done + else + echo "Helm serve already running" + fi + + if "${HELM}" repo list | grep -q "^stable" ; then + "${HELM}" repo remove stable + fi + + ${HELM} repo add local http://localhost:8879/charts +} + +mkdir -p "$BUILD_DIR" +pushd "$BUILD_DIR" +git clone $HTK_REPO || true +pushd openstack-helm-infra/$HTK_PATH +git reset --hard "${HTK_STABLE_COMMIT}" + +helm_serve +# OSH Makefile is bugged, so ensure helm is in the path +if [[ ${HELM} != "helm" ]] +then + export PATH=${PATH}:$(dirname ${HELM}) +fi + +make helm-toolkit +popd && popd +for c in $DEP_UP_LIST +do + ${HELM} dep up charts/$c +done diff --git a/tools/image_tags.py b/tools/image_tags.py new file mode 100644 index 00000000..1870b58a --- /dev/null +++ b/tools/image_tags.py @@ -0,0 +1,126 @@ +#!/bin/python +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import sys + +LOG = logging.getLogger(__name__) + +LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(funcName)s [%(lineno)3d] %(message)s' # noqa + + +class TagGenExeception(Exception): + pass + + +def read_config(stream, env): + config = {} + try: + config['tags'] = json.load(stream) + except ValueError: + LOG.exception('Failed to decode JSON from input stream') + config['tags'] = {} + + LOG.debug('Configuration after reading stream: %s', config) + + config['context'] = { + 'branch': env.get('BRANCH'), + 'change': env.get('CHANGE'), + 'commit': env.get('COMMIT'), + 'ps': env.get('PATCHSET'), + } + + LOG.info('Final configuration: %s', config) + + return config + + +def build_tags(config): + tags = config.get('tags', {}).get('static', []) + LOG.debug('Dynamic tags: %s', tags) + tags.extend(build_dynamic_tags(config)) + LOG.info('All tags: %s', tags) + return tags + + +def build_dynamic_tags(config): + dynamic_tags = [] + + dynamic_tags.extend(_build_branch_tag(config)) + dynamic_tags.extend(_build_commit_tag(config)) + dynamic_tags.extend(_build_ps_tag(config)) + + return dynamic_tags + + +def _build_branch_tag(config): + if _valid_dg(config, 'branch'): + return [config['context']['branch']] + else: + return [] + + +def _build_commit_tag(config): + if _valid_dg(config, 'commit'): + return [config['context']['commit']] + else: + return [] + + +def _build_ps_tag(config): + if _valid_dg(config, 'patch_set', 'change') and _valid_dg( + config, 'patch_set', 'ps'): + return [ + '%s-%s' % (config['context']['change'], config['context']['ps']) + ] + else: + return [] + + +def _valid_dg(config, dynamic_tag, context_name=None): + if context_name is None: + context_name = dynamic_tag + + if config.get('tags', {}).get('dynamic', {}).get(dynamic_tag): + if config.get('context', {}).get(context_name): + return True + else: + raise TagGenExeception('Dynamic tag "%s" requested, but "%s"' + ' not found in context' % (dynamic_tag, + context_name)) + else: + return False + + +def main(): + config = read_config(sys.stdin, os.environ) + tags = build_tags(config) + + for tag in tags: + print(tag) + + +if __name__ == '__main__': + logging.basicConfig(format=LOG_FORMAT, level=logging.WARNING) + try: + main() + except TagGenExeception: + LOG.exception('Failed to generate tags') + sys.exit(1) + except Exception: + LOG.exception('Unexpected exception') + sys.exit(2) diff --git a/zuul.d/calicoctl-utility.yaml b/zuul.d/calicoctl-utility.yaml new file mode 100644 index 00000000..465c6ddd --- /dev/null +++ b/zuul.d/calicoctl-utility.yaml @@ -0,0 +1,35 @@ +# Copyright 2019 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- project: + check: + jobs: + - porthole-calicoctl-utility-lint-ws + gate: + jobs: + - porthole-calicoctl-utility-lint-ws + +- nodeset: + name: porthole-calicoctl-utility-single-node + nodes: + - name: primary + label: ubuntu-xenial + +- job: + name: porthole-calicoctl-utility-lint-ws + description: | + Lints all files by checking them for whitespace. + run: tools/gate/playbooks/zuul-linter.yaml + timeout: 300 + nodeset: porthole-calicoctl-utility-single-node diff --git a/zuul.d/ceph-utility.yaml b/zuul.d/ceph-utility.yaml new file mode 100644 index 00000000..bd526125 --- /dev/null +++ b/zuul.d/ceph-utility.yaml @@ -0,0 +1,35 @@ +# Copyright 2019 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- project: + check: + jobs: + - porthole-ceph-utility-lint-ws + gate: + jobs: + - porthole-ceph-utility-lint-ws + +- nodeset: + name: porthole-ceph-utility-single-node + nodes: + - name: primary + label: ubuntu-xenial + +- job: + name: porthole-ceph-utility-lint-ws + description: | + Lints all files by checking them for whitespace. + run: tools/gate/playbooks/zuul-linter.yaml + timeout: 300 + nodeset: porthole-ceph-utility-single-node