Zuul gates setup for Utility Containers

Added zuul gate related changes for porthole repo and added zuul gates
for ceph and calicoctl utility containers.

Change-Id: Ifa24b69a4463de9d25e0958e8f2233eec26aeb6a

tools/gate/playbooks/zuul-linter.yaml

@@ -0,0 +1,20 @@
+# Copyright 2019 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Execute a Whitespace Linter check
+      command: find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -not -name "*.png" -type f -exec egrep -l " +$" {} \;
+      register: result
+      failed_when: result.stdout != ""

tools/helm_install.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -x
+
+HELM=$1
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://storage.googleapis.com/kubernetes-helm/helm-v2.14.0-linux-amd64.tar.gz"}
+
+
+function install_helm_binary {
+  if [[ -z "${HELM}" ]]
+  then
+    echo "No Helm binary target location."
+    exit -1
+  fi
+
+  if [[ -w "$(dirname ${HELM})" ]]
+  then
+    TMP_DIR=${BUILD_DIR:-$(mktemp -d)}
+    curl -o "${TMP_DIR}/helm.tar.gz" "${HELM_ARTIFACT_URL}"
+    pushd ${TMP_DIR}
+    tar -xvzf helm.tar.gz
+    cp "linux-amd64/helm" "helm"
+    popd
+  else
+    echo "Cannot write to ${HELM}"
+    exit -1
+  fi
+}
+
+install_helm_binary

tools/helm_tk.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Script to setup helm-toolkit and helm dep up the shipyard chart
+#
+HELM=$1
+HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"}
+HTK_PATH=${HTK_PATH:-""}
+HTK_STABLE_COMMIT=${HTK_COMMIT:-"200b5e902b3a176fbfbe669b6a10a254c9b50f5d"}
+DEP_UP_LIST=${DEP_UP_LIST:-"porthole"}
+BUILD_DIR=${BUILD_DIR:-$(mktemp -d)}
+
+if [[ ! -z $(echo $http_proxy) ]]
+then
+  export no_proxy=$no_proxy,127.0.0.1
+fi
+
+set -x
+
+# Use ./helm as we expect this to be run in a already
+# configured build directory
+
+function helm_serve {
+  if [[ -d "$HOME/.helm" ]]; then
+     echo ".helm directory found"
+  else
+     ${HELM} init --client-only --skip-refresh
+  fi
+  if [[ -z $(curl --noproxy '*' -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; then
+     "${HELM}" serve & > /dev/null
+     while [[ -z $(curl --noproxy '*' -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; do
+        sleep 1
+        echo "Waiting for Helm Repository"
+     done
+  else
+     echo "Helm serve already running"
+  fi
+
+  if "${HELM}" repo list | grep -q "^stable" ; then
+     "${HELM}" repo remove stable
+  fi
+
+  ${HELM} repo add local http://localhost:8879/charts
+}
+
+mkdir -p "$BUILD_DIR"
+pushd "$BUILD_DIR"
+git clone $HTK_REPO || true
+pushd openstack-helm-infra/$HTK_PATH
+git reset --hard "${HTK_STABLE_COMMIT}"
+
+helm_serve
+# OSH Makefile is bugged, so ensure helm is in the path
+if [[ ${HELM} != "helm" ]]
+then
+  export PATH=${PATH}:$(dirname ${HELM})
+fi
+
+make helm-toolkit
+popd && popd
+for c in $DEP_UP_LIST
+do
+  ${HELM} dep up charts/$c
+done

tools/image_tags.py

@@ -0,0 +1,126 @@
+#!/bin/python
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import os
+import sys
+
+LOG = logging.getLogger(__name__)
+
+LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(funcName)s [%(lineno)3d] %(message)s'  # noqa
+
+
+class TagGenExeception(Exception):
+    pass
+
+
+def read_config(stream, env):
+    config = {}
+    try:
+        config['tags'] = json.load(stream)
+    except ValueError:
+        LOG.exception('Failed to decode JSON from input stream')
+        config['tags'] = {}
+
+    LOG.debug('Configuration after reading stream: %s', config)
+
+    config['context'] = {
+        'branch': env.get('BRANCH'),
+        'change': env.get('CHANGE'),
+        'commit': env.get('COMMIT'),
+        'ps': env.get('PATCHSET'),
+    }
+
+    LOG.info('Final configuration: %s', config)
+
+    return config
+
+
+def build_tags(config):
+    tags = config.get('tags', {}).get('static', [])
+    LOG.debug('Dynamic tags: %s', tags)
+    tags.extend(build_dynamic_tags(config))
+    LOG.info('All tags: %s', tags)
+    return tags
+
+
+def build_dynamic_tags(config):
+    dynamic_tags = []
+
+    dynamic_tags.extend(_build_branch_tag(config))
+    dynamic_tags.extend(_build_commit_tag(config))
+    dynamic_tags.extend(_build_ps_tag(config))
+
+    return dynamic_tags
+
+
+def _build_branch_tag(config):
+    if _valid_dg(config, 'branch'):
+        return [config['context']['branch']]
+    else:
+        return []
+
+
+def _build_commit_tag(config):
+    if _valid_dg(config, 'commit'):
+        return [config['context']['commit']]
+    else:
+        return []
+
+
+def _build_ps_tag(config):
+    if _valid_dg(config, 'patch_set', 'change') and _valid_dg(
+            config, 'patch_set', 'ps'):
+        return [
+            '%s-%s' % (config['context']['change'], config['context']['ps'])
+        ]
+    else:
+        return []
+
+
+def _valid_dg(config, dynamic_tag, context_name=None):
+    if context_name is None:
+        context_name = dynamic_tag
+
+    if config.get('tags', {}).get('dynamic', {}).get(dynamic_tag):
+        if config.get('context', {}).get(context_name):
+            return True
+        else:
+            raise TagGenExeception('Dynamic tag "%s" requested, but "%s"'
+                                   ' not found in context' % (dynamic_tag,
+                                                              context_name))
+    else:
+        return False
+
+
+def main():
+    config = read_config(sys.stdin, os.environ)
+    tags = build_tags(config)
+
+    for tag in tags:
+        print(tag)
+
+
+if __name__ == '__main__':
+    logging.basicConfig(format=LOG_FORMAT, level=logging.WARNING)
+    try:
+        main()
+    except TagGenExeception:
+        LOG.exception('Failed to generate tags')
+        sys.exit(1)
+    except Exception:
+        LOG.exception('Unexpected exception')
+        sys.exit(2)

zuul.d/calicoctl-utility.yaml

@@ -0,0 +1,35 @@
+# Copyright 2019 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- project:
+    check:
+      jobs:
+        - porthole-calicoctl-utility-lint-ws
+    gate:
+      jobs:
+        - porthole-calicoctl-utility-lint-ws
+
+- nodeset:
+    name: porthole-calicoctl-utility-single-node
+    nodes:
+      - name: primary
+        label: ubuntu-xenial
+
+- job:
+    name: porthole-calicoctl-utility-lint-ws
+    description: |
+      Lints all files by checking them for whitespace.
+    run: tools/gate/playbooks/zuul-linter.yaml
+    timeout: 300
+    nodeset: porthole-calicoctl-utility-single-node

zuul.d/ceph-utility.yaml

@@ -0,0 +1,35 @@
+# Copyright 2019 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- project:
+    check:
+      jobs:
+        - porthole-ceph-utility-lint-ws
+    gate:
+      jobs:
+        - porthole-ceph-utility-lint-ws
+
+- nodeset:
+    name: porthole-ceph-utility-single-node
+    nodes:
+      - name: primary
+        label: ubuntu-xenial
+
+- job:
+    name: porthole-ceph-utility-lint-ws
+    description: |
+      Lints all files by checking them for whitespace.
+    run: tools/gate/playbooks/zuul-linter.yaml
+    timeout: 300
+    nodeset: porthole-ceph-utility-single-node