Create image build & publish zuul gate

public ranger artifactory needs to be updated
automatically rather than manually. This zuul
job will both build and publish images from
successful merges into the ranger repo.

Change-Id: I00667417cf9f11bd216ad7ea28ac29c11453adda

.zuul.yaml

@@ -4,15 +4,81 @@
         - openstack-tox-pep8
         - openstack-tox-py36
         - ranger-tox-bandit
+        - ranger-image-build
     gate:
       jobs:
         - openstack-tox-pep8
         - openstack-tox-py36
         - ranger-tox-bandit
+    post:
+      jobs:
+        - ranger-image-publish
+
 - job:
     name: ranger-tox-bandit
     parent: openstack-tox
     timeout: 600
-    pre-run: playbooks/run_unit_test_job.yaml
+    pre-run: tools/zuul/playbooks/run-unit-test-job.yaml
     vars:
       tox_envlist: bandit-baseline
+
+- job:
+    name: ranger-image-build
+    run: tools/zuul/playbooks/docker-image-build.yaml
+    nodeset: ubuntu-bionic
+    vars:
+      publish: false
+      tags:
+        dynamic:
+          patch_set: true
+    irrelevant-files:
+      - ^charts/.*$
+      - ^etc/.*$
+      - ^tests/.*$
+      - ^tools/.*$
+
+- job:
+    name: ranger-image-publish
+    run: tools/zuul/playbooks/docker-image-build.yaml
+    nodeset: ubuntu-bionic
+    secrets:
+      - ranger_quay_io_credentials
+    vars:
+      publish: true
+      tags:
+        dynamic:
+          branch: true
+          commit: true
+        static:
+          - latest
+    irrelevant-files:
+      - ^charts/.*$
+      - ^etc/.*$
+      - ^tests/.*$
+      - ^tools/.*$
+
+- secret:
+    name: ranger_quay_io_credentials
+    data:
+      username: !encrypted/pkcs1-oaep
+        - ZeiK85s+OWqeaDshARyWvlGjNjuE7USQlFn8ZPEzVlh1dSuCBTT9ygrRBe+IscSMvuRNN
+          GEVdJVWj/6afNpNL2IHm8OyjYfkypDi76gsQkla1fjMu4PWQLdCroJaKSn9U6ZLGD213e
+          OU/2ctMhdn5XCTKk3wuZ3EaxYZTPDT6Fizcd7cutCIC6j5stHwitfvAdLT30IZ7rHHvuf
+          zsdy7jInb7wpqz6Vq4h6ZaJLznfjlr6icWlcSL1F4Y86NnJFJKEA5NTAZDS9/qE6myU8D
+          bJHmbd3jCAi2l3lPvI4J3HJYJ4kT7CK+IESZ5zrT7ht8pYjdDrF8F3Fk1hbon4AwaNwYo
+          MXYA2YtKpPAYTk+WS+lMFyUPe6ASt341TJcmszIJHaDltLMfLI61VgyMXnYZH+nALsgJx
+          lNwv24SqfR+xsVLyFT25TkzroPJyZtrbadd+L0ugtm3LH7J5KI/qJ8fi/7aPsXl8v08fh
+          uGM4OsrQyXlOUElKuMdBCnxAdle8DI/oSubWSuFdrQdnfwEObNqlMcvr1IAjE8irDj2hz
+          SFrj3nn9IYMxoKLRrwDIslprZ3667T9GNZtN18IBCu5HoZDeJf/X2C/k5yZxvcVmiIbrS
+          P5zvbelxWvToa5pXuWE2MfvdI8GWuHtFNV9UTGCq7/UPES9k8EJkPG7CPGzboY=
+      password: !encrypted/pkcs1-oaep
+        - jm1OERqRgJ/ulvMJn3BKzjGeiHX4PaslTxscrUw36g+pYBh6JNV+5qf5kLST6ezOfL0zW
+          S3maDTqmlTcQbV0e1BxOkyu6VXS092jxEy8Meud+npG2Q6xBP3NgZ3Ktydf/F9qlZzKMu
+          lH0yZnstUISoKsnRIQndT+GgE6N9TSdd1oSZLNSDRT8j8U+zSXw9iFx3WSsHHcq2tZB+w
+          gkiUkfO1jwPpArHyETwc5gQ+07HXwl7RwhktBJO5qVMSaDh33LGPYrtYvuZT7R0m1GisB
+          A5E1gAHjI5NbeVTyOWsNT47TpdHXSrXuS+4837bJIY/2P9AR0CUHlGEJNO7XybVtIPAWn
+          n6nuPYqYnmmLzj1J4GmPFtUSXSEhUxQ9HDZx2wPnBA3rluR5UjqDHx/SPCzjXgh0j6Mxl
+          gedfSiFfMmmgpQvAo2XuouylulpK8ORzo/EPjCkflHxx1WFegxsVz6BRYFYDhkdI1OPR1
+          Y37kxWtyAAxeqtTcigAXM8x7ARbRKip4GYFv8U7bc4EKXpg+/lnHlCKyfc9lXetxNGdtX
+          z/2PEg1k1yqC7+3zXCJsGWUlCTuaU5mIG3AHk/nSQUil3dNuvXYJQ9B0X8OSHSGEMPbx2
+          GqyzGCxDJ9shmqINlj4d95WuC3SkqfgVpf/zQkHXKYfOxAgJYfbwjZWckVcjn0=

Makefile

@@ -18,7 +18,7 @@ IMAGE_PREFIX               ?= attcomdev
 IMAGE_TAG                  ?= ocata
 HELM                       ?= helm
 LABEL                      ?= commit-id
-PROXY                      ?= http://proxy.foo.com:8000
+PROXY                      ?= 
 NO_PROXY                   ?= localhost,127.0.0.1,.svc.cluster.local
 USE_PROXY                  ?= true
 RANGER_USER                := ranger

tools/image_tags.py

@@ -0,0 +1,113 @@
+#!/bin/python3
+
+import json
+import logging
+import os
+import sys
+
+LOG = logging.getLogger(__name__)
+
+LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(funcName)s [%(lineno)3d] %(message)s'  # noqa
+
+
+class TagGenExeception(Exception):
+    pass
+
+
+def read_config(stream, env):
+    config = {}
+    try:
+        config['tags'] = json.load(stream)
+    except ValueError:
+        LOG.exception('Failed to decode JSON from input stream')
+        config['tags'] = {}
+
+    LOG.debug('Configuration after reading stream: %s', config)
+
+    config['context'] = {
+        'branch': env.get('BRANCH'),
+        'change': env.get('CHANGE'),
+        'commit': env.get('COMMIT'),
+        'ps': env.get('PATCHSET'),
+    }
+
+    LOG.info('Final configuration: %s', config)
+
+    return config
+
+
+def build_tags(config):
+    tags = config.get('tags', {}).get('static', [])
+    LOG.debug('Dynamic tags: %s', tags)
+    tags.extend(build_dynamic_tags(config))
+    LOG.info('All tags: %s', tags)
+    return tags
+
+
+def build_dynamic_tags(config):
+    dynamic_tags = []
+
+    dynamic_tags.extend(_build_branch_tag(config))
+    dynamic_tags.extend(_build_commit_tag(config))
+    dynamic_tags.extend(_build_ps_tag(config))
+
+    return dynamic_tags
+
+
+def _build_branch_tag(config):
+    if _valid_dg(config, 'branch'):
+        return [config['context']['branch']]
+    else:
+        return []
+
+
+def _build_commit_tag(config):
+    if _valid_dg(config, 'commit'):
+        return [config['context']['commit']]
+    else:
+        return []
+
+
+def _build_ps_tag(config):
+    if _valid_dg(config, 'patch_set', 'change') and _valid_dg(
+            config, 'patch_set', 'ps'):
+        return [
+            '%s-%s' % (config['context']['change'], config['context']['ps'])
+        ]
+    else:
+        return []
+
+
+def _valid_dg(config, dynamic_tag, context_name=None):
+    if context_name is None:
+        context_name = dynamic_tag
+
+    if config.get('tags', {}).get('dynamic', {}).get(dynamic_tag):
+        if config.get('context', {}).get(context_name):
+            return True
+        else:
+            raise TagGenExeception(
+                'Dynamic tag "%s" requested, but "%s"'
+                ' not found in context' % (dynamic_tag, context_name))
+    else:
+        return False
+
+
+def main():
+    config = read_config(sys.stdin, os.environ)
+    tags = build_tags(config)
+
+    for tag in tags:
+        print(tag)
+
+
+if __name__ == '__main__':
+    logging.basicConfig(format=LOG_FORMAT, level=logging.WARNING)
+    try:
+        main()
+    except TagGenExeception:
+        LOG.exception('Failed to generate tags')
+        sys.exit(1)
+    except Exception:
+        LOG.exception('Unexpected exception')
+        sys.exit(2)

tools/zuul/playbooks/docker-image-build.yaml

@@ -0,0 +1,87 @@
+- hosts: all
+  tasks:
+    - include_vars: vars.yaml
+
+    - name: Install Docker (Debian)
+      when: ansible_os_family == 'Debian'
+      block:
+        - file:
+            path: "{{ item }}"
+            state: directory
+          with_items:
+            - /etc/docker/
+            - /etc/systemd/system/docker.service.d/
+            - /var/lib/docker/
+        - mount:
+            path: /var/lib/docker/
+            src: tmpfs
+            fstype: tmpfs
+            opts: size=25g
+            state: mounted
+        - copy: "{{ item }}"
+          with_items:
+            - content: "{{ docker_daemon | to_json }}"
+              dest: /etc/docker/daemon.json
+            - src: files/docker-systemd.conf
+              dest: /etc/systemd/system/docker.service.d/
+        - apt_key:
+            url: https://download.docker.com/linux/ubuntu/gpg
+        - apt_repository:
+            repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker bionic stable
+        - apt:
+            name: "{{ item }}"
+            allow_unauthenticated: True
+          with_items:
+            - docker-ce
+            - python-pip
+        - pip:
+            name: docker
+            version: 2.7.0
+        - iptables:
+            action: insert
+            chain: INPUT
+            in_interface: docker0
+            jump: ACCEPT
+      become: True
+
+    - name: Debug tag generation inputs
+      block:
+        - debug:
+            var: publish
+        - debug:
+            var: tags
+        - debug:
+            var: zuul
+        - debug:
+            msg: "{{ tags | to_json }}"
+
+    - name: Determine tags
+      shell: echo '{{ tags | to_json }}' | python {{ zuul.project.src_dir }}/tools/image_tags.py
+      environment:
+        BRANCH: "{{ zuul.branch | default('') }}"
+        CHANGE: "{{ zuul.change | default('')  }}"
+        COMMIT: "{{ zuul.newrev | default('') }}"
+        PATCHSET: "{{ zuul.patchset | default('')  }}"
+      register: image_tags
+
+    - name: Debug computed tags
+      debug:
+        var: image_tags
+
+    - name: Make images
+      when: not publish
+      block:
+        - make:
+            chdir: "{{ zuul.project.src_dir }}"
+            target: images
+            params:
+              IMAGE_TAG: "{{ item }}"
+          with_items: "{{ image_tags.stdout_lines }}"
+
+        - shell: "docker images"
+          register: docker_images
+
+        - debug:
+            var: docker_images
+
+      become: True

tools/zuul/playbooks/files/docker-systemd.conf

@@ -0,0 +1,3 @@
+[Service]
+ExecStart=
+ExecStart=/usr/bin/dockerd

tools/zuul/playbooks/run-unit-test-job.yaml

@@ -5,4 +5,4 @@
       bindep_dir: "{{ zuul_work_dir }}"
     - test-setup
     - ensure-tox
-    - tox
+    - tox

tools/zuul/playbooks/vars.yaml

@@ -0,0 +1,5 @@
+docker_daemon:
+  group: zuul
+  registry-mirrors:
+    - "http://{{ zuul_site_mirror_fqdn }}:8082/"
+  storage-driver: overlay2