From 62c9608eae7be80ad75684ffe8a60fe5a806a845 Mon Sep 17 00:00:00 2001 From: jh629g Date: Tue, 12 Nov 2019 09:16:35 -0600 Subject: [PATCH] Create image build & publish zuul gate public ranger artifactory needs to be updated automatically rather than manually. This zuul job will both build and publish images from successful merges into the ranger repo. Change-Id: I00667417cf9f11bd216ad7ea28ac29c11453adda --- .zuul.yaml | 68 ++++++++++- Makefile | 2 +- tools/image_tags.py | 113 ++++++++++++++++++ tools/zuul/playbooks/docker-image-build.yaml | 87 ++++++++++++++ .../zuul/playbooks/files/docker-systemd.conf | 3 + .../zuul/playbooks/run-unit-test-job.yaml | 2 +- tools/zuul/playbooks/vars.yaml | 5 + 7 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 tools/image_tags.py create mode 100644 tools/zuul/playbooks/docker-image-build.yaml create mode 100644 tools/zuul/playbooks/files/docker-systemd.conf rename playbooks/run_unit_test_job.yaml => tools/zuul/playbooks/run-unit-test-job.yaml (93%) create mode 100644 tools/zuul/playbooks/vars.yaml diff --git a/.zuul.yaml b/.zuul.yaml index 4c3dfc58..55c65b89 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -4,15 +4,81 @@ - openstack-tox-pep8 - openstack-tox-py36 - ranger-tox-bandit + - ranger-image-build gate: jobs: - openstack-tox-pep8 - openstack-tox-py36 - ranger-tox-bandit + post: + jobs: + - ranger-image-publish + - job: name: ranger-tox-bandit parent: openstack-tox timeout: 600 - pre-run: playbooks/run_unit_test_job.yaml + pre-run: tools/zuul/playbooks/run-unit-test-job.yaml vars: tox_envlist: bandit-baseline + +- job: + name: ranger-image-build + run: tools/zuul/playbooks/docker-image-build.yaml + nodeset: ubuntu-bionic + vars: + publish: false + tags: + dynamic: + patch_set: true + irrelevant-files: + - ^charts/.*$ + - ^etc/.*$ + - ^tests/.*$ + - ^tools/.*$ + +- job: + name: ranger-image-publish + run: tools/zuul/playbooks/docker-image-build.yaml + nodeset: ubuntu-bionic + secrets: + - ranger_quay_io_credentials + vars: + publish: true + tags: + dynamic: + branch: true + commit: true + static: + - latest + irrelevant-files: + - ^charts/.*$ + - ^etc/.*$ + - ^tests/.*$ + - ^tools/.*$ + +- secret: + name: ranger_quay_io_credentials + data: + username: !encrypted/pkcs1-oaep + - ZeiK85s+OWqeaDshARyWvlGjNjuE7USQlFn8ZPEzVlh1dSuCBTT9ygrRBe+IscSMvuRNN + GEVdJVWj/6afNpNL2IHm8OyjYfkypDi76gsQkla1fjMu4PWQLdCroJaKSn9U6ZLGD213e + OU/2ctMhdn5XCTKk3wuZ3EaxYZTPDT6Fizcd7cutCIC6j5stHwitfvAdLT30IZ7rHHvuf + zsdy7jInb7wpqz6Vq4h6ZaJLznfjlr6icWlcSL1F4Y86NnJFJKEA5NTAZDS9/qE6myU8D + bJHmbd3jCAi2l3lPvI4J3HJYJ4kT7CK+IESZ5zrT7ht8pYjdDrF8F3Fk1hbon4AwaNwYo + MXYA2YtKpPAYTk+WS+lMFyUPe6ASt341TJcmszIJHaDltLMfLI61VgyMXnYZH+nALsgJx + lNwv24SqfR+xsVLyFT25TkzroPJyZtrbadd+L0ugtm3LH7J5KI/qJ8fi/7aPsXl8v08fh + uGM4OsrQyXlOUElKuMdBCnxAdle8DI/oSubWSuFdrQdnfwEObNqlMcvr1IAjE8irDj2hz + SFrj3nn9IYMxoKLRrwDIslprZ3667T9GNZtN18IBCu5HoZDeJf/X2C/k5yZxvcVmiIbrS + P5zvbelxWvToa5pXuWE2MfvdI8GWuHtFNV9UTGCq7/UPES9k8EJkPG7CPGzboY= + password: !encrypted/pkcs1-oaep + - jm1OERqRgJ/ulvMJn3BKzjGeiHX4PaslTxscrUw36g+pYBh6JNV+5qf5kLST6ezOfL0zW + S3maDTqmlTcQbV0e1BxOkyu6VXS092jxEy8Meud+npG2Q6xBP3NgZ3Ktydf/F9qlZzKMu + lH0yZnstUISoKsnRIQndT+GgE6N9TSdd1oSZLNSDRT8j8U+zSXw9iFx3WSsHHcq2tZB+w + gkiUkfO1jwPpArHyETwc5gQ+07HXwl7RwhktBJO5qVMSaDh33LGPYrtYvuZT7R0m1GisB + A5E1gAHjI5NbeVTyOWsNT47TpdHXSrXuS+4837bJIY/2P9AR0CUHlGEJNO7XybVtIPAWn + n6nuPYqYnmmLzj1J4GmPFtUSXSEhUxQ9HDZx2wPnBA3rluR5UjqDHx/SPCzjXgh0j6Mxl + gedfSiFfMmmgpQvAo2XuouylulpK8ORzo/EPjCkflHxx1WFegxsVz6BRYFYDhkdI1OPR1 + Y37kxWtyAAxeqtTcigAXM8x7ARbRKip4GYFv8U7bc4EKXpg+/lnHlCKyfc9lXetxNGdtX + z/2PEg1k1yqC7+3zXCJsGWUlCTuaU5mIG3AHk/nSQUil3dNuvXYJQ9B0X8OSHSGEMPbx2 + GqyzGCxDJ9shmqINlj4d95WuC3SkqfgVpf/zQkHXKYfOxAgJYfbwjZWckVcjn0= diff --git a/Makefile b/Makefile index c6ac9869..3588b511 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ IMAGE_PREFIX ?= attcomdev IMAGE_TAG ?= ocata HELM ?= helm LABEL ?= commit-id -PROXY ?= http://proxy.foo.com:8000 +PROXY ?= NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local USE_PROXY ?= true RANGER_USER := ranger diff --git a/tools/image_tags.py b/tools/image_tags.py new file mode 100644 index 00000000..41f13a04 --- /dev/null +++ b/tools/image_tags.py @@ -0,0 +1,113 @@ +#!/bin/python3 + +import json +import logging +import os +import sys + +LOG = logging.getLogger(__name__) + +LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(funcName)s [%(lineno)3d] %(message)s' # noqa + + +class TagGenExeception(Exception): + pass + + +def read_config(stream, env): + config = {} + try: + config['tags'] = json.load(stream) + except ValueError: + LOG.exception('Failed to decode JSON from input stream') + config['tags'] = {} + + LOG.debug('Configuration after reading stream: %s', config) + + config['context'] = { + 'branch': env.get('BRANCH'), + 'change': env.get('CHANGE'), + 'commit': env.get('COMMIT'), + 'ps': env.get('PATCHSET'), + } + + LOG.info('Final configuration: %s', config) + + return config + + +def build_tags(config): + tags = config.get('tags', {}).get('static', []) + LOG.debug('Dynamic tags: %s', tags) + tags.extend(build_dynamic_tags(config)) + LOG.info('All tags: %s', tags) + return tags + + +def build_dynamic_tags(config): + dynamic_tags = [] + + dynamic_tags.extend(_build_branch_tag(config)) + dynamic_tags.extend(_build_commit_tag(config)) + dynamic_tags.extend(_build_ps_tag(config)) + + return dynamic_tags + + +def _build_branch_tag(config): + if _valid_dg(config, 'branch'): + return [config['context']['branch']] + else: + return [] + + +def _build_commit_tag(config): + if _valid_dg(config, 'commit'): + return [config['context']['commit']] + else: + return [] + + +def _build_ps_tag(config): + if _valid_dg(config, 'patch_set', 'change') and _valid_dg( + config, 'patch_set', 'ps'): + return [ + '%s-%s' % (config['context']['change'], config['context']['ps']) + ] + else: + return [] + + +def _valid_dg(config, dynamic_tag, context_name=None): + if context_name is None: + context_name = dynamic_tag + + if config.get('tags', {}).get('dynamic', {}).get(dynamic_tag): + if config.get('context', {}).get(context_name): + return True + else: + raise TagGenExeception( + 'Dynamic tag "%s" requested, but "%s"' + ' not found in context' % (dynamic_tag, context_name)) + else: + return False + + +def main(): + config = read_config(sys.stdin, os.environ) + tags = build_tags(config) + + for tag in tags: + print(tag) + + +if __name__ == '__main__': + logging.basicConfig(format=LOG_FORMAT, level=logging.WARNING) + try: + main() + except TagGenExeception: + LOG.exception('Failed to generate tags') + sys.exit(1) + except Exception: + LOG.exception('Unexpected exception') + sys.exit(2) diff --git a/tools/zuul/playbooks/docker-image-build.yaml b/tools/zuul/playbooks/docker-image-build.yaml new file mode 100644 index 00000000..33fda5db --- /dev/null +++ b/tools/zuul/playbooks/docker-image-build.yaml @@ -0,0 +1,87 @@ +- hosts: all + tasks: + - include_vars: vars.yaml + + - name: Install Docker (Debian) + when: ansible_os_family == 'Debian' + block: + - file: + path: "{{ item }}" + state: directory + with_items: + - /etc/docker/ + - /etc/systemd/system/docker.service.d/ + - /var/lib/docker/ + - mount: + path: /var/lib/docker/ + src: tmpfs + fstype: tmpfs + opts: size=25g + state: mounted + - copy: "{{ item }}" + with_items: + - content: "{{ docker_daemon | to_json }}" + dest: /etc/docker/daemon.json + - src: files/docker-systemd.conf + dest: /etc/systemd/system/docker.service.d/ + - apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + - apt_repository: + repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker bionic stable + - apt: + name: "{{ item }}" + allow_unauthenticated: True + with_items: + - docker-ce + - python-pip + - pip: + name: docker + version: 2.7.0 + - iptables: + action: insert + chain: INPUT + in_interface: docker0 + jump: ACCEPT + become: True + + - name: Debug tag generation inputs + block: + - debug: + var: publish + - debug: + var: tags + - debug: + var: zuul + - debug: + msg: "{{ tags | to_json }}" + + - name: Determine tags + shell: echo '{{ tags | to_json }}' | python {{ zuul.project.src_dir }}/tools/image_tags.py + environment: + BRANCH: "{{ zuul.branch | default('') }}" + CHANGE: "{{ zuul.change | default('') }}" + COMMIT: "{{ zuul.newrev | default('') }}" + PATCHSET: "{{ zuul.patchset | default('') }}" + register: image_tags + + - name: Debug computed tags + debug: + var: image_tags + + - name: Make images + when: not publish + block: + - make: + chdir: "{{ zuul.project.src_dir }}" + target: images + params: + IMAGE_TAG: "{{ item }}" + with_items: "{{ image_tags.stdout_lines }}" + + - shell: "docker images" + register: docker_images + + - debug: + var: docker_images + + become: True diff --git a/tools/zuul/playbooks/files/docker-systemd.conf b/tools/zuul/playbooks/files/docker-systemd.conf new file mode 100644 index 00000000..5bf16079 --- /dev/null +++ b/tools/zuul/playbooks/files/docker-systemd.conf @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd diff --git a/playbooks/run_unit_test_job.yaml b/tools/zuul/playbooks/run-unit-test-job.yaml similarity index 93% rename from playbooks/run_unit_test_job.yaml rename to tools/zuul/playbooks/run-unit-test-job.yaml index 59a3d05f..b5d7d238 100644 --- a/playbooks/run_unit_test_job.yaml +++ b/tools/zuul/playbooks/run-unit-test-job.yaml @@ -5,4 +5,4 @@ bindep_dir: "{{ zuul_work_dir }}" - test-setup - ensure-tox - - tox \ No newline at end of file + - tox diff --git a/tools/zuul/playbooks/vars.yaml b/tools/zuul/playbooks/vars.yaml new file mode 100644 index 00000000..6d00a744 --- /dev/null +++ b/tools/zuul/playbooks/vars.yaml @@ -0,0 +1,5 @@ +docker_daemon: + group: zuul + registry-mirrors: + - "http://{{ zuul_site_mirror_fqdn }}:8082/" + storage-driver: overlay2