diff --git a/.zuul.yaml b/.zuul.yaml index 4c3dfc58..55c65b89 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -4,15 +4,81 @@ - openstack-tox-pep8 - openstack-tox-py36 - ranger-tox-bandit + - ranger-image-build gate: jobs: - openstack-tox-pep8 - openstack-tox-py36 - ranger-tox-bandit + post: + jobs: + - ranger-image-publish + - job: name: ranger-tox-bandit parent: openstack-tox timeout: 600 - pre-run: playbooks/run_unit_test_job.yaml + pre-run: tools/zuul/playbooks/run-unit-test-job.yaml vars: tox_envlist: bandit-baseline + +- job: + name: ranger-image-build + run: tools/zuul/playbooks/docker-image-build.yaml + nodeset: ubuntu-bionic + vars: + publish: false + tags: + dynamic: + patch_set: true + irrelevant-files: + - ^charts/.*$ + - ^etc/.*$ + - ^tests/.*$ + - ^tools/.*$ + +- job: + name: ranger-image-publish + run: tools/zuul/playbooks/docker-image-build.yaml + nodeset: ubuntu-bionic + secrets: + - ranger_quay_io_credentials + vars: + publish: true + tags: + dynamic: + branch: true + commit: true + static: + - latest + irrelevant-files: + - ^charts/.*$ + - ^etc/.*$ + - ^tests/.*$ + - ^tools/.*$ + +- secret: + name: ranger_quay_io_credentials + data: + username: !encrypted/pkcs1-oaep + - ZeiK85s+OWqeaDshARyWvlGjNjuE7USQlFn8ZPEzVlh1dSuCBTT9ygrRBe+IscSMvuRNN + GEVdJVWj/6afNpNL2IHm8OyjYfkypDi76gsQkla1fjMu4PWQLdCroJaKSn9U6ZLGD213e + OU/2ctMhdn5XCTKk3wuZ3EaxYZTPDT6Fizcd7cutCIC6j5stHwitfvAdLT30IZ7rHHvuf + zsdy7jInb7wpqz6Vq4h6ZaJLznfjlr6icWlcSL1F4Y86NnJFJKEA5NTAZDS9/qE6myU8D + bJHmbd3jCAi2l3lPvI4J3HJYJ4kT7CK+IESZ5zrT7ht8pYjdDrF8F3Fk1hbon4AwaNwYo + MXYA2YtKpPAYTk+WS+lMFyUPe6ASt341TJcmszIJHaDltLMfLI61VgyMXnYZH+nALsgJx + lNwv24SqfR+xsVLyFT25TkzroPJyZtrbadd+L0ugtm3LH7J5KI/qJ8fi/7aPsXl8v08fh + uGM4OsrQyXlOUElKuMdBCnxAdle8DI/oSubWSuFdrQdnfwEObNqlMcvr1IAjE8irDj2hz + SFrj3nn9IYMxoKLRrwDIslprZ3667T9GNZtN18IBCu5HoZDeJf/X2C/k5yZxvcVmiIbrS + P5zvbelxWvToa5pXuWE2MfvdI8GWuHtFNV9UTGCq7/UPES9k8EJkPG7CPGzboY= + password: !encrypted/pkcs1-oaep + - jm1OERqRgJ/ulvMJn3BKzjGeiHX4PaslTxscrUw36g+pYBh6JNV+5qf5kLST6ezOfL0zW + S3maDTqmlTcQbV0e1BxOkyu6VXS092jxEy8Meud+npG2Q6xBP3NgZ3Ktydf/F9qlZzKMu + lH0yZnstUISoKsnRIQndT+GgE6N9TSdd1oSZLNSDRT8j8U+zSXw9iFx3WSsHHcq2tZB+w + gkiUkfO1jwPpArHyETwc5gQ+07HXwl7RwhktBJO5qVMSaDh33LGPYrtYvuZT7R0m1GisB + A5E1gAHjI5NbeVTyOWsNT47TpdHXSrXuS+4837bJIY/2P9AR0CUHlGEJNO7XybVtIPAWn + n6nuPYqYnmmLzj1J4GmPFtUSXSEhUxQ9HDZx2wPnBA3rluR5UjqDHx/SPCzjXgh0j6Mxl + gedfSiFfMmmgpQvAo2XuouylulpK8ORzo/EPjCkflHxx1WFegxsVz6BRYFYDhkdI1OPR1 + Y37kxWtyAAxeqtTcigAXM8x7ARbRKip4GYFv8U7bc4EKXpg+/lnHlCKyfc9lXetxNGdtX + z/2PEg1k1yqC7+3zXCJsGWUlCTuaU5mIG3AHk/nSQUil3dNuvXYJQ9B0X8OSHSGEMPbx2 + GqyzGCxDJ9shmqINlj4d95WuC3SkqfgVpf/zQkHXKYfOxAgJYfbwjZWckVcjn0= diff --git a/Makefile b/Makefile index c6ac9869..3588b511 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ IMAGE_PREFIX ?= attcomdev IMAGE_TAG ?= ocata HELM ?= helm LABEL ?= commit-id -PROXY ?= http://proxy.foo.com:8000 +PROXY ?= NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local USE_PROXY ?= true RANGER_USER := ranger diff --git a/tools/image_tags.py b/tools/image_tags.py new file mode 100644 index 00000000..41f13a04 --- /dev/null +++ b/tools/image_tags.py @@ -0,0 +1,113 @@ +#!/bin/python3 + +import json +import logging +import os +import sys + +LOG = logging.getLogger(__name__) + +LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(funcName)s [%(lineno)3d] %(message)s' # noqa + + +class TagGenExeception(Exception): + pass + + +def read_config(stream, env): + config = {} + try: + config['tags'] = json.load(stream) + except ValueError: + LOG.exception('Failed to decode JSON from input stream') + config['tags'] = {} + + LOG.debug('Configuration after reading stream: %s', config) + + config['context'] = { + 'branch': env.get('BRANCH'), + 'change': env.get('CHANGE'), + 'commit': env.get('COMMIT'), + 'ps': env.get('PATCHSET'), + } + + LOG.info('Final configuration: %s', config) + + return config + + +def build_tags(config): + tags = config.get('tags', {}).get('static', []) + LOG.debug('Dynamic tags: %s', tags) + tags.extend(build_dynamic_tags(config)) + LOG.info('All tags: %s', tags) + return tags + + +def build_dynamic_tags(config): + dynamic_tags = [] + + dynamic_tags.extend(_build_branch_tag(config)) + dynamic_tags.extend(_build_commit_tag(config)) + dynamic_tags.extend(_build_ps_tag(config)) + + return dynamic_tags + + +def _build_branch_tag(config): + if _valid_dg(config, 'branch'): + return [config['context']['branch']] + else: + return [] + + +def _build_commit_tag(config): + if _valid_dg(config, 'commit'): + return [config['context']['commit']] + else: + return [] + + +def _build_ps_tag(config): + if _valid_dg(config, 'patch_set', 'change') and _valid_dg( + config, 'patch_set', 'ps'): + return [ + '%s-%s' % (config['context']['change'], config['context']['ps']) + ] + else: + return [] + + +def _valid_dg(config, dynamic_tag, context_name=None): + if context_name is None: + context_name = dynamic_tag + + if config.get('tags', {}).get('dynamic', {}).get(dynamic_tag): + if config.get('context', {}).get(context_name): + return True + else: + raise TagGenExeception( + 'Dynamic tag "%s" requested, but "%s"' + ' not found in context' % (dynamic_tag, context_name)) + else: + return False + + +def main(): + config = read_config(sys.stdin, os.environ) + tags = build_tags(config) + + for tag in tags: + print(tag) + + +if __name__ == '__main__': + logging.basicConfig(format=LOG_FORMAT, level=logging.WARNING) + try: + main() + except TagGenExeception: + LOG.exception('Failed to generate tags') + sys.exit(1) + except Exception: + LOG.exception('Unexpected exception') + sys.exit(2) diff --git a/tools/zuul/playbooks/docker-image-build.yaml b/tools/zuul/playbooks/docker-image-build.yaml new file mode 100644 index 00000000..33fda5db --- /dev/null +++ b/tools/zuul/playbooks/docker-image-build.yaml @@ -0,0 +1,87 @@ +- hosts: all + tasks: + - include_vars: vars.yaml + + - name: Install Docker (Debian) + when: ansible_os_family == 'Debian' + block: + - file: + path: "{{ item }}" + state: directory + with_items: + - /etc/docker/ + - /etc/systemd/system/docker.service.d/ + - /var/lib/docker/ + - mount: + path: /var/lib/docker/ + src: tmpfs + fstype: tmpfs + opts: size=25g + state: mounted + - copy: "{{ item }}" + with_items: + - content: "{{ docker_daemon | to_json }}" + dest: /etc/docker/daemon.json + - src: files/docker-systemd.conf + dest: /etc/systemd/system/docker.service.d/ + - apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + - apt_repository: + repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker bionic stable + - apt: + name: "{{ item }}" + allow_unauthenticated: True + with_items: + - docker-ce + - python-pip + - pip: + name: docker + version: 2.7.0 + - iptables: + action: insert + chain: INPUT + in_interface: docker0 + jump: ACCEPT + become: True + + - name: Debug tag generation inputs + block: + - debug: + var: publish + - debug: + var: tags + - debug: + var: zuul + - debug: + msg: "{{ tags | to_json }}" + + - name: Determine tags + shell: echo '{{ tags | to_json }}' | python {{ zuul.project.src_dir }}/tools/image_tags.py + environment: + BRANCH: "{{ zuul.branch | default('') }}" + CHANGE: "{{ zuul.change | default('') }}" + COMMIT: "{{ zuul.newrev | default('') }}" + PATCHSET: "{{ zuul.patchset | default('') }}" + register: image_tags + + - name: Debug computed tags + debug: + var: image_tags + + - name: Make images + when: not publish + block: + - make: + chdir: "{{ zuul.project.src_dir }}" + target: images + params: + IMAGE_TAG: "{{ item }}" + with_items: "{{ image_tags.stdout_lines }}" + + - shell: "docker images" + register: docker_images + + - debug: + var: docker_images + + become: True diff --git a/tools/zuul/playbooks/files/docker-systemd.conf b/tools/zuul/playbooks/files/docker-systemd.conf new file mode 100644 index 00000000..5bf16079 --- /dev/null +++ b/tools/zuul/playbooks/files/docker-systemd.conf @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd diff --git a/playbooks/run_unit_test_job.yaml b/tools/zuul/playbooks/run-unit-test-job.yaml similarity index 93% rename from playbooks/run_unit_test_job.yaml rename to tools/zuul/playbooks/run-unit-test-job.yaml index 59a3d05f..b5d7d238 100644 --- a/playbooks/run_unit_test_job.yaml +++ b/tools/zuul/playbooks/run-unit-test-job.yaml @@ -5,4 +5,4 @@ bindep_dir: "{{ zuul_work_dir }}" - test-setup - ensure-tox - - tox \ No newline at end of file + - tox diff --git a/tools/zuul/playbooks/vars.yaml b/tools/zuul/playbooks/vars.yaml new file mode 100644 index 00000000..6d00a744 --- /dev/null +++ b/tools/zuul/playbooks/vars.yaml @@ -0,0 +1,5 @@ +docker_daemon: + group: zuul + registry-mirrors: + - "http://{{ zuul_site_mirror_fqdn }}:8082/" + storage-driver: overlay2