From 813c1af44e003852a9f1c4ed8d3b0d296c6319b4 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Mon, 30 Sep 2019 14:53:24 -0700 Subject: [PATCH] Fix container image build This corrects outstanding errors in the container image build and adds the build to the gate. Change-Id: I2c8b637d66a640b5eeb894eefd148ce61df4a847 --- .zuul.yaml | 61 ++++++++++++++++++- Dockerfile | 2 +- bindep.txt | 13 ++++ playbooks/functional-test/conf/registry.yaml | 15 +++++ playbooks/functional-test/docker-compose.yaml | 13 ++++ playbooks/functional-test/localtest.yaml | 7 +++ playbooks/functional-test/main.yaml | 56 +++++++++++++++++ playbooks/functional-test/run.yaml | 32 ++++++++++ requirements.txt | 2 + setup.cfg | 9 +-- zuul_registry/__init__.py | 0 zuul_registry/filesystem.py | 3 +- zuul_registry/main.py | 12 ++-- zuul_registry/swift.py | 2 +- 14 files changed, 210 insertions(+), 17 deletions(-) create mode 100644 bindep.txt create mode 100644 playbooks/functional-test/conf/registry.yaml create mode 100644 playbooks/functional-test/docker-compose.yaml create mode 100644 playbooks/functional-test/localtest.yaml create mode 100644 playbooks/functional-test/main.yaml create mode 100644 playbooks/functional-test/run.yaml create mode 100644 zuul_registry/__init__.py diff --git a/.zuul.yaml b/.zuul.yaml index e30e3ff..f7817d4 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,7 +1,64 @@ +# Image building jobs +- secret: + name: zuul-registry-dockerhub + data: + username: zuulzuul + password: !encrypted/pkcs1-oaep + - mc+bK6/dQJ/gzrR5jFYEm9eepVfWG0xiTxxPCunDN0tgbis+xbe1cBOAEdZmjkNrXO/0V + 6d7EcH0JP8mD9jgaTMVWKhVHUF4m/pikP5S4oKgfQYf9PKtzaSp298fhu+4RzuKR01ywj + ks8MWj2oZKJeikQVVbK6z8wMQYirxmaUvivKOU6eJmiPMYs3AaC+P6oc1vEGHMFaa+QpG + 3hD2z+y7Ix/YFKyDQPk4DOs1lTveE18RCBIiP4IRXuYxKkR5uSqj4KEmdcdxBZ6mQyTbw + uZwLMPlkFTNYkaKetobVMo79I3HEACPRN7ylhu96690DsJ7QkvEQD6LPqw82XODuM+T4o + u9kJLLhPhZEALGE9yga4RC6YxfWx1KbJRr74vu5ctxTffuDQGwHjYYf76B0TPuX72UuaP + MqG2F1M9Agr0WpTufyXD/33a+G4iybR0kY21cfv6TEYsock43d8k2ygaPgKqUHFQ5EEia + PiMzt/6vKATl+EUZEJIbqLlP0GyB6RTHylnzSjrInQEumOcsZPx0zhW0hvrs1M8L1WQqQ + VksKhsXrJPO9AS2+/7GUC3+Epsui7R/4A52JK2L73iNh4S+BwBIMMqruC9msbNVPtzEPd + HzKPwH9EkgIRZqvpu7oP9EHzKmsaYk/dRBWSwZ9f9E1GRiTuQ11DoI2km4iTJI= + +- job: + name: zuul-registry-build-image + description: Build and test a Docker image. + parent: opendev-build-docker-image + allowed-projects: zuul/zuul-registry + run: playbooks/functional-test/run.yaml + vars: &image_vars + docker_images: + - context: . + repository: zuul/zuul-registry + target: zuul-registry + +- job: + name: zuul-registry-upload-image + parent: opendev-upload-docker-image + description: Build and test a Docker image and upload to Docker Hub. + allowed-projects: zuul/zuul-registry + run: playbooks/functional-test/run.yaml + secrets: + name: docker_credentials + secret: zuul-registry-dockerhub + pass-to-parent: true + vars: *image_vars + +- job: + name: zuul-registry-promote-image + parent: opendev-promote-docker-image + description: Promote previously uploaded Docker image. + allowed-projects: zuul/zuul-registry + secrets: + name: docker_credentials + secret: zuul-registry-dockerhub + pass-to-parent: true + nodeset: + nodes: [] + vars: *image_vars + - project: check: jobs: - - noop + - zuul-registry-build-image gate: jobs: - - noop + - zuul-registry-upload-image + promote: + jobs: + - zuul-registry-promote-image diff --git a/Dockerfile b/Dockerfile index aee4a2a..3b26f3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,4 +24,4 @@ COPY --from=builder /output/ /output RUN /output/install-from-bindep VOLUME /storage -CMD ["/usr/local/bin/zuul-registry -c /conf/registry.conf serve"] +CMD ["/usr/local/bin/zuul-registry", "-c", "/conf/registry.yaml", "serve"] diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..a4176d7 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,13 @@ +musl-dev [compile test platform:apk] +make [compile test platform:apk] +linux-headers [compile test platform:apk] +gcc [compile test] +g++ [compile test platform:apk platform:dpkg] +gcc-c++ [compile test platform:rpm] +libssl-dev [compile test platform:dpkg] +openssl-devel [compile test platform:rpm] +libressl-dev [compile test platform:apk] +libffi-dev [compile test platform:dpkg platform:apk] +libffi-devel [compile test platform:rpm] +python3-dev [compile test platform:dpkg platform:apk] +python3-devel [compile test platform:rpm] diff --git a/playbooks/functional-test/conf/registry.yaml b/playbooks/functional-test/conf/registry.yaml new file mode 100644 index 0000000..295159c --- /dev/null +++ b/playbooks/functional-test/conf/registry.yaml @@ -0,0 +1,15 @@ +registry: + address: '0.0.0.0' + port: 9000 + tls-cert: /tls/cert.pem + tls-key: /tls/cert.key + users: + - name: testuser + pass: testpass + access: write + - name: anonymous + pass: '' + access: read + storage: + driver: filesystem + root: /storage diff --git a/playbooks/functional-test/docker-compose.yaml b/playbooks/functional-test/docker-compose.yaml new file mode 100644 index 0000000..7bb4965 --- /dev/null +++ b/playbooks/functional-test/docker-compose.yaml @@ -0,0 +1,13 @@ +# Version 2 is the latest that is supported by docker-compose in +# Ubuntu Xenial. +version: '2' + +services: + zuul-registry: + image: zuul/zuul-registry + volumes: + - "./conf/:/conf/:z" + - "/tmp/registry-test/storage/:/storage:z" + - "/tmp/registry-test/tls/:/tls:z" + ports: + - "9000:9000" diff --git a/playbooks/functional-test/localtest.yaml b/playbooks/functional-test/localtest.yaml new file mode 100644 index 0000000..4961395 --- /dev/null +++ b/playbooks/functional-test/localtest.yaml @@ -0,0 +1,7 @@ +- hosts: localhost + vars: + workspace: /tmp/registry-test + local: true + tasks: + - name: Run main tasks + include_tasks: main.yaml diff --git a/playbooks/functional-test/main.yaml b/playbooks/functional-test/main.yaml new file mode 100644 index 0000000..8aa7001 --- /dev/null +++ b/playbooks/functional-test/main.yaml @@ -0,0 +1,56 @@ +- name: Create workspace directory + file: + state: directory + path: "{{ workspace }}" + +- name: Create storage directory + file: + state: directory + path: "{{ workspace }}/storage" + +- name: Create TLS directory + file: + state: directory + path: "{{ workspace }}/tls" + +- name: Generate a TLS key for the registry + command: "openssl req -x509 -newkey rsa:2048 -keyout {{ workspace }}/tls/cert.key -out {{ workspace }}/tls/cert.pem -days 365 -nodes -subj '/C=US/ST=California/L=Oakland/O=Company Name/OU=Org/CN=127.0.0.1'" + +- name: Run docker-compose up + shell: + cmd: docker-compose up -d + chdir: "{{ ansible_user_dir }}/src/opendev.org/zuul/zuul-registry/playbooks/functional-test" + +- name: Print list of images + command: docker image ls --all --digests --no-trunc + +- name: Pull the test image from Docker Hub + command: docker pull registry + +- name: Tag the test image with a realistic name + command: docker tag registry localhost:9000/test/registry + +- name: Log in to local registry + command: docker login localhost:9000 -u testuser -p testpass + ignore_errors: true + +- name: Push the test image to zuul-registry + command: docker image push localhost:9000/test/registry + +- name: Remove the test image from the local cache + command: docker rmi localhost:9000/test/registry + +- name: Remove the test image from the local cache + command: docker rmi registry + +- name: Clean up the local image cache + command: docker image prune -f + +- name: Print list of images + command: docker image ls --all --digests --no-trunc + +- name: Pull the image from zuul-registry + command: docker image pull localhost:9000/test/registry + +- name: Print list of images + command: docker image ls --all --digests --no-trunc diff --git a/playbooks/functional-test/run.yaml b/playbooks/functional-test/run.yaml new file mode 100644 index 0000000..301f142 --- /dev/null +++ b/playbooks/functional-test/run.yaml @@ -0,0 +1,32 @@ +# This installs docker and runs a buildset registry +- hosts: all + roles: + - build-docker-image + +- hosts: all + vars: + workspace: /tmp/registry-test + local: false + tasks: + - name: Install packages + package: + name: + - docker-compose + - openssl + state: present + become: true + - name: Run main tasks + include_tasks: main.yaml + +# If buildset_registry is defined, that means a parent job is running it; +# only if it is not defined does it mean that we are running it. If we +# are running it, pause the job so that child jobs will automatically +# use it. +- hosts: localhost + tasks: + - name: Pause the job + when: buildset_registry is not defined + zuul_return: + data: + zuul: + pause: true diff --git a/requirements.txt b/requirements.txt index 25bc4e3..bf938b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ cherrypy routes requests openstacksdk +python-dateutil +rehash diff --git a/setup.cfg b/setup.cfg index e1ae57d..95d687a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,12 +16,9 @@ classifier = Programming Language :: Python :: 3 Programming Language :: Python :: 3.5 -[pbr] -warnerrors = True - -[global] -setup_hooks = - zuul_registry._setup_hook.setup_hook +[files] +packages = + zuul_registry [entry_points] console_scripts = diff --git a/zuul_registry/__init__.py b/zuul_registry/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zuul_registry/filesystem.py b/zuul_registry/filesystem.py index dd2e2c4..ddf91c0 100644 --- a/zuul_registry/filesystem.py +++ b/zuul_registry/filesystem.py @@ -15,7 +15,8 @@ import os import time -import storageutils + +from . import storageutils class FilesystemDriver(storageutils.StorageDriver): def __init__(self, conf): diff --git a/zuul_registry/main.py b/zuul_registry/main.py index dbf40af..2ba7813 100644 --- a/zuul_registry/main.py +++ b/zuul_registry/main.py @@ -17,15 +17,15 @@ import argparse import sys import logging import cherrypy -import filesystem -import storage -import swift import hashlib import json -import pprint import urllib import yaml +from . import filesystem +from . import storage +from . import swift + DRIVERS = { 'filesystem': filesystem.Driver, 'swift': swift.Driver, @@ -297,12 +297,12 @@ class RegistryServer: def prune(self): self.store.prune() -if __name__ == "__main__": +def main(): parser = argparse.ArgumentParser( description='Zuul registry server') parser.add_argument('-c', dest='config', help='Config file path', - default='/conf/registry.conf') + default='/conf/registry.yaml') parser.add_argument('-d', dest='debug', help='Debug log level', action='store_true') diff --git a/zuul_registry/swift.py b/zuul_registry/swift.py index 7523df3..b0775ea 100644 --- a/zuul_registry/swift.py +++ b/zuul_registry/swift.py @@ -23,7 +23,7 @@ import json import dateutil.parser -import storageutils +from . import storageutils POST_ATTEMPTS = 3