From 466aa92635eb55b7bb24126a14fafb56130295c9 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 22 Mar 2023 15:34:53 -0700 Subject: [PATCH] Add container build jobs These jobs use the container build roles. Change-Id: I13d1987980bc3d0b1c717878a4bc47edc6dcfe1c --- doc/source/container-jobs.rst | 6 ++ doc/source/jobs.rst | 1 + playbooks/container-image/README.rst | 125 ++++++++++++++++++++++ playbooks/container-image/credentials.rst | 40 +++++++ playbooks/container-image/pre.yaml | 10 ++ playbooks/container-image/promote.yaml | 3 + playbooks/container-image/run.yaml | 3 + playbooks/container-image/upload.yaml | 3 + zuul.d/container-jobs.yaml | 31 ++++++ 9 files changed, 222 insertions(+) create mode 100644 doc/source/container-jobs.rst create mode 100644 playbooks/container-image/README.rst create mode 100644 playbooks/container-image/credentials.rst create mode 100644 playbooks/container-image/pre.yaml create mode 100644 playbooks/container-image/promote.yaml create mode 100644 playbooks/container-image/run.yaml create mode 100644 playbooks/container-image/upload.yaml create mode 100644 zuul.d/container-jobs.yaml diff --git a/doc/source/container-jobs.rst b/doc/source/container-jobs.rst new file mode 100644 index 000000000..b19e9775a --- /dev/null +++ b/doc/source/container-jobs.rst @@ -0,0 +1,6 @@ +Container Jobs +============== + +.. zuul:autojob:: build-container-image +.. zuul:autojob:: upload-container-image +.. zuul:autojob:: promote-container-image diff --git a/doc/source/jobs.rst b/doc/source/jobs.rst index b051b97d8..e2029ccdd 100644 --- a/doc/source/jobs.rst +++ b/doc/source/jobs.rst @@ -8,6 +8,7 @@ Jobs python-jobs js-jobs docker-jobs + container-jobs go-jobs hashicorp-jobs haskell-jobs diff --git a/playbooks/container-image/README.rst b/playbooks/container-image/README.rst new file mode 100644 index 000000000..5eeea6444 --- /dev/null +++ b/playbooks/container-image/README.rst @@ -0,0 +1,125 @@ +This is one of a collection of jobs which are designed to work +together to build, upload, and promote container images in a gating +context: + + * :zuul:job:`build-container-image`: Build the images. + * :zuul:job:`upload-container-image`: Build and stage the images in a registry. + * :zuul:job:`promote-container-image`: Promote previously uploaded images. + +The :zuul:job:`build-container-image` job is designed to be used in +a `check` pipeline and simply builds the images to verify that +the build functions. + +The :zuul:job:`upload-container-image` job builds and uploads the +images to a registry, but only with a single tag corresponding to the +change ID. This job is designed in a `gate` pipeline so that the +build produced by the gate is staged and can later be promoted to +production if the change is successful. + +The :zuul:job:`promote-container-image` job is designed to be +used in a `promote` pipeline. It requires no nodes and runs very +quickly on the Zuul executor. It simply re-tags a previously uploaded +image for a change with whatever tags are supplied by +:zuul:jobvar:`build-container-image.container_images.tags`. +It also removes the change ID tag from the repository in the registry. +If any changes fail to merge, this cleanup will not run and those tags +will need to be deleted manually. + +They all accept the same input data, principally a list of +dictionaries representing the images to build. YAML anchors_ can be +used to supply the same data to all three jobs. + +**Job Variables** + +.. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + The project directory. Serves as the base for + :zuul:jobvar:`build-container-image.container_images.context`. + +.. zuul:jobvar:: container_filename + + The default container filename name to use. Serves as the base for + :zuul:jobvar:`build-container-image.container_images.container_filename`. + This allows a global overriding of the container filename name, for + example when building all images from different folders with + similarily named containerfiles. + + If omitted, the default depends on the container command used. + Typically, this is ``Dockerfile`` for ``docker`` and + ``Containerfile`` (with a fallback on ``Dockerfile``) for + ``podman``. + +.. zuul:jobvar:: container_command + :default: podman + + The command to use when building the image (E.g., ``docker``). + +.. zuul:jobvar:: container_images + :type: list + + A list of images to build. Each item in the list should have: + + .. zuul:jobvar:: context + + The build context; this should be a directory underneath + :zuul:jobvar:`build-container-image.zuul_work_dir`. + + .. zuul:jobvar:: container_filename + + The filename of the container file, present in the context + folder, used for building the image. Provide this if you are + using a non-standard filename for a specific image. + + .. zuul:jobvar:: registry + + The name of the target registry (E.g., ``quay.io``). Used by + the upload and promote roles. + + .. zuul:jobvar:: repository + + The name of the target repository in the registry for the image. + Supply this even if the image is not going to be uploaded (it + will be tagged with this in the local registry). This should + include the registry name. E.g., ``quay.io/example/image``. + + .. zuul:jobvar:: path + + Optional: the directory that should be passed to the build + command. Useful for building images with a container file in + the context directory but a source repository elsewhere. + + .. zuul:jobvar:: build_args + :type: list + + Optional: a list of values to pass to the ``--build-arg`` + parameter. + + .. zuul:jobvar:: target + + Optional: the target for a multi-stage build. + + .. zuul:jobvar:: tags + :type: list + :default: ['latest'] + + A list of tags to be added to the image when promoted. + + .. zuul:jobvar:: siblings + :type: list + :default: [] + + A list of sibling projects to be copied into + ``{{zuul_work_dir}}/.zuul-siblings``. This can be useful to + collect multiple projects to be installed within the same Docker + context. A ``-build-arg`` called ``ZUUL_SIBLINGS`` will be + added with each sibling project. Note that projects here must + be listed in ``required-projects``. + +.. zuul:jobvar:: container_build_extra_env + :type: dict + + A dictionary of key value pairs to add to the container build environment. + This may be useful to enable buildkit with docker builds for example. + +.. _anchors: https://yaml.org/spec/1.2/spec.html#&%20anchor// diff --git a/playbooks/container-image/credentials.rst b/playbooks/container-image/credentials.rst new file mode 100644 index 000000000..2461abf76 --- /dev/null +++ b/playbooks/container-image/credentials.rst @@ -0,0 +1,40 @@ +.. zuul:jobvar:: container_registry_credentials + :type: dict + + This is only required for the upload and promote roles. This is + expected to be a Zuul Secret in dictionary form. Each key is the + name of a registry, and its value a dictionary with information + about that registry. + + Example: + + .. code-block:: yaml + + container_registry_credentials: + quay.io: + username: foo + password: bar + + .. zuul:jobvar:: [registry name] + :type: dict + + Information about a registry. The key is the registry name, and + its value a dict as follows: + + .. zuul:jobvar:: username + + The registry username. + + .. zuul:jobvar:: password + + The registry password. + + .. zuul:jobvar:: repository + + Optional; if supplied this is a regular expression which + restricts to what repositories the image may be uploaded. + The registry name should be included. The following example + allows projects to upload images to repositories within an + organization based on their own names:: + + repository: "^quay.io/myorgname/{{ zuul.project.short_name }}.*" diff --git a/playbooks/container-image/pre.yaml b/playbooks/container-image/pre.yaml new file mode 100644 index 000000000..d90cda489 --- /dev/null +++ b/playbooks/container-image/pre.yaml @@ -0,0 +1,10 @@ +- hosts: all + tasks: + - name: Ensure docker is installed + when: "container_command == 'docker'" + include_role: + name: ensure-docker + - name: Ensure podman is installed + when: "container_command != 'docker'" + include_role: + name: ensure-podman diff --git a/playbooks/container-image/promote.yaml b/playbooks/container-image/promote.yaml new file mode 100644 index 000000000..5353bbd32 --- /dev/null +++ b/playbooks/container-image/promote.yaml @@ -0,0 +1,3 @@ +- hosts: localhost + roles: + - promote-container-image diff --git a/playbooks/container-image/run.yaml b/playbooks/container-image/run.yaml new file mode 100644 index 000000000..041b7eca3 --- /dev/null +++ b/playbooks/container-image/run.yaml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - build-container-image diff --git a/playbooks/container-image/upload.yaml b/playbooks/container-image/upload.yaml new file mode 100644 index 000000000..36f9909a0 --- /dev/null +++ b/playbooks/container-image/upload.yaml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - upload-container-image diff --git a/zuul.d/container-jobs.yaml b/zuul.d/container-jobs.yaml new file mode 100644 index 000000000..7d430d33d --- /dev/null +++ b/zuul.d/container-jobs.yaml @@ -0,0 +1,31 @@ +# Jobs listed in container-jobs.rst. + +- job: + name: build-container-image + description: | + Build a container image. + + .. include:: ../../playbooks/container-image/README.rst + pre-run: playbooks/container-image/pre.yaml + run: playbooks/container-image/run.yaml + +- job: + name: upload-container-image + parent: build-container-image + description: | + Build and upload a container image. + + .. include:: ../../playbooks/container-image/README.rst + .. include:: ../../playbooks/container-image/credentials.rst + post-run: playbooks/container-image/upload.yaml + +- job: + name: promote-container-image + description: | + Retag a previously-uploaded container image. + + .. include:: ../../playbooks/container-image/README.rst + .. include:: ../../playbooks/container-image/credentials.rst + run: playbooks/container-image/promote.yaml + nodeset: + nodes: []