diff --git a/README.md b/README.md index 2089efc..ec0d86e 100644 --- a/README.md +++ b/README.md @@ -101,4 +101,4 @@ Configuration management of the base OS is divided into several realms, each wit # FAQ Q: Why is the build target slow? -A: There is a `mksquashfs` command which runs as part of the build target, and performs slowly if your build environment lacks certain CPU flags which accelerate compression. Use "host-passthrough" or equivalent in your build environment to pass through these CPU flags. In libvirt domain XML, you would change your `cpu` mode element as follows: `` +A: There is a `mksquashfs` command which runs as part of the build target, and performs slowly if your build environment lacks certain CPU flags which accelerate compression. Use "host-passthrough" or equivalent in your build environment to pass through these CPU flags. In libvirt domain XML, you would change your `cpu` mode element as follows: `` \ No newline at end of file diff --git a/image-builder/Makefile b/image-builder/Makefile index 74cb8ee..6032c81 100644 --- a/image-builder/Makefile +++ b/image-builder/Makefile @@ -23,6 +23,9 @@ IMAGE_TYPE ?= iso # iso | qcow PUSH_IMAGE ?= false DISTRO ?= ubuntu_focal WORKDIR ?= ./manifests +# Specifiy if you want to use a different profile than the default. +# i.e. PROFILE ?= k8s-1.18 +PROFILE ?= QCOW_BUNDLE ?= ${WORKDIR}/qcow-bundle # Specify if you want to only build a certain subset of QCOW bundles QCOW_BUNDLE_DIRS ?= @@ -50,11 +53,17 @@ help: ## This help. images: build generate_iso package_qcow clean build: +ifneq ($(PROFILE), ) + set -ex + # Apply any user-defined profiles overrides to playbooks + rsync -rc ./../profiles/$(PROFILE)/manifests/ $(WORKDIR)/ +endif set -ex # Apply any user-defined rootfs overrides to playbooks cp $(WORKDIR)/rootfs/multistrap-vars.yaml assets/playbooks/roles/multistrap/vars/main.yaml cp $(WORKDIR)/rootfs/osconfig-vars.yaml assets/playbooks/roles/osconfig/vars/main.yaml cp $(WORKDIR)/rootfs/livecdcontent-vars.yaml assets/playbooks/roles/livecdcontent/vars/main.yaml + ifneq ($(PROXY), ) sudo -E ./tools/docker_proxy.sh $(PROXY) $(NO_PROXY) export http_proxy=$(PROXY) @@ -90,8 +99,8 @@ endif --label "org.opencontainers.image.title=$(IMAGE_NAME)" endif imgId=`sudo docker images | grep 'image-builder ' | awk '{print $$3}'` - sudo -E DOCKER_BUILDKIT=1 docker run $$imgId ls -ltra /build/usr/bin/sudo > /tmp/sticky_result - sudo grep '^-rws' /tmp/sticky_result >& /dev/null || \ + time sudo -E DOCKER_BUILDKIT=1 docker run $$imgId ls -ltra /build/usr/bin/sudo > /tmp/sticky_result + time sudo grep '^-rws' /tmp/sticky_result >& /dev/null || \ (echo Could not find sticky bit set on target image sudo binary. Are you using buildkit? && \ sudo cat /tmp/sticky_result && exit 1) ifeq ($(PUSH_IMAGE), true) diff --git a/image-builder/manifests/README.md b/image-builder/manifests/README.md index f95c4b2..3b6c890 100644 --- a/image-builder/manifests/README.md +++ b/image-builder/manifests/README.md @@ -21,6 +21,10 @@ |-- qcow ``` +## profiles + +To make modifications please consider using `profiles`. + ## iso The image-builder `generate_iso` makefile target can be used to build the diff --git a/playbooks/airship-image-builder-build.yaml b/playbooks/airship-image-builder-build.yaml index d2f3a2d..b081185 100644 --- a/playbooks/airship-image-builder-build.yaml +++ b/playbooks/airship-image-builder-build.yaml @@ -54,5 +54,6 @@ PROXY: "{{ proxy.http }}" QCOW_CONF_DIRS: "{{ qcow_conf_dirs | default('') }}" USE_PROXY: "{{ proxy.enabled | lower }}" - WORKDIR: "{{ image_config_dir | default('manifests') }}" + WORKDIR: "{{ image_work_dir | default('manifests') }}" + PROFILE: "{{ image_profile_dir | default('') }}" become: True diff --git a/playbooks/airship-image-builder-publish-latest.yaml b/playbooks/airship-image-builder-publish-latest.yaml index b4326a5..2e6c74c 100644 --- a/playbooks/airship-image-builder-publish-latest.yaml +++ b/playbooks/airship-image-builder-publish-latest.yaml @@ -60,5 +60,6 @@ PROXY: "{{ proxy.http }}" QCOW_CONF_DIRS: "{{ qcow_conf_dirs | default('') }}" USE_PROXY: "{{ proxy.enabled | lower }}" - WORKDIR: "{{ image_config_dir | default('manifests') }}" + WORKDIR: "{{ image_work_dir | default('manifests') }}" + PROFILE: "{{ image_profile_dir | default('') }}" become: True diff --git a/profiles/README.md b/profiles/README.md new file mode 100644 index 0000000..f0258a6 --- /dev/null +++ b/profiles/README.md @@ -0,0 +1,144 @@ +# Directory structure: + +``` +|-- profiles + |-- profile1 + |-- manifests + |-- iso + +-- network_data.json + +-- user_data + |-- qcow-bundle-[bundle name] + |-- control-plane + +-- osconfig-vars.yaml + +-- qcow-vars.yaml + |-- data-plane + +-- osconfig-vars.yaml + +-- qcow-vars.yaml + |-- rootfs + |-- livecdcontent-vars.yaml + |-- multistrap-vars.yaml + |-- osconfig-vars.yaml + |-- scripts + |-- common + |-- qcow +``` + +## profiles +This directory allows multiple bundle modifications to be created. It +serves the same purpose as the `manifests` directory just expands on +it's utility. + +The folder structure is the same as the `manifests` directory. Create +a new profile folder and copy the contents of `manifests` into it. Make your +modifications and add the `profile=profile1` flag to the `make images` command. + +The profile will copy only the modified files into the `manifests` folder. + +## iso + +The image-builder `generate_iso` makefile target can be used to build the +ephemeral ISO using the test config data stored under the `manifests/iso` +directory. + +This is *only* for testing. It is *not* an artifact promoted or published. The +final ISO is built by airshipctl, where the network\_data and user\_data are +sourced from airshipctl manifests. + +The following items are expected in the `manifests/iso` directory when using +the `generate_iso` makefile target: +- `user_data` - YAML file containing cloud-init user-data +- `network_data.json` - JSON file containing cloud-init network data + +## qcow-bundles + +The image-builder `package_qcow` makefile target can be used to build the QCOW +artifacts sourced from the manifests/qcow-bundle-\* directories. + +QCOWs are grouped into publishable "bundles", i.e. a container image where all +QCOWs needed for a given deployment are stored. A bundle will be built for each +`manifests/qcow-bundle*` directory. Each `manifests/qcow-bundle*` directory contains +one subdirectory per QCOW that is part of that bundle, where overrides for +those images can be placed. + +QCOWs expect the following files to be present in their directory: +- `osconfig-vars.yaml` - YAML file containing `osconfig` playbook overrides +- `qcow-vars.yaml` - YAML file containing `qcow` playboook overrides + +## rootfs + +This directory contains a number of image-builder ansible playbook overrides +which are applied to base-image inherited by all ISO and QCOWs. + +`livecdcontent-vars.yaml` contains overrides to the livecdcontent playbook. + +`multistrap-vars.yaml` contains overrides to the `multistrap` playbook. + +`osconfig-vars.yaml` contains overrides to the `osconfig` playbook. +NOTE: qcow-bundles contains another level of `osconfig-vars` overrides, which +are applied on top of these common overrides. This common `osconfig-vars` +overrides should be used for playbook overrides, except in cases where those +overrides are actually unique to a particular QCOW variation (e.g., hugepages, +cpu pinning, or other hardware-specific configs). + +## scripts + +This is a convenience directory for adding scripts that run when building images. +These scripts run in the chroot of the target image. For example, a script that +writes 'hello world' to `/hello-world.txt` will appear in the same path on the +target image. + +Use the `manifests/scripts/qcow` directory for scripts that should only run +when building the QCOWs. Use the `manifests/scripts/common` directory for +scripts that are applied to the base container image, which is inherited both by +the QCOWs as well as by the ephemeral ISO. + +No additional configuration is needed for these scripts to run. Just add your +script(s) to these directories as needed. + +# Customizing images in your environment + +Keep in mind that some tasks could also be accomplished by cloud-init or by +the hostconfig operator instead. Refer to the parent image-builder README to +understand the different use-cases for each and to determine the best option +for your use-case. These are lower-effort paths if they support your use-case. + +If you determine that you do require image customizations, start with a manual +image build to reduce complexity: + +1. Clone this repository in your environment. +1. Make any desired changes to the `manifests` directory to customize the + image, as described in prior sections. +1. Perform a `docker login` to the docker registry you will publish image + artifacts to. This should be a registry you have credentials for and that + is accessible by the environment which you plan to consume these artifacts, + (e.g., airshipctl). +1. Run the `make images` target to generate image artifacts. Ensure that the + `PUSH_IMAGE` environment variable is set to `true`, and that the + `DOCKER_REGISTRY` environment variable is set to the container image + repository you performed the login for in the previous step. + +Perform an end-to-end to deployment (e.g., with airshipctl) to verify your +customized image performs as you expect and works properly. + +Now after getting this working, there are several options to proceed depending +on the nature of the changes: +1. Some set of changes to defaults could be proposed upstream (e.g., package + install list). This may be appropriate for changes that are useful for + everyone. In this case, you don't need a custom image because the changes + will be reflected in the image produced upstream. +1. Some enhancements or additions to ansible playbooks to configure some other + aspects of the image, which are useful for everyone and proposed upstream. + In this case, you would be able to leverage ansible overrides to customize + your image with ansible playbooks that are maintained/adopted upstream. +1. Some change to image configuration that is specific to your needs and not + appropriate to be upstreamed. + +In the case of #2 or #3 where you have some portion of image config changes that +are specific to your use-case (i.e. not part of the default upstream image), +and you want to perform regular rebuilds with the latest upstream image-builder +plus your customized changes on top, then you can setup a Zuul child-job that +interfaces with the image-builder parent-job to accomplish this. + +By overriding the `image_config_dir` zuul variable in your child-job, the +image-builder Makefile will use use your customized manifests in place of the +`manifests` directory that is present in upstream image-builder. diff --git a/profiles/k8s-1.18/manifests/rootfs/README.md b/profiles/k8s-1.18/manifests/rootfs/README.md new file mode 100644 index 0000000..0ca0c48 --- /dev/null +++ b/profiles/k8s-1.18/manifests/rootfs/README.md @@ -0,0 +1 @@ +Playbook variable overrides used for building the image-builder container image. diff --git a/profiles/k8s-1.18/manifests/rootfs/multistrap-vars.yaml b/profiles/k8s-1.18/manifests/rootfs/multistrap-vars.yaml new file mode 100644 index 0000000..64e0913 --- /dev/null +++ b/profiles/k8s-1.18/manifests/rootfs/multistrap-vars.yaml @@ -0,0 +1,50 @@ +# Overrides to the multistrap playbook defaults may be defined in this file. +# +# The following are examples that show you how to override variables. +# +# Example 1: The following usage will *overwrite* the list of repos & packages +# defined under multistrap/defaults/main.yaml with the list here: +#repos: +# - register_repo_with_rootfs: true +# name: Ubuntu +# packages: +# - package1 +# - package2 +# source: http://archive.ubuntu.com/ubuntu/ +# keyring_pkg: ubuntu-keyring +# suite: focal +# components: main restricted universe +# +# You would do the above in the event you had a local or other controlled mirror +# you wanted to build the image from. In this case, you will have to redefine +# everything, including the package list(s) for each mirror. +# +# Example 2: The following usage will *append* to the list of default repos +# defined under multistrap/defaults/main.yaml with the list here: +#repos_append: +# - register_repo_with_rootfs: true +# name: DellUtilsRepo +# packages: +# - package1 +# - package2 +# source: http://dell.utils.example.com +# keyring_pkg: ubuntu-keyring +# suite: focal +# components: main restricted universe +# +# You would do the above in the event you are happy with the default mirrors, but +# need to add another mirror that has additional third-party packages you require, +# such as may be needed for HW manufacturer utilities that are not published to +# the community Ubuntu mirrors. +# +# Example 3: The following usage will *append* to the list of default packages +# installed from the default Ubuntu mirror defined in +# multistrap/defaults/main.yaml: +#ubuntu_packages_append: +# - package1 +# - package2 +# +# You would do the above if you are happy with the default mirrors, but need to +# install some additional packages from the same mirror, for example your preferred +# text editor, troubleshooting utilities, etc. +k8s_version: 1.18.20-00 \ No newline at end of file diff --git a/profiles/k8s-1.19/manifests/rootfs/README.md b/profiles/k8s-1.19/manifests/rootfs/README.md new file mode 100644 index 0000000..0ca0c48 --- /dev/null +++ b/profiles/k8s-1.19/manifests/rootfs/README.md @@ -0,0 +1 @@ +Playbook variable overrides used for building the image-builder container image. diff --git a/profiles/k8s-1.19/manifests/rootfs/multistrap-vars.yaml b/profiles/k8s-1.19/manifests/rootfs/multistrap-vars.yaml new file mode 100644 index 0000000..9fa6758 --- /dev/null +++ b/profiles/k8s-1.19/manifests/rootfs/multistrap-vars.yaml @@ -0,0 +1,50 @@ +# Overrides to the multistrap playbook defaults may be defined in this file. +# +# The following are examples that show you how to override variables. +# +# Example 1: The following usage will *overwrite* the list of repos & packages +# defined under multistrap/defaults/main.yaml with the list here: +#repos: +# - register_repo_with_rootfs: true +# name: Ubuntu +# packages: +# - package1 +# - package2 +# source: http://archive.ubuntu.com/ubuntu/ +# keyring_pkg: ubuntu-keyring +# suite: focal +# components: main restricted universe +# +# You would do the above in the event you had a local or other controlled mirror +# you wanted to build the image from. In this case, you will have to redefine +# everything, including the package list(s) for each mirror. +# +# Example 2: The following usage will *append* to the list of default repos +# defined under multistrap/defaults/main.yaml with the list here: +#repos_append: +# - register_repo_with_rootfs: true +# name: DellUtilsRepo +# packages: +# - package1 +# - package2 +# source: http://dell.utils.example.com +# keyring_pkg: ubuntu-keyring +# suite: focal +# components: main restricted universe +# +# You would do the above in the event you are happy with the default mirrors, but +# need to add another mirror that has additional third-party packages you require, +# such as may be needed for HW manufacturer utilities that are not published to +# the community Ubuntu mirrors. +# +# Example 3: The following usage will *append* to the list of default packages +# installed from the default Ubuntu mirror defined in +# multistrap/defaults/main.yaml: +#ubuntu_packages_append: +# - package1 +# - package2 +# +# You would do the above if you are happy with the default mirrors, but need to +# install some additional packages from the same mirror, for example your preferred +# text editor, troubleshooting utilities, etc. +k8s_version: 1.19.14-00 \ No newline at end of file diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 735a9f8..21c2b2c 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -9,8 +9,9 @@ secret: airship_image_builder_airshipit_github_key pass-to-parent: true +# Run the default build job 1.18 - job: - name: airship-image-builder-build + name: airship-image-builder-build-k8s-1.18 nodeset: airship-image-builder-single-node timeout: 7200 post-timeout: 7200 @@ -18,8 +19,22 @@ run: playbooks/airship-image-builder-build.yaml post-run: playbooks/airship-image-builder-collect-logs.yaml +# Run the build job 1.19 - job: - name: airship-image-builder-publish-latest + name: airship-image-builder-build-k8s-1.19 + nodeset: airship-image-builder-single-node + timeout: 7200 + post-timeout: 7200 + pre-run: playbooks/airship-image-builder-deploy-docker.yaml + run: playbooks/airship-image-builder-build.yaml + post-run: playbooks/airship-image-builder-collect-logs.yaml + vars: + image_profile_dir: k8s-1.19 + +# Tag default job with 1.18 +- job: + name: airship-image-builder-publish-k8s-1.18 + parent: airship-image-builder-build-k8s-1.18 nodeset: airship-image-builder-single-node timeout: 7200 post-timeout: 7200 @@ -28,12 +43,37 @@ secrets: - name: image_builder_quay_creds secret: image_builder_quay_creds - -- job: - name: airship-image-builder-publish-commit - parent: airship-image-builder-publish-latest vars: - image_tag: "{{ zuul.newrev }}" + image_tag: "k8s-1.18-latest" + +# Tag the 1.19 job with 1.19 +- job: + name: airship-image-builder-publish-k8s-1.19 + parent: airship-image-builder-build-k8s-1.19 + nodeset: airship-image-builder-single-node + timeout: 7200 + post-timeout: 7200 + pre-run: playbooks/airship-image-builder-deploy-docker.yaml + run: playbooks/airship-image-builder-publish-latest.yaml + secrets: + - name: image_builder_quay_creds + secret: image_builder_quay_creds + vars: + image_tag: "k8s-1.19-latest" + +# Tag the 1.18 job with zuul rev tag +- job: + name: airship-image-builder-publish-commit-k8s-1.18 + parent: airship-image-builder-publish-k8s-1.18 + vars: + image_tag: "k8s-1.18-{{ zuul.newrev }}" + +# Tag the 1.19 job with zuul rev tag +- job: + name: airship-image-builder-publish-commit-k8s-1.19 + parent: airship-image-builder-publish-k8s-1.19 + vars: + image_tag: "k8s-1.19-{{ zuul.newrev }}" - nodeset: name: airship-image-builder-single-node diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 5e306f4..0561cdc 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -15,15 +15,19 @@ image_tag: latest check: jobs: - - airship-image-builder-build + - airship-image-builder-build-k8s-1.18 + - airship-image-builder-build-k8s-1.19 gate: jobs: - - airship-image-builder-build + - airship-image-builder-build-k8s-1.18 + - airship-image-builder-build-k8s-1.19 post: jobs: - airship-image-builder-upload-git-mirror - - airship-image-builder-publish-latest - - airship-image-builder-publish-commit + - airship-image-builder-publish-k8s-1.18 + - airship-image-builder-publish-k8s-1.19 + - airship-image-builder-publish-commit-k8s-1.18 + - airship-image-builder-publish-commit-k8s-1.19 tag: jobs: - airship-image-builder-upload-git-mirror \ No newline at end of file