From 129d958a51bc0cc72790855381c7f01170b65cd2 Mon Sep 17 00:00:00 2001 From: "Anselme, Schubbert (sa246v)" Date: Thu, 1 Sep 2022 11:28:13 -0400 Subject: [PATCH] Upgrading MAAS to v3 Signed-off-by: Anselme, Schubert (sa246v) Change-Id: I4b5a5f6a7e21d790cce13a5ccff9819f517cad64 --- .zuul.yaml | 11 ++ Makefile | 4 +- charts/maas/templates/bin/_maas-test.sh.tpl | 18 +++ .../templates/deployment-maas-ingress.yaml | 1 + charts/maas/values.yaml | 10 +- .../2.8_ipmi_error.patch | 2 +- .../2.8_nic_filter.patch | 0 .../2.8_redfish_retries.patch | 2 +- .../2.8_secure_headers.patch | 4 +- images/maas-rack-controller-bionic/Dockerfile | 80 +++++++++++++ images/maas-rack-controller-bionic/README.md | 1 + .../scripts/journalctl-to-tty.service | 0 .../scripts/register-rack-controller.service | 0 images/maas-rack-controller/Dockerfile | 59 +++++----- images/maas-rack-controller/ipmi_error.patch | 27 +++++ .../journalctl-to-tty.service | 13 +++ images/maas-rack-controller/nic_filter.patch | 12 ++ .../redfish_retries.patch | 12 ++ .../register-rack-controller.service | 12 ++ .../2.8_bios_grub_partition.patch | 0 .../2.8_configure_ipmi_user.patch | 12 +- .../2.8_kernel_package.patch | 0 .../2.8_maas_ipmi_autodetect_tool.patch | 4 +- .../2.8_partitiontable_does_not_exist.patch | 0 .../2.8_proxy_acl.patch | 0 .../2.8_region_secret_rotate.patch | 3 +- .../2.8_route.patch | 4 +- .../2.8_secure_headers.patch | 4 +- .../maas-region-controller-bionic/Dockerfile | 88 +++++++++++++++ .../maas-region-controller-bionic/README.md | 1 + .../journalctl-to-tty.service | 13 +++ images/maas-region-controller/Dockerfile | 76 +++++++------ .../bios_grub_partition.patch | 15 +++ .../kernel_package.patch | 31 ++++++ .../partitiontable_does_not_exists.patch | 13 +++ images/maas-region-controller/proxy_acl.patch | 10 ++ .../region_secret_rotate.patch | 20 ++++ images/maas-region-controller/route.patch | 17 +++ images/sstream-cache-bionic/Dockerfile | 48 ++++++++ images/sstream-cache/Dockerfile | 23 ++-- tools/gate/playbooks/helm-deploy.yaml | 29 +++++ tools/maas/00-packages.sh | 10 ++ tools/maas/01-create-cluster.sh | 66 +++++++++++ tools/maas/02-cert-manager.sh | 37 ++++++ tools/maas/03-postgresql.sh | 19 ++++ tools/maas/04-load-images.sh | 18 +++ tools/maas/05-maas.sh | 105 ++++++++++++++++++ 47 files changed, 836 insertions(+), 98 deletions(-) rename images/{maas-rack-controller => maas-rack-controller-bionic}/2.8_ipmi_error.patch (99%) rename images/{maas-rack-controller => maas-rack-controller-bionic}/2.8_nic_filter.patch (100%) rename images/{maas-rack-controller => maas-rack-controller-bionic}/2.8_redfish_retries.patch (99%) rename images/{maas-rack-controller => maas-rack-controller-bionic}/2.8_secure_headers.patch (99%) create mode 100644 images/maas-rack-controller-bionic/Dockerfile create mode 100644 images/maas-rack-controller-bionic/README.md rename images/{maas-rack-controller => maas-rack-controller-bionic}/scripts/journalctl-to-tty.service (100%) rename images/{maas-rack-controller => maas-rack-controller-bionic}/scripts/register-rack-controller.service (100%) create mode 100644 images/maas-rack-controller/ipmi_error.patch create mode 100644 images/maas-rack-controller/journalctl-to-tty.service create mode 100644 images/maas-rack-controller/nic_filter.patch create mode 100644 images/maas-rack-controller/redfish_retries.patch create mode 100644 images/maas-rack-controller/register-rack-controller.service rename images/{maas-region-controller => maas-region-controller-bionic}/2.8_bios_grub_partition.patch (100%) rename images/{maas-region-controller => maas-region-controller-bionic}/2.8_configure_ipmi_user.patch (99%) rename images/{maas-region-controller => maas-region-controller-bionic}/2.8_kernel_package.patch (100%) rename images/{maas-region-controller => maas-region-controller-bionic}/2.8_maas_ipmi_autodetect_tool.patch (99%) rename images/{maas-region-controller => maas-region-controller-bionic}/2.8_partitiontable_does_not_exist.patch (100%) rename images/{maas-region-controller => maas-region-controller-bionic}/2.8_proxy_acl.patch (100%) rename images/{maas-region-controller => maas-region-controller-bionic}/2.8_region_secret_rotate.patch (99%) rename images/{maas-region-controller => maas-region-controller-bionic}/2.8_route.patch (99%) rename images/{maas-region-controller => maas-region-controller-bionic}/2.8_secure_headers.patch (99%) create mode 100644 images/maas-region-controller-bionic/Dockerfile create mode 100644 images/maas-region-controller-bionic/README.md create mode 100644 images/maas-region-controller-bionic/journalctl-to-tty.service create mode 100644 images/maas-region-controller/bios_grub_partition.patch create mode 100644 images/maas-region-controller/kernel_package.patch create mode 100644 images/maas-region-controller/partitiontable_does_not_exists.patch create mode 100644 images/maas-region-controller/proxy_acl.patch create mode 100644 images/maas-region-controller/region_secret_rotate.patch create mode 100644 images/maas-region-controller/route.patch create mode 100644 images/sstream-cache-bionic/Dockerfile create mode 100644 tools/gate/playbooks/helm-deploy.yaml create mode 100755 tools/maas/00-packages.sh create mode 100755 tools/maas/01-create-cluster.sh create mode 100755 tools/maas/02-cert-manager.sh create mode 100755 tools/maas/03-postgresql.sh create mode 100755 tools/maas/04-load-images.sh create mode 100755 tools/maas/05-maas.sh diff --git a/.zuul.yaml b/.zuul.yaml index 51b6cd0..4b4c1d4 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -18,6 +18,9 @@ - airship-maas-chart-build-latest-htk - airship-maas-docker-build-gate - airship-maas-lint-yaml + # NOTE(sanselme): This job is disabled until fixed + # waiting for coredns to be ready timeout + # - airship-maas-helm-deploy gate: jobs: - airship-maas-lint-ws @@ -82,6 +85,14 @@ irrelevant-files: - '^charts/maas/templates/.*' +- job: + name: airship-maas-helm-deploy + timeout: 7200 + run: tools/gate/playbooks/helm-deploy.yaml + nodeset: airship-maas-single-node + files: + - '^charts/.*' + - job: name: airship-maas-docker-publish timeout: 1800 diff --git a/Makefile b/Makefile index a475d3f..aa5dbcf 100644 --- a/Makefile +++ b/Makefile @@ -33,8 +33,8 @@ IMAGE_NAME := maas-rack-controller maas-region-controller sstream-cache BUILD_DIR := $(shell mktemp -d) HELM := $(BUILD_DIR)/helm SSTREAM_IMAGE := "https://images.maas.io/ephemeral-v3/stable/" -SSTREAM_RELEASE := "bionic" -UBUNTU_BASE_IMAGE ?= ubuntu:18.04 +SSTREAM_RELEASE := "focal" +UBUNTU_BASE_IMAGE ?= ubuntu:20.04 USE_CACHED_IMG ?= false DOCKER_EXTRA_ARGS ?= diff --git a/charts/maas/templates/bin/_maas-test.sh.tpl b/charts/maas/templates/bin/_maas-test.sh.tpl index 0257571..95daf52 100644 --- a/charts/maas/templates/bin/_maas-test.sh.tpl +++ b/charts/maas/templates/bin/_maas-test.sh.tpl @@ -17,6 +17,16 @@ set -ex +function check_admin_api { + if maas local version read; + then + echo 'Admin API is responding' + return 0 + else + return 1 + fi +} + function check_boot_images { if maas local boot-resources is-importing | grep -q 'true'; then @@ -74,5 +84,13 @@ then exit 1 fi +check_admin_api + +if [[ $? -eq 1 ]] +then + echo "Admin API response FAILED!" + exit 1 +fi + echo "MAAS Validation SUCCESS!" exit 0 diff --git a/charts/maas/templates/deployment-maas-ingress.yaml b/charts/maas/templates/deployment-maas-ingress.yaml index 4292f6b..53a859c 100644 --- a/charts/maas/templates/deployment-maas-ingress.yaml +++ b/charts/maas/templates/deployment-maas-ingress.yaml @@ -119,6 +119,7 @@ rules: - configmaps verbs: - create + - update - apiGroups: - "" resources: diff --git a/charts/maas/values.yaml b/charts/maas/values.yaml index 7daec84..9078f58 100644 --- a/charts/maas/values.yaml +++ b/charts/maas/values.yaml @@ -96,7 +96,7 @@ manifests: images: tags: - db_init: docker.io/postgres:9.5 + db_init: docker.io/postgres:14.5 db_sync: quay.io/airshipit/maas-region-controller:latest maas_rack: quay.io/airshipit/maas-rack-controller:latest maas_region: quay.io/airshipit/maas-region-controller:latest @@ -104,9 +104,9 @@ images: export_api_key: quay.io/airshipit/maas-region-controller:latest maas_cache: quay.io/airshipit/sstream-cache:latest dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 - ingress: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 + ingress: k8s.gcr.io/ingress-nginx/controller:v1.2.0 ingress_vip: docker.io/busybox:latest - error_pages: gcr.io/google_containers/ingress-gce-404-server-with-metrics-amd64:v1.6.0 + error_pages: k8s.gcr.io/defaultbackend-amd64:1.5 maas_syslog: quay.io/airshipit/maas-region-controller:latest pull_policy: IfNotPresent local_registry: @@ -258,8 +258,8 @@ conf: proxy_server: null images: default_os: 'ubuntu' - default_image: 'bionic' - default_kernel: 'ga-18.04' + default_image: 'focal' + default_kernel: 'ga-20.04' credentials: secret: namespace: maas diff --git a/images/maas-rack-controller/2.8_ipmi_error.patch b/images/maas-rack-controller-bionic/2.8_ipmi_error.patch similarity index 99% rename from images/maas-rack-controller/2.8_ipmi_error.patch rename to images/maas-rack-controller-bionic/2.8_ipmi_error.patch index 6c14876..215d167 100644 --- a/images/maas-rack-controller/2.8_ipmi_error.patch +++ b/images/maas-rack-controller-bionic/2.8_ipmi_error.patch @@ -22,6 +22,6 @@ index e99b807ce..8f56dc77a 100644 ip_extractor = make_ip_extractor("power_address") - wait_time = (4, 8, 16, 32) + wait_time = (4, 4, 8, 8, 16, 16, 32, 32) - + def detect_missing_packages(self): if not shell.has_command_available("ipmipower"): diff --git a/images/maas-rack-controller/2.8_nic_filter.patch b/images/maas-rack-controller-bionic/2.8_nic_filter.patch similarity index 100% rename from images/maas-rack-controller/2.8_nic_filter.patch rename to images/maas-rack-controller-bionic/2.8_nic_filter.patch diff --git a/images/maas-rack-controller/2.8_redfish_retries.patch b/images/maas-rack-controller-bionic/2.8_redfish_retries.patch similarity index 99% rename from images/maas-rack-controller/2.8_redfish_retries.patch rename to images/maas-rack-controller-bionic/2.8_redfish_retries.patch index ab55fc5..576ed42 100644 --- a/images/maas-rack-controller/2.8_redfish_retries.patch +++ b/images/maas-rack-controller-bionic/2.8_redfish_retries.patch @@ -7,6 +7,6 @@ index 27f63545a..9c39d577e 100644 ] ip_extractor = make_ip_extractor("power_address") + wait_time = (4, 8, 16, 32) - + def detect_missing_packages(self): # no required packages diff --git a/images/maas-rack-controller/2.8_secure_headers.patch b/images/maas-rack-controller-bionic/2.8_secure_headers.patch similarity index 99% rename from images/maas-rack-controller/2.8_secure_headers.patch rename to images/maas-rack-controller-bionic/2.8_secure_headers.patch index c7a520e..71c7f92 100644 --- a/images/maas-rack-controller/2.8_secure_headers.patch +++ b/images/maas-rack-controller-bionic/2.8_secure_headers.patch @@ -4,9 +4,9 @@ index 3a3f9f89b..1eb273816 100644 +++ b/src/twisted/web/server.py @@ -174,7 +174,6 @@ class Request(Copyable, http.Request, components.Componentized): self.site = self.channel.site - + # set various default headers - self.setHeader(b'server', version) self.setHeader(b'date', http.datetimeToString()) - + # Resource Identification diff --git a/images/maas-rack-controller-bionic/Dockerfile b/images/maas-rack-controller-bionic/Dockerfile new file mode 100644 index 0000000..7e22c02 --- /dev/null +++ b/images/maas-rack-controller-bionic/Dockerfile @@ -0,0 +1,80 @@ +FROM ubuntu:18.04 + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' +LABEL org.opencontainers.image.url='https://airshipit.org' +LABEL org.opencontainers.image.documentation='https://github.com/openstack/airship-maas' +LABEL org.opencontainers.image.source='https://git.openstack.org/openstack/airship-maas' +LABEL org.opencontainers.image.vendor='The Airship Authors' +LABEL org.opencontainers.image.licenses='Apache-2.0' + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY +ARG http_proxy +ARG https_proxy +ARG no_proxy + +ENV DEBIAN_FRONTEND noninteractive +ENV container docker + +ENV MAAS_VERSION 2.8.7-8611-g.f2514168f-0ubuntu1~18.04.1 + +RUN apt-get -qq update \ + && apt-get install -y \ + avahi-daemon \ + isc-dhcp-server \ + jq \ + libvirt-bin \ + patch \ + software-properties-common \ + sudo \ + systemd \ + ca-certificates \ + # Don't start any optional services except for the few we need. + # (specifically, don't start avahi-daemon, isc-dhcp-server, or libvirtd) + && find /etc/systemd/system \ + /lib/systemd/system \ + -path '*.wants/*' \ + -not -name '*journald*' \ + -not -name '*systemd-tmpfiles*' \ + -not -name '*systemd-user-sessions*' \ + -exec rm \{} \; \ + && systemctl set-default multi-user.target \ + # Install maas from the ppa + && add-apt-repository -yu ppa:maas/2.8 \ + && apt-get install -y \ + maas-rack-controller=$MAAS_VERSION \ + && rm -rf /var/lib/apt/lists/* + +# Preserve the directory structure, permissions, and contents of /var/lib/maas +RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas + +# register ourselves with the region controller +COPY scripts/register-rack-controller.service /lib/systemd/system/register-rack-controller.service +RUN systemctl enable register-rack-controller.service + +# Patch so that Calico interfaces are ignored +COPY 2.8_nic_filter.patch /tmp/2.8_nic_filter.patch +COPY 2.8_secure_headers.patch /tmp/2.8_secure_headers.patch +# Patch so maas knows that "BMC error" is retriable +COPY 2.8_ipmi_error.patch /tmp/2.8_ipmi_error.patch +# Patch to space redfish request retries apart a bit, to avoid overwhelming the BMC +COPY 2.8_redfish_retries.patch /tmp/2.8_redfish_retries.patch + +RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch network.py < /tmp/2.8_nic_filter.patch +RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.8_secure_headers.patch +RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch ipmi.py < /tmp/2.8_ipmi_error.patch +RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch redfish.py < /tmp/2.8_redfish_retries.patch + +# echo journalctl logs to the container's stdout +COPY scripts/journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service +RUN systemctl enable journalctl-to-tty.service + +# quiet sudo for the maas user +RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log + +# avoid triggering bind9 high cpu utilization bug +RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf + +# initalize systemd +CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"] diff --git a/images/maas-rack-controller-bionic/README.md b/images/maas-rack-controller-bionic/README.md new file mode 100644 index 0000000..0a11659 --- /dev/null +++ b/images/maas-rack-controller-bionic/README.md @@ -0,0 +1 @@ +[![Docker Repository on Quay](https://quay.io/repository/airshipit/maas-rack/status "Docker Repository on Quay")](https://quay.io/repository/airshipit/maas-rack) Ubuntu MaaS Rack Controller diff --git a/images/maas-rack-controller/scripts/journalctl-to-tty.service b/images/maas-rack-controller-bionic/scripts/journalctl-to-tty.service similarity index 100% rename from images/maas-rack-controller/scripts/journalctl-to-tty.service rename to images/maas-rack-controller-bionic/scripts/journalctl-to-tty.service diff --git a/images/maas-rack-controller/scripts/register-rack-controller.service b/images/maas-rack-controller-bionic/scripts/register-rack-controller.service similarity index 100% rename from images/maas-rack-controller/scripts/register-rack-controller.service rename to images/maas-rack-controller-bionic/scripts/register-rack-controller.service diff --git a/images/maas-rack-controller/Dockerfile b/images/maas-rack-controller/Dockerfile index 059071d..ca9c439 100644 --- a/images/maas-rack-controller/Dockerfile +++ b/images/maas-rack-controller/Dockerfile @@ -1,4 +1,4 @@ -ARG FROM=ubuntu:18.04 +ARG FROM=ubuntu:20.04 FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' @@ -18,57 +18,60 @@ ARG no_proxy ENV DEBIAN_FRONTEND noninteractive ENV container docker -ENV MAAS_VERSION 2.8.7-8611-g.f2514168f-0ubuntu1~18.04.1 +ENV MAAS_VERSION 1:3.0.0-10029-g.986ea3e45-0ubuntu1~20.04.1 RUN apt-get -qq update \ - && apt-get install -y \ + && apt-get install -y \ avahi-daemon \ isc-dhcp-server \ jq \ - libvirt-bin \ + libvirt-daemon-system \ + libvirt-clients \ patch \ software-properties-common \ sudo \ systemd \ ca-certificates \ -# Don't start any optional services except for the few we need. -# (specifically, don't start avahi-daemon, isc-dhcp-server, or libvirtd) - && find /etc/systemd/system \ - /lib/systemd/system \ - -path '*.wants/*' \ - -not -name '*journald*' \ - -not -name '*systemd-tmpfiles*' \ - -not -name '*systemd-user-sessions*' \ - -exec rm \{} \; \ - && systemctl set-default multi-user.target \ -# Install maas from the ppa - && add-apt-repository -yu ppa:maas/2.8 \ - && apt-get install -y \ + # Don't start any optional services except for the few we need. + # (specifically, don't start avahi-daemon, isc-dhcp-server, or libvirtd) + && find /etc/systemd/system \ + /lib/systemd/system \ + -path '*.wants/*' \ + -not -name '*journald*' \ + -not -name '*systemd-tmpfiles*' \ + -not -name '*systemd-user-sessions*' \ + -exec rm \{} \; \ + && systemctl set-default multi-user.target \ + # Install maas from the ppa + && add-apt-repository -yu ppa:maas/3.0 \ + && apt-get install -y \ maas-rack-controller=$MAAS_VERSION \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* + +# Update latest packages, including security updates +RUN apt-get -qq update \ + && apt-get upgrade -y # Preserve the directory structure, permissions, and contents of /var/lib/maas RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas # register ourselves with the region controller -COPY scripts/register-rack-controller.service /lib/systemd/system/register-rack-controller.service +COPY register-rack-controller.service /lib/systemd/system/register-rack-controller.service RUN systemctl enable register-rack-controller.service # Patch so that Calico interfaces are ignored -COPY 2.8_nic_filter.patch /tmp/2.8_nic_filter.patch -COPY 2.8_secure_headers.patch /tmp/2.8_secure_headers.patch +COPY nic_filter.patch /tmp/nic_filter.patch # Patch so maas knows that "BMC error" is retriable -COPY 2.8_ipmi_error.patch /tmp/2.8_ipmi_error.patch +COPY ipmi_error.patch /tmp/ipmi_error.patch # Patch to space redfish request retries apart a bit, to avoid overwhelming the BMC -COPY 2.8_redfish_retries.patch /tmp/2.8_redfish_retries.patch +COPY redfish_retries.patch /tmp/redfish_retries.patch -RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch network.py < /tmp/2.8_nic_filter.patch -RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.8_secure_headers.patch -RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch ipmi.py < /tmp/2.8_ipmi_error.patch -RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch redfish.py < /tmp/2.8_redfish_retries.patch +RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch network.py < /tmp/nic_filter.patch +RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch ipmi.py < /tmp/ipmi_error.patch +RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch redfish.py < /tmp/redfish_retries.patch # echo journalctl logs to the container's stdout -COPY scripts/journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service +COPY journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service RUN systemctl enable journalctl-to-tty.service # quiet sudo for the maas user diff --git a/images/maas-rack-controller/ipmi_error.patch b/images/maas-rack-controller/ipmi_error.patch new file mode 100644 index 0000000..4bf72cf --- /dev/null +++ b/images/maas-rack-controller/ipmi_error.patch @@ -0,0 +1,27 @@ +diff --git a/src/provisioningserver/drivers/power/ipmi.py b/src/provisioningserver/drivers/power/ipmi.py +index 70201e86e..26625e21d 100644 +--- a/src/provisioningserver/drivers/power/ipmi.py ++++ b/src/provisioningserver/drivers/power/ipmi.py +@@ -155,6 +155,13 @@ IPMI_ERRORS = { + ), + "exception": PowerConnError, + }, ++ "BMC error": { ++ "message": ( ++ "Device not responding correctly while performing power action." ++ " MAAS performed several retries. Please wait and try again." ++ ), ++ "exception": PowerConnError, ++ }, + "could not find inband device": { + "message": ( + "An inband device could not be found." +@@ -308,7 +315,7 @@ class IPMIPowerDriver(PowerDriver): + ), + ] + ip_extractor = make_ip_extractor("power_address") +- wait_time = (4, 8, 16, 32) ++ wait_time = (4, 4, 8, 8, 16, 16, 32, 32) + + def detect_missing_packages(self): + if not shell.has_command_available("ipmipower"): diff --git a/images/maas-rack-controller/journalctl-to-tty.service b/images/maas-rack-controller/journalctl-to-tty.service new file mode 100644 index 0000000..2725055 --- /dev/null +++ b/images/maas-rack-controller/journalctl-to-tty.service @@ -0,0 +1,13 @@ +[Unit] +Description=Journald console log streamer +Requires=systemd-journald.service +After=systemd-journald.service + +[Service] +Restart=always +RestartSec=0 +ExecStart=/bin/journalctl -f +StandardOutput=tty + +[Install] +WantedBy=basic.target diff --git a/images/maas-rack-controller/nic_filter.patch b/images/maas-rack-controller/nic_filter.patch new file mode 100644 index 0000000..7d3b083 --- /dev/null +++ b/images/maas-rack-controller/nic_filter.patch @@ -0,0 +1,12 @@ +diff --git a/src/provisioningserver/utils/network.py b/src/provisioningserver/utils/network.py +index 7895227c4..df83836f3 100644 +--- a/src/provisioningserver/utils/network.py ++++ b/src/provisioningserver/utils/network.py +@@ -1128,6 +1128,7 @@ def get_all_interfaces_definition( + # interfaces for guests. By themselves, they're not useful for MAAS to + # manage. + "tunnel", ++ "ethernet", + ] + if not running_in_container(): + # When not running in a container, we should be able to identify diff --git a/images/maas-rack-controller/redfish_retries.patch b/images/maas-rack-controller/redfish_retries.patch new file mode 100644 index 0000000..020f983 --- /dev/null +++ b/images/maas-rack-controller/redfish_retries.patch @@ -0,0 +1,12 @@ +diff --git a/src/provisioningserver/drivers/power/redfish.py b/src/provisioningserver/drivers/power/redfish.py +index 19d9ecd88..0075997dd 100644 +--- a/src/provisioningserver/drivers/power/redfish.py ++++ b/src/provisioningserver/drivers/power/redfish.py +@@ -170,6 +170,7 @@ class RedfishPowerDriver(RedfishPowerDriverBase): + make_setting_field("node_id", "Node ID", scope=SETTING_SCOPE.NODE), + ] + ip_extractor = make_ip_extractor("power_address") ++ wait_time = (4, 8, 16, 32) + + def detect_missing_packages(self): + # no required packages diff --git a/images/maas-rack-controller/register-rack-controller.service b/images/maas-rack-controller/register-rack-controller.service new file mode 100644 index 0000000..fb439d3 --- /dev/null +++ b/images/maas-rack-controller/register-rack-controller.service @@ -0,0 +1,12 @@ +[Unit] +Description=Register with MaaS Region Controller +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +PassEnvironment=MAAS_ENDPOINT MAAS_REGION_SECRET MAAS_API_KEY HOST_MOUNT_PATH +ExecStart=/usr/local/bin/register-rack-controller.sh + +[Install] +WantedBy=multi-user.target diff --git a/images/maas-region-controller/2.8_bios_grub_partition.patch b/images/maas-region-controller-bionic/2.8_bios_grub_partition.patch similarity index 100% rename from images/maas-region-controller/2.8_bios_grub_partition.patch rename to images/maas-region-controller-bionic/2.8_bios_grub_partition.patch diff --git a/images/maas-region-controller/2.8_configure_ipmi_user.patch b/images/maas-region-controller-bionic/2.8_configure_ipmi_user.patch similarity index 99% rename from images/maas-region-controller/2.8_configure_ipmi_user.patch rename to images/maas-region-controller-bionic/2.8_configure_ipmi_user.patch index f9e909f..e0363ec 100644 --- a/images/maas-region-controller/2.8_configure_ipmi_user.patch +++ b/images/maas-region-controller-bionic/2.8_configure_ipmi_user.patch @@ -4,8 +4,8 @@ index 13188ecb8..7b3dad4d4 100755 +++ b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py @@ -235,8 +235,30 @@ def make_ipmi_user_settings(username, password): return user_settings - - + + +def configure_ipmi_user_with_backoff(username): + """Create/configure an IPMI user, but with several tries""" + attempt = 1 @@ -45,15 +45,15 @@ index 13188ecb8..7b3dad4d4 100755 + raise IPMIError( + "Unable to set BMC password:\n{}".format(exceptions_caught) + ) - - + + def set_ipmi_lan_channel_settings(): @@ -389,7 +413,7 @@ def main(): IPMI_MAAS_USER = args.maas_ipmi_user IPMI_MAAS_PASSWORD = None - + - IPMI_MAAS_PASSWORD = configure_ipmi_user(IPMI_MAAS_USER) + IPMI_MAAS_PASSWORD = configure_ipmi_user_with_backoff(IPMI_MAAS_USER) - + # Attempt to enable IPMI Over Lan. If it is disabled, MAAS won't # be able to remotely communicate to the BMC. diff --git a/images/maas-region-controller/2.8_kernel_package.patch b/images/maas-region-controller-bionic/2.8_kernel_package.patch similarity index 100% rename from images/maas-region-controller/2.8_kernel_package.patch rename to images/maas-region-controller-bionic/2.8_kernel_package.patch diff --git a/images/maas-region-controller/2.8_maas_ipmi_autodetect_tool.patch b/images/maas-region-controller-bionic/2.8_maas_ipmi_autodetect_tool.patch similarity index 99% rename from images/maas-region-controller/2.8_maas_ipmi_autodetect_tool.patch rename to images/maas-region-controller-bionic/2.8_maas_ipmi_autodetect_tool.patch index 2175681..7444c4b 100644 --- a/images/maas-region-controller/2.8_maas_ipmi_autodetect_tool.patch +++ b/images/maas-region-controller-bionic/2.8_maas_ipmi_autodetect_tool.patch @@ -3,8 +3,8 @@ index f8ca88467..530bc7d15 100755 --- a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect_tool.py +++ b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect_tool.py @@ -33,7 +33,11 @@ def detect_ipmi(): - - + + def is_host_moonshot(): - output = subprocess.check_output(["ipmitool", "raw", "06", "01"]) + (status, output) = subprocess.getstatusoutput( diff --git a/images/maas-region-controller/2.8_partitiontable_does_not_exist.patch b/images/maas-region-controller-bionic/2.8_partitiontable_does_not_exist.patch similarity index 100% rename from images/maas-region-controller/2.8_partitiontable_does_not_exist.patch rename to images/maas-region-controller-bionic/2.8_partitiontable_does_not_exist.patch diff --git a/images/maas-region-controller/2.8_proxy_acl.patch b/images/maas-region-controller-bionic/2.8_proxy_acl.patch similarity index 100% rename from images/maas-region-controller/2.8_proxy_acl.patch rename to images/maas-region-controller-bionic/2.8_proxy_acl.patch diff --git a/images/maas-region-controller/2.8_region_secret_rotate.patch b/images/maas-region-controller-bionic/2.8_region_secret_rotate.patch similarity index 99% rename from images/maas-region-controller/2.8_region_secret_rotate.patch rename to images/maas-region-controller-bionic/2.8_region_secret_rotate.patch index 92d795c..6637413 100644 --- a/images/maas-region-controller/2.8_region_secret_rotate.patch +++ b/images/maas-region-controller-bionic/2.8_region_secret_rotate.patch @@ -16,6 +16,5 @@ index f92529265..542970009 100644 + # secret and set it in the database (set_config function) + secret = secret_on_fs + Config.objects.set_config("rpc_shared_secret", to_hex(secret)) - + return secret - diff --git a/images/maas-region-controller/2.8_route.patch b/images/maas-region-controller-bionic/2.8_route.patch similarity index 99% rename from images/maas-region-controller/2.8_route.patch rename to images/maas-region-controller-bionic/2.8_route.patch index e7a3b34..e06f00f 100644 --- a/images/maas-region-controller/2.8_route.patch +++ b/images/maas-region-controller-bionic/2.8_route.patch @@ -3,7 +3,7 @@ index 99a3ce309..2a9e72d88 100644 --- a/src/maasserver/preseed_network.py +++ b/src/maasserver/preseed_network.py @@ -308,7 +308,11 @@ class InterfaceConfiguration: - + def _get_matching_routes(self, source): """Return all route objects matching `source`.""" - return {route for route in self.routes if route.source == source} @@ -12,6 +12,6 @@ index 99a3ce309..2a9e72d88 100644 + for route in self.routes + if str(route.source.cidr) == str(source.cidr) + } - + def _generate_addresses(self, version=1): """Generate the various addresses needed for this interface.""" diff --git a/images/maas-region-controller/2.8_secure_headers.patch b/images/maas-region-controller-bionic/2.8_secure_headers.patch similarity index 99% rename from images/maas-region-controller/2.8_secure_headers.patch rename to images/maas-region-controller-bionic/2.8_secure_headers.patch index c7a520e..71c7f92 100644 --- a/images/maas-region-controller/2.8_secure_headers.patch +++ b/images/maas-region-controller-bionic/2.8_secure_headers.patch @@ -4,9 +4,9 @@ index 3a3f9f89b..1eb273816 100644 +++ b/src/twisted/web/server.py @@ -174,7 +174,6 @@ class Request(Copyable, http.Request, components.Componentized): self.site = self.channel.site - + # set various default headers - self.setHeader(b'server', version) self.setHeader(b'date', http.datetimeToString()) - + # Resource Identification diff --git a/images/maas-region-controller-bionic/Dockerfile b/images/maas-region-controller-bionic/Dockerfile new file mode 100644 index 0000000..2384def --- /dev/null +++ b/images/maas-region-controller-bionic/Dockerfile @@ -0,0 +1,88 @@ +FROM ubuntu:18.04 + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' +LABEL org.opencontainers.image.url='https://airshipit.org' +LABEL org.opencontainers.image.documentation='https://github.com/openstack/airship-maas' +LABEL org.opencontainers.image.source='https://git.openstack.org/openstack/airship-maas' +LABEL org.opencontainers.image.vendor='The Airship Authors' +LABEL org.opencontainers.image.licenses='Apache-2.0' + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY +ARG http_proxy +ARG https_proxy +ARG no_proxy + +ENV DEBIAN_FRONTEND noninteractive +ENV container docker + +ENV MAAS_VERSION 2.8.7-8611-g.f2514168f-0ubuntu1~18.04.1 + +RUN apt-get -qq update \ + && apt-get install -y \ + avahi-daemon \ + jq \ + patch \ + software-properties-common \ + sudo \ + systemd \ + ca-certificates \ + # Don't start any optional services except for the few we need. + # (specifically, don't start avahi-daemon) + && find /etc/systemd/system \ + /lib/systemd/system \ + -path '*.wants/*' \ + -not -name '*journald*' \ + -not -name '*systemd-tmpfiles*' \ + -not -name '*systemd-user-sessions*' \ + -exec rm \{} \; \ + && systemctl set-default multi-user.target \ + # Install maas from the ppa + && add-apt-repository -yu ppa:maas/2.8 \ + && apt-get install -y \ + maas-region-api=$MAAS_VERSION \ + # tcpdump is required by /usr/lib/maas/beacon-monitor + tcpdump \ + && rm -rf /var/lib/apt/lists/* + +# Preserve the directory structure, permissions, and contents of /var/lib/maas +RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas + +# MAAS workarounds +COPY 2.8_route.patch /tmp/2.8_route.patch +COPY 2.8_kernel_package.patch /tmp/2.8_kernel_package.patch +COPY 2.8_bios_grub_partition.patch /tmp/2.8_bios_grub_partition.patch +# sh8121att: allow all requests via the proxy to allow it to work +# behind ingress +COPY 2.8_proxy_acl.patch /tmp/2.8_proxy_acl.patch +# Patch to add retrying to MaaS BMC user setup, and improve exception handling +COPY 2.8_configure_ipmi_user.patch /tmp/2.8_configure_ipmi_user.patch +COPY 2.8_secure_headers.patch /tmp/2.8_secure_headers.patch +COPY 2.8_region_secret_rotate.patch /tmp/2.8_region_secret_rotate.patch +COPY 2.8_partitiontable_does_not_exist.patch /tmp/2.8_partitiontable_does_not_exist.patch +# Avoid enlistment failures due to exceptions during moonshot detect attempts +COPY 2.8_maas_ipmi_autodetect_tool.patch /tmp/2.8_maas_ipmi_autodetect_tool.patch + +RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/2.8_route.patch +RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/2.8_kernel_package.patch +RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/2.8_bios_grub_partition.patch +RUN cd /usr/lib/python3/dist-packages/maasserver && patch security.py < /tmp/2.8_region_secret_rotate.patch +RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_ipmi_autodetect.py < /tmp/2.8_configure_ipmi_user.patch +RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/2.8_proxy_acl.patch +RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.8_secure_headers.patch +RUN cd /usr/lib/python3/dist-packages/maasserver/api && patch partitions.py < /tmp/2.8_partitiontable_does_not_exist.patch +RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets/ && patch maas_ipmi_autodetect_tool.py < /tmp/2.8_maas_ipmi_autodetect_tool.patch + +# echo journalctl logs to the container's stdout +COPY journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service +RUN systemctl enable journalctl-to-tty.service + +# quiet sudo for the maas user +RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log + +# avoid triggering bind9 high cpu utilization bug +RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf + +# initalize systemd +CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"] diff --git a/images/maas-region-controller-bionic/README.md b/images/maas-region-controller-bionic/README.md new file mode 100644 index 0000000..cd33b83 --- /dev/null +++ b/images/maas-region-controller-bionic/README.md @@ -0,0 +1 @@ +[![Docker Repository on Quay](https://quay.io/repository/airshipit/maas-rack/status "Docker Repository on Quay")](https://quay.io/repository/airshipit/maas-region) Ubuntu MaaS Region Controller diff --git a/images/maas-region-controller-bionic/journalctl-to-tty.service b/images/maas-region-controller-bionic/journalctl-to-tty.service new file mode 100644 index 0000000..2725055 --- /dev/null +++ b/images/maas-region-controller-bionic/journalctl-to-tty.service @@ -0,0 +1,13 @@ +[Unit] +Description=Journald console log streamer +Requires=systemd-journald.service +After=systemd-journald.service + +[Service] +Restart=always +RestartSec=0 +ExecStart=/bin/journalctl -f +StandardOutput=tty + +[Install] +WantedBy=basic.target diff --git a/images/maas-region-controller/Dockerfile b/images/maas-region-controller/Dockerfile index 77d58b5..06a907a 100644 --- a/images/maas-region-controller/Dockerfile +++ b/images/maas-region-controller/Dockerfile @@ -1,4 +1,4 @@ -ARG FROM=ubuntu:18.04 +ARG FROM=ubuntu:20.04 FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' @@ -18,10 +18,10 @@ ARG no_proxy ENV DEBIAN_FRONTEND noninteractive ENV container docker -ENV MAAS_VERSION 2.8.7-8611-g.f2514168f-0ubuntu1~18.04.1 +ENV MAAS_VERSION 1:3.0.0-10029-g.986ea3e45-0ubuntu1~20.04.1 RUN apt-get -qq update \ - && apt-get install -y \ + && apt-get install -y \ avahi-daemon \ jq \ patch \ @@ -29,51 +29,53 @@ RUN apt-get -qq update \ sudo \ systemd \ ca-certificates \ -# Don't start any optional services except for the few we need. -# (specifically, don't start avahi-daemon) - && find /etc/systemd/system \ - /lib/systemd/system \ - -path '*.wants/*' \ - -not -name '*journald*' \ - -not -name '*systemd-tmpfiles*' \ - -not -name '*systemd-user-sessions*' \ - -exec rm \{} \; \ - && systemctl set-default multi-user.target \ -# Install maas from the ppa - && add-apt-repository -yu ppa:maas/2.8 \ - && apt-get install -y \ + # NOTE: required for maas-syslog + # Error: failed to create containerd task: + # failed to create shim: OCI runtime create failed: container_linux.go:380: + # starting container process caused: exec: "cron": + # executable file not found in $PATH: unknown + cron \ + # Don't start any optional services except for the few we need. + # (specifically, don't start avahi-daemon) + && find /etc/systemd/system \ + /lib/systemd/system \ + -path '*.wants/*' \ + -not -name '*journald*' \ + -not -name '*systemd-tmpfiles*' \ + -not -name '*systemd-user-sessions*' \ + -exec rm \{} \; \ + && systemctl set-default multi-user.target \ + # Install maas from the ppa + && add-apt-repository -yu ppa:maas/3.0 \ + && apt-get install -y \ maas-region-api=$MAAS_VERSION \ # tcpdump is required by /usr/lib/maas/beacon-monitor tcpdump \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* + +# Update latest packages, including security updates +RUN apt-get -qq update \ + && apt-get upgrade -y # Preserve the directory structure, permissions, and contents of /var/lib/maas RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas # MAAS workarounds -COPY 2.8_route.patch /tmp/2.8_route.patch -COPY 2.8_kernel_package.patch /tmp/2.8_kernel_package.patch -COPY 2.8_bios_grub_partition.patch /tmp/2.8_bios_grub_partition.patch +COPY route.patch /tmp/route.patch +COPY kernel_package.patch /tmp/kernel_package.patch +COPY bios_grub_partition.patch /tmp/bios_grub_partition.patch # sh8121att: allow all requests via the proxy to allow it to work # behind ingress -COPY 2.8_proxy_acl.patch /tmp/2.8_proxy_acl.patch -# Patch to add retrying to MaaS BMC user setup, and improve exception handling -COPY 2.8_configure_ipmi_user.patch /tmp/2.8_configure_ipmi_user.patch -COPY 2.8_secure_headers.patch /tmp/2.8_secure_headers.patch -COPY 2.8_region_secret_rotate.patch /tmp/2.8_region_secret_rotate.patch -COPY 2.8_partitiontable_does_not_exist.patch /tmp/2.8_partitiontable_does_not_exist.patch -# Avoid enlistment failures due to exceptions during moonshot detect attempts -COPY 2.8_maas_ipmi_autodetect_tool.patch /tmp/2.8_maas_ipmi_autodetect_tool.patch +COPY proxy_acl.patch /tmp/proxy_acl.patch +COPY region_secret_rotate.patch /tmp/region_secret_rotate.patch +COPY partitiontable_does_not_exists.patch /tmp/partitiontable_does_not_exists.patch -RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/2.8_route.patch -RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/2.8_kernel_package.patch -RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/2.8_bios_grub_partition.patch -RUN cd /usr/lib/python3/dist-packages/maasserver && patch security.py < /tmp/2.8_region_secret_rotate.patch -RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_ipmi_autodetect.py < /tmp/2.8_configure_ipmi_user.patch -RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/2.8_proxy_acl.patch -RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.8_secure_headers.patch -RUN cd /usr/lib/python3/dist-packages/maasserver/api && patch partitions.py < /tmp/2.8_partitiontable_does_not_exist.patch -RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets/ && patch maas_ipmi_autodetect_tool.py < /tmp/2.8_maas_ipmi_autodetect_tool.patch +RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/route.patch +# RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/kernel_package.patch +# RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/bios_grub_partition.patch +RUN cd /usr/lib/python3/dist-packages/maasserver && patch security.py < /tmp/region_secret_rotate.patch +RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/proxy_acl.patch +RUN cd /usr/lib/python3/dist-packages/maasserver/api && patch partitions.py < /tmp/partitiontable_does_not_exists.patch # echo journalctl logs to the container's stdout COPY journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service diff --git a/images/maas-region-controller/bios_grub_partition.patch b/images/maas-region-controller/bios_grub_partition.patch new file mode 100644 index 0000000..3e64404 --- /dev/null +++ b/images/maas-region-controller/bios_grub_partition.patch @@ -0,0 +1,15 @@ +diff --git a/src/maasserver/models/partition.py b/src/maasserver/models/partition.py +index 84a8fba98..50f6d915f 100644 +--- a/src/maasserver/models/partition.py ++++ b/src/maasserver/models/partition.py +@@ -205,7 +205,9 @@ class Partition(CleanSave, TimestampedModel): + block_device = self.partition_table.block_device + + need_prep_partition = ( +- arch == "ppc64el" and block_device.id == boot_disk.id ++ arch == "amd64" ++ and bios_boot_method != "uefi" ++ and block_device.id == boot_disk.id + ) + need_bios_grub = ( + arch == "amd64" diff --git a/images/maas-region-controller/kernel_package.patch b/images/maas-region-controller/kernel_package.patch new file mode 100644 index 0000000..a8708b8 --- /dev/null +++ b/images/maas-region-controller/kernel_package.patch @@ -0,0 +1,31 @@ +diff --git a/src/maasserver/preseed.py b/src/maasserver/preseed.py +index c69296983..5b63327b1 100644 +--- a/src/maasserver/preseed.py ++++ b/src/maasserver/preseed.py +@@ -250,7 +250,26 @@ def compose_curtin_kernel_preseed(node): + if node.get_osystem() == "custom": + return [] + ++ # previous logic to retrieve kpackage parameter + kpackage = BootResource.objects.get_kpackage_for_node(node) ++ ++ # determine if this node has kernel parameters applied by drydock ++ # and override kpackage if we discover the right properties ++ kernel_opt_tag = "%s_kp" % (node.hostname) ++ if kernel_opt_tag in node.tag_names(): ++ ++ # the tag exists, retrieve it ++ kernel_opts = node.tags.get(name=kernel_opt_tag).kernel_opts ++ ++ # parse the string and find our package param value ++ # e.g. kernel_package=linux-image-4.15.0-34-generic ++ kparams = kernel_opts.split() ++ kdict = dict( ++ kparam.split("=", 1) for kparam in kparams if "=" in kparam ++ ) ++ if "kernel_package" in kdict: ++ kpackage = kdict["kernel_package"] ++ + if kpackage: + kernel_config = {"kernel": {"package": kpackage, "mapping": {}}} + return [yaml.safe_dump(kernel_config)] diff --git a/images/maas-region-controller/partitiontable_does_not_exists.patch b/images/maas-region-controller/partitiontable_does_not_exists.patch new file mode 100644 index 0000000..6baa332 --- /dev/null +++ b/images/maas-region-controller/partitiontable_does_not_exists.patch @@ -0,0 +1,13 @@ +diff --git a/src/maasserver/api/partitions.py b/src/maasserver/api/partitions.py +index dd1b4316c..235c03f45 100644 +--- a/src/maasserver/api/partitions.py ++++ b/src/maasserver/api/partitions.py +@@ -99,7 +99,7 @@ class PartitionsHandler(OperationsHandler): + device = BlockDevice.objects.get_block_device_or_404( + system_id, device_id, request.user, NodePermission.view + ) +- partition_table = device.partitiontable_set.get() ++ partition_table = device.get_partitiontable() + if partition_table is None: + return [] + else: diff --git a/images/maas-region-controller/proxy_acl.patch b/images/maas-region-controller/proxy_acl.patch new file mode 100644 index 0000000..0de535e --- /dev/null +++ b/images/maas-region-controller/proxy_acl.patch @@ -0,0 +1,10 @@ +18,24c18 +< http_access allow maas_proxy_manager localhost +< http_access deny maas_proxy_manager +< http_access deny !Safe_ports +< http_access deny CONNECT !SSL_ports +< http_access allow localnet +< http_access allow localhost +< http_access deny all +--- +> http_access allow all diff --git a/images/maas-region-controller/region_secret_rotate.patch b/images/maas-region-controller/region_secret_rotate.patch new file mode 100644 index 0000000..2768006 --- /dev/null +++ b/images/maas-region-controller/region_secret_rotate.patch @@ -0,0 +1,20 @@ +diff --git a/src/maasserver/security.py b/src/maasserver/security.py +index a9420e504..a8399f1a4 100644 +--- a/src/maasserver/security.py ++++ b/src/maasserver/security.py +@@ -96,11 +96,11 @@ def get_shared_secret_txn(): + elif secret_in_db == secret_on_fs: + secret = secret_in_db # or secret_on_fs. + else: +- raise AssertionError( +- "The secret stored in the database does not match the secret " +- "stored on the filesystem at %s. Please investigate." +- % get_shared_secret_filesystem_path() +- ) ++ # (nk613n): When we rotate secrets we only update the filesystem ++ # so if the secrets don't match we will default to the FS ++ # secret and set it in the database (set_config function) ++ secret = secret_on_fs ++ Config.objects.set_config("rpc_shared_secret", to_hex(secret)) + + return secret diff --git a/images/maas-region-controller/route.patch b/images/maas-region-controller/route.patch new file mode 100644 index 0000000..0b95116 --- /dev/null +++ b/images/maas-region-controller/route.patch @@ -0,0 +1,17 @@ +diff --git a/src/maasserver/preseed_network.py b/src/maasserver/preseed_network.py +index 7660feba1..dae412d01 100644 +--- a/src/maasserver/preseed_network.py ++++ b/src/maasserver/preseed_network.py +@@ -308,7 +308,11 @@ class InterfaceConfiguration: + + def _get_matching_routes(self, source): + """Return all route objects matching `source`.""" +- return {route for route in self.routes if route.source == source} ++ return { ++ route ++ for route in self.routes ++ if str(route.source.cidr) == str(source.cidr) ++ } + + def _generate_addresses(self, version=1): + """Generate the various addresses needed for this interface.""" diff --git a/images/sstream-cache-bionic/Dockerfile b/images/sstream-cache-bionic/Dockerfile new file mode 100644 index 0000000..2e88b52 --- /dev/null +++ b/images/sstream-cache-bionic/Dockerfile @@ -0,0 +1,48 @@ +FROM ubuntu:18.04 + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' +LABEL org.opencontainers.image.url='https://airshipit.org' +LABEL org.opencontainers.image.documentation='https://github.com/openstack/airship-maas' +LABEL org.opencontainers.image.source='https://git.openstack.org/openstack/airship-maas' +LABEL org.opencontainers.image.vendor='The Airship Authors' +LABEL org.opencontainers.image.licenses='Apache-2.0' + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY +ARG http_proxy +ARG https_proxy +ARG no_proxy + +ARG SSTREAM_IMAGE=https://images.maas.io/ephemeral-v3/stable/ +ARG SSTREAM_RELEASE=bionic + +ENV DEBIAN_FRONTEND noninteractive +ENV container docker + +RUN apt-get -qq update && \ + apt install -y simplestreams \ + apache2 \ + gpgv \ + ubuntu-cloudimage-keyring \ + python-certifi --no-install-recommends \ + file + +RUN sstream-mirror --keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg ${SSTREAM_IMAGE} \ + /var/www/html/maas/images/ephemeral-v3/daily 'arch=amd64' "release~${SSTREAM_RELEASE}" --max=1 --progress + +RUN sstream-mirror --keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg ${SSTREAM_IMAGE} \ + /var/www/html/maas/images/ephemeral-v3/daily 'os~(grub*|pxelinux)' --max=1 --progress + +RUN sh -c 'echo "" > /etc/apache2/ports.conf' + +ENV APACHE_RUN_USER www-data +ENV APACHE_RUN_GROUP www-data +ENV APACHE_PID_FILE /var/run/apache2.pid +ENV APACHE_RUN_DIR /var/run/ +ENV APACHE_LOCK_DIR /var/lock +ENV APACHE_LOG_DIR /var/log/ +ENV LANG C + +ENTRYPOINT ["/usr/sbin/apache2"] +CMD ["-E", "/dev/stderr","-c","ErrorLog /dev/stderr","-c","Listen 8888","-c","ServerRoot /etc/apache2","-c","DocumentRoot /var/www/html","-D","FOREGROUND"] diff --git a/images/sstream-cache/Dockerfile b/images/sstream-cache/Dockerfile index d5b3b24..ab2c01f 100644 --- a/images/sstream-cache/Dockerfile +++ b/images/sstream-cache/Dockerfile @@ -1,4 +1,4 @@ -ARG FROM=ubuntu:18.04 +ARG FROM=ubuntu:20.04 FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' @@ -16,21 +16,28 @@ ARG https_proxy ARG no_proxy ARG SSTREAM_IMAGE=https://images.maas.io/ephemeral-v3/stable/ -ARG SSTREAM_RELEASE=bionic +ARG SSTREAM_RELEASE=focal + +ENV DEBIAN_FRONTEND noninteractive +ENV container docker RUN apt-get -qq update && \ apt install -y simplestreams \ - apache2 \ - gpgv \ - ubuntu-cloudimage-keyring \ - python-certifi --no-install-recommends \ - file + apache2 \ + gpgv \ + ubuntu-cloudimage-keyring \ + python-certifi --no-install-recommends \ + file + +# Update latest packages, including security updates +RUN apt-get -qq update \ + && apt-get upgrade -y RUN sstream-mirror --keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg ${SSTREAM_IMAGE} \ /var/www/html/maas/images/ephemeral-v3/daily 'arch=amd64' "release~${SSTREAM_RELEASE}" --max=1 --progress RUN sstream-mirror --keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg ${SSTREAM_IMAGE} \ - /var/www/html/maas/images/ephemeral-v3/daily 'os~(grub*|pxelinux)' --max=1 --progress + /var/www/html/maas/images/ephemeral-v3/daily 'os~(grub*|pxelinux)' --max=1 --progress RUN sh -c 'echo "" > /etc/apache2/ports.conf' diff --git a/tools/gate/playbooks/helm-deploy.yaml b/tools/gate/playbooks/helm-deploy.yaml new file mode 100644 index 0000000..d8e62bc --- /dev/null +++ b/tools/gate/playbooks/helm-deploy.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + tasks: + - name: Deploy MAAS helm chart + shell: | + set -ex; + ./tools/maas/00-packages.sh + ./tools/maas/01-create-cluster.sh + ./tools/maas/02-cert-manager.sh + ./tools/maas/03-postgresql.sh + ./tools/maas/05-maas.sh + args: + chdir: "{{ zuul.project.src_dir }}" + # TODO(sa246v): add container images for MAAS + # environment: + # MAAS_REGION_CONTROLLER: "" + # MAAS_RACK_CONTROLLER: "" + # MAAS_SSTREAM_CACHE: "" diff --git a/tools/maas/00-packages.sh b/tools/maas/00-packages.sh new file mode 100755 index 0000000..4866c33 --- /dev/null +++ b/tools/maas/00-packages.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -ex + +# clone osh-infra +git clone https://opendev.org/openstack/openstack-helm-infra.git + +# install packages +./openstack-helm-infra/tools/deployment/common/000-install-packages.sh +./openstack-helm-infra/tools/deployment/common/001-setup-apparmor-profiles.sh diff --git a/tools/maas/01-create-cluster.sh b/tools/maas/01-create-cluster.sh new file mode 100755 index 0000000..aeec151 --- /dev/null +++ b/tools/maas/01-create-cluster.sh @@ -0,0 +1,66 @@ +#!/bin/sh + +set -ex + +# create cluster +sed -i 's/timeout=240s/timeout=900s/g' ./openstack-helm-infra/tools/deployment/common/005-deploy-k8s.sh +sed -i 's/make all/#make all/g' ./openstack-helm-infra/tools/deployment/common/005-deploy-k8s.sh + +./openstack-helm-infra/tools/deployment/common/005-deploy-k8s.sh +sleep 5 + +# add node labels +kubectl label node --all openstack-control-plane=enabled --overwrite +kubectl label node --all ucp-control-plane=enabled --overwrite + +# create maas namespace +kubectl create namespace ucp --dry-run=client -o yaml | kubectl apply -f - + +# configure storageclass +cat </tmp/ingress.yaml +controller: + admissionWebhooks: + enabled: false + config: + enable-underscores-in-headers: "true" + ssl-reject-handshake: "true" + ingressClass: maas-ingress + ingressClassByName: true + ingressClassResource: + controllerValue: k8s.io/maas-ingress + enabled: true + name: maas-ingress + kind: DaemonSet + nodeSelector: + ucp-control-plane: enabled +defaultBackend: + enabled: true + nodeSelector: + ucp-control-plane: enabled +fullnameOverride: maas-ingress +udp: + "53": ucp/maas-region:region-dns + "514": ucp/maas-syslog:syslog +EOF + +helm dependency update ./openstack-helm-infra/ingress +helm upgrade --install ingress-ucp ./openstack-helm-infra/ingress \ + --namespace=ucp \ + --values /tmp/ingress.yaml \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_INGRESS_OPENSTACK} + +./openstack-helm-infra/tools/deployment/common/wait-for-pods.sh ucp diff --git a/tools/maas/02-cert-manager.sh b/tools/maas/02-cert-manager.sh new file mode 100755 index 0000000..706964d --- /dev/null +++ b/tools/maas/02-cert-manager.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +set -ex + +# deploy cert-manager +helm upgrade --install cert-manager cert-manager \ + --repo=https://charts.jetstack.io \ + --namespace=cert-manager \ + --create-namespace \ + --set installCRDs=true + +./openstack-helm-infra/tools/deployment/common/wait-for-pods.sh cert-manager + +# generate ca cert +openssl req -x509 \ + -sha256 -days 356 \ + -nodes \ + -newkey rsa:2048 \ + -subj "/CN=MAAS CA" \ + -keyout /tmp/tls.key \ + -out /tmp/tls.crt + +kubectl create secret generic \ + --namespace=cert-manager \ + --from-file=/tmp/tls.key \ + --from-file=/tmp/tls.crt \ + ca-clusterissuer-creds \ + --dry-run=client -o yaml | kubectl apply -f - + +# deploy cluster-ca-issuer +helm dependency update ./openstack-helm-infra/ca-clusterissuer +helm upgrade --install cluster-issuer \ + --namespace=cert-manager \ + ./openstack-helm-infra/ca-clusterissuer \ + --set conf.ca.issuer.name=ca-issuer \ + --set conf.ca.secret.name=ca-clusterissuer-creds \ + --set manifests.secret_ca=false diff --git a/tools/maas/03-postgresql.sh b/tools/maas/03-postgresql.sh new file mode 100755 index 0000000..fcba994 --- /dev/null +++ b/tools/maas/03-postgresql.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -ex + +: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} +: ${OSH_INFRA_EXTRA_HELM_ARGS_POSTGRESQL:="$(./tools/deployment/common/get-values-overrides.sh postgresql)"} + +# deploy postgresql +helm dependency update ./openstack-helm-infra/postgresql +helm upgrade --install postgresql ./openstack-helm-infra/postgresql \ + --namespace=ucp \ + --set monitoring.prometheus.enabled=true \ + --set storage.pvc.size=1Gi \ + --set storage.pvc.enabled=true \ + --set pod.replicas.server=1 \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_POSTGRESQL} + +./openstack-helm-infra/tools/deployment/common/wait-for-pods.sh ucp diff --git a/tools/maas/04-load-images.sh b/tools/maas/04-load-images.sh new file mode 100755 index 0000000..b1cb606 --- /dev/null +++ b/tools/maas/04-load-images.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -ex + +# import region controller +sudo -E docker image import \ + ${MAAS_REGION_CONTROLLER} \ + quay.io/airshipit/maas-region-controller:latest + +# import rack controller +sudo -E docker image import \ + ${MAAS_RACK_CONTROLLER} \ + quay.io/airshipit/maas-rack-controller:latest + +# import sstream cache +sudo -E docker image import \ + ${MAAS_SSTREAM_CACHE} \ + quay.io/airshipit/sstream-cache:latest diff --git a/tools/maas/05-maas.sh b/tools/maas/05-maas.sh new file mode 100755 index 0000000..6982959 --- /dev/null +++ b/tools/maas/05-maas.sh @@ -0,0 +1,105 @@ +#!/bin/sh + +set -ex + +# maas +cat </tmp/maas.yaml +conf: + cache: + enabled: true + cloudconfig: + override: true + sections: + bootcmd: + - rm -fr /var/lib/apt/lists + - sysctl net.ipv6.conf.all.disable_ipv6=1 + - sysctl net.ipv6.conf.default.disable_ipv6=1 + - sysctl net.ipv6.conf.lo.disable_ipv6=0 + maas: + url: + maas_url: http://maas-region.ucp.svc.cluster.local/MAAS + credentials: + secret: + namespace: ucp + dns: + require_dnssec: "no" + dns_servers: + - 10.96.0.10 + - 8.8.8.8 + - 8.8.4.4 + extra_settings: + active_discovery_interval: 0 + enlist_commissioning: false + force_v1_network_yaml: true + network_discovery: disabled + images: + default_os: ubuntu + default_image: focal + default_kernel: ga-20.04 + ntp: + disable_ntpd_rack: true + disable_ntpd_region: true + use_external_only: "true" + ntp_servers: + - 209.115.181.110 + - 216.197.228.230 + - 207.210.46.249 + - 216.232.132.95 + proxy: + peer_proxy_enabled: false + proxy_enabled: false + system_passwd: null + system_user: null + syslog: + log_level: DEBUG + maas_region: + host_fqdn_override: + default: null + public: + host: maas.ucp.svc.cluster.local + hosts: + default: maas-region + name: maas-region + path: + default: /MAAS + port: + region_api: + default: 80 + nodeport: 31900 + podport: 5240 + public: 80 + region_proxy: + default: 8000 + scheme: + default: http + maas_syslog: + host_fqdn_override: + public: + host: maas.ucp.svc.cluster.local +manifests: + configmap_ingress: false + maas_ingress: false +network: + proxy: + node_port: + enabled: false +pod: + replicas: + rack: 1 + region: 1 + syslog: 1 +endpoints: + maas_ingress: + hosts: + default: ingress + error_pages: ingress-error-pages + monitor: ingress-exporter +EOF + +# deploy maas +helm upgrade --install maas \ + --namespace=ucp \ + --values /tmp/maas.yaml \ + ./charts/maas + +./openstack-helm-infra/tools/deployment/common/wait-for-pods.sh ucp