Upgrading MAAS to v3

Signed-off-by: Anselme, Schubert (sa246v) <sa246v@att.com>
Change-Id: I4b5a5f6a7e21d790cce13a5ccff9819f517cad64

.zuul.yaml

@@ -18,6 +18,9 @@
         - airship-maas-chart-build-latest-htk
         - airship-maas-docker-build-gate
         - airship-maas-lint-yaml
+        # NOTE(sanselme): This job is disabled until fixed
+        # waiting for coredns to be ready timeout
+        # - airship-maas-helm-deploy
     gate:
       jobs:
         - airship-maas-lint-ws
@@ -82,6 +85,14 @@
     irrelevant-files:
       - '^charts/maas/templates/.*'
 
+- job:
+    name: airship-maas-helm-deploy
+    timeout: 7200
+    run: tools/gate/playbooks/helm-deploy.yaml
+    nodeset: airship-maas-single-node
+    files:
+      - '^charts/.*'
+
 - job:
     name: airship-maas-docker-publish
     timeout: 1800

Makefile

@@ -33,8 +33,8 @@ IMAGE_NAME        := maas-rack-controller maas-region-controller sstream-cache
 BUILD_DIR         := $(shell mktemp -d)
 HELM              := $(BUILD_DIR)/helm
 SSTREAM_IMAGE     := "https://images.maas.io/ephemeral-v3/stable/"
-SSTREAM_RELEASE   := "bionic"
+SSTREAM_RELEASE   := "focal"
-UBUNTU_BASE_IMAGE ?= ubuntu:18.04
+UBUNTU_BASE_IMAGE ?= ubuntu:20.04
 USE_CACHED_IMG    ?= false
 DOCKER_EXTRA_ARGS ?=
 

charts/maas/templates/bin/_maas-test.sh.tpl

@@ -17,6 +17,16 @@
 
 set -ex
 
+function check_admin_api {
+    if maas local version read;
+    then
+        echo 'Admin API is responding'
+        return 0
+    else
+        return 1
+    fi
+}
+
 function check_boot_images {
     if maas local boot-resources is-importing | grep -q 'true';
     then
@@ -74,5 +84,13 @@ then
     exit 1
 fi
 
+check_admin_api
+
+if [[ $? -eq 1 ]]
+then
+    echo "Admin API response FAILED!"
+    exit 1
+fi
+
 echo "MAAS Validation SUCCESS!"
 exit 0

charts/maas/templates/deployment-maas-ingress.yaml

@@ -119,6 +119,7 @@ rules:
       - configmaps
     verbs:
       - create
+      - update
   - apiGroups:
       - ""
     resources:

charts/maas/values.yaml

@@ -96,7 +96,7 @@ manifests:
 
 images:
   tags:
-    db_init: docker.io/postgres:9.5
+    db_init: docker.io/postgres:14.5
     db_sync: quay.io/airshipit/maas-region-controller:latest
     maas_rack: quay.io/airshipit/maas-rack-controller:latest
     maas_region: quay.io/airshipit/maas-region-controller:latest
@@ -104,9 +104,9 @@ images:
     export_api_key: quay.io/airshipit/maas-region-controller:latest
     maas_cache: quay.io/airshipit/sstream-cache:latest
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
-    ingress: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1
+    ingress: k8s.gcr.io/ingress-nginx/controller:v1.2.0
     ingress_vip: docker.io/busybox:latest
-    error_pages: gcr.io/google_containers/ingress-gce-404-server-with-metrics-amd64:v1.6.0
+    error_pages: k8s.gcr.io/defaultbackend-amd64:1.5
     maas_syslog: quay.io/airshipit/maas-region-controller:latest
   pull_policy: IfNotPresent
   local_registry:
@@ -258,8 +258,8 @@ conf:
       proxy_server: null
     images:
       default_os: 'ubuntu'
-      default_image: 'bionic'
+      default_image: 'focal'
-      default_kernel: 'ga-18.04'
+      default_kernel: 'ga-20.04'
     credentials:
       secret:
         namespace: maas

images/maas-rack-controller-bionic/Dockerfile

@@ -0,0 +1,80 @@
+FROM ubuntu:18.04
+
+LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode'
+LABEL org.opencontainers.image.url='https://airshipit.org'
+LABEL org.opencontainers.image.documentation='https://github.com/openstack/airship-maas'
+LABEL org.opencontainers.image.source='https://git.openstack.org/openstack/airship-maas'
+LABEL org.opencontainers.image.vendor='The Airship Authors'
+LABEL org.opencontainers.image.licenses='Apache-2.0'
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+ARG http_proxy
+ARG https_proxy
+ARG no_proxy
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV container docker
+
+ENV MAAS_VERSION 2.8.7-8611-g.f2514168f-0ubuntu1~18.04.1
+
+RUN apt-get -qq update \
+        && apt-get install -y \
+        avahi-daemon \
+        isc-dhcp-server \
+        jq \
+        libvirt-bin \
+        patch \
+        software-properties-common \
+        sudo \
+        systemd \
+        ca-certificates \
+        # Don't start any optional services except for the few we need.
+        # (specifically, don't start avahi-daemon, isc-dhcp-server, or libvirtd)
+        && find /etc/systemd/system \
+        /lib/systemd/system \
+        -path '*.wants/*' \
+        -not -name '*journald*' \
+        -not -name '*systemd-tmpfiles*' \
+        -not -name '*systemd-user-sessions*' \
+        -exec rm \{} \; \
+        && systemctl set-default multi-user.target \
+        # Install maas from the ppa
+        && add-apt-repository -yu ppa:maas/2.8 \
+        && apt-get install -y \
+        maas-rack-controller=$MAAS_VERSION \
+        && rm -rf /var/lib/apt/lists/*
+
+# Preserve the directory structure, permissions, and contents of /var/lib/maas
+RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas
+
+# register ourselves with the region controller
+COPY scripts/register-rack-controller.service /lib/systemd/system/register-rack-controller.service
+RUN systemctl enable register-rack-controller.service
+
+# Patch so that Calico interfaces are ignored
+COPY 2.8_nic_filter.patch /tmp/2.8_nic_filter.patch
+COPY 2.8_secure_headers.patch /tmp/2.8_secure_headers.patch
+# Patch so maas knows that "BMC error" is retriable
+COPY 2.8_ipmi_error.patch /tmp/2.8_ipmi_error.patch
+# Patch to space redfish request retries apart a bit, to avoid overwhelming the BMC
+COPY 2.8_redfish_retries.patch /tmp/2.8_redfish_retries.patch
+
+RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch network.py < /tmp/2.8_nic_filter.patch
+RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.8_secure_headers.patch
+RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch ipmi.py < /tmp/2.8_ipmi_error.patch
+RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch redfish.py < /tmp/2.8_redfish_retries.patch
+
+# echo journalctl logs to the container's stdout
+COPY scripts/journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service
+RUN systemctl enable journalctl-to-tty.service
+
+# quiet sudo for the maas user
+RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log
+
+# avoid triggering bind9 high cpu utilization bug
+RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf
+
+# initalize systemd
+CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]

images/maas-rack-controller-bionic/README.md

@@ -0,0 +1 @@
+[![Docker Repository on Quay](https://quay.io/repository/airshipit/maas-rack/status "Docker Repository on Quay")](https://quay.io/repository/airshipit/maas-rack) Ubuntu MaaS Rack Controller

images/maas-rack-controller/Dockerfile

@@ -1,4 +1,4 @@
-ARG FROM=ubuntu:18.04
+ARG FROM=ubuntu:20.04
 FROM ${FROM}
 
 LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode'
@@ -18,57 +18,60 @@ ARG no_proxy
 ENV DEBIAN_FRONTEND noninteractive
 ENV container docker
 
-ENV MAAS_VERSION 2.8.7-8611-g.f2514168f-0ubuntu1~18.04.1
+ENV MAAS_VERSION 1:3.0.0-10029-g.986ea3e45-0ubuntu1~20.04.1
 
 RUN apt-get -qq update \
- && apt-get install -y \
+        && apt-get install -y \
         avahi-daemon \
         isc-dhcp-server \
         jq \
-        libvirt-bin \
+        libvirt-daemon-system \
+        libvirt-clients \
         patch \
         software-properties-common \
         sudo \
         systemd \
         ca-certificates \
-# Don't start any optional services except for the few we need.
+        # Don't start any optional services except for the few we need.
-# (specifically, don't start avahi-daemon, isc-dhcp-server, or libvirtd)
+        # (specifically, don't start avahi-daemon, isc-dhcp-server, or libvirtd)
- && find /etc/systemd/system \
+        && find /etc/systemd/system \
-         /lib/systemd/system \
+        /lib/systemd/system \
-         -path '*.wants/*' \
+        -path '*.wants/*' \
-         -not -name '*journald*' \
+        -not -name '*journald*' \
-         -not -name '*systemd-tmpfiles*' \
+        -not -name '*systemd-tmpfiles*' \
-         -not -name '*systemd-user-sessions*' \
+        -not -name '*systemd-user-sessions*' \
-         -exec rm \{} \; \
+        -exec rm \{} \; \
- && systemctl set-default multi-user.target \
+        && systemctl set-default multi-user.target \
-# Install maas from the ppa
+        # Install maas from the ppa
- && add-apt-repository -yu ppa:maas/2.8 \
+        && add-apt-repository -yu ppa:maas/3.0 \
- && apt-get install -y \
+        && apt-get install -y \
         maas-rack-controller=$MAAS_VERSION \
- && rm -rf /var/lib/apt/lists/*
+        && rm -rf /var/lib/apt/lists/*
+
+# Update latest packages, including security updates
+RUN apt-get -qq update \
+        && apt-get upgrade -y
 
 # Preserve the directory structure, permissions, and contents of /var/lib/maas
 RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas
 
 # register ourselves with the region controller
-COPY scripts/register-rack-controller.service /lib/systemd/system/register-rack-controller.service
+COPY register-rack-controller.service /lib/systemd/system/register-rack-controller.service
 RUN systemctl enable register-rack-controller.service
 
 # Patch so that Calico interfaces are ignored
-COPY 2.8_nic_filter.patch /tmp/2.8_nic_filter.patch
+COPY nic_filter.patch /tmp/nic_filter.patch
-COPY 2.8_secure_headers.patch /tmp/2.8_secure_headers.patch
 # Patch so maas knows that "BMC error" is retriable
-COPY 2.8_ipmi_error.patch /tmp/2.8_ipmi_error.patch
+COPY ipmi_error.patch /tmp/ipmi_error.patch
 # Patch to space redfish request retries apart a bit, to avoid overwhelming the BMC
-COPY 2.8_redfish_retries.patch /tmp/2.8_redfish_retries.patch
+COPY redfish_retries.patch /tmp/redfish_retries.patch
 
-RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch network.py < /tmp/2.8_nic_filter.patch
+RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch network.py < /tmp/nic_filter.patch
-RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.8_secure_headers.patch
+RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch ipmi.py < /tmp/ipmi_error.patch
-RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch ipmi.py < /tmp/2.8_ipmi_error.patch
+RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch redfish.py < /tmp/redfish_retries.patch
-RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch redfish.py < /tmp/2.8_redfish_retries.patch
 
 # echo journalctl logs to the container's stdout
-COPY scripts/journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service
+COPY journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service
 RUN systemctl enable journalctl-to-tty.service
 
 # quiet sudo for the maas user

images/maas-rack-controller/ipmi_error.patch

@@ -0,0 +1,27 @@
+diff --git a/src/provisioningserver/drivers/power/ipmi.py b/src/provisioningserver/drivers/power/ipmi.py
+index 70201e86e..26625e21d 100644
+--- a/src/provisioningserver/drivers/power/ipmi.py
++++ b/src/provisioningserver/drivers/power/ipmi.py
+@@ -155,6 +155,13 @@ IPMI_ERRORS = {
+         ),
+         "exception": PowerConnError,
+     },
++    "BMC error": {
++        "message": (
++            "Device not responding correctly while performing power action."
++            "  MAAS performed several retries.  Please wait and try again."
++        ),
++        "exception": PowerConnError,
++    },
+     "could not find inband device": {
+         "message": (
+             "An inband device could not be found."
+@@ -308,7 +315,7 @@ class IPMIPowerDriver(PowerDriver):
+         ),
+     ]
+     ip_extractor = make_ip_extractor("power_address")
+-    wait_time = (4, 8, 16, 32)
++    wait_time = (4, 4, 8, 8, 16, 16, 32, 32)
+
+     def detect_missing_packages(self):
+         if not shell.has_command_available("ipmipower"):

images/maas-rack-controller/journalctl-to-tty.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Journald console log streamer
+Requires=systemd-journald.service
+After=systemd-journald.service
+
+[Service]
+Restart=always
+RestartSec=0
+ExecStart=/bin/journalctl -f
+StandardOutput=tty
+
+[Install]
+WantedBy=basic.target

images/maas-rack-controller/nic_filter.patch

@@ -0,0 +1,12 @@
+diff --git a/src/provisioningserver/utils/network.py b/src/provisioningserver/utils/network.py
+index 7895227c4..df83836f3 100644
+--- a/src/provisioningserver/utils/network.py
++++ b/src/provisioningserver/utils/network.py
+@@ -1128,6 +1128,7 @@ def get_all_interfaces_definition(
+         # interfaces for guests. By themselves, they're not useful for MAAS to
+         # manage.
+         "tunnel",
++        "ethernet",
+     ]
+     if not running_in_container():
+         # When not running in a container, we should be able to identify

images/maas-rack-controller/redfish_retries.patch

@@ -0,0 +1,12 @@
+diff --git a/src/provisioningserver/drivers/power/redfish.py b/src/provisioningserver/drivers/power/redfish.py
+index 19d9ecd88..0075997dd 100644
+--- a/src/provisioningserver/drivers/power/redfish.py
++++ b/src/provisioningserver/drivers/power/redfish.py
+@@ -170,6 +170,7 @@ class RedfishPowerDriver(RedfishPowerDriverBase):
+         make_setting_field("node_id", "Node ID", scope=SETTING_SCOPE.NODE),
+     ]
+     ip_extractor = make_ip_extractor("power_address")
++    wait_time = (4, 8, 16, 32)
+
+     def detect_missing_packages(self):
+         # no required packages

images/maas-rack-controller/register-rack-controller.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Register with MaaS Region Controller
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+PassEnvironment=MAAS_ENDPOINT MAAS_REGION_SECRET MAAS_API_KEY HOST_MOUNT_PATH
+ExecStart=/usr/local/bin/register-rack-controller.sh
+
+[Install]
+WantedBy=multi-user.target

images/maas-region-controller-bionic/2.8_region_secret_rotate.patch

@@ -18,4 +18,3 @@ index f92529265..542970009 100644
 +        Config.objects.set_config("rpc_shared_secret", to_hex(secret))
 
      return secret
- 

images/maas-region-controller-bionic/Dockerfile

@@ -0,0 +1,88 @@
+FROM ubuntu:18.04
+
+LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode'
+LABEL org.opencontainers.image.url='https://airshipit.org'
+LABEL org.opencontainers.image.documentation='https://github.com/openstack/airship-maas'
+LABEL org.opencontainers.image.source='https://git.openstack.org/openstack/airship-maas'
+LABEL org.opencontainers.image.vendor='The Airship Authors'
+LABEL org.opencontainers.image.licenses='Apache-2.0'
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+ARG http_proxy
+ARG https_proxy
+ARG no_proxy
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV container docker
+
+ENV MAAS_VERSION 2.8.7-8611-g.f2514168f-0ubuntu1~18.04.1
+
+RUN apt-get -qq update \
+        && apt-get install -y \
+        avahi-daemon \
+        jq \
+        patch \
+        software-properties-common \
+        sudo \
+        systemd \
+        ca-certificates \
+        # Don't start any optional services except for the few we need.
+        # (specifically, don't start avahi-daemon)
+        && find /etc/systemd/system \
+        /lib/systemd/system \
+        -path '*.wants/*' \
+        -not -name '*journald*' \
+        -not -name '*systemd-tmpfiles*' \
+        -not -name '*systemd-user-sessions*' \
+        -exec rm \{} \; \
+        && systemctl set-default multi-user.target \
+        # Install maas from the ppa
+        && add-apt-repository -yu ppa:maas/2.8 \
+        && apt-get install -y \
+        maas-region-api=$MAAS_VERSION \
+        # tcpdump is required by /usr/lib/maas/beacon-monitor
+        tcpdump \
+        && rm -rf /var/lib/apt/lists/*
+
+# Preserve the directory structure, permissions, and contents of /var/lib/maas
+RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas
+
+# MAAS workarounds
+COPY 2.8_route.patch /tmp/2.8_route.patch
+COPY 2.8_kernel_package.patch /tmp/2.8_kernel_package.patch
+COPY 2.8_bios_grub_partition.patch /tmp/2.8_bios_grub_partition.patch
+# sh8121att: allow all requests via the proxy to allow it to work
+# behind ingress
+COPY 2.8_proxy_acl.patch /tmp/2.8_proxy_acl.patch
+# Patch to add retrying to MaaS BMC user setup, and improve exception handling
+COPY 2.8_configure_ipmi_user.patch /tmp/2.8_configure_ipmi_user.patch
+COPY 2.8_secure_headers.patch /tmp/2.8_secure_headers.patch
+COPY 2.8_region_secret_rotate.patch /tmp/2.8_region_secret_rotate.patch
+COPY 2.8_partitiontable_does_not_exist.patch /tmp/2.8_partitiontable_does_not_exist.patch
+# Avoid enlistment failures due to exceptions during moonshot detect attempts
+COPY 2.8_maas_ipmi_autodetect_tool.patch /tmp/2.8_maas_ipmi_autodetect_tool.patch
+
+RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/2.8_route.patch
+RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/2.8_kernel_package.patch
+RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/2.8_bios_grub_partition.patch
+RUN cd /usr/lib/python3/dist-packages/maasserver && patch security.py < /tmp/2.8_region_secret_rotate.patch
+RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_ipmi_autodetect.py < /tmp/2.8_configure_ipmi_user.patch
+RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/2.8_proxy_acl.patch
+RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.8_secure_headers.patch
+RUN cd /usr/lib/python3/dist-packages/maasserver/api && patch partitions.py < /tmp/2.8_partitiontable_does_not_exist.patch
+RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets/ && patch maas_ipmi_autodetect_tool.py < /tmp/2.8_maas_ipmi_autodetect_tool.patch
+
+# echo journalctl logs to the container's stdout
+COPY journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service
+RUN systemctl enable journalctl-to-tty.service
+
+# quiet sudo for the maas user
+RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log
+
+# avoid triggering bind9 high cpu utilization bug
+RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf
+
+# initalize systemd
+CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]

images/maas-region-controller-bionic/README.md

@@ -0,0 +1 @@
+[![Docker Repository on Quay](https://quay.io/repository/airshipit/maas-rack/status "Docker Repository on Quay")](https://quay.io/repository/airshipit/maas-region) Ubuntu MaaS Region Controller

images/maas-region-controller-bionic/journalctl-to-tty.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Journald console log streamer
+Requires=systemd-journald.service
+After=systemd-journald.service
+
+[Service]
+Restart=always
+RestartSec=0
+ExecStart=/bin/journalctl -f
+StandardOutput=tty
+
+[Install]
+WantedBy=basic.target

images/maas-region-controller/Dockerfile

@@ -1,4 +1,4 @@
-ARG FROM=ubuntu:18.04
+ARG FROM=ubuntu:20.04
 FROM ${FROM}
 
 LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode'
@@ -18,10 +18,10 @@ ARG no_proxy
 ENV DEBIAN_FRONTEND noninteractive
 ENV container docker
 
-ENV MAAS_VERSION 2.8.7-8611-g.f2514168f-0ubuntu1~18.04.1
+ENV MAAS_VERSION 1:3.0.0-10029-g.986ea3e45-0ubuntu1~20.04.1
 
 RUN apt-get -qq update \
- && apt-get install -y \
+        && apt-get install -y \
         avahi-daemon \
         jq \
         patch \
@@ -29,51 +29,53 @@ RUN apt-get -qq update \
         sudo \
         systemd \
         ca-certificates \
-# Don't start any optional services except for the few we need.
+        # NOTE: required for maas-syslog
-# (specifically, don't start avahi-daemon)
+        # Error: failed to create containerd task:
- && find /etc/systemd/system \
+        # failed to create shim: OCI runtime create failed: container_linux.go:380:
-         /lib/systemd/system \
+        # starting container process caused: exec: "cron":
-         -path '*.wants/*' \
+        # executable file not found in $PATH: unknown
-         -not -name '*journald*' \
+        cron \
-         -not -name '*systemd-tmpfiles*' \
+        # Don't start any optional services except for the few we need.
-         -not -name '*systemd-user-sessions*' \
+        # (specifically, don't start avahi-daemon)
-         -exec rm \{} \; \
+        && find /etc/systemd/system \
- && systemctl set-default multi-user.target \
+        /lib/systemd/system \
-# Install maas from the ppa
+        -path '*.wants/*' \
- && add-apt-repository -yu ppa:maas/2.8 \
+        -not -name '*journald*' \
- && apt-get install -y \
+        -not -name '*systemd-tmpfiles*' \
+        -not -name '*systemd-user-sessions*' \
+        -exec rm \{} \; \
+        && systemctl set-default multi-user.target \
+        # Install maas from the ppa
+        && add-apt-repository -yu ppa:maas/3.0 \
+        && apt-get install -y \
         maas-region-api=$MAAS_VERSION \
         # tcpdump is required by /usr/lib/maas/beacon-monitor
         tcpdump \
- && rm -rf /var/lib/apt/lists/*
+        && rm -rf /var/lib/apt/lists/*
+
+# Update latest packages, including security updates
+RUN apt-get -qq update \
+        && apt-get upgrade -y
 
 # Preserve the directory structure, permissions, and contents of /var/lib/maas
 RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas
 
 # MAAS workarounds
-COPY 2.8_route.patch /tmp/2.8_route.patch
+COPY route.patch /tmp/route.patch
-COPY 2.8_kernel_package.patch /tmp/2.8_kernel_package.patch
+COPY kernel_package.patch /tmp/kernel_package.patch
-COPY 2.8_bios_grub_partition.patch /tmp/2.8_bios_grub_partition.patch
+COPY bios_grub_partition.patch /tmp/bios_grub_partition.patch
 # sh8121att: allow all requests via the proxy to allow it to work
 # behind ingress
-COPY 2.8_proxy_acl.patch /tmp/2.8_proxy_acl.patch
+COPY proxy_acl.patch /tmp/proxy_acl.patch
-# Patch to add retrying to MaaS BMC user setup, and improve exception handling
+COPY region_secret_rotate.patch /tmp/region_secret_rotate.patch
-COPY 2.8_configure_ipmi_user.patch /tmp/2.8_configure_ipmi_user.patch
+COPY partitiontable_does_not_exists.patch /tmp/partitiontable_does_not_exists.patch
-COPY 2.8_secure_headers.patch /tmp/2.8_secure_headers.patch
-COPY 2.8_region_secret_rotate.patch /tmp/2.8_region_secret_rotate.patch
-COPY 2.8_partitiontable_does_not_exist.patch /tmp/2.8_partitiontable_does_not_exist.patch
-# Avoid enlistment failures due to exceptions during moonshot detect attempts
-COPY 2.8_maas_ipmi_autodetect_tool.patch /tmp/2.8_maas_ipmi_autodetect_tool.patch
 
-RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/2.8_route.patch
+RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/route.patch
-RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/2.8_kernel_package.patch
+# RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/kernel_package.patch
-RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/2.8_bios_grub_partition.patch
+# RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/bios_grub_partition.patch
-RUN cd /usr/lib/python3/dist-packages/maasserver && patch security.py < /tmp/2.8_region_secret_rotate.patch
+RUN cd /usr/lib/python3/dist-packages/maasserver && patch security.py < /tmp/region_secret_rotate.patch
-RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_ipmi_autodetect.py < /tmp/2.8_configure_ipmi_user.patch
+RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/proxy_acl.patch
-RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/2.8_proxy_acl.patch
+RUN cd /usr/lib/python3/dist-packages/maasserver/api && patch partitions.py < /tmp/partitiontable_does_not_exists.patch
-RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.8_secure_headers.patch
-RUN cd /usr/lib/python3/dist-packages/maasserver/api && patch partitions.py < /tmp/2.8_partitiontable_does_not_exist.patch
-RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets/ && patch maas_ipmi_autodetect_tool.py < /tmp/2.8_maas_ipmi_autodetect_tool.patch
 
 # echo journalctl logs to the container's stdout
 COPY journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service

images/maas-region-controller/bios_grub_partition.patch

@@ -0,0 +1,15 @@
+diff --git a/src/maasserver/models/partition.py b/src/maasserver/models/partition.py
+index 84a8fba98..50f6d915f 100644
+--- a/src/maasserver/models/partition.py
++++ b/src/maasserver/models/partition.py
+@@ -205,7 +205,9 @@ class Partition(CleanSave, TimestampedModel):
+             block_device = self.partition_table.block_device
+
+             need_prep_partition = (
+-                arch == "ppc64el" and block_device.id == boot_disk.id
++                arch == "amd64"
++                and bios_boot_method != "uefi"
++                and block_device.id == boot_disk.id
+             )
+             need_bios_grub = (
+                 arch == "amd64"

images/maas-region-controller/kernel_package.patch

@@ -0,0 +1,31 @@
+diff --git a/src/maasserver/preseed.py b/src/maasserver/preseed.py
+index c69296983..5b63327b1 100644
+--- a/src/maasserver/preseed.py
++++ b/src/maasserver/preseed.py
+@@ -250,7 +250,26 @@ def compose_curtin_kernel_preseed(node):
+     if node.get_osystem() == "custom":
+         return []
+
++    # previous logic to retrieve kpackage parameter
+     kpackage = BootResource.objects.get_kpackage_for_node(node)
++
++    # determine if this node has kernel parameters applied by drydock
++    # and override kpackage if we discover the right properties
++    kernel_opt_tag = "%s_kp" % (node.hostname)
++    if kernel_opt_tag in node.tag_names():
++
++        # the tag exists, retrieve it
++        kernel_opts = node.tags.get(name=kernel_opt_tag).kernel_opts
++
++        # parse the string and find our package param value
++        # e.g. kernel_package=linux-image-4.15.0-34-generic
++        kparams = kernel_opts.split()
++        kdict = dict(
++            kparam.split("=", 1) for kparam in kparams if "=" in kparam
++        )
++        if "kernel_package" in kdict:
++            kpackage = kdict["kernel_package"]
++
+     if kpackage:
+         kernel_config = {"kernel": {"package": kpackage, "mapping": {}}}
+         return [yaml.safe_dump(kernel_config)]

images/maas-region-controller/partitiontable_does_not_exists.patch

@@ -0,0 +1,13 @@
+diff --git a/src/maasserver/api/partitions.py b/src/maasserver/api/partitions.py
+index dd1b4316c..235c03f45 100644
+--- a/src/maasserver/api/partitions.py
++++ b/src/maasserver/api/partitions.py
+@@ -99,7 +99,7 @@ class PartitionsHandler(OperationsHandler):
+         device = BlockDevice.objects.get_block_device_or_404(
+             system_id, device_id, request.user, NodePermission.view
+         )
+-        partition_table = device.partitiontable_set.get()
++        partition_table = device.get_partitiontable()
+         if partition_table is None:
+             return []
+         else:

images/maas-region-controller/proxy_acl.patch

@@ -0,0 +1,10 @@
+18,24c18
+< http_access allow maas_proxy_manager localhost
+< http_access deny maas_proxy_manager
+< http_access deny !Safe_ports
+< http_access deny CONNECT !SSL_ports
+< http_access allow localnet
+< http_access allow localhost
+< http_access deny all
+---
+> http_access allow all

images/maas-region-controller/region_secret_rotate.patch

@@ -0,0 +1,20 @@
+diff --git a/src/maasserver/security.py b/src/maasserver/security.py
+index a9420e504..a8399f1a4 100644
+--- a/src/maasserver/security.py
++++ b/src/maasserver/security.py
+@@ -96,11 +96,11 @@ def get_shared_secret_txn():
+     elif secret_in_db == secret_on_fs:
+         secret = secret_in_db  # or secret_on_fs.
+     else:
+-        raise AssertionError(
+-            "The secret stored in the database does not match the secret "
+-            "stored on the filesystem at %s. Please investigate."
+-            % get_shared_secret_filesystem_path()
+-        )
++        # (nk613n): When we rotate secrets we only update the filesystem
++        #           so if the secrets don't match we will default to the FS
++        #           secret and set it in the database (set_config function)
++        secret = secret_on_fs
++        Config.objects.set_config("rpc_shared_secret", to_hex(secret))
+
+     return secret

images/maas-region-controller/route.patch

@@ -0,0 +1,17 @@
+diff --git a/src/maasserver/preseed_network.py b/src/maasserver/preseed_network.py
+index 7660feba1..dae412d01 100644
+--- a/src/maasserver/preseed_network.py
++++ b/src/maasserver/preseed_network.py
+@@ -308,7 +308,11 @@ class InterfaceConfiguration:
+
+     def _get_matching_routes(self, source):
+         """Return all route objects matching `source`."""
+-        return {route for route in self.routes if route.source == source}
++        return {
++            route
++            for route in self.routes
++            if str(route.source.cidr) == str(source.cidr)
++        }
+
+     def _generate_addresses(self, version=1):
+         """Generate the various addresses needed for this interface."""

images/sstream-cache-bionic/Dockerfile

@@ -0,0 +1,48 @@
+FROM ubuntu:18.04
+
+LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode'
+LABEL org.opencontainers.image.url='https://airshipit.org'
+LABEL org.opencontainers.image.documentation='https://github.com/openstack/airship-maas'
+LABEL org.opencontainers.image.source='https://git.openstack.org/openstack/airship-maas'
+LABEL org.opencontainers.image.vendor='The Airship Authors'
+LABEL org.opencontainers.image.licenses='Apache-2.0'
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+ARG http_proxy
+ARG https_proxy
+ARG no_proxy
+
+ARG SSTREAM_IMAGE=https://images.maas.io/ephemeral-v3/stable/
+ARG SSTREAM_RELEASE=bionic
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV container docker
+
+RUN apt-get -qq update && \
+    apt install -y simplestreams \
+    apache2 \
+    gpgv \
+    ubuntu-cloudimage-keyring \
+    python-certifi --no-install-recommends \
+    file
+
+RUN sstream-mirror --keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg ${SSTREAM_IMAGE} \
+    /var/www/html/maas/images/ephemeral-v3/daily 'arch=amd64' "release~${SSTREAM_RELEASE}" --max=1 --progress
+
+RUN sstream-mirror --keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg ${SSTREAM_IMAGE} \
+    /var/www/html/maas/images/ephemeral-v3/daily 'os~(grub*|pxelinux)' --max=1 --progress
+
+RUN sh -c 'echo "" > /etc/apache2/ports.conf'
+
+ENV APACHE_RUN_USER www-data
+ENV APACHE_RUN_GROUP www-data
+ENV APACHE_PID_FILE /var/run/apache2.pid
+ENV APACHE_RUN_DIR /var/run/
+ENV APACHE_LOCK_DIR /var/lock
+ENV APACHE_LOG_DIR /var/log/
+ENV LANG C
+
+ENTRYPOINT ["/usr/sbin/apache2"]
+CMD ["-E", "/dev/stderr","-c","ErrorLog /dev/stderr","-c","Listen 8888","-c","ServerRoot /etc/apache2","-c","DocumentRoot /var/www/html","-D","FOREGROUND"]

images/sstream-cache/Dockerfile

@@ -1,4 +1,4 @@
-ARG FROM=ubuntu:18.04
+ARG FROM=ubuntu:20.04
 FROM ${FROM}
 
 LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode'
@@ -16,21 +16,28 @@ ARG https_proxy
 ARG no_proxy
 
 ARG SSTREAM_IMAGE=https://images.maas.io/ephemeral-v3/stable/
-ARG SSTREAM_RELEASE=bionic
+ARG SSTREAM_RELEASE=focal
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV container docker
 
 RUN apt-get -qq update && \
     apt install -y simplestreams \
-                   apache2 \
+    apache2 \
-                   gpgv \
+    gpgv \
-                   ubuntu-cloudimage-keyring \
+    ubuntu-cloudimage-keyring \
-                   python-certifi --no-install-recommends \
+    python-certifi --no-install-recommends \
-                   file
+    file
+
+# Update latest packages, including security updates
+RUN apt-get -qq update \
+    && apt-get upgrade -y
 
 RUN sstream-mirror --keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg ${SSTREAM_IMAGE} \
     /var/www/html/maas/images/ephemeral-v3/daily 'arch=amd64' "release~${SSTREAM_RELEASE}" --max=1 --progress
 
 RUN sstream-mirror --keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg ${SSTREAM_IMAGE} \
-   /var/www/html/maas/images/ephemeral-v3/daily 'os~(grub*|pxelinux)' --max=1 --progress
+    /var/www/html/maas/images/ephemeral-v3/daily 'os~(grub*|pxelinux)' --max=1 --progress
 
 RUN sh -c 'echo "" > /etc/apache2/ports.conf'
 

tools/gate/playbooks/helm-deploy.yaml

@@ -0,0 +1,29 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Deploy MAAS helm chart
+      shell: |
+        set -ex;
+        ./tools/maas/00-packages.sh
+        ./tools/maas/01-create-cluster.sh
+        ./tools/maas/02-cert-manager.sh
+        ./tools/maas/03-postgresql.sh
+        ./tools/maas/05-maas.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      # TODO(sa246v): add container images for MAAS
+      #   environment:
+      #     MAAS_REGION_CONTROLLER: ""
+      #     MAAS_RACK_CONTROLLER: ""
+      #     MAAS_SSTREAM_CACHE: ""

tools/maas/00-packages.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -ex
+
+# clone osh-infra
+git clone https://opendev.org/openstack/openstack-helm-infra.git
+
+# install packages
+./openstack-helm-infra/tools/deployment/common/000-install-packages.sh
+./openstack-helm-infra/tools/deployment/common/001-setup-apparmor-profiles.sh

tools/maas/01-create-cluster.sh

@@ -0,0 +1,66 @@
+#!/bin/sh
+
+set -ex
+
+# create cluster
+sed -i 's/timeout=240s/timeout=900s/g' ./openstack-helm-infra/tools/deployment/common/005-deploy-k8s.sh
+sed -i 's/make all/#make all/g' ./openstack-helm-infra/tools/deployment/common/005-deploy-k8s.sh
+
+./openstack-helm-infra/tools/deployment/common/005-deploy-k8s.sh
+sleep 5
+
+# add node labels
+kubectl label node --all openstack-control-plane=enabled --overwrite
+kubectl label node --all ucp-control-plane=enabled --overwrite
+
+# create maas namespace
+kubectl create namespace ucp --dry-run=client -o yaml | kubectl apply -f -
+
+# configure storageclass
+cat <<EOF | kubectl apply -f -
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: general
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+provisioner: k8s.io/minikube-hostpath
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+EOF
+
+# deploy ingress
+cat <<EOF >/tmp/ingress.yaml
+controller:
+  admissionWebhooks:
+    enabled: false
+  config:
+    enable-underscores-in-headers: "true"
+    ssl-reject-handshake: "true"
+  ingressClass: maas-ingress
+  ingressClassByName: true
+  ingressClassResource:
+    controllerValue: k8s.io/maas-ingress
+    enabled: true
+    name: maas-ingress
+  kind: DaemonSet
+  nodeSelector:
+    ucp-control-plane: enabled
+defaultBackend:
+  enabled: true
+  nodeSelector:
+    ucp-control-plane: enabled
+fullnameOverride: maas-ingress
+udp:
+  "53": ucp/maas-region:region-dns
+  "514": ucp/maas-syslog:syslog
+EOF
+
+helm dependency update ./openstack-helm-infra/ingress
+helm upgrade --install ingress-ucp ./openstack-helm-infra/ingress \
+  --namespace=ucp \
+  --values /tmp/ingress.yaml \
+  ${OSH_INFRA_EXTRA_HELM_ARGS} \
+  ${OSH_INFRA_EXTRA_HELM_ARGS_INGRESS_OPENSTACK}
+
+./openstack-helm-infra/tools/deployment/common/wait-for-pods.sh ucp

tools/maas/02-cert-manager.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+set -ex
+
+# deploy cert-manager
+helm upgrade --install cert-manager cert-manager \
+  --repo=https://charts.jetstack.io \
+  --namespace=cert-manager \
+  --create-namespace \
+  --set installCRDs=true
+
+./openstack-helm-infra/tools/deployment/common/wait-for-pods.sh cert-manager
+
+# generate ca cert
+openssl req -x509 \
+  -sha256 -days 356 \
+  -nodes \
+  -newkey rsa:2048 \
+  -subj "/CN=MAAS CA" \
+  -keyout /tmp/tls.key \
+  -out /tmp/tls.crt
+
+kubectl create secret generic \
+  --namespace=cert-manager \
+  --from-file=/tmp/tls.key \
+  --from-file=/tmp/tls.crt \
+  ca-clusterissuer-creds \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# deploy cluster-ca-issuer
+helm dependency update ./openstack-helm-infra/ca-clusterissuer
+helm upgrade --install cluster-issuer \
+  --namespace=cert-manager \
+  ./openstack-helm-infra/ca-clusterissuer \
+  --set conf.ca.issuer.name=ca-issuer \
+  --set conf.ca.secret.name=ca-clusterissuer-creds \
+  --set manifests.secret_ca=false

tools/maas/03-postgresql.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -ex
+
+: ${OSH_INFRA_EXTRA_HELM_ARGS:=""}
+: ${OSH_INFRA_EXTRA_HELM_ARGS_POSTGRESQL:="$(./tools/deployment/common/get-values-overrides.sh postgresql)"}
+
+# deploy postgresql
+helm dependency update ./openstack-helm-infra/postgresql
+helm upgrade --install postgresql ./openstack-helm-infra/postgresql \
+  --namespace=ucp \
+  --set monitoring.prometheus.enabled=true \
+  --set storage.pvc.size=1Gi \
+  --set storage.pvc.enabled=true \
+  --set pod.replicas.server=1 \
+  ${OSH_INFRA_EXTRA_HELM_ARGS} \
+  ${OSH_INFRA_EXTRA_HELM_ARGS_POSTGRESQL}
+
+./openstack-helm-infra/tools/deployment/common/wait-for-pods.sh ucp

tools/maas/04-load-images.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -ex
+
+# import region controller
+sudo -E docker image import \
+  ${MAAS_REGION_CONTROLLER} \
+  quay.io/airshipit/maas-region-controller:latest
+
+# import rack controller
+sudo -E docker image import \
+  ${MAAS_RACK_CONTROLLER} \
+  quay.io/airshipit/maas-rack-controller:latest
+
+# import sstream cache
+sudo -E docker image import \
+  ${MAAS_SSTREAM_CACHE} \
+  quay.io/airshipit/sstream-cache:latest

tools/maas/05-maas.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+set -ex
+
+# maas
+cat <<EOF >/tmp/maas.yaml
+conf:
+  cache:
+    enabled: true
+  cloudconfig:
+    override: true
+    sections:
+      bootcmd:
+      - rm -fr /var/lib/apt/lists
+      - sysctl net.ipv6.conf.all.disable_ipv6=1
+      - sysctl net.ipv6.conf.default.disable_ipv6=1
+      - sysctl net.ipv6.conf.lo.disable_ipv6=0
+  maas:
+    url:
+      maas_url: http://maas-region.ucp.svc.cluster.local/MAAS
+    credentials:
+      secret:
+        namespace: ucp
+    dns:
+      require_dnssec: "no"
+      dns_servers:
+        - 10.96.0.10
+        - 8.8.8.8
+        - 8.8.4.4
+    extra_settings:
+      active_discovery_interval: 0
+      enlist_commissioning: false
+      force_v1_network_yaml: true
+      network_discovery: disabled
+    images:
+      default_os: ubuntu
+      default_image: focal
+      default_kernel: ga-20.04
+    ntp:
+      disable_ntpd_rack: true
+      disable_ntpd_region: true
+      use_external_only: "true"
+      ntp_servers:
+        - 209.115.181.110
+        - 216.197.228.230
+        - 207.210.46.249
+        - 216.232.132.95
+    proxy:
+      peer_proxy_enabled: false
+      proxy_enabled: false
+    system_passwd: null
+    system_user: null
+  syslog:
+    log_level: DEBUG
+  maas_region:
+    host_fqdn_override:
+      default: null
+      public:
+        host: maas.ucp.svc.cluster.local
+    hosts:
+      default: maas-region
+    name: maas-region
+    path:
+      default: /MAAS
+    port:
+      region_api:
+        default: 80
+        nodeport: 31900
+        podport: 5240
+        public: 80
+      region_proxy:
+        default: 8000
+    scheme:
+      default: http
+  maas_syslog:
+    host_fqdn_override:
+      public:
+        host: maas.ucp.svc.cluster.local
+manifests:
+  configmap_ingress: false
+  maas_ingress: false
+network:
+  proxy:
+    node_port:
+      enabled: false
+pod:
+  replicas:
+    rack: 1
+    region: 1
+    syslog: 1
+endpoints:
+  maas_ingress:
+    hosts:
+      default: ingress
+      error_pages: ingress-error-pages
+      monitor: ingress-exporter
+EOF
+
+# deploy maas
+helm upgrade --install maas \
+  --namespace=ucp \
+  --values /tmp/maas.yaml \
+  ./charts/maas
+
+./openstack-helm-infra/tools/deployment/common/wait-for-pods.sh ucp