Use precreated minikube-aio image

With this commit minikube is installed using contents of precreated
minikube-aio image containing installation script, all required binaries
and images inside. Pulling a single image from dockerhub via opendev
dockerhub proxy and loading images allows to save up to 6 minutes in
minikube installation.

Change-Id: I5936f440eb0567b8dcba2fdae614e4c5e88a7b9a
Signed-off-by: Andrii Ostapenko <andrii.ostapenko@att.com>

tools/deployment/common/005-deploy-k8s.sh

@@ -15,51 +15,11 @@
 
 set -xe
 
-: ${HELM_VERSION:="v2.14.1"}
+: ${MINIKUBE_AIO:="docker.io/openstackhelm/minikube-aio:latest-ubuntu_bionic"}
-: ${KUBE_VERSION:="v1.16.2"}
-: ${MINIKUBE_VERSION:="v1.3.1"}
-: ${CALICO_VERSION:="v3.9"}
-
-: "${HTTP_PROXY:=""}"
-: "${HTTPS_PROXY:=""}"
 
 export DEBCONF_NONINTERACTIVE_SEEN=true
 export DEBIAN_FRONTEND=noninteractive
 
-function configure_resolvconf {
-  # Setup resolv.conf to use the k8s api server, which is required for the
-  # kubelet to resolve cluster services.
-  sudo mv /etc/resolv.conf /etc/resolv.conf.backup
-
-  # Create symbolic link to the resolv.conf file managed by systemd-resolved, as
-  # the kubelet.resolv-conf extra-config flag is automatically executed by the
-  # minikube start command, regardless of being passed in here
-  sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf
-
-  sudo bash -c "echo 'nameserver 10.96.0.10' >> /etc/resolv.conf"
-
-  # NOTE(drewwalters96): Use the Google DNS servers to prevent local addresses in
-  # the resolv.conf file unless using a proxy, then use the existing DNS servers,
-  # as custom DNS nameservers are commonly required when using a proxy server.
-  if [ -z "${HTTP_PROXY}" ]; then
-    sudo bash -c "echo 'nameserver 8.8.8.8' >> /etc/resolv.conf"
-    sudo bash -c "echo 'nameserver 8.8.4.4' >> /etc/resolv.conf"
-  else
-    sed -ne "s/nameserver //p" /etc/resolv.conf.backup | while read -r ns; do
-      sudo bash -c "echo 'nameserver ${ns}' >> /etc/resolv.conf"
-    done
-  fi
-
-  sudo bash -c "echo 'search svc.cluster.local cluster.local' >> /etc/resolv.conf"
-  sudo bash -c "echo 'options ndots:5 timeout:1 attempts:1' >> /etc/resolv.conf"
-
-  sudo rm /etc/resolv.conf.backup
-}
-
-# NOTE: Clean Up hosts file
-sudo sed -i '/^127.0.0.1/c\127.0.0.1 localhost localhost.localdomain localhost4localhost4.localdomain4' /etc/hosts
-sudo sed -i '/^::1/c\::1 localhost6 localhost6.localdomain6' /etc/hosts
-
 # Install required packages for K8s on host
 wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
 RELEASE_NAME=$(grep 'CODENAME' /etc/lsb-release | awk -F= '{print $2}')
@@ -67,7 +27,12 @@ sudo add-apt-repository "deb https://download.ceph.com/debian-nautilus/
 ${RELEASE_NAME} main"
 sudo -E apt-get update
 sudo -E apt-get install -y \
-    docker.io \
+    docker.io
+
+# Starting to pull early in parallel
+sudo -E docker pull -q ${MINIKUBE_AIO} &
+
+sudo -E apt-get install -y \
     socat \
     jq \
     util-linux \
@@ -81,165 +46,21 @@ sudo -E tee /etc/modprobe.d/rbd.conf << EOF
 install rbd /bin/true
 EOF
 
-configure_resolvconf
+set +x;
-
+# give 2 minutes to pull the image (usually takes less than 30-60s) and proceed. If something bad
-# Prepare tmpfs for etcd
+# happens we'll see it on 'docker create'
-sudo mkdir -p /data
+echo "Waiting for ${MINIKUBE_AIO} image is pulled"
-sudo mount -t tmpfs -o size=512m tmpfs /data
+i=0
-
+while [ "$i" -le "60" ]; do
-# Install minikube and kubectl
+  (( ++i ))
-URL="https://storage.googleapis.com"
+  sudo docker inspect ${MINIKUBE_AIO} && break || sleep 2;
-sudo -E curl -sSLo /usr/local/bin/minikube \
+done &> /dev/null; set -x
-  "${URL}"/minikube/releases/"${MINIKUBE_VERSION}"/minikube-linux-amd64
-
-sudo -E curl -sSLo /usr/local/bin/kubectl \
-  "${URL}"/kubernetes-release/release/"${KUBE_VERSION}"/bin/linux/amd64/kubectl
-
-sudo -E chmod +x /usr/local/bin/minikube
-sudo -E chmod +x /usr/local/bin/kubectl
-
-# Install Helm
 TMP_DIR=$(mktemp -d)
-sudo -E bash -c \
+sudo docker create --name minikube-aio ${MINIKUBE_AIO} bash
-  "curl -sSL ${URL}/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | \
+sudo docker export minikube-aio | tar x -C ${TMP_DIR}
-    tar -zxv --strip-components=1 -C ${TMP_DIR}"
+sudo docker rm minikube-aio
+sudo docker rmi ${MINIKUBE_AIO}
+${TMP_DIR}/install.sh
+rm ${TMP_DIR} -rf
 
-sudo -E mv "${TMP_DIR}"/helm /usr/local/bin/helm
-rm -rf "${TMP_DIR}"
-
-# NOTE: Deploy kubenetes using minikube. A CNI that supports network policy is
-# required for validation; use calico for simplicity.
-sudo -E minikube config set kubernetes-version "${KUBE_VERSION}"
-sudo -E minikube config set vm-driver none
-sudo -E minikube config set embed-certs true
-
-export CHANGE_MINIKUBE_NONE_USER=true
-export MINIKUBE_IN_STYLE=false
-sudo -E minikube start \
-  --docker-env HTTP_PROXY="${HTTP_PROXY}" \
-  --docker-env HTTPS_PROXY="${HTTPS_PROXY}" \
-  --docker-env NO_PROXY="${NO_PROXY},10.96.0.0/12" \
-  --network-plugin=cni \
-  --extra-config=controller-manager.allocate-node-cidrs=true \
-  --extra-config=controller-manager.cluster-cidr=192.168.0.0/16
-
-curl https://docs.projectcalico.org/"${CALICO_VERSION}"/manifests/calico.yaml -o /tmp/calico.yaml
-kubectl apply -f /tmp/calico.yaml
-
-# Note: Patch calico daemonset to enable Prometheus metrics and annotations
-tee /tmp/calico-node.yaml << EOF
-spec:
-  template:
-    metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "9091"
-    spec:
-      containers:
-        - name: calico-node
-          env:
-            - name: FELIX_PROMETHEUSMETRICSENABLED
-              value: "true"
-            - name: FELIX_PROMETHEUSMETRICSPORT
-              value: "9091"
-EOF
-kubectl patch daemonset calico-node -n kube-system --patch "$(cat /tmp/calico-node.yaml)"
-
-# NOTE: Wait for dns to be running.
-END=$(($(date +%s) + 240))
-until kubectl --namespace=kube-system \
-        get pods -l k8s-app=kube-dns --no-headers -o name | grep -q "^pod/coredns"; do
-  NOW=$(date +%s)
-  [ "${NOW}" -gt "${END}" ] && exit 1
-  echo "still waiting for dns"
-  sleep 10
-done
-kubectl --namespace=kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns
-
-# Deploy helm/tiller into the cluster
-kubectl create -n kube-system serviceaccount helm-tiller
-cat <<EOF | kubectl apply -f -
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: helm-tiller
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin
-subjects:
-  - kind: ServiceAccount
-    name: helm-tiller
-    namespace: kube-system
-EOF
-
-# NOTE(srwilkers): Required due to tiller deployment spec using extensions/v1beta1
-# which has been removed in Kubernetes 1.16.0.
-# See: https://github.com/helm/helm/issues/6374
-helm init --service-account helm-tiller --output yaml \
-  | sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' \
-  | sed 's@  replicas: 1@  replicas: 1\n  selector: {"matchLabels": {"app": "helm", "name": "tiller"}}@' \
-  | kubectl apply -f -
-
-  # Patch tiller-deploy service to expose metrics port
-tee /tmp/tiller-deploy.yaml << EOF
-metadata:
-  annotations:
-    prometheus.io/scrape: "true"
-    prometheus.io/port: "44135"
-spec:
-  ports:
-  - name: http
-    port: 44135
-    targetPort: http
-EOF
-
-kubectl patch service tiller-deploy -n kube-system --patch "$(cat /tmp/tiller-deploy.yaml)"
-kubectl --namespace=kube-system wait --timeout=240s --for=condition=Ready pod -l app=helm,name=tiller
-
-helm init --client-only
-
-# Set up local helm server
-sudo -E tee /etc/systemd/system/helm-serve.service << EOF
-[Unit]
-Description=Helm Server
-After=network.target
-
-[Service]
-User=$(id -un 2>&1)
-Restart=always
-ExecStart=/usr/local/bin/helm serve
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-sudo chmod 0640 /etc/systemd/system/helm-serve.service
-
-sudo systemctl daemon-reload
-sudo systemctl restart helm-serve
-sudo systemctl enable helm-serve
-
-# Remove stable repo, if present, to improve build time
-helm repo remove stable || true
-
-# Set up local helm repo
-helm repo add local http://localhost:8879/charts
-helm repo update
 make
-
-# Set required labels on host(s)
-kubectl label nodes --all openstack-control-plane=enabled
-kubectl label nodes --all openstack-compute-node=enabled
-kubectl label nodes --all openvswitch=enabled
-kubectl label nodes --all linuxbridge=enabled
-kubectl label nodes --all ceph-mon=enabled
-kubectl label nodes --all ceph-osd=enabled
-kubectl label nodes --all ceph-mds=enabled
-kubectl label nodes --all ceph-rgw=enabled
-kubectl label nodes --all ceph-mgr=enabled
-
-# Add labels to the core namespaces
-kubectl label --overwrite namespace default name=default
-kubectl label --overwrite namespace kube-system name=kube-system
-kubectl label --overwrite namespace kube-public name=kube-public

tools/deployment/common/validate-minikube-aio.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -ex
+cd /tmp
+DIFF=$(diff loaded_images images_after_installation)
+if [ ! -z ${DIFF} ]; then
+  echo -e "Looks like minikube-aio does not contain all images required for minikube installation:\n${DIFF}"
+  exit 1
+fi

zuul.d/jobs.yaml

@@ -595,4 +595,16 @@
         - ./tools/deployment/osh-infra-local-storage/040-prometheus.sh
         - ./tools/deployment/osh-infra-local-storage/050-elasticsearch.sh
         - ./tools/deployment/osh-infra-local-storage/060-volume-info.sh
+
+- job:
+    name: openstack-helm-infra-validate-minikube-aio
+    pre-run:
+      - playbooks/osh-infra-upgrade-host.yaml
+    post-run: playbooks/osh-infra-collect-logs.yaml
+    vars:
+      gate_scripts_relative_path: ../openstack-helm-infra
+      gate_scripts:
+        - ./tools/deployment/common/000-install-packages.sh
+        - ./tools/deployment/common/005-deploy-k8s.sh
+        - ./tools/deployment/common/validate-minikube-aio.sh
 ...

zuul.d/project.yaml

@@ -43,8 +43,9 @@
         - openstack-helm-infra-openstack-support
     # NOTE(srwilkers): Disabling all periodic and experimental jobs until
     # issues with the kubeadm-aio based deployments are addressed
-    # periodic:
+    periodic:
-    #   jobs:
+      jobs:
+        - openstack-helm-infra-validate-minikube-aio
         # - openstack-helm-infra-tenant-ceph
         # - openstack-helm-infra-five-ubuntu
         # - openstack-helm-infra-armada-deploy