From 10019004da8df67ce6b9bef554334d85676d6492 Mon Sep 17 00:00:00 2001 From: Andrii Ostapenko Date: Sat, 1 Aug 2020 19:14:16 -0500 Subject: [PATCH] Use precreated minikube-aio image With this commit minikube is installed using contents of precreated minikube-aio image containing installation script, all required binaries and images inside. Pulling a single image from dockerhub via opendev dockerhub proxy and loading images allows to save up to 6 minutes in minikube installation. Change-Id: I5936f440eb0567b8dcba2fdae614e4c5e88a7b9a Signed-off-by: Andrii Ostapenko --- tools/deployment/common/005-deploy-k8s.sh | 223 ++---------------- .../common/validate-minikube-aio.sh | 9 + zuul.d/jobs.yaml | 12 + zuul.d/project.yaml | 5 +- 4 files changed, 46 insertions(+), 203 deletions(-) create mode 100644 tools/deployment/common/validate-minikube-aio.sh diff --git a/tools/deployment/common/005-deploy-k8s.sh b/tools/deployment/common/005-deploy-k8s.sh index 6269b87a7..f1e9e82e8 100755 --- a/tools/deployment/common/005-deploy-k8s.sh +++ b/tools/deployment/common/005-deploy-k8s.sh @@ -15,51 +15,11 @@ set -xe -: ${HELM_VERSION:="v2.14.1"} -: ${KUBE_VERSION:="v1.16.2"} -: ${MINIKUBE_VERSION:="v1.3.1"} -: ${CALICO_VERSION:="v3.9"} - -: "${HTTP_PROXY:=""}" -: "${HTTPS_PROXY:=""}" +: ${MINIKUBE_AIO:="docker.io/openstackhelm/minikube-aio:latest-ubuntu_bionic"} export DEBCONF_NONINTERACTIVE_SEEN=true export DEBIAN_FRONTEND=noninteractive -function configure_resolvconf { - # Setup resolv.conf to use the k8s api server, which is required for the - # kubelet to resolve cluster services. - sudo mv /etc/resolv.conf /etc/resolv.conf.backup - - # Create symbolic link to the resolv.conf file managed by systemd-resolved, as - # the kubelet.resolv-conf extra-config flag is automatically executed by the - # minikube start command, regardless of being passed in here - sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf - - sudo bash -c "echo 'nameserver 10.96.0.10' >> /etc/resolv.conf" - - # NOTE(drewwalters96): Use the Google DNS servers to prevent local addresses in - # the resolv.conf file unless using a proxy, then use the existing DNS servers, - # as custom DNS nameservers are commonly required when using a proxy server. - if [ -z "${HTTP_PROXY}" ]; then - sudo bash -c "echo 'nameserver 8.8.8.8' >> /etc/resolv.conf" - sudo bash -c "echo 'nameserver 8.8.4.4' >> /etc/resolv.conf" - else - sed -ne "s/nameserver //p" /etc/resolv.conf.backup | while read -r ns; do - sudo bash -c "echo 'nameserver ${ns}' >> /etc/resolv.conf" - done - fi - - sudo bash -c "echo 'search svc.cluster.local cluster.local' >> /etc/resolv.conf" - sudo bash -c "echo 'options ndots:5 timeout:1 attempts:1' >> /etc/resolv.conf" - - sudo rm /etc/resolv.conf.backup -} - -# NOTE: Clean Up hosts file -sudo sed -i '/^127.0.0.1/c\127.0.0.1 localhost localhost.localdomain localhost4localhost4.localdomain4' /etc/hosts -sudo sed -i '/^::1/c\::1 localhost6 localhost6.localdomain6' /etc/hosts - # Install required packages for K8s on host wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add - RELEASE_NAME=$(grep 'CODENAME' /etc/lsb-release | awk -F= '{print $2}') @@ -67,7 +27,12 @@ sudo add-apt-repository "deb https://download.ceph.com/debian-nautilus/ ${RELEASE_NAME} main" sudo -E apt-get update sudo -E apt-get install -y \ - docker.io \ + docker.io + +# Starting to pull early in parallel +sudo -E docker pull -q ${MINIKUBE_AIO} & + +sudo -E apt-get install -y \ socat \ jq \ util-linux \ @@ -81,165 +46,21 @@ sudo -E tee /etc/modprobe.d/rbd.conf << EOF install rbd /bin/true EOF -configure_resolvconf - -# Prepare tmpfs for etcd -sudo mkdir -p /data -sudo mount -t tmpfs -o size=512m tmpfs /data - -# Install minikube and kubectl -URL="https://storage.googleapis.com" -sudo -E curl -sSLo /usr/local/bin/minikube \ - "${URL}"/minikube/releases/"${MINIKUBE_VERSION}"/minikube-linux-amd64 - -sudo -E curl -sSLo /usr/local/bin/kubectl \ - "${URL}"/kubernetes-release/release/"${KUBE_VERSION}"/bin/linux/amd64/kubectl - -sudo -E chmod +x /usr/local/bin/minikube -sudo -E chmod +x /usr/local/bin/kubectl - -# Install Helm +set +x; +# give 2 minutes to pull the image (usually takes less than 30-60s) and proceed. If something bad +# happens we'll see it on 'docker create' +echo "Waiting for ${MINIKUBE_AIO} image is pulled" +i=0 +while [ "$i" -le "60" ]; do + (( ++i )) + sudo docker inspect ${MINIKUBE_AIO} && break || sleep 2; +done &> /dev/null; set -x TMP_DIR=$(mktemp -d) -sudo -E bash -c \ - "curl -sSL ${URL}/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | \ - tar -zxv --strip-components=1 -C ${TMP_DIR}" +sudo docker create --name minikube-aio ${MINIKUBE_AIO} bash +sudo docker export minikube-aio | tar x -C ${TMP_DIR} +sudo docker rm minikube-aio +sudo docker rmi ${MINIKUBE_AIO} +${TMP_DIR}/install.sh +rm ${TMP_DIR} -rf -sudo -E mv "${TMP_DIR}"/helm /usr/local/bin/helm -rm -rf "${TMP_DIR}" - -# NOTE: Deploy kubenetes using minikube. A CNI that supports network policy is -# required for validation; use calico for simplicity. -sudo -E minikube config set kubernetes-version "${KUBE_VERSION}" -sudo -E minikube config set vm-driver none -sudo -E minikube config set embed-certs true - -export CHANGE_MINIKUBE_NONE_USER=true -export MINIKUBE_IN_STYLE=false -sudo -E minikube start \ - --docker-env HTTP_PROXY="${HTTP_PROXY}" \ - --docker-env HTTPS_PROXY="${HTTPS_PROXY}" \ - --docker-env NO_PROXY="${NO_PROXY},10.96.0.0/12" \ - --network-plugin=cni \ - --extra-config=controller-manager.allocate-node-cidrs=true \ - --extra-config=controller-manager.cluster-cidr=192.168.0.0/16 - -curl https://docs.projectcalico.org/"${CALICO_VERSION}"/manifests/calico.yaml -o /tmp/calico.yaml -kubectl apply -f /tmp/calico.yaml - -# Note: Patch calico daemonset to enable Prometheus metrics and annotations -tee /tmp/calico-node.yaml << EOF -spec: - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9091" - spec: - containers: - - name: calico-node - env: - - name: FELIX_PROMETHEUSMETRICSENABLED - value: "true" - - name: FELIX_PROMETHEUSMETRICSPORT - value: "9091" -EOF -kubectl patch daemonset calico-node -n kube-system --patch "$(cat /tmp/calico-node.yaml)" - -# NOTE: Wait for dns to be running. -END=$(($(date +%s) + 240)) -until kubectl --namespace=kube-system \ - get pods -l k8s-app=kube-dns --no-headers -o name | grep -q "^pod/coredns"; do - NOW=$(date +%s) - [ "${NOW}" -gt "${END}" ] && exit 1 - echo "still waiting for dns" - sleep 10 -done -kubectl --namespace=kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns - -# Deploy helm/tiller into the cluster -kubectl create -n kube-system serviceaccount helm-tiller -cat <&1) -Restart=always -ExecStart=/usr/local/bin/helm serve - -[Install] -WantedBy=multi-user.target -EOF - -sudo chmod 0640 /etc/systemd/system/helm-serve.service - -sudo systemctl daemon-reload -sudo systemctl restart helm-serve -sudo systemctl enable helm-serve - -# Remove stable repo, if present, to improve build time -helm repo remove stable || true - -# Set up local helm repo -helm repo add local http://localhost:8879/charts -helm repo update make - -# Set required labels on host(s) -kubectl label nodes --all openstack-control-plane=enabled -kubectl label nodes --all openstack-compute-node=enabled -kubectl label nodes --all openvswitch=enabled -kubectl label nodes --all linuxbridge=enabled -kubectl label nodes --all ceph-mon=enabled -kubectl label nodes --all ceph-osd=enabled -kubectl label nodes --all ceph-mds=enabled -kubectl label nodes --all ceph-rgw=enabled -kubectl label nodes --all ceph-mgr=enabled - -# Add labels to the core namespaces -kubectl label --overwrite namespace default name=default -kubectl label --overwrite namespace kube-system name=kube-system -kubectl label --overwrite namespace kube-public name=kube-public diff --git a/tools/deployment/common/validate-minikube-aio.sh b/tools/deployment/common/validate-minikube-aio.sh new file mode 100644 index 000000000..8aa05deb6 --- /dev/null +++ b/tools/deployment/common/validate-minikube-aio.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -ex +cd /tmp +DIFF=$(diff loaded_images images_after_installation) +if [ ! -z ${DIFF} ]; then + echo -e "Looks like minikube-aio does not contain all images required for minikube installation:\n${DIFF}" + exit 1 +fi diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 04f96028a..8ea98ae1d 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -595,4 +595,16 @@ - ./tools/deployment/osh-infra-local-storage/040-prometheus.sh - ./tools/deployment/osh-infra-local-storage/050-elasticsearch.sh - ./tools/deployment/osh-infra-local-storage/060-volume-info.sh + +- job: + name: openstack-helm-infra-validate-minikube-aio + pre-run: + - playbooks/osh-infra-upgrade-host.yaml + post-run: playbooks/osh-infra-collect-logs.yaml + vars: + gate_scripts_relative_path: ../openstack-helm-infra + gate_scripts: + - ./tools/deployment/common/000-install-packages.sh + - ./tools/deployment/common/005-deploy-k8s.sh + - ./tools/deployment/common/validate-minikube-aio.sh ... diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 46f5e53cc..034d2a45c 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -43,8 +43,9 @@ - openstack-helm-infra-openstack-support # NOTE(srwilkers): Disabling all periodic and experimental jobs until # issues with the kubeadm-aio based deployments are addressed - # periodic: - # jobs: + periodic: + jobs: + - openstack-helm-infra-validate-minikube-aio # - openstack-helm-infra-tenant-ceph # - openstack-helm-infra-five-ubuntu # - openstack-helm-infra-armada-deploy