From fd7add74ac93fdde6eeb9aa47b28b1798cdb54cd Mon Sep 17 00:00:00 2001
From: Drew Walters <drewwalters96@gmail.com>
Date: Mon, 21 Jan 2019 17:30:31 -0600
Subject: [PATCH] Add proxy support to Minikube gate script

This commit introduces proxy support to the Minikube gate script by
leveraging existing `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY`
environment variables.  Additionally, this adds the ability to interpret
DNS nameservers when running behind a proxy server and use those in
`/etc/resolv.conf` over the Google DNS servers.

Change-Id: I508dd00fb7df33945e8ee96af250a8eff9db389a
---
 tools/deployment/common/005-deploy-k8s.sh | 111 ++++++++++++++--------
 1 file changed, 74 insertions(+), 37 deletions(-)

diff --git a/tools/deployment/common/005-deploy-k8s.sh b/tools/deployment/common/005-deploy-k8s.sh
index 10948f06a..50509f5c9 100755
--- a/tools/deployment/common/005-deploy-k8s.sh
+++ b/tools/deployment/common/005-deploy-k8s.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 # Copyright 2017 The Openstack-Helm Authors.
+# Copyright 2019, AT&T Intellectual Property
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -15,20 +16,50 @@
 #    under the License.
 
 set -xe
+
 : ${HELM_VERSION:="v2.12.3"}
 : ${KUBE_VERSION:="v1.12.2"}
 : ${MINIKUBE_VERSION:="v0.30.0"}
 : ${CALICO_VERSION:="v3.3"}
-export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true
+
+: "${HTTP_PROXY:=""}"
+
+export DEBCONF_NONINTERACTIVE_SEEN=true
+export DEBIAN_FRONTEND=noninteractive
+
+function configure_resolvconf {
+  # Setup resolv.conf to use the k8s api server, which is required for the
+  # kubelet to resolve cluster services.
+  sudo mv /etc/resolv.conf /etc/resolv.conf.backup
+
+  sudo bash -c "echo 'search svc.cluster.local cluster.local' > /etc/resolv.conf"
+  sudo bash -c "echo 'nameserver 10.96.0.10' >> /etc/resolv.conf"
+
+  # NOTE(drewwalters96): Use the Google DNS servers to prevent local addresses in
+  # the resolv.conf file unless using a proxy, then use the existing DNS servers,
+  # as custom DNS nameservers are commonly required when using a proxy server.
+  if [ -z "${HTTP_PROXY}" ]; then
+    sudo bash -c "echo 'nameserver 8.8.8.8' >> /etc/resolv.conf"
+    sudo bash -c "echo 'nameserver 8.8.4.4' >> /etc/resolv.conf"
+  else
+    sed -ne "s/nameserver //p" /etc/resolv.conf.backup | while read -r ns; do
+      sudo bash -c "echo 'nameserver ${ns}' >> /etc/resolv.conf"
+    done
+  fi
+
+  sudo bash -c "echo 'options ndots:5 timeout:1 attempts:1' >> /etc/resolv.conf"
+  sudo rm /etc/resolv.conf.backup
+}
 
 # NOTE: Clean Up hosts file
-sudo sed -i '/^127.0.0.1/c\127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4' /etc/hosts
+sudo sed -i '/^127.0.0.1/c\127.0.0.1 localhost localhost.localdomain localhost4localhost4.localdomain4' /etc/hosts
 sudo sed -i '/^::1/c\::1 localhost6 localhost6.localdomain6' /etc/hosts
 
-# NOTE: Install required packages on host
+# Install required packages for K8s on host
 sudo apt-key adv --keyserver keyserver.ubuntu.com  --recv 460F3994
 RELEASE_NAME=$(grep 'CODENAME' /etc/lsb-release | awk -F= '{print $2}')
-sudo add-apt-repository "deb https://download.ceph.com/debian-mimic/ ${RELEASE_NAME} main"
+sudo add-apt-repository "deb https://download.ceph.com/debian-mimic/
+${RELEASE_NAME} main"
 sudo -E apt-get update
 sudo -E apt-get install -y \
     docker.io \
@@ -40,34 +71,54 @@ sudo -E apt-get install -y \
     nfs-common \
     bridge-utils \
     libxtables11
-sudo -E tee /etc/modprobe.d/rbd.conf <<EOF
+
+sudo -E tee /etc/modprobe.d/rbd.conf << EOF
 install rbd /bin/true
 EOF
 
-sudo -E curl -sSLo /usr/local/bin/minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
-sudo -E chmod +x /usr/local/bin/minikube
+configure_resolvconf
 
-sudo -E curl -sSLo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64/kubectl
+# Install minikube and kubectl
+URL="https://storage.googleapis.com"
+sudo -E curl -sSLo /usr/local/bin/minikube \
+  "${URL}"/minikube/releases/"${MINIKUBE_VERSION}"/minikube-linux-amd64
+
+sudo -E curl -sSLo /usr/local/bin/kubectl \
+  "${URL}"/kubernetes-release/release/"${KUBE_VERSION}"/bin/linux/amd64/kubectl
+
+sudo -E chmod +x /usr/local/bin/minikube
 sudo -E chmod +x /usr/local/bin/kubectl
 
+# Install Helm
 TMP_DIR=$(mktemp -d)
-sudo -E bash -c "curl -sSL https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR}"
-sudo -E mv ${TMP_DIR}/helm /usr/local/bin/helm
-rm -rf ${TMP_DIR}
+sudo -E bash -c \
+  "curl -sSL ${URL}/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | \
+    tar -zxv --strip-components=1 -C ${TMP_DIR}"
+
+sudo -E mv "${TMP_DIR}"/helm /usr/local/bin/helm
+rm -rf "${TMP_DIR}"
 
 # NOTE: Deploy kubenetes using minikube. A CNI that supports network policy is
-# required for validation, lets just use calico for simplicity
-sudo -E minikube config set kubernetes-version ${KUBE_VERSION}
+# required for validation; use calico for simplicity.
+sudo -E minikube config set embed-certs true
+sudo -E minikube config set kubernetes-version "${KUBE_VERSION}"
 sudo -E minikube config set vm-driver none
 sudo -E minikube addons disable addon-manager
 sudo -E minikube addons disable dashboard
+
 export CHANGE_MINIKUBE_NONE_USER=true
 sudo -E minikube start \
+  --docker-env HTTP_PROXY="${HTTP_PROXY}" \
+  --docker-env HTTPS_PROXY="${HTTPS_PROXY}" \
+  --docker-env NO_PROXY="${NO_PROXY},10.96.0.0/12" \
   --extra-config=kubelet.network-plugin=cni \
   --extra-config=controller-manager.allocate-node-cidrs=true \
   --extra-config=controller-manager.cluster-cidr=192.168.0.0/16
-kubectl apply -f https://docs.projectcalico.org/${CALICO_VERSION}/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
-kubectl apply -f https://docs.projectcalico.org/${CALICO_VERSION}/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
+
+kubectl apply -f \
+  https://docs.projectcalico.org/"${CALICO_VERSION}"/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
+kubectl apply -f \
+  https://docs.projectcalico.org/"${CALICO_VERSION}"/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
 
 # NOTE: Wait for node to be ready.
 kubectl wait --timeout=240s --for=condition=Ready nodes/minikube
@@ -79,10 +130,11 @@ until kubectl --namespace=kube-system \
   NOW=$(date +%s)
   [ "${NOW}" -gt "${END}" ] && exit -1
   echo "still waiting for dns"
+  sleep 10
 done
 kubectl --namespace=kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns
 
-# NOTE: Deploy helm/tiller into the cluster
+# Deploy helm/tiller into the cluster
 kubectl create -n kube-system serviceaccount helm-tiller
 cat <<EOF | kubectl apply -f -
 apiVersion: rbac.authorization.k8s.io/v1
@@ -106,9 +158,8 @@ kubectl --namespace=kube-system wait \
   --for=condition=Ready \
   pod -l app=helm,name=tiller
 
-
-# NOTE: Set up local helm server
-sudo -E tee /etc/systemd/system/helm-serve.service <<EOF
+# Set up local helm server
+sudo -E tee /etc/systemd/system/helm-serve.service << EOF
 [Unit]
 Description=Helm Server
 After=network.target
@@ -123,19 +174,17 @@ WantedBy=multi-user.target
 EOF
 
 sudo chmod 0640 /etc/systemd/system/helm-serve.service
-sudo systemctl restart helm-serve
+
 sudo systemctl daemon-reload
+sudo systemctl restart helm-serve
 sudo systemctl enable helm-serve
 
-
-# NOTE: Set up local helm repo
+# Set up local helm repo
 helm repo add local http://localhost:8879/charts
 helm repo update
-uptime
 make
 
-
-# NOTE: Set required labels on host(s)
+# Set required labels on host(s)
 kubectl label nodes --all openstack-control-plane=enabled
 kubectl label nodes --all openstack-compute-node=enabled
 kubectl label nodes --all openvswitch=enabled
@@ -145,15 +194,3 @@ kubectl label nodes --all ceph-osd=enabled
 kubectl label nodes --all ceph-mds=enabled
 kubectl label nodes --all ceph-rgw=enabled
 kubectl label nodes --all ceph-mgr=enabled
-
-
-# NOTE: Setup resolv.conf to use the k8s api server, which is required for the
-# kubelet to resolve cluster services.
-sudo -E rm -rf /etc/resolv.conf
-sudo -E tee /etc/resolv.conf <<EOF
-search svc.cluster.local cluster.local
-nameserver 10.96.0.10
-nameserver 8.8.8.8
-nameserver 8.8.4.4
-options ndots:5 timeout:1 attempts:1
-EOF