Enable CI for SIP

* Install Dependency packages like make,wget,debconf and snapd * Deploy Kubernetes and SIP * Setup roles for Docker and for firewall for Kubernetes * Updated Readme and makefile to reflect minikube Installation Change-Id: I6e8bbe011a00ded0b8b047c1fc095c901a18e1db
2021-01-14 21:38:31 +00:00 · 2021-01-14 21:38:31 +00:00 · 09d76d3e97
commit 09d76d3e97
parent fa7d655868
7 changed files with 208 additions and 33 deletions
--- a/25
+++ b/25
@ -1,13 +1,19 @@
+# Docker Image Options
+DOCKER_REGISTRY     ?= quay.io
+DOCKER_FORCE_CLEAN  ?= true
+DOCKER_IMAGE_NAME   ?= sip
+DOCKER_IMAGE_PREFIX ?= airshipit
+DOCKER_IMAGE_TAG    ?= latest
+DOCKER_TARGET_STAGE ?= release
+PUBLISH             ?= false
+
 # Image URL to use all building/pushing image targets
 #IMG ?= controller:latest
-IMG ?= quay.io/airshipit/sip
+IMG ?= $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)

 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"

-# Name of the kind cluster that will be created by kind-create target
-KIND_CLUSTER_NAME ?= sip-cluster
-
 TOOLBINDIR          := tools/bin

 # linting
@ -26,6 +32,9 @@ DOCKER_PROXY_FLAGS  := --build-arg http_proxy=$(HTTP_PROXY)
 DOCKER_PROXY_FLAGS  += --build-arg https_proxy=$(HTTPS_PROXY)
 DOCKER_PROXY_FLAGS  += --build-arg NO_PROXY=$(NO_PROXY)

+kubernetes:
+	./tools/deployment/install-k8s.sh
+
 all: manager

 # Run tests
@ -78,14 +87,6 @@ docker-build:
 docker-push:
 	docker push ${IMG}

-# Create kind cluster
-kind-create:
-	kind create cluster --name ${KIND_CLUSTER_NAME}
-
-# Build docker container and load it into running kind cluster
-kind-load-image: docker-build
-	kind load docker-image ${IMG} --name ${KIND_CLUSTER_NAME}
-
 # find or download controller-gen
 # download controller-gen if necessary
 controller-gen:
--- a/README.md
+++ b/README.md
@ -72,36 +72,68 @@ SIPCluster CR will exists within the Control phase for a Tenant cluster.

 ## Development environment

-### Kind kubernetes cluster
-Fastest way to set up a k8s cluster for development env is to use kind to set it up
+### Pre-requisites

-#### Install kind on linux (amd64 arch)
+#### Install Golang 1.15+
+
+SIP is a project written in Go, and the make targets used to deploy SIP leverage both Go and
+Kustomize commands which require Golang be installed.
+
+For detailed installation instructions, please see the [Golang installation guide](https://golang.org/doc/install).
+
+#### Install Kustomize v3.2.3+
+
+In order to apply manifests to your cluster via Make targets we suggest the use of Kustomize.
+
+For detailed installation instructions, please see the [Kustomize installation guide](https://kubectl.docs.kubernetes.io/installation/kustomize/).
+
+#### Proxy Setup
+
+If your organization requires development behind a proxy server, you will need to define the
+following environment variables with your organization's information:

 ```
-# curl -Lo kind https://kind.sigs.k8s.io/dl/v0.9.0/kind-linux-amd64
-# sudo install  -m 755 --owner=root --group=root kind /usr/local/bin
-# rm kind
+HTTP_PROXY=http://username:password@host:port
+HTTPS_PROXY=http://username:password@host:port
+NO_PROXY="localhost,127.0.0.1,10.96.0.0/12"
+PROXY=http://username:password@host:port
+USE_PROXY=true
 ```

-More information on how to install kind binary can be found be found [here](https://kind.sigs.k8s.io/docs/user/quick-start/#installation)
+10.96.0.0/12 is the Kubernetes service CIDR.

-#### Create k8s cluster with kind
+#### Deploy kubernetes using minikube and create k8s cluster

 ```
-# make kind-create
+# ./tools/deployment/install-k8s.sh
+```
+
+### Deploy SIP
+
+```
+# make docker-build
 # kubectl get nodes
-```
-
-### Deploy SIP operator on top of kind cluster
-kind-load-image target will build docker image from the current state of your local
-git repository and upload it to kind cluster to be available for kubelet.
-
-```
-# make kind-load-image
 # make deploy
 ```

-Now you have a working k8s cluster with sip installed on it with your changes to SIP operator
+By now, you should have a working cluster with ViNO deployed on top of it.
+
+```
+kubectl get pods -A
+NAMESPACE           NAME                                            READY   STATUS    RESTARTS   AGE
+kube-system         calico-kube-controllers-744cfdf676-428vp        1/1     Running   0          4h30m
+kube-system         calico-node-pgr4k                               1/1     Running   0          4h30m
+kube-system         coredns-f9fd979d6-qk2dc                         1/1     Running   0          4h30m
+kube-system         etcd-govino                                     1/1     Running   0          4h30m
+kube-system         kube-apiserver-govino                           1/1     Running   0          4h30m
+kube-system         kube-controller-manager-govino                  1/1     Running   0          4h30m
+kube-system         kube-proxy-6wx46                                1/1     Running   0          4h30m
+kube-system         kube-scheduler-govino                           1/1     Running   0          4h30m
+kube-system         storage-provisioner                             1/1     Running   0          4h30m
+sipcluster-system   sipcluster-controller-manager-59c7dddcb-65lcb   2/2     Running   0          3h47m
+```
+
+

 ### Deliver SIP CRs to kubernetes

--- a/playbooks/integration-tests.yaml
+++ b/playbooks/integration-tests.yaml
@ -0,0 +1,23 @@
+- hosts: ubuntu-bionic
+  tasks:
+    - name: Run ensure-docker
+      include_role:
+        name: ensure-docker
+    - name: Install Dependent Packages
+      apt:
+        pkg:
+          - debconf
+          - make
+          - wget
+          - snapd
+      become: yes
+    - name:  Set up requirements for kubernetes
+      include_role:
+        name: clear-firewall
+    - name: Install kubernetes and Deploy Sip
+      shell: |
+        set -xe;
+        ./tools/deployment/install-k8s.sh
+        ./tools/deployment/deploy-sip.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
--- a/tools/deployment/deploy-sip.sh
+++ b/tools/deployment/deploy-sip.sh
@ -0,0 +1,9 @@
+#!/bin/bash
+set -xe
+sudo snap install kustomize && sudo snap install go --classic
+make docker-build
+kubectl wait --for=condition=Ready pods --all -A --timeout=180s
+make deploy
+#Wait for sip controller manager Pod.
+kubectl wait -n sipcluster-system pod -l control-plane=controller-manager --for=condition=ready --timeout=240s
+kubectl get po -A
--- a/tools/deployment/install-k8s.sh
+++ b/tools/deployment/install-k8s.sh
@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -ex
+
+: ${KUBE_VERSION:="v1.19.2"}
+: ${MINIKUBE_VERSION:="v1.16.0"}
+: ${UPSTREAM_DNS_SERVER:="8.8.4.4"}
+: ${DNS_DOMAIN:="cluster.local"}
+: ${CALICO_VERSION:="v3.17"}
+: ${CNI_MANIFEST_PATH:="/tmp/calico.yaml"}
+
+export DEBCONF_NONINTERACTIVE_SEEN=true
+export DEBIAN_FRONTEND=noninteractive
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+
+sudo apt-key fingerprint 0EBFCD88
+
+sudo add-apt-repository \
+  "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) \
+  stable"
+
+sudo -E apt-get update
+
+sudo -E apt-get install -y \
+  docker-ce \
+  docker-ce-cli \
+  containerd.io \
+  socat \
+  jq \
+  util-linux \
+  nfs-common \
+  bridge-utils \
+  iptables \
+  conntrack \
+  libffi-dev
+
+# Prepare tmpfs for etcd
+sudo mkdir -p /data
+sudo mount -t tmpfs -o size=512m tmpfs /data
+
+# Download calico manifest
+if [ ! -f "$CNI_MANIFEST_PATH" ]; then
+  curl -Ss https://docs.projectcalico.org/"${CALICO_VERSION}"/manifests/calico.yaml -o ${CNI_MANIFEST_PATH}
+fi
+
+# Install minikube and kubectl
+URL="https://storage.googleapis.com"
+sudo -E curl -sSLo /usr/local/bin/minikube "${URL}"/minikube/releases/"${MINIKUBE_VERSION}"/minikube-linux-amd64
+sudo -E curl -sSLo /usr/local/bin/kubectl "${URL}"/kubernetes-release/release/"${KUBE_VERSION}"/bin/linux/amd64/kubectl
+sudo -E chmod +x /usr/local/bin/minikube
+sudo -E chmod +x /usr/local/bin/kubectl
+
+export CHANGE_MINIKUBE_NONE_USER=true
+export MINIKUBE_IN_STYLE=false
+
+sudo -E minikube start \
+  --kubernetes-version="${KUBE_VERSION}" \
+  --embed-certs=true \
+  --interactive=false \
+  --driver=none \
+  --wait=apiserver,system_pods,node_ready \
+  --wait-timeout=6m0s \
+  --network-plugin=cni \
+  --cni=${CNI_MANIFEST_PATH} \
+  --extra-config=kube-proxy.mode=ipvs \
+  --extra-config=controller-manager.allocate-node-cidrs=true \
+  --extra-config=controller-manager.cluster-cidr=192.168.0.0/16 \
+  --extra-config=kubeadm.pod-network-cidr=192.168.0.0/16 \
+  --extra-config=kubelet.resolv-conf=/run/systemd/resolve/resolv.conf
+
+kubectl get nodes -o wide
+kubectl get pod -A
+
+cat <<EOF | kubectl replace -f -
+apiVersion: v1
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+           lameduck 5s
+        }
+        ready
+        kubernetes ${DNS_DOMAIN} in-addr.arpa ip6.arpa {
+           pods insecure
+           fallthrough in-addr.arpa ip6.arpa
+           ttl 30
+        }
+        prometheus :9153
+        forward . ${UPSTREAM_DNS_SERVER} {
+           max_concurrent 1000
+        }
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+EOF
+
+kubectl wait --for=condition=Ready pods --all -A --timeout=180s
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@ -8,3 +8,8 @@
      - name: git_mirror_credentials
        secret: airship_sip_airshipit_github_secret
        pass-to-parent: true
+- job:
+    name: airship-deploy-sip
+    run: playbooks/integration-tests.yaml
+    description: Deploys kubernetes and sip
+    timeout: 9600
--- a/zuul.d/projects.yaml
+++ b/zuul.d/projects.yaml
@ -1,10 +1,10 @@
 - project:
    check:
      jobs:
-        - noop
+        - airship-deploy-sip
    gate:
      jobs:
-        - noop
+        - airship-deploy-sip
    post:
      jobs:
-        - airship-sip-upload-git-mirror
+        - airship-sip-upload-git-mirror