From 1df9ba7978835d2f0414f58264c20fa12e5c6349 Mon Sep 17 00:00:00 2001 From: Kostiantyn Kalynovskyi Date: Mon, 11 Jan 2021 11:05:59 -0600 Subject: [PATCH] Add basic integration test for vino CR This commit introduces a script to instal k8s with minukube, with basic test environment for VINO. And runs basic test that makes sure that if daemonset is succefully deployed vino CR is marked as ready. Ensure docker role is left intentionally because it also adds current user to docker group which allows other scripts to invoke docker commands without sudo. It will be removed in next patchsets. Change-Id: Iff7c956aa88ffbcf0d85956da4057fce526b67fc --- config/samples/daemonset_template.yaml | 32 +------ .../samples/vino_cr_daemonset_template.yaml | 4 +- controllers/vino_controller.go | 13 +-- ...{deploy-k8s.yaml => integration-test.yaml} | 7 +- tools/deployment/deploy-vino.sh | 6 +- tools/deployment/install-k8s.sh | 96 +++++++++++++++++++ tools/deployment/test-cr.sh | 25 +++++ zuul.d/jobs.yaml | 2 +- 8 files changed, 140 insertions(+), 45 deletions(-) rename playbooks/{deploy-k8s.yaml => integration-test.yaml} (70%) create mode 100755 tools/deployment/install-k8s.sh create mode 100755 tools/deployment/test-cr.sh diff --git a/config/samples/daemonset_template.yaml b/config/samples/daemonset_template.yaml index d2f7242..ea25ff4 100644 --- a/config/samples/daemonset_template.yaml +++ b/config/samples/daemonset_template.yaml @@ -12,11 +12,11 @@ data: spec: selector: matchLabels: - kubernetes.io/os: linux + vino-test: cr-with-ds-template template: metadata: labels: - kubernetes.io/os: linux + vino-test: cr-with-ds-template spec: tolerations: - key: node-role.kubernetes.io/master @@ -28,24 +28,12 @@ data: - name: libvirt command: - /tmp/libvirt.sh - image: docker.io/openstackhelm/libvirt:ubuntu_xenial-20190903 + image: quay.io/teoyaomiqui/libvirt securityContext: privileged: true runAsUser: 0 readOnlyRootFilesystem: false volumeMounts: - - name: pod-tmp - mountPath: /tmp - - name: libvirt-bin - mountPath: /tmp/libvirt.sh - subPath: libvirt.sh - readOnly: true - - name: libvirt-etc - mountPath: /etc/libvirt/libvirtd.conf - subPath: libvirtd.conf - - name: libvirt-qemu - mountPath: /etc/libvirt/qemu.conf - subPath: qemu.conf - mountPath: /lib/modules name: libmodules readOnly: true @@ -63,20 +51,6 @@ data: - name: logs mountPath: /var/log/libvirt volumes: - - name: pod-tmp - emptyDir: {} - - name: libvirt-bin - configMap: - name: libvirt-bin - defaultMode: 0555 - - name: libvirt-etc - configMap: - name: libvirt-etc - defaultMode: 0444 - - name: libvirt-qemu - configMap: - name: libvirt-qemu - defaultMode: 0444 - name: libmodules hostPath: path: /lib/modules diff --git a/config/samples/vino_cr_daemonset_template.yaml b/config/samples/vino_cr_daemonset_template.yaml index debc8aa..db12601 100644 --- a/config/samples/vino_cr_daemonset_template.yaml +++ b/config/samples/vino_cr_daemonset_template.yaml @@ -1,7 +1,7 @@ apiVersion: airship.airshipit.org/v1 kind: Vino metadata: - name: vino + name: vino-with-template spec: daemonSetOptions: namespacedName: @@ -9,7 +9,7 @@ spec: namespace: default nodeSelector: matchLabels: - node-type: worker + beta.kubernetes.io/os: linux configuration: cpuExclude: 0-4,54-60 redfishCredentialSecret: diff --git a/controllers/vino_controller.go b/controllers/vino_controller.go index 8da02b0..5dd17be 100644 --- a/controllers/vino_controller.go +++ b/controllers/vino_controller.go @@ -40,11 +40,11 @@ import ( const ( DaemonSetTemplateDefaultDataKey = "template" - ContainerNameLibvirt = "libvirt" - ConfigMapKeyVinoSpec = "vino-spec" + ContainerNameLibvirt = "libvirt" + ConfigMapKeyVinoSpec = "vino-spec" // TODO (kkalynovskyi) remove this, when moving to default libvirt template. - DefaultImageLibvirt = "quay.io/teoyaomiqui/libvirt" + DefaultImageLibvirt = "quay.io/teoyaomiqui/libvirt" ) // VinoReconciler reconciles a Vino object @@ -184,8 +184,8 @@ func (r *VinoReconciler) getCurrentConfigMap(ctx context.Context, vino *vinov1.V r.Log.Info("Getting current config map for vino object") cm := &corev1.ConfigMap{} err := r.Get(ctx, types.NamespacedName{ - Name: vino.Name, - Namespace: vino.Namespace, + Name: vino.Name, + Namespace: vino.Namespace, }, cm) if err != nil { if !apierror.IsNotFound(err) { @@ -315,7 +315,8 @@ func (r *VinoReconciler) waitDaemonSet(timeout int, ds *appsv1.DaemonSet) error "error", err.Error()) } else { logger.Info("checking daemonset status", "status", getDS.Status) - if getDS.Status.DesiredNumberScheduled == getDS.Status.NumberReady { + if getDS.Status.DesiredNumberScheduled == getDS.Status.NumberReady && + getDS.Status.DesiredNumberScheduled != 0 { logger.Info("daemonset is in ready status") return nil } diff --git a/playbooks/deploy-k8s.yaml b/playbooks/integration-test.yaml similarity index 70% rename from playbooks/deploy-k8s.yaml rename to playbooks/integration-test.yaml index d199e91..bd6cdb1 100644 --- a/playbooks/deploy-k8s.yaml +++ b/playbooks/integration-test.yaml @@ -6,7 +6,6 @@ - name: Install Dependent Packages apt: pkg: - - docker - debconf - make - wget @@ -17,7 +16,9 @@ name: clear-firewall - name: Install kubernetes and Deploy Vino shell: | - set -xe; - ./tools/deployment/deploy-vino.sh + set -xe; + ./tools/deployment/install-k8s.sh + ./tools/deployment/deploy-vino.sh + ./tools/deployment/test-cr.sh args: chdir: "{{ zuul.project.src_dir }}" \ No newline at end of file diff --git a/tools/deployment/deploy-vino.sh b/tools/deployment/deploy-vino.sh index 3db5e37..a5dd7b5 100755 --- a/tools/deployment/deploy-vino.sh +++ b/tools/deployment/deploy-vino.sh @@ -1,11 +1,9 @@ #!/bin/bash + set -xe -curl -s -L https://opendev.org/airship/charts/raw/branch/master/tools/gate/deploy-k8s.sh | bash sudo snap install kustomize && sudo snap install go --classic -#Wait for all pods to be ready before starting Vino Image build. -kubectl wait --for=condition=Ready pods --all -A --timeout=180s make docker-build -kustomize build config/default | kubectl apply -f - +make deploy kubectl get po -A #Wait for vino controller manager Pod. kubectl wait -n vino-system pod -l control-plane=controller-manager --for=condition=ready --timeout=240s \ No newline at end of file diff --git a/tools/deployment/install-k8s.sh b/tools/deployment/install-k8s.sh new file mode 100755 index 0000000..1b7461f --- /dev/null +++ b/tools/deployment/install-k8s.sh @@ -0,0 +1,96 @@ +#!/bin/bash +set -ex + +: ${KUBE_VERSION:="v1.19.2"} +: ${MINIKUBE_VERSION:="v1.16.0"} +: ${UPSTREAM_DNS_SERVER:="8.8.4.4"} +: ${DNS_DOMAIN:="cluster.local"} + +export DEBCONF_NONINTERACTIVE_SEEN=true +export DEBIAN_FRONTEND=noninteractive + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo apt-key fingerprint 0EBFCD88 +sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + +sudo -E apt-get update +sudo -E apt-get install -y \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + socat \ + jq \ + util-linux \ + nfs-common \ + bridge-utils \ + iptables \ + conntrack \ + libffi-dev + +# Prepare tmpfs for etcd +sudo mkdir -p /data +sudo mount -t tmpfs -o size=512m tmpfs /data + +# Install minikube and kubectl +URL="https://storage.googleapis.com" +sudo -E curl -sSLo /usr/local/bin/minikube "${URL}"/minikube/releases/"${MINIKUBE_VERSION}"/minikube-linux-amd64 +sudo -E curl -sSLo /usr/local/bin/kubectl "${URL}"/kubernetes-release/release/"${KUBE_VERSION}"/bin/linux/amd64/kubectl + +sudo -E chmod +x /usr/local/bin/minikube +sudo -E chmod +x /usr/local/bin/kubectl + +export CHANGE_MINIKUBE_NONE_USER=true +export MINIKUBE_IN_STYLE=false +sudo -E minikube start \ + --kubernetes-version="${KUBE_VERSION}" \ + --embed-certs=true \ + --interactive=false \ + --driver=none \ + --wait=apiserver,system_pods,node_ready \ + --wait-timeout=6m0s \ + --extra-config=kube-proxy.mode=ipvs \ + --extra-config=controller-manager.allocate-node-cidrs=true \ + --extra-config=controller-manager.cluster-cidr=192.168.0.0/16 \ + --extra-config=kubelet.resolv-conf=/run/systemd/resolve/resolv.conf + +cat <