From b0217a8ba11e982ec8e2661e420e0827d878c64e Mon Sep 17 00:00:00 2001 From: Rishabh Kumar Jain Date: Wed, 7 Oct 2020 08:30:06 -0700 Subject: [PATCH] Integrate Sonobuoy Conformance Test Scripts Add jobs in experimental pipeline to do the following: - install Sonobuoy - run CNCF Conformace Tests - run CIS Benchmarks Tests Conformance tests include: - CNCF Compliance: uses sonobuoy end-to-end (e2e) and systemd-logs plugins - CIS Benchmarks: utilizes the kube-bench implementation of the CIS security benchmarks plugin Pipeline can be triggered by comment - "check experimental" Change-Id: I7d08ae42512dc4c83e2f550c4809ce1f8ddccc7b Change-Id: I2e6469f5b8e229828532ce5499498da639d23fe6 --- .../sonobuoy/01-install_sonobuoy.sh | 26 +++++++++ tools/deployment/sonobuoy/02-run_default.sh | 50 ++++++++++++++++ tools/deployment/sonobuoy/03-kubebench.sh | 53 +++++++++++++++++ zuul.d/jobs.yaml | 58 ++++++++++++++++++- zuul.d/projects.yaml | 2 + 5 files changed, 188 insertions(+), 1 deletion(-) create mode 100755 tools/deployment/sonobuoy/01-install_sonobuoy.sh create mode 100755 tools/deployment/sonobuoy/02-run_default.sh create mode 100755 tools/deployment/sonobuoy/03-kubebench.sh diff --git a/tools/deployment/sonobuoy/01-install_sonobuoy.sh b/tools/deployment/sonobuoy/01-install_sonobuoy.sh new file mode 100755 index 000000000..1e780e118 --- /dev/null +++ b/tools/deployment/sonobuoy/01-install_sonobuoy.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +: ${SONOBUOY_VERSION:="0.18.2"} +: ${KUBECONFIG:="$HOME/.airship/kubeconfig"} +URL="https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz" +rm -rf /tmp/sonobuoy +mkdir /tmp/sonobuoy +sudo -E curl -sSLo "/tmp/sonobuoy/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz" ${URL} +tar xvf /tmp/sonobuoy/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz -C /tmp/sonobuoy/ +sudo install -m 755 -o root /tmp/sonobuoy/sonobuoy /usr/local/bin +echo ${KUBECONFIG} +sonobuoy version --kubeconfig ${KUBECONFIG} diff --git a/tools/deployment/sonobuoy/02-run_default.sh b/tools/deployment/sonobuoy/02-run_default.sh new file mode 100755 index 000000000..502a4f9c8 --- /dev/null +++ b/tools/deployment/sonobuoy/02-run_default.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe +: ${KUBECONFIG:="$HOME/.airship/kubeconfig"} +# Available Modes: quick, certified-conformance, non-disruptive-conformance. +# (default quick) +: ${CONFORMANCE_MODE:="quick"} +: ${KUBE_CONFORMANCE_IMAGE_VERSION:="v1.18.6"} +: ${TIMEOUT:=10800} +: ${TARGET_CLUSTER_CONTEXT:="target-cluster"} + +mkdir -p /tmp/sonobuoy_snapshots/e2e +cd /tmp/sonobuoy_snapshots/e2e + +# Run aggregator, and default plugins e2e and systemd-logs +sonobuoy run --plugin e2e --plugin systemd-logs -m ${CONFORMANCE_MODE} \ +--context "$TARGET_CLUSTER_CONTEXT" \ +--kube-conformance-image gcr.io/google-containers/conformance:${KUBE_CONFORMANCE_IMAGE_VERSION} \ +--kubeconfig ${KUBECONFIG} \ +--wait --timeout ${TIMEOUT} \ +--log_dir /tmp/sonobuoy_snapshots/e2e + +# Get information on pods +kubectl get all -n sonobuoy --kubeconfig ${KUBECONFIG} --context "$TARGET_CLUSTER_CONTEXT" + +# Check sonobuoy status +sonobuoy status --kubeconfig ${KUBECONFIG} --context "$TARGET_CLUSTER_CONTEXT" + +# Get logs +sonobuoy logs + +# Store Results +results=$(sonobuoy retrieve --kubeconfig ${KUBECONFIG} --context $TARGET_CLUSTER_CONTEXT) +echo "Results: ${results}" + +# Display Results +sonobuoy results $results +ls -ltr /tmp/sonobuoy_snapshots/e2e \ No newline at end of file diff --git a/tools/deployment/sonobuoy/03-kubebench.sh b/tools/deployment/sonobuoy/03-kubebench.sh new file mode 100755 index 000000000..df6399923 --- /dev/null +++ b/tools/deployment/sonobuoy/03-kubebench.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe +: ${KUBECONFIG:="$HOME/.airship/kubeconfig"} +: ${KUBEBENCH_MASTER_PLUGIN:="https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml"} +: ${KUBEBENCH_WORKER_PLUGIN:="https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml"} +: ${TARGET_CLUSTER_CONTEXT:="target-cluster"} +# This shouldnot include minor version +: ${KUBEBENCH_K8S_VERSION:=1.18} +: ${TIMEOUT:=300} + +mkdir -p /tmp/sonobuoy_snapshots/kubebench +cd /tmp/sonobuoy_snapshots/kubebench + +# Run aggregator, and default plugins e2e and systemd-logs +sonobuoy run \ +--kubeconfig ${KUBECONFIG} \ +--context ${TARGET_CLUSTER_CONTEXT} \ +--plugin ${KUBEBENCH_MASTER_PLUGIN} \ +--plugin ${KUBEBENCH_WORKER_PLUGIN} \ +--plugin-env kube-bench-master.KUBERNETES_VERSION=${KUBEBENCH_K8S_VERSION} \ +--plugin-env kube-bench-master.KUBERNETES_VERSION=${KUBEBENCH_K8S_VERSION} \ +--wait --timeout ${TIMEOUT} \ +--log_dir /tmp/sonobuoy_snapshots/kubebench + +# Get information on pods +kubectl get all -n sonobuoy --kubeconfig ${KUBECONFIG} --context ${TARGET_CLUSTER_CONTEXT} + +# Check sonobuoy status +sonobuoy status --kubeconfig ${KUBECONFIG} --context ${TARGET_CLUSTER_CONTEXT} + +# Get logs +sonobuoy logs + +# Store Results +results=$(sonobuoy retrieve --kubeconfig ${KUBECONFIG} --context ${TARGET_CLUSTER_CONTEXT}) +echo "Results: ${results}" + +# Display Results +sonobuoy results $results +ls -ltr /tmp/sonobuoy_snapshots/kubebench \ No newline at end of file diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index a6c959222..19ccd8e44 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -175,7 +175,7 @@ soft: true vars: site_name: docker-test-site - gate_scripts: + gate_scripts: &docker_gate_scripts - ./tools/deployment/21_systemwide_executable.sh - ./tools/deployment/01_install_kubectl.sh - ./tools/deployment/provider_common/01_install_kind.sh @@ -188,6 +188,62 @@ - ./tools/deployment/provider_common/33_cluster_move_target_node.sh - WORKERS_COUNT=2 KUBECONFIG=/tmp/target-cluster.kubeconfig SITE=docker-test-site ./tools/deployment/provider_common/34_deploy_worker_node.sh voting: false +- job: + name: airship-airshipctl-docker-kubebench-conformance + attempts: 1 + timeout: 10800 + pre-run: playbooks/airship-airshipctl-deploy-docker.yaml + run: playbooks/airshipctl-gate-runner.yaml + nodeset: airship-airshipctl-single-node + irrelevant-files: *noncodefiles + dependencies: + - name: openstack-tox-docs + soft: true + - name: airship-airshipctl-lint + soft: true + - name: airship-airshipctl-unit + soft: true + - name: airship-airshipctl-golint + soft: true + - name: airship-airshipctl-build-image + soft: true + - name: airship-airshipctl-validate-site-docs + soft: true + vars: + site_name: docker-test-site + gate_scripts: + - *docker_gate_scripts + - KUBECONFIG=/tmp/target-cluster.kubeconfig TARGET_CLUSTER_CONTEXT=target-cluster ./tools/deployment/sonobuoy/01-install_sonobuoy.sh + - KUBECONFIG=/tmp/target-cluster.kubeconfig TARGET_CLUSTER_CONTEXT=target-cluster ./tools/deployment/sonobuoy/03-kubebench.sh + voting: false +- job: + name: airship-airshipctl-docker-cncf-conformance + attempts: 1 + timeout: 10800 + pre-run: playbooks/airship-airshipctl-deploy-docker.yaml + run: playbooks/airshipctl-gate-runner.yaml + nodeset: airship-airshipctl-single-node + irrelevant-files: *noncodefiles + dependencies: + - name: openstack-tox-docs + soft: true + - name: airship-airshipctl-lint + soft: true + - name: airship-airshipctl-unit + soft: true + - name: airship-airshipctl-golint + soft: true + - name: airship-airshipctl-build-image + soft: true + - name: airship-airshipctl-validate-site-docs + soft: true + vars: + site_name: docker-test-site + gate_scripts: + - *docker_gate_scripts + - KUBECONFIG=/tmp/target-cluster.kubeconfig TARGET_CLUSTER_CONTEXT=target-cluster ./tools/deployment/sonobuoy/01-install_sonobuoy.sh + - KUBECONFIG=/tmp/target-cluster.kubeconfig TARGET_CLUSTER_CONTEXT=target-cluster CONFORMANCE_MODE=certified-conformance ./tools/deployment/sonobuoy/02-run_default.sh + voting: false - job: name: airship-airshipctl-publish-image parent: airship-airshipctl-build-image diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 0c07c8463..04bfd02d8 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -35,6 +35,8 @@ - airship-airshipctl-gate-script-runner-docker experimental: jobs: + - airship-airshipctl-docker-kubebench-conformance + - airship-airshipctl-docker-cncf-conformance - airship-airshipctl-gate-script-runner gate: jobs: