From c3e085b800bb3236a0c68dd74ffa8c2ec2896e53 Mon Sep 17 00:00:00 2001 From: Gage Hugo Date: Wed, 11 Sep 2019 11:56:08 -0500 Subject: [PATCH] Add network policy nonvoting checks This change adds two network policy zuul checks, one for the compute-kit, and one for cinder/ceph, to test network policy for each OpenStack service. These checks will be non-voting initially. The network policy rules for each service will initially allow all traffic. These ingress/egress rules will be defined in future changes to only explicitly allow traffic between services that are explicitly allowed to communicate, other traffic will be denied. Depends-On: https://review.opendev.org/#/c/685130/ Change-Id: Ide2998ebb2af2832f24ca7abc398a82e4a6d70e3 --- cinder/values_overrides/netpol.yaml | 2 + glance/values.yaml | 30 +------ glance/values_overrides/netpol.yaml | 35 ++++++++ heat/values.yaml | 34 +------- heat/values_overrides/netpol.yaml | 39 +++++++++ horizon/values.yaml | 16 +--- horizon/values_overrides/netpol.yaml | 2 + keystone/values.yaml | 86 ++----------------- keystone/values_overrides/netpol.yaml | 84 ++++++++++++++++++ neutron/values_overrides/netpol.yaml | 2 + nova/values.yaml | 16 ---- nova/values_overrides/netpol.yaml | 2 + .../lockdown-netpol.sh} | 0 tools/deployment/common/openstack-exporter.sh | 36 ++++++++ tools/deployment/common/test-networkpolicy.sh | 48 +++++++---- zuul.d/jobs-openstack-helm.yaml | 59 +++++++++++++ zuul.d/project.yaml | 4 + 17 files changed, 308 insertions(+), 187 deletions(-) create mode 100644 cinder/values_overrides/netpol.yaml create mode 100644 glance/values_overrides/netpol.yaml create mode 100644 heat/values_overrides/netpol.yaml create mode 100644 horizon/values_overrides/netpol.yaml create mode 100644 keystone/values_overrides/netpol.yaml create mode 100644 neutron/values_overrides/netpol.yaml create mode 100644 nova/values_overrides/netpol.yaml rename tools/deployment/{developer/common/049-lockdown.sh => common/lockdown-netpol.sh} (100%) create mode 100755 tools/deployment/common/openstack-exporter.sh diff --git a/cinder/values_overrides/netpol.yaml b/cinder/values_overrides/netpol.yaml new file mode 100644 index 0000000000..7a85753209 --- /dev/null +++ b/cinder/values_overrides/netpol.yaml @@ -0,0 +1,2 @@ +manifests: + network_policy: true diff --git a/glance/values.yaml b/glance/values.yaml index 17bdf4dfe2..7e5fab674f 100644 --- a/glance/values.yaml +++ b/glance/values.yaml @@ -88,35 +88,7 @@ ceph_client: network_policy: glance: ingress: - - from: - - podSelector: - matchLabels: - application: glance - - podSelector: - matchLabels: - application: nova - - podSelector: - matchLabels: - application: horizon - - podSelector: - matchLabels: - application: ingress - - podSelector: - matchLabels: - application: heat - - podSelector: - matchLabels: - application: ironic - - podSelector: - matchLabels: - application: cinder - ports: - - protocol: TCP - port: 80 - - protocol: TCP - port: 9191 - - protocol: TCP - port: 9292 + - {} egress: - {} diff --git a/glance/values_overrides/netpol.yaml b/glance/values_overrides/netpol.yaml new file mode 100644 index 0000000000..4c6afc2b1c --- /dev/null +++ b/glance/values_overrides/netpol.yaml @@ -0,0 +1,35 @@ +manifests: + network_policy: true +#NOTE(gagehugo): Test this whitelist when the netpol gate works +#network_policy: +# glance: +# ingress: +# - from: +# - podSelector: +# matchLabels: +# application: glance +# - podSelector: +# matchLabels: +# application: nova +# - podSelector: +# matchLabels: +# application: horizon +# - podSelector: +# matchLabels: +# application: ingress +# - podSelector: +# matchLabels: +# application: heat +# - podSelector: +# matchLabels: +# application: ironic +# - podSelector: +# matchLabels: +# application: cinder +# ports: +# - protocol: TCP +# port: 80 +# - protocol: TCP +# port: 9191 +# - protocol: TCP +# port: 9292 diff --git a/heat/values.yaml b/heat/values.yaml index f9ac5e7b76..26bd874220 100644 --- a/heat/values.yaml +++ b/heat/values.yaml @@ -1249,39 +1249,9 @@ pod: network_policy: heat: ingress: - - from: - - podSelector: - matchLabels: - application: heat - - podSelector: - matchLabels: - application: ingress - - podSelector: - matchLabels: - application: horizon - ports: - - protocol: TCP - port: 80 - - protocol: TCP - port: 8000 - - protocol: TCP - port: 8003 - - protocol: TCP - port: 8004 + - {} egress: - - to: - - podSelector: - matchLabels: - application: neutron - - podSelector: - matchLabels: - application: nova - - podSelector: - matchLabels: - application: glance - - podSelector: - matchLabels: - application: cinder + - {} manifests: configmap_bin: true diff --git a/heat/values_overrides/netpol.yaml b/heat/values_overrides/netpol.yaml new file mode 100644 index 0000000000..e16bc97c5e --- /dev/null +++ b/heat/values_overrides/netpol.yaml @@ -0,0 +1,39 @@ +manifests: + network_policy: true +#NOTE(gagehugo): Test these once the netpol gate works +#network_policy: +# heat: +# ingress: +# - from: +# - podSelector: +# matchLabels: +# application: heat +# - podSelector: +# matchLabels: +# application: ingress +# - podSelector: +# matchLabels: +# application: horizon +# ports: +# - protocol: TCP +# port: 80 +# - protocol: TCP +# port: 8000 +# - protocol: TCP +# port: 8003 +# - protocol: TCP +# port: 8004 +# egress: +# - to: +# - podSelector: +# matchLabels: +# application: neutron +# - podSelector: +# matchLabels: +# application: nova +# - podSelector: +# matchLabels: +# application: glance +# - podSelector: +# matchLabels: +# application: cinder diff --git a/horizon/values.yaml b/horizon/values.yaml index c16b865997..7a66e051bb 100644 --- a/horizon/values.yaml +++ b/horizon/values.yaml @@ -2237,19 +2237,9 @@ endpoints: network_policy: horizon: ingress: - - from: - - podSelector: - matchLabels: - application: horizon - - podSelector: - matchLabels: - application: ingress - - namespaceSelector: - matchLabels: - name: kube-system - - ports: - - protocol: TCP - port: 80 + - {} + egress: + - {} manifests: configmap_bin: true diff --git a/horizon/values_overrides/netpol.yaml b/horizon/values_overrides/netpol.yaml new file mode 100644 index 0000000000..7a85753209 --- /dev/null +++ b/horizon/values_overrides/netpol.yaml @@ -0,0 +1,2 @@ +manifests: + network_policy: true diff --git a/keystone/values.yaml b/keystone/values.yaml index 6ff6daef67..c5134bfcbe 100644 --- a/keystone/values.yaml +++ b/keystone/values.yaml @@ -384,86 +384,12 @@ jobs: failed: 1 network_policy: - keystone: - ingress: - - from: - - podSelector: - matchLabels: - application: ceph - - podSelector: - matchLabels: - application: ingress - - podSelector: - matchLabels: - application: keystone - - podSelector: - matchLabels: - application: heat - - podSelector: - matchLabels: - application: glance - - podSelector: - matchLabels: - application: cinder - - podSelector: - matchLabels: - application: congress - - podSelector: - matchLabels: - application: barbican - - podSelector: - matchLabels: - application: ceilometer - - podSelector: - matchLabels: - application: horizon - - podSelector: - matchLabels: - application: ironic - - podSelector: - matchLabels: - application: magnum - - podSelector: - matchLabels: - application: mistral - - podSelector: - matchLabels: - application: nova - - podSelector: - matchLabels: - application: neutron - - podSelector: - matchLabels: - application: senlin - - podSelector: - matchLabels: - application: placement - - podSelector: - matchLabels: - application: prometheus-openstack-exporter - ports: - - protocol: TCP - port: 80 - - protocol: TCP - port: 443 - - protocol: TCP - port: 5000 - - protocol: TCP - port: 35357 - egress: - - to: - - namespaceSelector: - matchLabels: - name: ceph - - to: - - podSelector: - matchLabels: - application: ceph - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP + keystone: + ingress: + - {} + egress: + - {} + conf: security: | # diff --git a/keystone/values_overrides/netpol.yaml b/keystone/values_overrides/netpol.yaml new file mode 100644 index 0000000000..d80d31bed2 --- /dev/null +++ b/keystone/values_overrides/netpol.yaml @@ -0,0 +1,84 @@ +manifests: + network_policy: true +#NOTE(gagehugo): Test the below whitelist after netpol gate works +#network_policy: +# keystone: +# ingress: +# - from: +# - podSelector: +# matchLabels: +# application: ceph +# - podSelector: +# matchLabels: +# application: ingress +# - podSelector: +# matchLabels: +# application: keystone +# - podSelector: +# matchLabels: +# application: heat +# - podSelector: +# matchLabels: +# application: glance +# - podSelector: +# matchLabels: +# application: cinder +# - podSelector: +# matchLabels: +# application: congress +# - podSelector: +# matchLabels: +# application: barbican +# - podSelector: +# matchLabels: +# application: ceilometer +# - podSelector: +# matchLabels: +# application: horizon +# - podSelector: +# matchLabels: +# application: ironic +# - podSelector: +# matchLabels: +# application: magnum +# - podSelector: +# matchLabels: +# application: mistral +# - podSelector: +# matchLabels: +# application: nova +# - podSelector: +# matchLabels: +# application: neutron +# - podSelector: +# matchLabels: +# application: senlin +# - podSelector: +# matchLabels: +# application: placement +# - podSelector: +# matchLabels: +# application: prometheus-openstack-exporter +# ports: +# - protocol: TCP +# port: 80 +# - protocol: TCP +# port: 443 +# - protocol: TCP +# port: 5000 +# - protocol: TCP +# port: 35357 +# egress: +# - to: +# - namespaceSelector: +# matchLabels: +# name: ceph +# - to: +# - podSelector: +# matchLabels: +# application: ceph +# - ports: +# - port: 53 +# protocol: UDP +# - port: 53 +# protocol: TCP diff --git a/neutron/values_overrides/netpol.yaml b/neutron/values_overrides/netpol.yaml new file mode 100644 index 0000000000..7a85753209 --- /dev/null +++ b/neutron/values_overrides/netpol.yaml @@ -0,0 +1,2 @@ +manifests: + network_policy: true diff --git a/nova/values.yaml b/nova/values.yaml index 29512caaac..124e4ca15d 100644 --- a/nova/values.yaml +++ b/nova/values.yaml @@ -2489,22 +2489,6 @@ network_policy: - {} egress: - {} - - to: - - podSelector: - matchLabels: - application: ceph - - podSelector: - matchLabels: - application: ingress - - podSelector: - matchLabels: - application: openvswitch - - podSelector: - matchLabels: - application: libvirt - - podSelector: - matchLabels: - application: cinder placement: # TODO(lamt): Need to tighten this ingress for security. ingress: diff --git a/nova/values_overrides/netpol.yaml b/nova/values_overrides/netpol.yaml new file mode 100644 index 0000000000..7a85753209 --- /dev/null +++ b/nova/values_overrides/netpol.yaml @@ -0,0 +1,2 @@ +manifests: + network_policy: true diff --git a/tools/deployment/developer/common/049-lockdown.sh b/tools/deployment/common/lockdown-netpol.sh similarity index 100% rename from tools/deployment/developer/common/049-lockdown.sh rename to tools/deployment/common/lockdown-netpol.sh diff --git a/tools/deployment/common/openstack-exporter.sh b/tools/deployment/common/openstack-exporter.sh new file mode 100755 index 0000000000..8c6d07a550 --- /dev/null +++ b/tools/deployment/common/openstack-exporter.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright 2019 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Get the over-rides to use +export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${OSH_INFRA_PATH:="../openstack-helm-infra"}"}" +: ${OSH_EXTRA_HELM_ARGS_OSEXPORTER:="$(./tools/deployment/common/get-values-overrides.sh prometheus-openstack-exporter)"} + +#NOTE: Lint and package chart +make -C ${HELM_CHART_ROOT_PATH} prometheus-openstack-exporter + +: ${OSH_EXTRA_HELM_ARGS:=""} +helm upgrade --install prometheus-openstack-exporter ${HELM_CHART_ROOT_PATH}/prometheus-openstack-exporter \ + --namespace=openstack \ + ${OSH_EXTRA_HELM_ARGS} \ + ${OSH_EXTRA_HELM_ARGS_OSEXPORTER} + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh openstack + +#NOTE: Validate Deployment info +helm status prometheus-openstack-exporter diff --git a/tools/deployment/common/test-networkpolicy.sh b/tools/deployment/common/test-networkpolicy.sh index 04e7c337fa..b7f5db7759 100755 --- a/tools/deployment/common/test-networkpolicy.sh +++ b/tools/deployment/common/test-networkpolicy.sh @@ -15,14 +15,15 @@ # under the License. set -xe -# test_netpol(namespace, component, target_host, expected_result{fail,success}) +# test_netpol(namespace, application, component, target_host, expected_result{fail,success}) function test_netpol { NS=$1 - COMPONENT=$2 - HOST=$3 - STATUS=$4 - echo Testing connection from $COMPONENT to host $HOST with namespace $NS - POD=$(kubectl -n $NS get pod | grep $COMPONENT | grep Running | awk '{print $1}') + APP=$2 + COMPONENT=$3 + HOST=$4 + STATUS=$5 + echo Testing connection from $APP - $COMPONENT to host $HOST with namespace $NS + POD=$(kubectl -n $NS get pod -l application=$APP,component=$COMPONENT | grep Running | cut -f 1 -d " " | head -n 1) PID=$(sudo docker inspect --format '{{ .State.Pid }}' $(kubectl get pods --namespace $NS $POD -o jsonpath='{.status.containerStatuses[0].containerID}' | cut -c 10-21)) if [ "x${STATUS}" == "xfail" ]; then if ! sudo nsenter -t $PID -n wget --spider --timeout=5 --tries=1 $HOST ; then @@ -34,17 +35,30 @@ function test_netpol { sudo nsenter -t $PID -n wget --spider --timeout=5 --tries=1 $HOST fi } + +#NOTE(gagehugo): Enable the negative tests once the services policy is defined + +# General Netpol Tests # Doing negative tests -test_netpol openstack keystone-api heat-api.openstack.svc.cluster.local fail -test_netpol openstack keystone-api glance-api.openstack.svc.cluster.local fail -test_netpol openstack mariadb-server rabbitmq.openstack.svc.cluster.local:5672 fail -test_netpol openstack rabbitmq-rabbitmq memcached.openstack.svc.cluster.local:11211 fail -test_netpol openstack memcached mariadb.openstack.svc.cluster.local:3306 fail - +#test_netpol openstack mariadb server rabbitmq.openstack.svc.cluster.local:5672 fail +#test_netpol openstack rabbitmq-rabbitmq server memcached.openstack.svc.cluster.local:11211 fail +#test_netpol openstack memcached server mariadb.openstack.svc.cluster.local:3306 fail # Doing positive tests -test_netpol openstack keystone-api mariadb.openstack.svc.cluster.local:3306 success -test_netpol openstack keystone-api rabbitmq.openstack.svc.cluster.local:5672 success -test_netpol openstack heat-api mariadb.openstack.svc.cluster.local:3306 success -test_netpol openstack glance-api mariadb.openstack.svc.cluster.local:3306 success +test_netpol openstack keystone api mariadb.openstack.svc.cluster.local:3306 success +test_netpol openstack keystone api rabbitmq.openstack.svc.cluster.local:5672 success -echo Test successfully +if kubectl -n openstack get pod -l application=cinder | grep Running ; then +# Negative Cinder Tests + #test_netpol openstack keystone api cinder-api.openstack.svc.cluster.local fail +# Positive Cinder Tests + test_netpol openstack cinder api rabbitmq.openstack.svc.cluster.local:5672 success +else +# Negative Compute-Kit Tests + #test_netpol openstack keystone api heat-api.openstack.svc.cluster.local fail + #test_netpol openstack keystone api glance-api.openstack.svc.cluster.local fail +# Positive Compute-Kit Tests + test_netpol openstack heat api mariadb.openstack.svc.cluster.local:3306 success + test_netpol openstack glance api mariadb.openstack.svc.cluster.local:3306 success +fi + +echo Test Success diff --git a/zuul.d/jobs-openstack-helm.yaml b/zuul.d/jobs-openstack-helm.yaml index 94a1122aa0..ed21a9cdc7 100644 --- a/zuul.d/jobs-openstack-helm.yaml +++ b/zuul.d/jobs-openstack-helm.yaml @@ -266,6 +266,65 @@ - ./tools/deployment/developer/common/170-setup-gateway.sh - ./tools/deployment/developer/common/900-use-it.sh +- job: + name: openstack-helm-netpol-compute-kit + parent: openstack-helm-chart-deploy + timeout: 7200 + run: tools/gate/playbooks/osh-gate-runner.yaml + vars: + osh_params: + openstack_release: ocata + container_distro_name: ubuntu + container_distro_version: xenial + feature_gates: netpol + gate_scripts: + - ./tools/deployment/common/install-packages.sh + - ./tools/deployment/common/deploy-k8s.sh + - ./tools/deployment/common/setup-client.sh + - ./tools/deployment/component/common/ingress.sh + - ./tools/deployment/common/lockdown-netpol.sh + - ./tools/deployment/component/common/mariadb.sh + - ./tools/deployment/component/common/memcached.sh + - ./tools/deployment/component/common/rabbitmq.sh + - ./tools/deployment/component/nfs-provisioner/nfs-provisioner.sh + - ./tools/deployment/component/keystone/keystone.sh + - ./tools/deployment/component/heat/heat.sh + - ./tools/deployment/component/glance/glance.sh + - ./tools/deployment/component/compute-kit/openvswitch.sh + - ./tools/deployment/component/compute-kit/libvirt.sh + - ./tools/deployment/component/compute-kit/compute-kit.sh + - ./tools/deployment/developer/common/170-setup-gateway.sh + - ./tools/deployment/common/openstack-exporter.sh + - ./tools/deployment/developer/common/900-use-it.sh + - ./tools/deployment/common/test-networkpolicy.sh + +- job: + name: openstack-helm-netpol-cinder + parent: openstack-helm-chart-deploy + timeout: 7200 + run: tools/gate/playbooks/osh-gate-runner.yaml + vars: + osh_params: + openstack_release: ocata + container_distro_name: ubuntu + container_distro_version: xenial + feature_gates: netpol + gate_scripts: + - ./tools/deployment/common/install-packages.sh + - ./tools/deployment/common/deploy-k8s.sh + - ./tools/deployment/common/setup-client.sh + - ./tools/deployment/component/ceph/ceph.sh + - ./tools/deployment/component/ceph/ceph-ns-activate.sh + - ./tools/deployment/common/lockdown-netpol.sh + - ./tools/deployment/component/common/ingress.sh + - ./tools/deployment/component/common/mariadb.sh + - ./tools/deployment/component/common/memcached.sh + - ./tools/deployment/component/common/rabbitmq.sh + - ./tools/deployment/component/keystone/keystone.sh + - ./tools/deployment/component/cinder/cinder.sh + - ./tools/deployment/common/openstack-exporter.sh + - ./tools/deployment/common/test-networkpolicy.sh + - job: name: openstack-helm-multinode-temp parent: openstack-helm-functional-temp diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index a570a3bd82..6d99f44dff 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -42,6 +42,10 @@ - openstack-helm-horizon - openstack-helm-apparmor: voting: false + - openstack-helm-netpol-compute-kit: + voting: false + - openstack-helm-netpol-cinder: + voting: false gate: jobs: - openstack-helm-lint