openstack/openstack-helm
Code Issues Proposed changes

Add network policy nonvoting checks

Browse Source 
This change adds two network policy zuul checks, one for the compute-kit,
and one for cinder/ceph, to test network policy for each OpenStack
service. These checks will be non-voting initially.

The network policy rules for each service will initially allow all
traffic. These ingress/egress rules will be defined in future changes
to only explicitly allow traffic between services that are explicitly
allowed to communicate, other traffic will be denied.

Depends-On: https://review.opendev.org/#/c/685130/

Change-Id: Ide2998ebb2af2832f24ca7abc398a82e4a6d70e3
This commit is contained in:
Gage Hugo 2019-09-11 11:56:08 -05:00
parent b4d673a90e
commit c3e085b800
17 changed files with 308 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cinder/values_overrides/netpol.yaml Normal file
View File

@ -0,0 +1,2 @@
manifests:
network_policy: true

30
glance/values.yaml
View File

@ -88,35 +88,7 @@ ceph_client:
network_policy:
glance:
ingress:
- from:
- podSelector:
matchLabels:
application: glance
- podSelector:
matchLabels:
application: nova
- podSelector:
matchLabels:
application: horizon
- podSelector:
matchLabels:
application: ingress
- podSelector:
matchLabels:
application: heat
- podSelector:
matchLabels:
application: ironic
- podSelector:
matchLabels:
application: cinder
ports:
- protocol: TCP
port: 80
- protocol: TCP
port: 9191
- protocol: TCP
port: 9292
- {}
egress:
- {}

35
glance/values_overrides/netpol.yaml Normal file
View File

@ -0,0 +1,35 @@
manifests:
network_policy: true
#NOTE(gagehugo): Test this whitelist when the netpol gate works
#network_policy:
# glance:
# ingress:
# - from:
# - podSelector:
# matchLabels:
# application: glance
# - podSelector:
# matchLabels:
# application: nova
# - podSelector:
# matchLabels:
# application: horizon
# - podSelector:
# matchLabels:
# application: ingress
# - podSelector:
# matchLabels:
# application: heat
# - podSelector:
# matchLabels:
# application: ironic
# - podSelector:
# matchLabels:
# application: cinder
# ports:
# - protocol: TCP
# port: 80
# - protocol: TCP
# port: 9191
# - protocol: TCP
# port: 9292

34
heat/values.yaml
View File

@ -1249,39 +1249,9 @@ pod:
network_policy:
heat:
ingress:
- from:
- podSelector:
matchLabels:
application: heat
- podSelector:
matchLabels:
application: ingress
- podSelector:
matchLabels:
application: horizon
ports:
- protocol: TCP
port: 80
- protocol: TCP
port: 8000
- protocol: TCP
port: 8003
- protocol: TCP
port: 8004
- {}
egress:
- to:
- podSelector:
matchLabels:
application: neutron
- podSelector:
matchLabels:
application: nova
- podSelector:
matchLabels:
application: glance
- podSelector:
matchLabels:
application: cinder
- {}
manifests:
configmap_bin: true

39
heat/values_overrides/netpol.yaml Normal file
View File

@ -0,0 +1,39 @@
manifests:
network_policy: true
#NOTE(gagehugo): Test these once the netpol gate works
#network_policy:
# heat:
# ingress:
# - from:
# - podSelector:
# matchLabels:
# application: heat
# - podSelector:
# matchLabels:
# application: ingress
# - podSelector:
# matchLabels:
# application: horizon
# ports:
# - protocol: TCP
# port: 80
# - protocol: TCP
# port: 8000
# - protocol: TCP
# port: 8003
# - protocol: TCP
# port: 8004
# egress:
# - to:
# - podSelector:
# matchLabels:
# application: neutron
# - podSelector:
# matchLabels:
# application: nova
# - podSelector:
# matchLabels:
# application: glance
# - podSelector:
# matchLabels:
# application: cinder

16
horizon/values.yaml
View File

@ -2237,19 +2237,9 @@ endpoints:
network_policy:
horizon:
ingress:
- from:
- podSelector:
matchLabels:
application: horizon
- podSelector:
matchLabels:
application: ingress
- namespaceSelector:
matchLabels:
name: kube-system
- ports:
- protocol: TCP
port: 80
- {}
egress:
- {}
manifests:
configmap_bin: true

2
horizon/values_overrides/netpol.yaml Normal file
View File

@ -0,0 +1,2 @@
manifests:
network_policy: true

86
keystone/values.yaml
View File

@ -384,86 +384,12 @@ jobs:
failed: 1
network_policy:
keystone:
ingress:
- from:
- podSelector:
matchLabels:
application: ceph
- podSelector:
matchLabels:
application: ingress
- podSelector:
matchLabels:
application: keystone
- podSelector:
matchLabels:
application: heat
- podSelector:
matchLabels:
application: glance
- podSelector:
matchLabels:
application: cinder
- podSelector:
matchLabels:
application: congress
- podSelector:
matchLabels:
application: barbican
- podSelector:
matchLabels:
application: ceilometer
- podSelector:
matchLabels:
application: horizon
- podSelector:
matchLabels:
application: ironic
- podSelector:
matchLabels:
application: magnum
- podSelector:
matchLabels:
application: mistral
- podSelector:
matchLabels:
application: nova
- podSelector:
matchLabels:
application: neutron
- podSelector:
matchLabels:
application: senlin
- podSelector:
matchLabels:
application: placement
- podSelector:
matchLabels:
application: prometheus-openstack-exporter
ports:
- protocol: TCP
port: 80
- protocol: TCP
port: 443
- protocol: TCP
port: 5000
- protocol: TCP
port: 35357
egress:
- to:
- namespaceSelector:
matchLabels:
name: ceph
- to:
- podSelector:
matchLabels:
application: ceph
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
keystone:
ingress:
- {}
egress:
- {}
conf:
security: |
#

84
keystone/values_overrides/netpol.yaml Normal file
View File

@ -0,0 +1,84 @@
manifests:
network_policy: true
#NOTE(gagehugo): Test the below whitelist after netpol gate works
#network_policy:
# keystone:
# ingress:
# - from:
# - podSelector:
# matchLabels:
# application: ceph
# - podSelector:
# matchLabels:
# application: ingress
# - podSelector:
# matchLabels:
# application: keystone
# - podSelector:
# matchLabels:
# application: heat
# - podSelector:
# matchLabels:
# application: glance
# - podSelector:
# matchLabels:
# application: cinder
# - podSelector:
# matchLabels:
# application: congress
# - podSelector:
# matchLabels:
# application: barbican
# - podSelector:
# matchLabels:
# application: ceilometer
# - podSelector:
# matchLabels:
# application: horizon
# - podSelector:
# matchLabels:
# application: ironic
# - podSelector:
# matchLabels:
# application: magnum
# - podSelector:
# matchLabels:
# application: mistral
# - podSelector:
# matchLabels:
# application: nova
# - podSelector:
# matchLabels:
# application: neutron
# - podSelector:
# matchLabels:
# application: senlin
# - podSelector:
# matchLabels:
# application: placement
# - podSelector:
# matchLabels:
# application: prometheus-openstack-exporter
# ports:
# - protocol: TCP
# port: 80
# - protocol: TCP
# port: 443
# - protocol: TCP
# port: 5000
# - protocol: TCP
# port: 35357
# egress:
# - to:
# - namespaceSelector:
# matchLabels:
# name: ceph
# - to:
# - podSelector:
# matchLabels:
# application: ceph
# - ports:
# - port: 53
# protocol: UDP
# - port: 53
# protocol: TCP

2
neutron/values_overrides/netpol.yaml Normal file
View File

@ -0,0 +1,2 @@
manifests:
network_policy: true

16
nova/values.yaml
View File

@ -2489,22 +2489,6 @@ network_policy:
- {}
egress:
- {}
- to:
- podSelector:
matchLabels:
application: ceph
- podSelector:
matchLabels:
application: ingress
- podSelector:
matchLabels:
application: openvswitch
- podSelector:
matchLabels:
application: libvirt
- podSelector:
matchLabels:
application: cinder
placement:
# TODO(lamt): Need to tighten this ingress for security.
ingress:

2
nova/values_overrides/netpol.yaml Normal file
View File

@ -0,0 +1,2 @@
manifests:
network_policy: true

0
tools/deployment/developer/common/049-lockdown.sh → tools/deployment/common/lockdown-netpol.sh
View File

36
tools/deployment/common/openstack-exporter.sh Executable file
View File

@ -0,0 +1,36 @@
#!/bin/bash
# Copyright 2019 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Get the over-rides to use
export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${OSH_INFRA_PATH:="../openstack-helm-infra"}"}"
: ${OSH_EXTRA_HELM_ARGS_OSEXPORTER:="$(./tools/deployment/common/get-values-overrides.sh prometheus-openstack-exporter)"}
#NOTE: Lint and package chart
make -C ${HELM_CHART_ROOT_PATH} prometheus-openstack-exporter
: ${OSH_EXTRA_HELM_ARGS:=""}
helm upgrade --install prometheus-openstack-exporter ${HELM_CHART_ROOT_PATH}/prometheus-openstack-exporter \
--namespace=openstack \
${OSH_EXTRA_HELM_ARGS} \
${OSH_EXTRA_HELM_ARGS_OSEXPORTER}
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh openstack
#NOTE: Validate Deployment info
helm status prometheus-openstack-exporter

48
tools/deployment/common/test-networkpolicy.sh
View File

@ -15,14 +15,15 @@
# under the License.
set -xe
# test_netpol(namespace, component, target_host, expected_result{fail,success})
# test_netpol(namespace, application, component, target_host, expected_result{fail,success})
function test_netpol {
NS=$1
COMPONENT=$2
HOST=$3
STATUS=$4
echo Testing connection from $COMPONENT to host $HOST with namespace $NS
POD=$(kubectl -n $NS get pod | grep $COMPONENT | grep Running | awk '{print $1}')
APP=$2
COMPONENT=$3
HOST=$4
STATUS=$5
echo Testing connection from $APP - $COMPONENT to host $HOST with namespace $NS
POD=$(kubectl -n $NS get pod -l application=$APP,component=$COMPONENT | grep Running | cut -f 1 -d " " | head -n 1)
PID=$(sudo docker inspect --format '{{ .State.Pid }}' $(kubectl get pods --namespace $NS $POD -o jsonpath='{.status.containerStatuses[0].containerID}' | cut -c 10-21))
if [ "x${STATUS}" == "xfail" ]; then
if ! sudo nsenter -t $PID -n wget --spider --timeout=5 --tries=1 $HOST ; then
@ -34,17 +35,30 @@ function test_netpol {
sudo nsenter -t $PID -n wget --spider --timeout=5 --tries=1 $HOST
fi
}
#NOTE(gagehugo): Enable the negative tests once the services policy is defined
# General Netpol Tests
# Doing negative tests
test_netpol openstack keystone-api heat-api.openstack.svc.cluster.local fail
test_netpol openstack keystone-api glance-api.openstack.svc.cluster.local fail
test_netpol openstack mariadb-server rabbitmq.openstack.svc.cluster.local:5672 fail
test_netpol openstack rabbitmq-rabbitmq memcached.openstack.svc.cluster.local:11211 fail
test_netpol openstack memcached mariadb.openstack.svc.cluster.local:3306 fail
#test_netpol openstack mariadb server rabbitmq.openstack.svc.cluster.local:5672 fail
#test_netpol openstack rabbitmq-rabbitmq server memcached.openstack.svc.cluster.local:11211 fail
#test_netpol openstack memcached server mariadb.openstack.svc.cluster.local:3306 fail
# Doing positive tests
test_netpol openstack keystone-api mariadb.openstack.svc.cluster.local:3306 success
test_netpol openstack keystone-api rabbitmq.openstack.svc.cluster.local:5672 success
test_netpol openstack heat-api mariadb.openstack.svc.cluster.local:3306 success
test_netpol openstack glance-api mariadb.openstack.svc.cluster.local:3306 success
test_netpol openstack keystone api mariadb.openstack.svc.cluster.local:3306 success
test_netpol openstack keystone api rabbitmq.openstack.svc.cluster.local:5672 success
echo Test successfully
if kubectl -n openstack get pod -l application=cinder | grep Running ; then
# Negative Cinder Tests
#test_netpol openstack keystone api cinder-api.openstack.svc.cluster.local fail
# Positive Cinder Tests
test_netpol openstack cinder api rabbitmq.openstack.svc.cluster.local:5672 success
else
# Negative Compute-Kit Tests
#test_netpol openstack keystone api heat-api.openstack.svc.cluster.local fail
#test_netpol openstack keystone api glance-api.openstack.svc.cluster.local fail
# Positive Compute-Kit Tests
test_netpol openstack heat api mariadb.openstack.svc.cluster.local:3306 success
test_netpol openstack glance api mariadb.openstack.svc.cluster.local:3306 success
fi
echo Test Success

59
zuul.d/jobs-openstack-helm.yaml
View File

@ -266,6 +266,65 @@
- ./tools/deployment/developer/common/170-setup-gateway.sh
- ./tools/deployment/developer/common/900-use-it.sh
- job:
name: openstack-helm-netpol-compute-kit
parent: openstack-helm-chart-deploy
timeout: 7200
run: tools/gate/playbooks/osh-gate-runner.yaml
vars:
osh_params:
openstack_release: ocata
container_distro_name: ubuntu
container_distro_version: xenial
feature_gates: netpol
gate_scripts:
- ./tools/deployment/common/install-packages.sh
- ./tools/deployment/common/deploy-k8s.sh
- ./tools/deployment/common/setup-client.sh
- ./tools/deployment/component/common/ingress.sh
- ./tools/deployment/common/lockdown-netpol.sh
- ./tools/deployment/component/common/mariadb.sh
- ./tools/deployment/component/common/memcached.sh
- ./tools/deployment/component/common/rabbitmq.sh
- ./tools/deployment/component/nfs-provisioner/nfs-provisioner.sh
- ./tools/deployment/component/keystone/keystone.sh
- ./tools/deployment/component/heat/heat.sh
- ./tools/deployment/component/glance/glance.sh
- ./tools/deployment/component/compute-kit/openvswitch.sh
- ./tools/deployment/component/compute-kit/libvirt.sh
- ./tools/deployment/component/compute-kit/compute-kit.sh
- ./tools/deployment/developer/common/170-setup-gateway.sh
- ./tools/deployment/common/openstack-exporter.sh
- ./tools/deployment/developer/common/900-use-it.sh
- ./tools/deployment/common/test-networkpolicy.sh
- job:
name: openstack-helm-netpol-cinder
parent: openstack-helm-chart-deploy
timeout: 7200
run: tools/gate/playbooks/osh-gate-runner.yaml
vars:
osh_params:
openstack_release: ocata
container_distro_name: ubuntu
container_distro_version: xenial
feature_gates: netpol
gate_scripts:
- ./tools/deployment/common/install-packages.sh
- ./tools/deployment/common/deploy-k8s.sh
- ./tools/deployment/common/setup-client.sh
- ./tools/deployment/component/ceph/ceph.sh
- ./tools/deployment/component/ceph/ceph-ns-activate.sh
- ./tools/deployment/common/lockdown-netpol.sh
- ./tools/deployment/component/common/ingress.sh
- ./tools/deployment/component/common/mariadb.sh
- ./tools/deployment/component/common/memcached.sh
- ./tools/deployment/component/common/rabbitmq.sh
- ./tools/deployment/component/keystone/keystone.sh
- ./tools/deployment/component/cinder/cinder.sh
- ./tools/deployment/common/openstack-exporter.sh
- ./tools/deployment/common/test-networkpolicy.sh
- job:
name: openstack-helm-multinode-temp
parent: openstack-helm-functional-temp

4
zuul.d/project.yaml
View File

@ -42,6 +42,10 @@
- openstack-helm-horizon
- openstack-helm-apparmor:
voting: false
- openstack-helm-netpol-compute-kit:
voting: false
- openstack-helm-netpol-cinder:
voting: false
gate:
jobs:
- openstack-helm-lint