Add Egress Helm-toolkit function & enforce the nework policy at OSH-INFRA

This PS implements the helm toolkit function to generate the
Egress in kubernetes network policy manifest based on overrideable values.
It also enbale the K8s network policy at Osh-infra gate.

Change-Id: Icbe2a18c98dba795d15398dcdcac64228f6a7b4c

ceph-rgw/templates/network_policy.yaml

@@ -0,0 +1,21 @@
+{{/*
+Copyright 2017-2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+{{- if .Values.manifests.network_policy -}}
+{{- $netpol_opts := dict "envAll" . "name" "application" "label" "ceph-rgw" -}}
+{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
+{{- $netpol_opts := dict "envAll" . "name" "application" "label" "ceph" }}
+{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
+{{- end -}}

ceph-rgw/values.yaml

@@ -474,6 +474,18 @@ endpoints:
       mon:
         default: 6789
 
+network_policy:
+  ceph-rgw:
+    ingress:
+      - {}
+    egress:
+      - {}
+  ceph:
+    ingress:
+      - {}
+    egress:
+      - {}
+
 
 manifests:
   configmap_ceph_templates: true
@@ -483,6 +495,7 @@ manifests:
   configmap_etc: true
   deployment_rgw: true
   ingress_rgw: true
+  network_policy: false
   job_ceph_rgw_storage_init: true
   job_image_repo_sync: true
   job_ks_endpoints: true

elasticsearch/values.yaml

@@ -586,6 +586,21 @@ endpoints:
       api:
         default: 8088
         public: 80
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
 
 monitoring:
   prometheus:
@@ -606,6 +621,13 @@ network:
       enabled: false
       port: 30920
 
+network_policy:
+  elasticsearch:
+    ingress:
+      - {}
+    egress:
+      - {}
+
 storage:
   enabled: true
   pvc:
@@ -623,6 +645,7 @@ manifests:
   deployment_client: true
   deployment_master: true
   ingress: true
+  network_policy: false
   job_image_repo_sync: true
   job_snapshot_repository: true
   job_s3_user: true

fluent-logging/values.yaml

@@ -481,6 +481,43 @@ endpoints:
     port:
       metrics:
         default: 9309
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
+
+network_policy:
+  fluentbit:
+    ingress:
+      - {}
+    egress:
+      - {}
+  fluentd:
+    ingress:
+      - {}
+    egress:
+      - {}
+  fluent:
+    ingress:
+      - {}
+    egress:
+      - {}
+  fluent-logging:
+    ingress:
+      - {}
+    egress:
+      - {}
 
 monitoring:
   prometheus:

grafana/values.yaml

@@ -232,6 +232,26 @@ endpoints:
     port:
       ldap:
         default: 389
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
+
+network_policy:
+  grafana:
+    egress:
+      - {}
 
 dependencies:
   dynamic:

helm-toolkit/templates/manifests/_network_policy.tpl

@@ -11,12 +11,28 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */}}
-
 {{/*
 abstract: |
   Creates a network policy manifest for services.
 values: |
-  network_policy:
+endpoints:
+  kube_dns:
+    namespace: kube-system
+    name: kubernetes-dns
+    hosts:
+      default: kube-dns
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme: http
+    port:
+      dns_tcp:
+        default: 53
+      dns:
+        default: 53
+        protocol: UDP
+network_policy:
     myLabel:
       ingress:
         - from:
@@ -26,6 +42,14 @@ values: |
           ports:
           - protocol: TCP
             port: 80
+      egress:
+        - to:
+          - namespaceSelector:
+              matchLabels:
+                name: default
+          - namespaceSelector:
+              matchLabels:
+                name: kube-public
 usage: |
   {{ dict "envAll" . "name" "application" "label" "myLabel" | include "helm-toolkit.manifests.kubernetes_network_policy" }}
 return: |
@@ -51,7 +75,25 @@ return: |
         - protocol: TCP
           port: 80
     egress:
-      - {}
+      - to:
+          - podSelector:
+              matchLabels:
+                application: kube-dns
+          - namespaceSelector:
+              matchLabels:
+                 name: kube-system
+          ports:
+          - protocol: TCP
+            port: 53
+          - protocol: UDP
+            port: 53
+      - to:
+          - namespaceSelector:
+              matchLabels:
+                 name: kube-public
+          - namespaceSelector:
+              matchLabels:
+                 name: default
 */}}
 
 {{- define "helm-toolkit.manifests.kubernetes_network_policy" -}}
@@ -76,8 +118,47 @@ spec:
     matchLabels:
       {{ $name }}: {{ $label }}
   egress:
-    - {}
+{{- range $key, $value := $envAll.Values.endpoints }}
+{{- if kindIs "map" $value }}
+    - to:
+{{- if index $value "namespace" }}
+      - namespaceSelector:
+          matchLabels:
+            name: {{ index $value "namespace" }}
+{{- else if index $value "hosts" }}
+{{- $defaultValue := index $value "hosts" "internal" }}
+{{- if hasKey (index $value "hosts") "internal" }}
+{{- $a := split "-" $defaultValue }}
+      - podSelector:
+          matchLabels:
+            application: {{ printf "%s" (index $a._0) | default $defaultValue }}
+{{- else }}
+{{- $defaultValue := index $value "hosts" "default" }}
+{{- $a := split "-" $defaultValue }}
+      - podSelector:
+          matchLabels:
+            application: {{ printf "%s" (index $a._0) | default $defaultValue }}
+{{- end }}
+{{- end }}
+      ports:
+{{- if index $value "port" }}
+{{- range $k, $v := index $value "port" }}
+{{- if $k }}
+{{- range $pk, $pv := $v }}
+{{- if (ne $pk "protocol") }}
+      - port: {{ $pv }}
+        protocol: {{ $v.protocol | default "TCP" }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
 {{- if hasKey (index $envAll.Values "network_policy") $label }}
+{{- if index $envAll.Values.network_policy $label "egress" }}
+{{ index $envAll.Values.network_policy $label "egress" | toYaml | indent 4 }}
+{{- end }}
 {{- if index $envAll.Values.network_policy $label "ingress" }}
   ingress:
 {{ index $envAll.Values.network_policy $label "ingress" | toYaml | indent 4 }}

ingress/values.yaml

@@ -198,11 +198,28 @@ endpoints:
       dns:
         default: 53
         protocol: UDP
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
 
 network_policy:
   ingress:
     ingress:
       - {}
+    egress:
+      - {}
 
 conf:
   controller:

kibana/values.yaml

@@ -294,7 +294,26 @@ endpoints:
     port:
       ldap:
         default: 389
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
 
+network_policy:
+  kibana:
+    egress:
+      - {}
 network:
   kibana:
     ingress:

ldap/values.yaml

@@ -146,11 +146,28 @@ endpoints:
     port:
       ldap:
         default: 389
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
 
 network_policy:
   ldap:
     ingress:
       - {}
+    egress:
+      - {}
 
 data:
   sample: |

libvirt/values.yaml

@@ -57,11 +57,26 @@ endpoints:
     port:
       registry:
         node: 5000
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
 
 network_policy:
   libvirt:
     ingress:
       - {}
+    egress:
+      - {}
 
 ceph_client:
   configmap: ceph-etc

mariadb/values.yaml

@@ -275,6 +275,21 @@ endpoints:
       dns:
         default: 53
         protocol: UDP
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
 
 network_policy:
   mariadb:

memcached/values.yaml

@@ -98,6 +98,21 @@ endpoints:
       dns:
         default: 53
         protocol: UDP
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
 
 network_policy:
   memcached:

nagios/values.yaml

@@ -168,6 +168,21 @@ endpoints:
         default: 9283
     scheme:
       default: http
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
 
 network:
   nagios:

openvswitch/values.yaml

@@ -90,6 +90,19 @@ endpoints:
     port:
       registry:
         node: 5000
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
 
 network_policy:
   openvswitch:

postgresql/values.yaml

@@ -198,10 +198,32 @@ endpoints:
     port:
       metrics:
         default: 9187
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
+
+network_policy:
+  postgresql:
+    ingress:
+      - {}
+
 
 manifests:
   configmap_bin: true
   job_image_repo_sync: true
+  network_policy: false
   secret_admin: true
   service: true
   statefulset: true

prometheus/values.yaml

@@ -167,6 +167,21 @@ endpoints:
     port:
       ldap:
         default: 389
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
 
 dependencies:
   dynamic:

rabbitmq/values.yaml

@@ -265,6 +265,21 @@ endpoints:
       dns:
         default: 53
         protocol: UDP
+  #NOTE(tp6510): these endpoints allow for things like DNS lookups and apiserver access.
+  # They are using to enable the Egress K8s network policy.
+  k8s:
+    port:
+      api:
+        default: 6443
+        internal: 5000
+      http:
+        default: 80
+  default:
+    namespace: default
+  kube_system:
+    namespace: kube-system
+  kube_public:
+    namespace: kube-public
 
 network_policy:
   rabbitmq:

tools/deployment/network-policy/040-ldap.sh

@@ -23,28 +23,29 @@ tee /tmp/ldap.yaml <<EOF
 manifests:
   network_policy: true
 network_policy:
-  ingress:
+  ldap:
-    - from:
+    ingress:
-      - podSelector:
+      - from:
-          matchLabels:
+        - podSelector:
-            application: ldap
+            matchLabels:
-      - podSelector:
+              application: ldap
-          matchLabels:
+        - podSelector:
-            application: grafana
+            matchLabels:
-      - podSelector:
+              application: grafana
-          matchLabels:
+        - podSelector:
-            application: nagios
+            matchLabels:
-      - podSelector:
+              application: nagios
-          matchLabels:
+        - podSelector:
-            application: elasticsearch
+            matchLabels:
-      - podSelector:
+              application: elasticsearch
-          matchLabels:
+        - podSelector:
-            application: kibana
+            matchLabels:
-      ports:
+              application: kibana
-      - protocol: TCP
+        ports:
-        port: 389
+        - protocol: TCP
-      - protocol: TCP
+          port: 389
-        port: 80
+        - protocol: TCP
+          port: 80
 EOF
 
 #NOTE: Deploy command

tools/deployment/network-policy/045-mariadb.sh

@@ -39,6 +39,11 @@ network_policy:
           port: 4567
         - protocol: TCP
           port: 80
+    egress:
+      - from:
+        - podSelector:
+            matchLabels:
+              application: ingress
 EOF
 
 #NOTE: Deploy command

tools/deployment/network-policy/120-elasticsearch.sh

@@ -28,16 +28,11 @@ pod:
   replicas:
     data: 1
     master: 2
-manifests:
-  network_policy: true
-network_policy:
-  elasticsearch:
-    ingress:
-      - from:
 EOF
 
 helm upgrade --install elasticsearch ./elasticsearch \
     --namespace=osh-infra \
+    --set manifests.network_policy=true \
     --values=/tmp/elasticsearch.yaml
 
 #NOTE: Wait for deploy

tools/deployment/network-policy/130-fluent-logging.sh

@@ -19,29 +19,10 @@ set -xe
 #NOTE: Lint and package chart
 make fluent-logging
 
-tee /tmp/fluent-logging.yaml <<EOF
-manifests:
-  network_policy: true
-network_policy:
-  fluentbit:
-    ingress:
-      - from:
-  fluentd:
-    ingress:
-      - from:
-  fluent:
-    ingress:
-      - from:
-  fluent-logging:
-    ingress:
-      - from:
-EOF
-
-
 #NOTE: Deploy command
 helm upgrade --install fluent-logging ./fluent-logging \
     --namespace=osh-infra \
-    --values=/tmp/fluent-logging.yaml \
+    --set manifests.network_policy=true \
     --set pod.replicas.fluentd=1
 
 #NOTE: Wait for deploy