From 88a85ae7d5289625f2200f445e313bc699d73381 Mon Sep 17 00:00:00 2001 From: Alan Meadows Date: Mon, 9 Jan 2017 07:49:44 -0800 Subject: [PATCH] Implement requested feedback in PR#91 - Split out osapi and metadata api to remove unnecessary security privileges - Prune dependencies - Fix static imagePullPolicies - Remove extraneous start.sh imported accidently from keystone chart --- nova/templates/bin/_start-osapi.sh.tpl | 21 ++++++ nova/templates/bin/_start.sh.tpl | 12 --- nova/templates/configmap-bin.yaml | 4 +- nova/templates/daemonset-compute.yaml | 9 --- ...-api.yaml => deployment-api-metadata.yaml} | 13 ++-- nova/templates/deployment-api-osapi.yaml | 75 +++++++++++++++++++ nova/templates/etc/_nova.conf.tpl | 2 +- nova/templates/job-db-sync.yaml | 4 +- nova/templates/job-init.yaml | 2 +- nova/templates/job-post.yaml | 2 +- ...service-api.yaml => service-metadata.yaml} | 4 +- nova/templates/service-osapi.yaml | 13 ++++ nova/values.yaml | 35 ++++----- 13 files changed, 136 insertions(+), 60 deletions(-) create mode 100644 nova/templates/bin/_start-osapi.sh.tpl delete mode 100644 nova/templates/bin/_start.sh.tpl rename nova/templates/{deployment-api.yaml => deployment-api-metadata.yaml} (88%) create mode 100644 nova/templates/deployment-api-osapi.yaml rename nova/templates/{service-api.yaml => service-metadata.yaml} (66%) create mode 100644 nova/templates/service-osapi.yaml diff --git a/nova/templates/bin/_start-osapi.sh.tpl b/nova/templates/bin/_start-osapi.sh.tpl new file mode 100644 index 0000000000..d1f0313299 --- /dev/null +++ b/nova/templates/bin/_start-osapi.sh.tpl @@ -0,0 +1,21 @@ +#!/bin/sh +# +# start nova-api-osapi service +# +# this helper script ensures our osapi service does not try to call iptables which requires privileged or NET_ADMIN privileges +# by stubbing in a fake iptables scripts + +echo </tmp/iptables +#!/bin/sh +# nova-api-metadata trys to run some iptables commands +# This enables the api-only container to run without NET_ADMIN privileges +true +EOF + +# make it executable and copy it over whatever iptables may be underneath in this image +chmod +x /tmp/iptables +cp -p /tmp/iptables /sbin/iptables +cp -p /tmp/iptables /sbin/iptables-restore +cp -p /tmp/iptables /sbin/iptables-save + +exec nova-api--config-file /etc/nova/nova.conf \ No newline at end of file diff --git a/nova/templates/bin/_start.sh.tpl b/nova/templates/bin/_start.sh.tpl deleted file mode 100644 index 7c802358bc..0000000000 --- a/nova/templates/bin/_start.sh.tpl +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -ex - -# link our keystone wsgi to apaches running config -ln -s /configmaps/wsgi-keystone.conf /etc/apache2/sites-enabled/wsgi-keystone.conf - -# Loading Apache2 ENV variables -source /etc/apache2/envvars -rm -rf /var/run/apache2/* -APACHE_DIR="apache2" - -apache2 -DFOREGROUND diff --git a/nova/templates/configmap-bin.yaml b/nova/templates/configmap-bin.yaml index 6c7d10cebd..f3d1fca6f3 100644 --- a/nova/templates/configmap-bin.yaml +++ b/nova/templates/configmap-bin.yaml @@ -7,8 +7,8 @@ data: {{ tuple "bin/_db-sync.sh.tpl" . | include "template" | indent 4 }} init.sh: | {{ tuple "bin/_init.sh.tpl" . | include "template" | indent 4 }} - start.sh: | -{{ tuple "bin/_start.sh.tpl" . | include "template" | indent 4 }} + start-osapi.sh: | +{{ tuple "bin/_start-osapi.sh.tpl" . | include "template" | indent 4 }} post.sh: | {{ tuple "bin/_post.sh.tpl" . | include "template" | indent 4 }} libvirt.sh: | diff --git a/nova/templates/daemonset-compute.yaml b/nova/templates/daemonset-compute.yaml index 53f4c6ce8b..60529cfbb0 100644 --- a/nova/templates/daemonset-compute.yaml +++ b/nova/templates/daemonset-compute.yaml @@ -59,15 +59,6 @@ spec: - nova-compute - --config-file - /etc/nova/nova.conf - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace volumeMounts: - name: novaconf mountPath: /etc/nova/nova.conf diff --git a/nova/templates/deployment-api.yaml b/nova/templates/deployment-api-metadata.yaml similarity index 88% rename from nova/templates/deployment-api.yaml rename to nova/templates/deployment-api-metadata.yaml index 4ce90155aa..a81444e5c5 100644 --- a/nova/templates/deployment-api.yaml +++ b/nova/templates/deployment-api-metadata.yaml @@ -46,17 +46,16 @@ spec: image: {{ .Values.image.api }} # https://bugs.launchpad.net/kolla-mesos/+bug/1546007 securityContext: - privileged: true + capabilities: + add: + - NET_ADMIN command: - - nova-api - - --config-file - - /etc/nova/nova.conf + - nova-api-metadata --config-file=/etc/nova/nova.conf ports: - - containerPort: {{ .Values.network.port.api }} - containerPort: {{ .Values.network.port.metadata }} readinessProbe: tcpSocket: - port: {{ .Values.network.port.api }} + port: {{ .Values.network.port.osapi }} volumeMounts: - name: novaconf mountPath: /etc/nova/nova.conf @@ -67,4 +66,4 @@ spec: name: nova-etc items: - key: nova.conf - path: nova.conf + path: nova.conf \ No newline at end of file diff --git a/nova/templates/deployment-api-osapi.yaml b/nova/templates/deployment-api-osapi.yaml new file mode 100644 index 0000000000..b9a50bbf8b --- /dev/null +++ b/nova/templates/deployment-api-osapi.yaml @@ -0,0 +1,75 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nova-osapi +spec: + replicas: {{ .Values.control_replicas }} + template: + metadata: + labels: + app: nova-osapi + annotations: + pod.beta.kubernetes.io/init-containers: '[ + { + "name": "init", + "image": "{{ .Values.image.entrypoint }}", + "imagePullPolicy": "{{ .Values.image.pull_policy }}", + "env": [ + { + "name": "NAMESPACE", + "value": "{{ .Release.Namespace }}" + }, + { + "name": "INTERFACE_NAME", + "value": "eth0" + }, + { + "name": "DEPENDENCY_SERVICE", + "value": "{{ include "joinListWithColon" .Values.dependencies.api.service }}" + }, + { + "name": "DEPENDENCY_JOBS", + "value": "{{ include "joinListWithColon" .Values.dependencies.api.jobs }}" + }, + { + "name": "COMMAND", + "value": "echo done" + } + ] + } + ]' + spec: + nodeSelector: + {{ .Values.labels.control_node_selector_key }}: {{ .Values.labels.control_node_selector_value }} + containers: + - name: nova-osapi + image: {{ .Values.image.api }} + command: + - bash + - /tmp/start-osapi.sh + ports: + - containerPort: {{ .Values.network.port.osapi }} + readinessProbe: + tcpSocket: + port: {{ .Values.network.port.osapi }} + volumeMounts: + - name: novaconf + mountPath: /etc/nova/nova.conf + subPath: nova.conf + volumeMounts: + - name: startsh + mountPath: /tmp/start-osapi.sh + subPath: start-osapi.sh + volumes: + - name: novaconf + configMap: + name: nova-etc + items: + - key: nova.conf + path: nova.conf + - name: startsh + configMap: + name: nova-bin + items: + - key: start-osapi.sh + path: start-osapi.sh diff --git a/nova/templates/etc/_nova.conf.tpl b/nova/templates/etc/_nova.conf.tpl index e3a46eef20..900e341ae1 100644 --- a/nova/templates/etc/_nova.conf.tpl +++ b/nova/templates/etc/_nova.conf.tpl @@ -9,7 +9,7 @@ force_config_drive = {{ .Values.nova.default.force_config_drive }} state_path = /var/lib/nova osapi_compute_listen = {{ .Values.network.ip_address }} -osapi_compute_listen_port = {{ .Values.network.port.api }} +osapi_compute_listen_port = {{ .Values.network.port.osapi }} osapi_compute_workers = {{ .Values.nova.default.osapi_workers }} workers = {{ .Values.nova.default.osapi_workers }} diff --git a/nova/templates/job-db-sync.yaml b/nova/templates/job-db-sync.yaml index 0e5c2102a0..013ad07569 100644 --- a/nova/templates/job-db-sync.yaml +++ b/nova/templates/job-db-sync.yaml @@ -10,7 +10,7 @@ spec: { "name": "init", "image": "{{ .Values.image.entrypoint }}", - "imagePullPolicy": "{{ .Values.image.pull_policy }}", + "imagePullPolicy": "{{ .Values.image.pull_policy }}", "env": [ { "name": "NAMESPACE", @@ -36,7 +36,7 @@ spec: containers: - name: nova-db-sync image: {{ .Values.image.db_sync }} - imagePullPolicy: Always + imagePullPolicy: {{ .Values.image.pull_policy }} command: - bash - /tmp/db-sync.sh diff --git a/nova/templates/job-init.yaml b/nova/templates/job-init.yaml index 16a2201ad8..70c873641e 100644 --- a/nova/templates/job-init.yaml +++ b/nova/templates/job-init.yaml @@ -36,7 +36,7 @@ spec: containers: - name: nova-init image: {{ .Values.image.init }} - imagePullPolicy: Always + imagePullPolicy: {{ .Values.image.pull_policy }} command: - bash - /tmp/init.sh diff --git a/nova/templates/job-post.yaml b/nova/templates/job-post.yaml index 48b0f3e52d..2c6fd237eb 100644 --- a/nova/templates/job-post.yaml +++ b/nova/templates/job-post.yaml @@ -36,7 +36,7 @@ spec: containers: - name: nova-post image: {{ .Values.image.post }} - imagePullPolicy: Always + imagePullPolicy: {{ .Values.image.pull_policy }} command: - bash - /tmp/post.sh diff --git a/nova/templates/service-api.yaml b/nova/templates/service-metadata.yaml similarity index 66% rename from nova/templates/service-api.yaml rename to nova/templates/service-metadata.yaml index 6aa4144d2a..0e0bac07f6 100644 --- a/nova/templates/service-api.yaml +++ b/nova/templates/service-metadata.yaml @@ -1,11 +1,9 @@ apiVersion: v1 kind: Service metadata: - name: nova-api + name: nova-metadata spec: ports: - - name: nova-api - port: {{ .Values.network.port.api }} - name: nova-metadata port: {{ .Values.network.port.metadata }} selector: diff --git a/nova/templates/service-osapi.yaml b/nova/templates/service-osapi.yaml new file mode 100644 index 0000000000..3821bdd633 --- /dev/null +++ b/nova/templates/service-osapi.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + # alanmeadows(TODO): refactor to nova-osapi + # which requires updating other charts + # dependencies + name: nova-api +spec: + ports: + - name: nova-osapi + port: {{ .Values.network.port.osapi }} + selector: + app: nova-osapi diff --git a/nova/values.yaml b/nova/values.yaml index 2530bc1c06..d4f315f94a 100644 --- a/nova/values.yaml +++ b/nova/values.yaml @@ -13,16 +13,16 @@ control_replicas: 1 compute_replicas: 1 image: - init: quay.io/stackanetes/stackanetes-kolla-toolbox:barcelona - db_sync: quay.io/stackanetes/stackanetes-nova-api:barcelona - api: quay.io/stackanetes/stackanetes-nova-api:barcelona - conductor: quay.io/stackanetes/stackanetes-nova-conductor:barcelona - scheduler: quay.io/stackanetes/stackanetes-nova-scheduler:barcelona - novncproxy: quay.io/stackanetes/stackanetes-nova-novncproxy:barcelona - consoleauth: quay.io/stackanetes/stackanetes-nova-consoleauth:barcelona - compute: quay.io/stackanetes/stackanetes-nova-compute:barcelona - libvirt: quay.io/stackanetes/stackanetes-nova-libvirt:barcelona - post: quay.io/stackanetes/stackanetes-kolla-toolbox:barcelona + init: quay.io/stackanetes/stackanetes-kolla-toolbox:newton + db_sync: quay.io/stackanetes/stackanetes-nova-api:newton + api: quay.io/stackanetes/stackanetes-nova-api:newton + conductor: quay.io/stackanetes/stackanetes-nova-conductor:newton + scheduler: quay.io/stackanetes/stackanetes-nova-scheduler:newton + novncproxy: quay.io/stackanetes/stackanetes-nova-novncproxy:newton + consoleauth: quay.io/stackanetes/stackanetes-nova-consoleauth:newton + compute: quay.io/stackanetes/stackanetes-nova-compute:newton + libvirt: quay.io/stackanetes/stackanetes-nova-libvirt:newton + post: quay.io/stackanetes/stackanetes-kolla-toolbox:newton entrypoint: quay.io/stackanetes/kubernetes-entrypoint:v0.1.0 pull_policy: "IfNotPresent" @@ -40,7 +40,7 @@ network: other_domains: "" port: - api: 8774 + osapi: 8774 metadata: 8775 novncproxy: 6080 @@ -51,7 +51,6 @@ nova: metadata_workers: 8 conductor_workers: 8 force_config_drive: True - drain_timeout: 60 database: address: "mariadb" @@ -103,7 +102,6 @@ memcached: dependencies: api: jobs: - - mariadb-seed - keystone-db-sync - nova-init - nova-db-sync @@ -121,6 +119,7 @@ dependencies: - nova-init - keystone-init - mariadb-seed + - keystone-db-sync service: - mariadb post: @@ -130,7 +129,7 @@ dependencies: - mariadb-seed service: - mariadb - - nova-api + - keystone-api init: jobs: - mariadb-seed @@ -138,9 +137,7 @@ dependencies: - mariadb compute: jobs: - - nova-init - nova-post - - nova-db-sync service: - keystone-api - nova-api @@ -164,17 +161,11 @@ dependencies: - mariadb scheduler: jobs: - - mariadb-seed - - keystone-db-sync - - nova-init - nova-db-sync service: - mariadb conductor: jobs: - - mariadb-seed - - keystone-db-sync - - nova-init - nova-db-sync service: - mariadb