fix(neutron): fixes tls issue

Updated neutron to use an Nginx sidecar to terminate internal TLS rather
than using Apache with a separate RPC servers. Multiple RPC servers (in
sidecar) causes communication issues with RabbitMQ causing expected
errors.

Change-Id: Iaa6d3d64b730a54b1b85a338517bcb5be1842bda
Signed-off-by: Tin Lam <tin@irrational.io>
This commit is contained in:
Tin Lam 2020-10-20 21:04:13 -05:00
parent 0699a2c162
commit 6895a5ba7a
9 changed files with 128 additions and 251 deletions

View File

@ -14,7 +14,7 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Neutron
name: neutron
version: 0.1.1
version: 0.1.2
home: https://docs.openstack.org/neutron/latest/
icon: https://www.openstack.org/themes/openstack/images/project-mascots/Neutron/OpenStack_Project_Neutron_vertical.png
sources:

View File

@ -1,43 +0,0 @@
#!/bin/bash
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
COMMAND="${@:-start}"
function start () {
exec neutron-rpc-server \
--config-file /etc/neutron/neutron.conf \
{{- if ( has "tungstenfabric" .Values.network.backend ) }}
--config-file /etc/neutron/plugins/tungstenfabric/tf_plugin.ini
{{- else }}
--config-file /etc/neutron/plugins/ml2/ml2_conf.ini
{{- end }}
{{- if .Values.conf.plugins.taas.taas.enabled }} \
--config-file /etc/neutron/taas_plugin.ini
{{- end }}
{{- if ( has "sriov" .Values.network.backend ) }} \
--config-file /etc/neutron/plugins/ml2/sriov_agent.ini
{{- end }}
{{- if .Values.conf.plugins.l2gateway }} \
--config-file /etc/neutron/l2gw_plugin.ini
{{- end }}
}
function stop () {
kill -TERM 1
}
$COMMAND

View File

@ -18,60 +18,6 @@ set -ex
COMMAND="${@:-start}"
function start () {
{{- if .Values.manifests.certificates }}
add_config="neutron.conf;"
{{- if ( has "tungstenfabric" .Values.network.backend ) }}
add_config+='plugins/tungstenfabric/tf_plugin.ini;'
{{- else }}
add_config+='plugins/ml2/ml2_conf.ini;'
{{- end }}
{{- if .Values.conf.plugins.taas.taas.enabled }}
add_config+='taas_plugin.ini;'
{{- end }}
{{- if ( has "sriov" .Values.network.backend ) }}
add_config+='plugins/ml2/sriov_agent.ini;'
{{- end }}
{{- if .Values.conf.plugins.l2gateway }}
add_config+='l2gw_plugin.ini;'
{{- end }}
export OS_NEUTRON_CONFIG_FILES=${add_config}
for WSGI_SCRIPT in neutron-api; do
cp -a $(type -p ${WSGI_SCRIPT}) /var/www/cgi-bin/neutron/
done
if [ -f /etc/apache2/envvars ]; then
# Loading Apache2 ENV variables
source /etc/apache2/envvars
mkdir -p ${APACHE_RUN_DIR}
fi
{{- if .Values.conf.software.apache2.a2enmod }}
{{- range .Values.conf.software.apache2.a2enmod }}
a2enmod {{ . }}
{{- end }}
{{- end }}
{{- if .Values.conf.software.apache2.a2ensite }}
{{- range .Values.conf.software.apache2.a2ensite }}
a2ensite {{ . }}
{{- end }}
{{- end }}
{{- if .Values.conf.software.apache2.a2dismod }}
{{- range .Values.conf.software.apache2.a2dismod }}
a2dismod {{ . }}
{{- end }}
{{- end }}
if [ -f /var/run/apache2/apache2.pid ]; then
# Remove the stale pid for debian/ubuntu images
rm -f /var/run/apache2/apache2.pid
fi
# Starts Apache2
exec {{ .Values.conf.software.apache2.binary }} {{ .Values.conf.software.apache2.start_parameters }}
{{- else }}
exec neutron-server \
--config-file /etc/neutron/neutron.conf \
{{- if ( has "tungstenfabric" .Values.network.backend ) }}
@ -88,18 +34,10 @@ function start () {
{{- if .Values.conf.plugins.l2gateway }} \
--config-file /etc/neutron/l2gw_plugin.ini
{{- end }}
{{- end }}
}
function stop () {
{{- if .Values.manifests.certificates }}
if [ -f /etc/apache2/envvars ]; then
source /etc/apache2/envvars
fi
{{ .Values.conf.software.apache2.binary }} -k graceful-stop
{{- else }}
kill -TERM 1
{{- end }}
}
$COMMAND

View File

@ -0,0 +1,17 @@
#!/bin/sh
set -xe
COMMAND="${@:-start}"
start () {
envsubst < /etc/nginx/nginx.conf > /tmp/nginx.conf
cat /tmp/nginx.conf
nginx -t -c /tmp/nginx.conf
exec nginx -c /tmp/nginx.conf
}
stop () {
nginx -s stop
}
$COMMAND

View File

@ -82,8 +82,8 @@ data:
neutron-bagpipe-bgp-init.sh: |
{{ tuple "bin/_neutron-bagpipe-bgp-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- if .Values.manifests.certificates }}
neutron-rpc-server.sh: |
{{ tuple "bin/_neutron-rpc-server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
nginx.sh: |
{{ tuple "bin/_nginx.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}
neutron-server.sh: |
{{ tuple "bin/_neutron-server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

View File

@ -279,8 +279,7 @@ data:
dpdk.conf: {{ toJson $envAll.Values.conf.ovs_dpdk | b64enc }}
update_dpdk_bond_config: {{ $envAll.Values.conf.ovs_dpdk.update_dpdk_bond_config | toString | b64enc }}
{{- if .Values.manifests.certificates }}
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.mpm_event "key" "mpm_event.conf" "format" "Secret" ) | indent 2 }}
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.wsgi_neutron_server "key" "wsgi-server.conf" "format" "Secret" ) | indent 2 }}
{{- include "helm-toolkit.snippets.values_template_renderer" ( dict "envAll" $envAll "template" .Values.conf.nginx "key" "nginx.conf" "format" "Secret" ) | indent 2 }}
{{- end }}
{{- range $key, $value := $envAll.Values.conf.rootwrap_filters }}
{{- $filePrefix := replace "_" "-" $key }}

View File

@ -13,13 +13,31 @@ limitations under the License.
*/}}
{{- define "serverReadinessProbeTemplate" }}
{{- if .Values.manifests.certificates }}
exec:
command:
- python
- -c
- "import requests; requests.get('http://127.0.0.1:{{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}')"
initialDelaySeconds: 30
{{- else }}
tcpSocket:
port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- end }}
{{- end }}
{{- define "serverLivenessProbeTemplate" }}
{{- if .Values.manifests.certificates }}
exec:
command:
- python
- -c
- "import requests; requests.get('http://127.0.0.1:{{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}')"
initialDelaySeconds: 30
{{- else }}
tcpSocket:
port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- end }}
{{- end }}
{{- if .Values.manifests.deployment_server }}
{{- $envAll := . }}
@ -81,6 +99,46 @@ spec:
mountPath: /opt/plugin
{{- end }}
containers:
{{- if $envAll.Values.manifests.certificates }}
- name: nginx
{{ tuple $envAll "nginx" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.nginx | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "neutron" "container" "nginx" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
ports:
- name: q-api
containerPort: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
env:
- name: PORT
value: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: SHORTNAME
value: {{ tuple "network" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
readinessProbe:
tcpSocket:
port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
command:
- /tmp/nginx.sh
- start
lifecycle:
preStop:
exec:
command:
- /tmp/nginx.sh
- stop
volumeMounts:
- name: neutron-bin
mountPath: /tmp/nginx.sh
subPath: nginx.sh
readOnly: true
- name: neutron-etc
mountPath: /etc/nginx/nginx.conf
subPath: nginx.conf
readOnly: true
{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.network.server.internal "path" "/etc/nginx/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
{{- end }}
- name: neutron-server
{{ tuple $envAll "neutron_server" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@ -177,83 +235,9 @@ spec:
mountPath: /etc/neutron/policy.json
subPath: policy.json
readOnly: true
{{- if .Values.manifests.certificates }}
- name: wsgi-neutron
mountPath: /var/www/cgi-bin/neutron
- name: neutron-etc
mountPath: {{ .Values.conf.software.apache2.site_dir }}/wsgi-server.conf
subPath: wsgi-server.conf
readOnly: true
- name: neutron-etc
mountPath: {{ .Values.conf.software.apache2.mods_dir }}/mpm_event.conf
subPath: mpm_event.conf
readOnly: true
{{ end }}
{{- dict "enabled" .Values.manifests.certificates "name" .Values.endpoints.oslo_db.auth.admin.secret.tls.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.network.server.internal "path" "/etc/neutron/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
{{ if $mounts_neutron_server.volumeMounts }}{{ toYaml $mounts_neutron_server.volumeMounts | indent 12 }}{{ end }}
{{- if .Values.manifests.certificates }}
- name: neutron-rpc-server
{{ tuple $envAll "neutron_rpc_server" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.rpc_server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "neutron_server" "container" "neutron_rpc_server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
{{ dict "envAll" $envAll "component" "server" "container" "server" "type" "readiness" "probeTemplate" (include "serverReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
{{ dict "envAll" $envAll "component" "server" "container" "server" "type" "liveness" "probeTemplate" (include "serverLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
command:
- /tmp/neutron-rpc-server.sh
- start
lifecycle:
preStop:
exec:
command:
- /tmp/neutron-rpc-server.sh
- stop
volumeMounts:
- name: neutron-bin
mountPath: /tmp/neutron-rpc-server.sh
subPath: neutron-rpc-server.sh
readOnly: true
- name: neutron-etc
mountPath: /etc/neutron/neutron.conf
subPath: neutron.conf
readOnly: true
{{- if .Values.conf.neutron.DEFAULT.log_config_append }}
- name: neutron-etc
mountPath: {{ .Values.conf.neutron.DEFAULT.log_config_append }}
subPath: {{ base .Values.conf.neutron.DEFAULT.log_config_append }}
readOnly: true
{{- end }}
- name: neutron-etc
mountPath: /etc/neutron/plugins/ml2/ml2_conf.ini
subPath: ml2_conf.ini
readOnly: true
{{- if( has "tungstenfabric" .Values.network.backend ) }}
- name: neutron-etc
mountPath: /etc/neutron/plugins/tungstenfabric/tf_plugin.ini
subPath: tf_plugin.ini
readOnly: true
{{ end }}
{{ if ( has "sriov" .Values.network.backend ) }}
- name: neutron-etc
mountPath: /etc/neutron/plugins/ml2/sriov_agent.ini
subPath: sriov_agent.ini
readOnly: true
{{ end }}
{{- if .Values.conf.plugins.taas.taas.enabled }}
- name: neutron-etc
mountPath: /etc/neutron/taas_plugin.ini
subPath: taas_plugin.ini
readOnly: true
{{ end }}
{{- if .Values.conf.plugins.l2gateway }}
- name: neutron-etc
mountPath: /etc/neutron/l2gw_plugin.ini
subPath: l2gw_plugin.ini
readOnly: true
{{ end }}
{{- dict "enabled" .Values.manifests.certificates "name" .Values.endpoints.oslo_db.auth.admin.secret.tls.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.network.server.internal "path" "/etc/neutron/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
{{- end }}
volumes:
- name: pod-tmp
emptyDir: {}

View File

@ -31,7 +31,6 @@ images:
ks_service: docker.io/openstackhelm/heat:stein-ubuntu_bionic
ks_endpoints: docker.io/openstackhelm/heat:stein-ubuntu_bionic
neutron_server: docker.io/openstackhelm/neutron:stein-ubuntu_bionic
neutron_rpc_server: docker.io/openstackhelm/neutron:stein-ubuntu_bionic
neutron_dhcp: docker.io/openstackhelm/neutron:stein-ubuntu_bionic
neutron_metadata: docker.io/openstackhelm/neutron:stein-ubuntu_bionic
neutron_l3: docker.io/openstackhelm/neutron:stein-ubuntu_bionic
@ -520,9 +519,6 @@ pod:
neutron_server:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
neutron_rpc_server:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
neutron_sriov_agent:
pod:
runAsUser: 42424

View File

@ -1,4 +1,7 @@
---
images:
tags:
nginx: docker.io/nginx:1.18.0
network:
server:
ingress:
@ -12,11 +15,8 @@ pod:
container:
neutron_server:
readOnlyRootFilesystem: false
neutron_rpc_server:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: false
resources:
rpc_server:
nginx:
requests:
memory: "128Mi"
cpu: "100m"
@ -24,69 +24,55 @@ pod:
memory: "1024Mi"
cpu: "2000m"
conf:
software:
apache2:
binary: apache2
start_parameters: -DFOREGROUND
conf_dir: /etc/apache2/conf-enabled
site_dir: /etc/apache2/sites-available
mods_dir: /etc/apache2/mods-available
a2enmod:
- ssl
a2dismod: null
a2ensite:
- wsgi-server
mpm_event: |
<IfModule mpm_event_module>
ServerLimit 1024
StartServers 32
MinSpareThreads 32
MaxSpareThreads 256
ThreadsPerChild 25
MaxRequestsPerChild 128
ThreadLimit 720
</IfModule>
wsgi_neutron_server: |
<Directory /usr/local/bin>
Require all granted
</Directory>
nginx: |
worker_processes 1;
daemon off;
user nginx;
{{- $portInt := tuple "network" "internal" "api" $ | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
Listen {{ $portInt }}
<VirtualHost *:{{ $portInt }}>
ServerName {{ printf "%s.%s.svc.%s" "neutron-server" .Release.Namespace .Values.endpoints.cluster_domain_suffix }}
WSGIDaemonProcess neutron-server processes=1 threads=1 user=neutron display-name=%{GROUP}
WSGIProcessGroup neutron-server
WSGIScriptAlias / /var/www/cgi-bin/neutron/neutron-api
WSGIApplicationGroup %{GLOBAL}
WSGIPassAuthorization On
AllowEncodedSlashes On
<IfVersion >= 2.4>
ErrorLogFormat "%{cu}t %M"
</IfVersion>
SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
ErrorLog /dev/stdout
CustomLog /dev/stdout combined env=!forwarded
CustomLog /dev/stdout proxy env=forwarded
events {
worker_connections 1024;
}
SSLEngine on
SSLCertificateFile /etc/neutron/certs/tls.crt
SSLCertificateKeyFile /etc/neutron/certs/tls.key
SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1
SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
SSLHonorCipherOrder on
</VirtualHost>
Alias /networking /var/www/cgi-bin/neutron/neutron-api
<Location /networking>
SetHandler wsgi-script
Options +ExecCGI
WSGIProcessGroup neutron-server
WSGIApplicationGroup %{GLOBAL}
WSGIPassAuthorization On
</Location>
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
WSGISocketPrefix /var/run/apache2
sendfile on;
keepalive_timeout 65s;
tcp_nodelay on;
log_format main '[nginx] method=$request_method path=$request_uri '
'status=$status upstream_status=$upstream_status duration=$request_time size=$body_bytes_sent '
'"$remote_user" "$http_referer" "$http_user_agent"';
access_log /dev/stdout main;
upstream websocket {
server 127.0.0.1:$PORT;
}
server {
server_name {{ printf "%s.%s.svc.%s" "${SHORTNAME}" .Release.Namespace .Values.endpoints.cluster_domain_suffix }};
listen $POD_IP:$PORT ssl;
client_max_body_size 0;
ssl_certificate /etc/nginx/certs/tls.crt;
ssl_certificate_key /etc/nginx/certs/tls.key;
ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384;
location / {
proxy_pass_request_headers on;
proxy_http_version 1.1;
proxy_pass http://websocket;
proxy_read_timeout 90;
}
}
}
neutron:
DEFAULT:
bind_host: 127.0.0.1
nova:
cafile: /etc/neutron/certs/ca.crt
keystone_authtoken: