From 50b3d689058e4ecb4927e314124a4c2e5189bd0f Mon Sep 17 00:00:00 2001 From: Phil Sphicas Date: Wed, 3 Nov 2021 17:02:32 -0700 Subject: [PATCH] Control bind9 and nginx resource usage The named and nginx processes both try to use all available CPUs. In addition, there is a bug in named that sometimes causes it to spin on a FUTEX, pegging the CPU. This change constrains those processes to a single CPU (overridable in values.yaml), and includes /etc/bind/bind.keys in named.conf to avoid the CPU spike. Change-Id: I4a278023f5c0dd5e7bdee46891591b278f2ddcad --- charts/maas/templates/bin/_start.sh.tpl | 4 +- charts/maas/templates/configmap-etc.yaml | 4 ++ charts/maas/templates/etc/_bind9.tpl | 8 ++++ charts/maas/templates/etc/_nginx.conf.tpl | 37 +++++++++++++++++++ charts/maas/templates/statefulset-rack.yaml | 8 ++++ charts/maas/templates/statefulset-region.yaml | 4 ++ charts/maas/values.yaml | 8 ++++ images/maas-rack-controller/Dockerfile | 3 ++ images/maas-region-controller/Dockerfile | 3 ++ 9 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 charts/maas/templates/etc/_bind9.tpl create mode 100644 charts/maas/templates/etc/_nginx.conf.tpl diff --git a/charts/maas/templates/bin/_start.sh.tpl b/charts/maas/templates/bin/_start.sh.tpl index ad2a879..6161f7d 100644 --- a/charts/maas/templates/bin/_start.sh.tpl +++ b/charts/maas/templates/bin/_start.sh.tpl @@ -21,11 +21,11 @@ env > /tmp/env # Ensure PVC volumes have correct ownership # Also restore the subdirectory structure and any default files -# (i.e. /var/lib/maas/http/nginx.conf) +# that are not overridden chown maas:maas ~maas/ chown maas:maas /etc/maas -[[ -r /opt/maas/var-lib-maas.tgz ]] && tar -C/ -xvzf /opt/maas/var-lib-maas.tgz +[[ -r /opt/maas/var-lib-maas.tgz ]] && tar -C/ -xvzf /opt/maas/var-lib-maas.tgz || true [[ -d ~maas/boot-resources ]] && chown -R maas:maas ~maas/boot-resources # MAAS must be able to ssh to libvirt hypervisors diff --git a/charts/maas/templates/configmap-etc.yaml b/charts/maas/templates/configmap-etc.yaml index 09ba719..04d2bf0 100644 --- a/charts/maas/templates/configmap-etc.yaml +++ b/charts/maas/templates/configmap-etc.yaml @@ -36,6 +36,10 @@ data: {{- if .Values.conf.bind.append -}} {{ .Values.conf.bind.append | indent 4 }} {{- end }} + bind9: | +{{ tuple "etc/_bind9.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + nginx.conf: | +{{ tuple "etc/_nginx.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} regiond.conf: | {{ tuple "etc/_regiond.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} nsswitch.conf: | diff --git a/charts/maas/templates/etc/_bind9.tpl b/charts/maas/templates/etc/_bind9.tpl new file mode 100644 index 0000000..52679f3 --- /dev/null +++ b/charts/maas/templates/etc/_bind9.tpl @@ -0,0 +1,8 @@ +{{/* file location: /etc/default/bind9 */}} +{{- $cpus := index .Values.conf.bind "cpus" -}} +# +# run resolvconf? +RESOLVCONF=no + +# startup options for the server +OPTIONS="-u bind {{- if $cpus }} -n {{ $cpus }}{{ end }}" diff --git a/charts/maas/templates/etc/_nginx.conf.tpl b/charts/maas/templates/etc/_nginx.conf.tpl new file mode 100644 index 0000000..6ec41a4 --- /dev/null +++ b/charts/maas/templates/etc/_nginx.conf.tpl @@ -0,0 +1,37 @@ +{{/* file location: /var/lib/maas/http/nginx.conf */}} +{{- $worker_processes := index .Values.conf.nginx "worker_processes" | default "auto" -}} +{{- $worker_connections := index .Values.conf.nginx "worker_connections" | default 768 -}} +pid /run/maas-http.pid; +worker_processes {{ $worker_processes }}; + +error_log /var/log/maas/http/error.log; + +events { + worker_connections {{ $worker_connections }}; +} + +http { + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + client_max_body_size 10M; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /var/log/maas/http/access.log; + + gzip on; + + include /var/lib/maas/http/*.nginx.conf; + + # LP: #1796224 and #1869067 - Use different paths otherwise this will + # conflict with the system's nginx daemon. + client_body_temp_path /var/lib/maas/http/body; + fastcgi_temp_path /var/lib/maas/http/fastcgi; + proxy_temp_path /var/lib/maas/http/proxy; + scgi_temp_path /var/lib/maas/http/scgi; + uwsgi_temp_path /var/lib/maas/http/uwsgi; +} diff --git a/charts/maas/templates/statefulset-rack.yaml b/charts/maas/templates/statefulset-rack.yaml index f92305e..bbf8b6d 100644 --- a/charts/maas/templates/statefulset-rack.yaml +++ b/charts/maas/templates/statefulset-rack.yaml @@ -138,6 +138,14 @@ spec: subPath: PRIVATE_KEY mountPath: /var/lib/maas/id_rsa {{- end }} + - name: maas-etc + mountPath: /var/lib/maas/http/nginx.conf + subPath: nginx.conf + readOnly: true + - name: maas-etc + mountPath: /etc/default/bind9 + subPath: bind9 + readOnly: true {{ if $mounts_maas_rack.volumeMounts }}{{ toYaml $mounts_maas_rack.volumeMounts | indent 12 }}{{ end }} volumes: - name: host-sys-fs-cgroup diff --git a/charts/maas/templates/statefulset-region.yaml b/charts/maas/templates/statefulset-region.yaml index 5426e58..f8c6562 100644 --- a/charts/maas/templates/statefulset-region.yaml +++ b/charts/maas/templates/statefulset-region.yaml @@ -169,6 +169,10 @@ spec: subPath: curtin readOnly: true {{- end }} + - name: maas-etc + mountPath: /etc/default/bind9 + subPath: bind9 + readOnly: true {{- if $mounts_maas_region.volumeMounts }}{{ toYaml $mounts_maas_region.volumeMounts | indent 12 }}{{ end }} volumes: - name: host-sys-fs-cgroup diff --git a/charts/maas/values.yaml b/charts/maas/values.yaml index 2f00a71..8f006d7 100644 --- a/charts/maas/values.yaml +++ b/charts/maas/values.yaml @@ -181,6 +181,14 @@ conf: bind: override: append: + # 'cpus: n' number of CPUs for bind to use + # 'cpus: ""' to revert to the default (all of them) + cpus: 1 + nginx: + # 'worker_processes: auto' (the maas default) launches one worker per core + worker_processes: 1 + # 'worker_connections: 768' is the MAAS default, 512 is the nginx default + worker_connections: 768 curtin: override: true late_commands: {} diff --git a/images/maas-rack-controller/Dockerfile b/images/maas-rack-controller/Dockerfile index 594a00e..059071d 100644 --- a/images/maas-rack-controller/Dockerfile +++ b/images/maas-rack-controller/Dockerfile @@ -74,5 +74,8 @@ RUN systemctl enable journalctl-to-tty.service # quiet sudo for the maas user RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log +# avoid triggering bind9 high cpu utilization bug +RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf + # initalize systemd CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"] diff --git a/images/maas-region-controller/Dockerfile b/images/maas-region-controller/Dockerfile index c47aca3..77d58b5 100644 --- a/images/maas-region-controller/Dockerfile +++ b/images/maas-region-controller/Dockerfile @@ -82,5 +82,8 @@ RUN systemctl enable journalctl-to-tty.service # quiet sudo for the maas user RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log +# avoid triggering bind9 high cpu utilization bug +RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf + # initalize systemd CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]