Control bind9 and nginx resource usage

The named and nginx processes both try to use all available CPUs. In
addition, there is a bug in named that sometimes causes it to spin on a
FUTEX, pegging the CPU.

This change constrains those processes to a single CPU (overridable in
values.yaml), and includes /etc/bind/bind.keys in named.conf to avoid
the CPU spike.

Change-Id: I4a278023f5c0dd5e7bdee46891591b278f2ddcad

charts/maas/templates/bin/_start.sh.tpl

@@ -21,11 +21,11 @@ env > /tmp/env
 
 # Ensure PVC volumes have correct ownership
 # Also restore the subdirectory structure and any default files
-# (i.e. /var/lib/maas/http/nginx.conf)
+# that are not overridden
 
 chown maas:maas ~maas/
 chown maas:maas /etc/maas
-[[ -r /opt/maas/var-lib-maas.tgz ]] && tar -C/ -xvzf /opt/maas/var-lib-maas.tgz
+[[ -r /opt/maas/var-lib-maas.tgz ]] && tar -C/ -xvzf /opt/maas/var-lib-maas.tgz || true
 [[ -d ~maas/boot-resources ]] && chown -R maas:maas ~maas/boot-resources
 
 # MAAS must be able to ssh to libvirt hypervisors

charts/maas/templates/configmap-etc.yaml

@@ -36,6 +36,10 @@ data:
 {{- if .Values.conf.bind.append -}}
 {{ .Values.conf.bind.append | indent 4 }}
 {{- end }}
+  bind9: |
+{{ tuple "etc/_bind9.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  nginx.conf: |
+{{ tuple "etc/_nginx.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   regiond.conf: |
 {{ tuple "etc/_regiond.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   nsswitch.conf: |

charts/maas/templates/etc/_bind9.tpl

@@ -0,0 +1,8 @@
+{{/* file location: /etc/default/bind9 */}}
+{{- $cpus := index .Values.conf.bind "cpus" -}}
+#
+# run resolvconf?
+RESOLVCONF=no
+
+# startup options for the server
+OPTIONS="-u bind {{- if $cpus }} -n {{ $cpus }}{{ end }}"

charts/maas/templates/etc/_nginx.conf.tpl

@@ -0,0 +1,37 @@
+{{/* file location: /var/lib/maas/http/nginx.conf */}}
+{{- $worker_processes := index .Values.conf.nginx "worker_processes" | default "auto" -}}
+{{- $worker_connections := index .Values.conf.nginx "worker_connections" | default 768 -}}
+pid /run/maas-http.pid;
+worker_processes {{ $worker_processes }};
+
+error_log /var/log/maas/http/error.log;
+
+events {
+    worker_connections {{ $worker_connections }};
+}
+
+http {
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+    client_max_body_size 10M;
+
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    access_log /var/log/maas/http/access.log;
+
+    gzip on;
+
+    include /var/lib/maas/http/*.nginx.conf;
+
+    # LP: #1796224 and #1869067 - Use different paths otherwise this will
+    # conflict with the system's nginx daemon.
+    client_body_temp_path /var/lib/maas/http/body;
+    fastcgi_temp_path /var/lib/maas/http/fastcgi;
+    proxy_temp_path /var/lib/maas/http/proxy;
+    scgi_temp_path /var/lib/maas/http/scgi;
+    uwsgi_temp_path /var/lib/maas/http/uwsgi;
+}

charts/maas/templates/statefulset-rack.yaml

@@ -138,6 +138,14 @@ spec:
               subPath: PRIVATE_KEY
               mountPath: /var/lib/maas/id_rsa
 {{- end }}
+            - name: maas-etc
+              mountPath: /var/lib/maas/http/nginx.conf
+              subPath: nginx.conf
+              readOnly: true
+            - name: maas-etc
+              mountPath: /etc/default/bind9
+              subPath: bind9
+              readOnly: true
 {{ if $mounts_maas_rack.volumeMounts }}{{ toYaml $mounts_maas_rack.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: host-sys-fs-cgroup

charts/maas/templates/statefulset-region.yaml

@@ -169,6 +169,10 @@ spec:
               subPath: curtin
               readOnly: true
 {{- end }}
+            - name: maas-etc
+              mountPath: /etc/default/bind9
+              subPath: bind9
+              readOnly: true
 {{- if $mounts_maas_region.volumeMounts }}{{ toYaml $mounts_maas_region.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: host-sys-fs-cgroup

charts/maas/values.yaml

@@ -181,6 +181,14 @@ conf:
   bind:
     override:
     append:
+    # 'cpus: n' number of CPUs for bind to use
+    # 'cpus: ""' to revert to the default (all of them)
+    cpus: 1
+  nginx:
+    # 'worker_processes: auto' (the maas default) launches one worker per core
+    worker_processes: 1
+    # 'worker_connections: 768' is the MAAS default, 512 is the nginx default
+    worker_connections: 768
   curtin:
     override: true
     late_commands: {}

images/maas-rack-controller/Dockerfile

@@ -74,5 +74,8 @@ RUN systemctl enable journalctl-to-tty.service
 # quiet sudo for the maas user
 RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log
 
+# avoid triggering bind9 high cpu utilization bug
+RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf
+
 # initalize systemd
 CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]

images/maas-region-controller/Dockerfile

@@ -82,5 +82,8 @@ RUN systemctl enable journalctl-to-tty.service
 # quiet sudo for the maas user
 RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log
 
+# avoid triggering bind9 high cpu utilization bug
+RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf
+
 # initalize systemd
 CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]