Allow terminationGracePeriodSeconds to be configurable

Set a default of 6 hours, but allow users to tell us to wait a
longer or shorter time for executors to gracefully stop.

Change-Id: I12aada7a3c3aaff7b8bbe51744a8dd4d367b6e5a

doc/source/index.rst

@@ -404,6 +404,16 @@ verbatim):
 
                The key name in the secret should be ``sshkey``.
 
+         .. attr:: terminationGracePeriodSeconds
+            :default: 21600
+
+            When performing a rolling restart of the executors, wait
+            this long for jobs to finish normally.  If an executor
+            takes longer than this amount of time, it will be
+            hard-stopped (and jobs will abort and retry).  The default
+            is 6 hours, but depending on the workload, a higher or
+            lower value may be appropriate.
+
       .. attr:: merger
 
          .. attr:: count

zuul_operator/templates/zuul.yaml

@@ -347,7 +347,7 @@ spec:
         {%- endfor %}
         securityContext:
           privileged: true
-        terminationGracePeriodSeconds: 3600
+        terminationGracePeriodSeconds: {{ spec.executor.terminationGracePeriodSeconds }}
         lifecycle:
           preStop:
             exec:

zuul_operator/zuul.py

@@ -69,6 +69,8 @@ class Zuul:
             '/etc/zuul/tenant/main.yaml'
 
         self.spec.setdefault('executor', {}).setdefault('count', 1)
+        self.spec.setdefault('executor', {}).setdefault(
+            'terminationGracePeriodSeconds', 21600)
         self.spec.setdefault('merger', {}).setdefault('count', 0)
         self.spec.setdefault('web', {}).setdefault('count', 1)
         self.spec.setdefault('fingergw', {}).setdefault('count', 1)