Elasticsearch: Add job history to Curator, update schedule key

This updates the Elastic Curator cron job to include configuration
for successful and failed job history limits, similar to the other
cron jobs we deploy. This also moves the key for configuring the
cron schedule from under .Values.conf.curator to a new top level
jobs key to maintain consistency

This also fixes an indentation issue with the deployment overrides
for Curator as well as adds the overrides for the Armada job

Change-Id: I9c720df9677215bdd2bf18be77959bd5f671c0ca
Signed-off-by: Steve Wilkerson <sw5822@att.com>

elasticsearch/templates/cron-job-curator.yaml

@@ -29,7 +29,10 @@ metadata:
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
 spec:
-  schedule: {{ .Values.conf.curator.schedule | quote }}
+  schedule: {{ .Values.jobs.curator.cron | quote }}
+  successfulJobsHistoryLimit: {{ .Values.jobs.curator.history.success }}
+  failedJobsHistoryLimit: {{ .Values.jobs.curator.history.failed }}
+  concurrencyPolicy: Forbid
   jobTemplate:
     metadata:
       labels:

elasticsearch/values.yaml

@@ -352,6 +352,13 @@ secrets:
       elasticsearch:
         public: elasticsearch-tls-public
 
+jobs:
+  curator:
+    cron: "* */6 * * *"
+    history:
+      success: 3
+      failed: 1
+
 conf:
   httpd: |
     ServerRoot "/usr/local/apache2"
@@ -490,8 +497,6 @@ conf:
   ceph:
     admin_keyring: null
   curator:
-    #run every 6th hour
-    schedule:  "0 */6 * * *"
     action_file:
       # Remember, leave a key empty if there is no value.  None will be a string,
       # not a Python "NoneType"

tools/deployment/armada/manifests/armada-lma.yaml

@@ -403,6 +403,28 @@ data:
                     pod_name:
                       type: keyword
                       index: false
+      curator:
+        action_file:
+          actions:
+            1:
+              action: delete_indices
+              description: >-
+                "Delete indices older than 365 days"
+              options:
+                timeout_override:
+                continue_if_exception: False
+                ignore_empty_list: True
+                disable_action: True
+              filters:
+              - filtertype: pattern
+                kind: prefix
+                value: logstash-
+              - filtertype: age
+                source: name
+                direction: older
+                timestring: '%Y.%m.%d'
+                unit: days
+                unit_count: 365
   source:
     type: local
     location: ${OSH_INFRA_PATH}

tools/deployment/multinode/120-elasticsearch.sh

@@ -34,29 +34,28 @@ conf:
         master: "-Xms512m -Xmx512m"
     snapshots:
       enabled: true
-    curator:
-      schedule:  "0 */6 * * *"
-      action_file:
-        actions:
-          1:
-            action: delete_indices
-            description: >-
-              "Delete indices older than 365 days"
-            options:
-              timeout_override:
-              continue_if_exception: False
-              ignore_empty_list: True
-              disable_action: True
-            filters:
-            - filtertype: pattern
-              kind: prefix
-              value: logstash-
-            - filtertype: age
-              source: name
-              direction: older
-              timestring: '%Y.%m.%d'
-              unit: days
-              unit_count: 365
+  curator:
+    action_file:
+      actions:
+        1:
+          action: delete_indices
+          description: >-
+            "Delete indices older than 365 days"
+          options:
+            timeout_override:
+            continue_if_exception: False
+            ignore_empty_list: True
+            disable_action: True
+          filters:
+          - filtertype: pattern
+            kind: prefix
+            value: logstash-
+          - filtertype: age
+            source: name
+            direction: older
+            timestring: '%Y.%m.%d'
+            unit: days
+            unit_count: 365
 monitoring:
   prometheus:
     enabled: true

tools/deployment/osh-infra-logging/050-elasticsearch.sh

@@ -32,29 +32,28 @@ conf:
   elasticsearch:
     snapshots:
       enabled: true
-    curator:
-      schedule:  "0 */6 * * *"
-      action_file:
-        actions:
-          1:
-            action: delete_indices
-            description: >-
-              "Delete indices older than 365 days"
-            options:
-              timeout_override:
-              continue_if_exception: False
-              ignore_empty_list: True
-              disable_action: True
-            filters:
-            - filtertype: pattern
-              kind: prefix
-              value: logstash-
-            - filtertype: age
-              source: name
-              direction: older
-              timestring: '%Y.%m.%d'
-              unit: days
-              unit_count: 365
+  curator:
+    action_file:
+      actions:
+        1:
+          action: delete_indices
+          description: >-
+            "Delete indices older than 365 days"
+          options:
+            timeout_override:
+            continue_if_exception: False
+            ignore_empty_list: True
+            disable_action: True
+          filters:
+          - filtertype: pattern
+            kind: prefix
+            value: logstash-
+          - filtertype: age
+            source: name
+            direction: older
+            timestring: '%Y.%m.%d'
+            unit: days
+            unit_count: 365
 
 EOF
 helm upgrade --install elasticsearch ./elasticsearch \