Upgrade ElasticSearch and Kibana to v8.9.0

Change-Id: I5ce965a2abf40bad14f0a8a505c8f3000f110d37

elasticsearch/Chart.yaml

@@ -12,10 +12,10 @@
 
 ---
 apiVersion: v1
-appVersion: v7.6.2
+appVersion: v8.9.0
 description: OpenStack-Helm ElasticSearch
 name: elasticsearch
-version: 0.2.24
+version: 0.2.25
 home: https://www.elastic.co/
 sources:
   - https://github.com/elastic/elasticsearch

elasticsearch/templates/bin/_elasticsearch.sh.tpl

@@ -19,26 +19,26 @@ set -e
 COMMAND="${@:-start}"
 
 function initiate_keystore () {
-  bin/elasticsearch-keystore create
+  elasticsearch-keystore create
-
   {{- if .Values.conf.elasticsearch.snapshots.enabled }}
   {{- range $client, $settings := .Values.storage.s3.clients -}}
   {{- $access_key := printf "%s_S3_ACCESS_KEY" ( $client | replace "-" "_" | upper) }}
   {{- $secret_key := printf "%s_S3_SECRET_KEY" ( $client | replace "-" "_" | upper) }}
-  echo ${{$access_key}} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.{{ $client }}.access_key
+  echo ${{$access_key}} | elasticsearch-keystore add -xf s3.client.{{ $client }}.access_key
-  echo ${{$secret_key}} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.{{ $client }}.secret_key
+  echo ${{$secret_key}} | elasticsearch-keystore add -xf s3.client.{{ $client }}.secret_key
   {{- end }}
   {{- end }}
 
   {{- if .Values.manifests.certificates }}
   {{- $alias := .Values.secrets.tls.elasticsearch.elasticsearch.internal }}
-  /usr/share/elasticsearch/jdk/bin/keytool -storepasswd -cacerts -new ${ELASTICSEARCH_PASSWORD} -storepass changeit
+  JAVA_KEYTOOL_PATH=/usr/share/elasticsearch/jdk/bin/keytool
-  /usr/share/elasticsearch/jdk/bin/keytool -importcert -alias {{$alias}} -cacerts -trustcacerts -noprompt -file ${JAVA_KEYSTORE_CERT_PATH} -storepass ${ELASTICSEARCH_PASSWORD}
+  TRUSTSTORE_PATH=/usr/share/elasticsearch/config/elasticsearch-java-truststore
+  ${JAVA_KEYTOOL_PATH} -importcert -alias {{$alias}} -keystore ${TRUSTSTORE_PATH} -trustcacerts -noprompt -file ${JAVA_KEYSTORE_CERT_PATH} -storepass ${ELASTICSEARCH_PASSWORD}
+  ${JAVA_KEYTOOL_PATH} -storepasswd -keystore ${TRUSTSTORE_PATH} -new ${ELASTICSEARCH_PASSWORD} -storepass ${ELASTICSEARCH_PASSWORD}
   {{- end }}
 }
 
 function start () {
-  ulimit -l unlimited
   initiate_keystore
   exec /usr/local/bin/docker-entrypoint.sh elasticsearch
 }
@@ -76,7 +76,6 @@ function allocate_data_node () {
 }
 
 function start_master_node () {
-  ulimit -l unlimited
   initiate_keystore
   if [ ! -f {{ $envAll.Values.conf.elasticsearch.config.path.data }}/cluster-bootstrap.txt ];
   then
@@ -97,7 +96,6 @@ function start_master_node () {
 }
 
 function start_data_node () {
-  ulimit -l unlimited
   initiate_keystore
   allocate_data_node &
   /usr/local/bin/docker-entrypoint.sh elasticsearch &

elasticsearch/templates/deployment-client.yaml

@@ -159,14 +159,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-            - name: NODE_MASTER
+            - name: node.roles
-              value: "false"
+              value: "[ingest]"
-            - name: NODE_INGEST
-              value: "true"
-            - name: NODE_DATA
-              value: "false"
-            - name: NODE_GATEWAY
-              value: "false"
             - name: HTTP_ENABLE
               value: "true"
             - name: DISCOVERY_SERVICE

elasticsearch/templates/deployment-gateway.yaml

@@ -101,14 +101,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-            - name: NODE_MASTER
+            - name: node.roles
-              value: "false"
+              value: "[ingest, gateway]"
-            - name: NODE_INGEST
-              value: "true"
-            - name: NODE_DATA
-              value: "false"
-            - name: NODE_GATEWAY
-              value: "true"
             - name: HTTP_ENABLE
               value: "false"
             - name: DISCOVERY_SERVICE

elasticsearch/templates/statefulset-data.yaml

@@ -76,7 +76,7 @@ spec:
           command:
             - chown
             - -R
-            - "elasticsearch:"
+            - "1000:1000"
             - {{ .Values.conf.elasticsearch.config.path.data }}
           volumeMounts:
             - name: storage
@@ -124,14 +124,8 @@ spec:
             - name: JAVA_KEYSTORE_CERT_PATH
               value: "/usr/share/elasticsearch/config/ca.crt"
 {{- end }}
-            - name: NODE_MASTER
+            - name: node.roles
-              value: "false"
+              value: "[data]"
-            - name: NODE_INGEST
-              value: "false"
-            - name: NODE_DATA
-              value: "true"
-            - name: NODE_GATEWAY
-              value: "false"
             - name: HTTP_ENABLE
               value: "false"
             - name: ES_JAVA_OPTS

elasticsearch/templates/statefulset-master.yaml

@@ -73,7 +73,7 @@ spec:
           command:
             - chown
             - -R
-            - "elasticsearch:"
+            - "1000:1000"
             - {{ .Values.conf.elasticsearch.config.path.data }}
           volumeMounts:
             - name: storage
@@ -109,14 +109,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-            - name: NODE_MASTER
+            - name: node.roles
-              value: "true"
+              value: "[master]"
-            - name: NODE_INGEST
-              value: "false"
-            - name: NODE_DATA
-              value: "false"
-            - name: NODE_GATEWAY
-              value: "false"
             - name: HTTP_ENABLE
               value: "false"
             - name: DISCOVERY_SERVICE

elasticsearch/values.yaml

@@ -19,16 +19,16 @@ images:
   tags:
     apache_proxy: docker.io/library/httpd:2.4
     memory_init: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
-    elasticsearch: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
+    elasticsearch: docker.io/openstackhelm/elasticsearch-s3:latest-8_9_0
     curator: docker.io/bobrik/curator:5.8.1
     ceph_key_placement: docker.io/openstackhelm/ceph-config-helper:ubuntu_focal_17.2.6-1-20230508
     s3_bucket: docker.io/openstackhelm/ceph-daemon:ubuntu_focal_17.2.6-1-20230508
     s3_user: docker.io/openstackhelm/ceph-config-helper:ubuntu_focal_17.2.6-1-20230508
-    helm_tests: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
+    helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
     prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.1.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     snapshot_repository: docker.io/openstackhelm/ceph-config-helper:ubuntu_focal_17.2.6-1-20230508
-    elasticsearch_templates: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
+    elasticsearch_templates: docker.io/openstackhelm/elasticsearch-s3:latest-8_9_0
     image_repo_sync: docker.io/library/docker:17.07.0
   pull_policy: "IfNotPresent"
   local_registry:
@@ -170,11 +170,8 @@ pod:
         apache_proxy:
           readOnlyRootFilesystem: false
         elasticsearch_client:
-          privileged: true
+          runAsUser: 1000
-          capabilities:
+          runAsGroup: 1000
-            add:
-              - IPC_LOCK
-              - SYS_RESOURCE
           readOnlyRootFilesystem: false
     master:
       pod:
@@ -186,11 +183,8 @@ pod:
         elasticsearch_perms:
           readOnlyRootFilesystem: true
         elasticsearch_master:
-          privileged: true
+          runAsUser: 1000
-          capabilities:
+          runAsGroup: 1000
-            add:
-              - IPC_LOCK
-              - SYS_RESOURCE
           readOnlyRootFilesystem: false
     snapshot_repository:
       pod:
@@ -214,11 +208,8 @@ pod:
         elasticsearch_perms:
           readOnlyRootFilesystem: true
         elasticsearch_data:
-          privileged: true
+          runAsUser: 1000
-          capabilities:
+          runAsGroup: 1000
-            add:
-              - IPC_LOCK
-              - SYS_RESOURCE
           # NOTE: This was changed from true to false to account for
           # recovery scenarios when the data pods are unexpectedly lost due to
           # node outages and shard/index recovery is required
@@ -233,11 +224,8 @@ pod:
         apache_proxy:
           readOnlyRootFilesystem: false
         elasticsearch_gateway:
-          privileged: true
+          runAsUser: 1000
-          capabilities:
+          runAsGroup: 1000
-            add:
-              - IPC_LOCK
-              - SYS_RESOURCE
           readOnlyRootFilesystem: false
     curator:
       pod:
@@ -300,7 +288,7 @@ pod:
         liveness:
           enabled: true
           params:
-            initialDelaySeconds: 30
+            initialDelaySeconds: 60
             periodSeconds: 10
   mounts:
     elasticsearch:
@@ -581,6 +569,10 @@ conf:
     -Dlog4j.shutdownHookEnabled=false
     -Dlog4j2.disable.jmx=true
     -Djava.io.tmpdir=${ES_TMPDIR}
+    {{- if .Values.manifests.certificates }}
+    -Djavax.net.ssl.trustStore=/usr/share/elasticsearch/config/elasticsearch-java-truststore
+    -Djavax.net.ssl.trustStorePassword={{ .Values.endpoints.elasticsearch.auth.admin.password }}
+    {{- end }}
     -XX:+HeapDumpOnOutOfMemoryError
     -XX:HeapDumpPath=data
     -XX:ErrorFile=logs/hs_err_pid%p.log
@@ -713,12 +705,15 @@ conf:
         blacklist: ['elasticsearch', 'urllib3']
   elasticsearch:
     config:
+      xpack:
+        security:
+          enabled: false
       bootstrap:
-        memory_lock: true
+        # As far as we run the pod as non-root, we can't make locking memory unlimited.
+        # configure the memory locking limits on host itself of disable swap completely.
+        memory_lock: false
       cluster:
         name: elasticsearch
-        remote:
-          connect: ${NODE_GATEWAY}
       discovery:
         # NOTE(srwilkers): This gets configured dynamically via endpoint lookups
         seed_hosts: null
@@ -726,12 +721,6 @@ conf:
         host: 0.0.0.0
       s3:
         client: {}
-      node:
-        ingest: ${NODE_INGEST}
-        master: ${NODE_MASTER}
-        data: ${NODE_DATA}
-        name: ${NODE_NAME}
-        max_local_storage_nodes: 3
       path:
         data: /data
         logs: /logs

elasticsearch/values_overrides/tls.yaml

@@ -137,6 +137,7 @@ conf:
     config:
       xpack:
         security:
+          enabled: true
           transport:
             ssl:
               enabled: true

kibana/Chart.yaml

@@ -12,10 +12,10 @@
 
 ---
 apiVersion: v1
-appVersion: v7.1.0
+appVersion: v8.9.0
 description: OpenStack-Helm Kibana
 name: kibana
-version: 0.1.12
+version: 0.1.13
 home: https://www.elastic.co/products/kibana
 sources:
   - https://github.com/elastic/kibana

kibana/values.yaml

@@ -22,7 +22,7 @@ labels:
 images:
   tags:
     apache_proxy: docker.io/library/httpd:2.4
-    kibana: docker.elastic.co/kibana/kibana-oss:7.1.0
+    kibana: docker.elastic.co/kibana/kibana:8.9.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/library/docker:17.07.0
     register_kibana_indexes: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
@@ -286,29 +286,19 @@ conf:
   kibana:
     elasticsearch:
       pingTimeout: 1500
-      preserveHost: true
       requestTimeout: 30000
       shardTimeout: 0
-      startupTimeout: 5000
-    kibana:
-      defaultAppId: discover
-    logging:
-      quiet: false
-      silent: false
-      verbose: false
     ops:
       interval: 5000
     server:
       rewriteBasePath: false
       host: localhost
       name: kibana
-      maxPayloadBytes: 1048576
+      maxPayload: 1048576
       port: 5601
       ssl:
         enabled: false
   create_kibana_indexes:
-    enabled: true
-    version: 7.1.0
     indexes:
       base:
         - logstash

releasenotes/notes/elasticsearch.yaml

@@ -34,4 +34,5 @@ elasticsearch:
   - 0.2.22 Update all Ceph images to Focal
   - 0.2.23 Add configurable liveness probe for elasticsearch client
   - 0.2.24 Update Ceph to 17.2.6
+  - 0.2.25 Update ElasticSearch to 8.9.0
 ...

releasenotes/notes/kibana.yaml

@@ -13,4 +13,5 @@ kibana:
   - 0.1.10 Update image defaults
   - 0.1.11 Added OCI registry authentication
   - 0.1.12 Added feedback http_code 200 for kibana indexes
+  - 0.1.13 Update Kibana to 8.9.0
 ...

tools/deployment/osh-infra-logging/050-elasticsearch.sh

@@ -45,7 +45,7 @@ conf:
     slm_policy:
       endpoint: _slm/policy/snapshots
       body:
-        schedule: "0 */3 * * * ?"
+        schedule: "0 */15 * * * ?"
         name: "<snapshot-{now/d}>"
         repository: ceph-rgw
         config: