LMA Configuration Updates

This change fixes some configurations of LMA CRs and values,
such that logs and metrics are properly visible on kibana
and grafana after workload and workload-config phase.

Also:
- Added dashboards to grafana
- Added a container to kibana which sets up the discover page
- Added the DependsOn directive was added to the workload-lma
  prometheus-elasticsearch-exporter HR
- Added NodePort bindings to LMA services in multi-tenant/lma
  subcluster

Change-Id: I14338f5cab0808654173a09494f5fde540397dcc

manifests/composite/lma-infra/patches/kube-prometheus-stack.yaml

@@ -3,9 +3,9 @@ kind: HelmRelease
 metadata:
   name: kube-prometheus-stack
 spec:
-  releaseName: lma-infra-monitoring
+  releaseName: monitoring
   values:
-    fullnameOverride: lma-infra-monitoring
+    fullnameOverride: monitoring
     grafana:
       enabled: false
     prometheus:

manifests/composite/lma-infra/patches/logging-operator.yaml

@@ -6,7 +6,7 @@ spec:
   dependsOn:
     - name: kube-prometheus-stack
   values:
-    fullnameOverride: lma-infra-logging-operator
+    fullnameOverride: logging-operator
     monitoring:
       serviceMonitor:
         enabled: true

manifests/composite/lma-infra/patches/logging-spec.yaml

@@ -10,9 +10,8 @@ spec:
   # https://github.com/banzaicloud/logging-operator/blob/master/charts/logging-operator-logging/values.yaml
   values:
     controlNamespace: lma
-    loggingRef: lma-infra-logging
+    nameOverride: logging
-    nameOverride: lma-infra-logging
+    fullnameOverride: logging
-    fullnameOverride: lma-infra-logging
     fluentbit:
       tolerations:
         - key: node-role.kubernetes.io/master

manifests/composite/monitoring-stack/patches/grafana.yaml

@@ -13,7 +13,8 @@ spec:
         datasources:
         - name: lma-infra-thanos
           type: prometheus
-          url: lma-infra-thanos-query.lma.svc.cluster.local:10902
+          url: http://monitoring-thanos-query.lma.svc.cluster.local:10902
+          access: server
     dashboardProviders:
       dashboardproviders.yaml:
         apiVersion: 1
@@ -26,6 +27,18 @@ spec:
               path: /var/lib/grafana/dashboards/default
     dashboards:
       default:
+        node-exporter:
+          gnetId: 7249
+          revision: 1
+          datasource: lma-infra-thanos
+        elasticsearch:
+          gnetId: 2322
+          revision: 4
+          datasource: lma-infra-thanos
+        kube-state-metrics:
+          gnetId: 13332
+          revision: 6
+          datasource: lma-infra-thanos
         logging:
           gnetId: 7752
           revision: 4

manifests/composite/monitoring-stack/patches/thanos-operator.yaml

@@ -4,4 +4,4 @@ metadata:
   name: thanos-operator
 data:
   values:
-    nameOverride: lma-infra-thanos
+    nameOverride: monitoring-thanos

manifests/function/kibana/helmrelease.yaml

@@ -13,4 +13,31 @@ spec:
         name: collator
       interval: 1m
   timeout: 5m
-  values: {}
+  values:
+    extraContainers:
+      - name: index-patterns
+        image: IMAGE:TAG
+        command:
+          - bash
+          - -c
+          - |
+            #!/bin/bash
+            set -ex
+            KIBANA_URL=http://localhost:5601
+
+            while [[ "$(curl -s -o /dev/null -w '%{http_code}\n' $KIBANA_URL/app/kibana)" != "200" ]]; do
+              echo "Waiting for API"
+              sleep 1;
+            done
+
+            curl -XPOST $KIBANA_URL/api/index_patterns/index_pattern \
+              -H 'kbn-xsrf: true' \
+              -H 'Content-Type: application/json' \
+              -d '{ "index_pattern": { "title": "logstash-*", "timeFieldName":"@timestamp" }}'
+
+            curl -XPOST $KIBANA_URL/api/kibana/settings/defaultIndex \
+              -H 'Content-Type: application/json' \
+              -H 'kbn-xsrf: true' \
+              -d '{"value" : "logstash-*"}'
+
+            tail -f /dev/null

manifests/function/kibana/replacements/versions.yaml

@@ -47,3 +47,23 @@ replacements:
       kind: HelmRelease
       name: kibana
     fieldrefs: ["{.spec.values.imageTag}"]
+- source:
+    objref:
+      kind: VersionsCatalogue
+      name: versions-treasuremap
+    fieldref: "{.spec.image_components.kibana.kibana.repository}"
+  target:
+    objref:
+      kind: HelmRelease
+      name: kibana
+    fieldrefs: ["{.spec.values.extraContainers[?(@.name == 'index-patterns')].image}%IMAGE%"]
+- source:
+    objref:
+      kind: VersionsCatalogue
+      name: versions-treasuremap
+    fieldref: "{.spec.image_components.kibana.kibana.tag}"
+  target:
+    objref:
+      kind: HelmRelease
+      name: kibana
+    fieldrefs: ["{.spec.values.extraContainers[?(@.name == 'index-patterns')].image}%TAG%"]

manifests/function/lma-configs/logging/flows/cluster-flow-all.yaml

@@ -5,17 +5,6 @@ metadata:
 spec:
   match:
     - select: {}
-  filters:
+  filters: []
-    - enhanceK8s: {}
-    - tag_normaliser:
-        format: ${namespace_name}.${pod_name}.${container_name}
-    - parser:
-        reserve_data: true
-        parse:
-          type: multi-format
-          patterns:
-            - format: nginx
-            - format: json
-            - format: multiline
   globalOutputRefs:
     - elasticsearch

manifests/function/lma-configs/logging/outputs/cluster-output-elasticsearch.yaml

@@ -3,12 +3,12 @@ kind: ClusterOutput
 metadata:
   name: elasticsearch
 spec:
-  loggingRef: lma-infra-logging
   elasticsearch:
-    # TODO: Replace this with the real endpoint
     host: elasticsearch-ingest.lma.svc.cluster.local
     port: 9200
-    scheme: https
+    scheme: http
+    logstash_format: true
+    include_timestamp: true
     ssl_verify: false
     ssl_version: TLSv1_2
     buffer:

manifests/function/lma-configs/monitoring/thanos/lma-infra-thanos.yaml

@@ -1,7 +1,7 @@
 apiVersion: monitoring.banzaicloud.io/v1alpha1
 kind: Thanos
 metadata:
-  name: lma-infra-thanos
+  name: monitoring-thanos
 spec:
   queryDiscovery: true
   query: {}

manifests/function/lma-configs/monitoring/thanos/storeendpoints/lma-infra-prometheus.yaml

@@ -3,7 +3,7 @@ kind: StoreEndpoint
 metadata:
   name: lma-infra-prometheus
 spec:
-  thanos: lma-infra-thanos
+  thanos: monitoring-thanos
   selector:
     labels:
       app: prometheus

manifests/site/reference-multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml

@@ -1,3 +1,6 @@
 resources:
   - ../../../../../type/multi-tenant/sub-clusters/lma/workload-config
   - ../catalogues
+
+transformers:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/workload-config/replacements

manifests/type/multi-tenant/shared/catalogues/subcluster-networking.yaml

@@ -36,6 +36,14 @@ spec:
         nodePort: 30003
       - name: kibana
         nodePort: 30004
+      - name: grafana
+        nodePort: 30005
+      - name: prometheus
+        nodePort: 30006
+      - name: alertmanager
+        nodePort: 30007
+      - name: thanos-query
+        nodePort: 30008
       # Potential ports that can be used by sub-cluster services.
       - name: loadBalancerWorker
         nodePortRange:

manifests/type/multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml

@@ -6,3 +6,7 @@ patches:
     target:
       kind: ClusterFlow
       name: all-pods
+  - path: patches/thanos-query.yaml
+    target:
+      kind: Thanos
+      name: monitoring-thanos

manifests/type/multi-tenant/sub-clusters/lma/workload-config/patches/thanos-query.yaml

@@ -0,0 +1,11 @@
+apiVersion: monitoring.banzaicloud.io/v1alpha1
+kind: Thanos
+metadata:
+  name: monitoring-thanos
+spec:
+  query:
+    serviceOverrides:
+      type: NodePort
+      ports:
+        - name: http
+          port: 10902

manifests/type/multi-tenant/sub-clusters/lma/workload-config/replacements/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - lma-service-ports.yaml

manifests/type/multi-tenant/sub-clusters/lma/workload-config/replacements/lma-service-ports.yaml

@@ -0,0 +1,19 @@
+apiVersion: airshipit.org/v1alpha1
+kind: ReplacementTransformer
+metadata:
+  name: lma-service-ports
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: localhost/replacement-transformer
+replacements:
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: subcluster-networking
+    fieldref: "{.spec.lma.exposed_services[?(.name == 'thanos-query')].nodePort}"
+  target:
+    objref:
+      kind: Thanos
+      name: monitoring-thanos
+    fieldrefs: ["{.spec.query.serviceOverrides.ports[?(.name == 'http')].nodePort}"]

manifests/type/multi-tenant/sub-clusters/lma/workload/services/kustomization.yaml

@@ -3,10 +3,6 @@ resources:
   - ../../../../../../composite/elastic-stack
   - ../../../../../../composite/monitoring-stack
 
-# This kustomization exists be able to set the namespace here without
-# affecting the namespaces of anything else in the parent directory
-namespace: lma
-
 patches:
   - path: patches/elasticsearch-ingest.yaml
     target:
@@ -20,3 +16,19 @@ patches:
     target:
       kind: HelmRelease
       name: kibana
+  - path: patches/grafana.yaml
+    target:
+      kind: HelmRelease
+      name: grafana
+  - path: patches/kube-prometheus-stack.yaml
+    target:
+      kind: HelmRelease
+      name: kube-prometheus-stack
+  - patch: |-
+      - op: add
+        path: /spec/dependsOn
+        value:
+          - name: kube-prometheus-stack
+    target:
+      kind: HelmRelease
+      name: prometheus-elasticsearch-exporter

manifests/type/multi-tenant/sub-clusters/lma/workload/services/patches/grafana.yaml

@@ -0,0 +1,8 @@
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: grafana
+spec:
+  values:
+    service:
+      type: NodePort

manifests/type/multi-tenant/sub-clusters/lma/workload/services/patches/kube-prometheus-stack.yaml

@@ -0,0 +1,12 @@
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: kube-prometheus-stack
+spec:
+  values:
+    alertmanager:
+      service:
+        type: NodePort
+    prometheus:
+      service:
+        type: NodePort

manifests/type/multi-tenant/sub-clusters/lma/workload/services/replacements/lma-service-ports.yaml

@@ -27,3 +27,33 @@ replacements:
       kind: HelmRelease
       name: kibana
     fieldrefs: ["{.spec.values.service.nodePort}"]
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: subcluster-networking
+    fieldref: "{.spec.lma.exposed_services[?(.name == 'grafana')].nodePort}"
+  target:
+    objref:
+      kind: HelmRelease
+      name: grafana
+    fieldrefs: ["{.spec.values.service.nodePort}"]
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: subcluster-networking
+    fieldref: "{.spec.lma.exposed_services[?(.name == 'prometheus')].nodePort}"
+  target:
+    objref:
+      kind: HelmRelease
+      name: kube-prometheus-stack
+    fieldrefs: ["{.spec.values.prometheus.service.nodePort}"]
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: subcluster-networking
+    fieldref: "{.spec.lma.exposed_services[?(.name == 'alertmanager')].nodePort}"
+  target:
+    objref:
+      kind: HelmRelease
+      name: kube-prometheus-stack
+    fieldrefs: ["{.spec.values.alertmanager.service.nodePort}"]

manifests/type/multi-tenant/target/workload-config/replacements/lma-elasticsearch-output.yaml

@@ -21,7 +21,7 @@ replacements:
     objref:
       kind: VariableCatalogue
       name: subcluster-networking
-    fieldref: "{.spec.lma.exposed_services[?(@.name == 'elasticsearch-ingest')].nodePort}"
+    fieldref: "{.spec.lma.exposed_services[?(.name == 'elasticsearch-ingest')].nodePort}"
   target:
     objref:
       kind: ClusterOutput