From 4bf27930e9e43dc8bf96ba834bd43e37b6422b05 Mon Sep 17 00:00:00 2001 From: gs909v Date: Thu, 21 Jan 2021 14:30:13 -0600 Subject: [PATCH] Treasuremap Elasticsearch & Kibana This change adds a composite to treasuremap for deploying Elasticsearch & Kibana in the lma-stack phase Co-Authored-By: gs909v Change-Id: I6b4a9b1787f9daae9889084cb2dde483b815186e --- .../elasticsearch-data/kustomization.yaml | 20 +++++++ .../patches/elasticsearch-data.yaml | 27 +++++++++ .../replacements/kustomization.yaml | 2 + .../replacements/versions.yaml | 59 +++++++++++++++++++ .../elasticsearch-ingest/kustomization.yaml | 19 ++++++ .../patches/elasticsearch-ingest.yaml | 26 ++++++++ .../replacements/kustomization.yaml | 2 + .../replacements/versions.yaml | 59 +++++++++++++++++++ .../elastic-stack/kustomization.yaml | 9 +++ .../elastic-stack/patches/kibana.yaml | 24 ++++++++ .../replacements/kustomization.yaml | 4 ++ .../lma-infra/patches/logging-spec.yaml | 2 +- .../function/elasticsearch/helmrelease.yaml | 16 +++++ .../elasticsearch/helmrepository.yaml | 6 ++ .../function/elasticsearch/kustomization.yaml | 3 + .../replacements/kustomization.yaml | 2 + .../elasticsearch/replacements/versions.yaml | 59 +++++++++++++++++++ manifests/function/kibana/helmrelease.yaml | 16 +++++ manifests/function/kibana/helmrepository.yaml | 6 ++ manifests/function/kibana/kustomization.yaml | 3 + .../kibana/replacements/kustomization.yaml | 2 + .../kibana/replacements/versions.yaml | 59 +++++++++++++++++++ .../versions-treasuremap.yaml | 16 +++++ .../target/lma-stack/kustomization.yaml | 2 + 24 files changed, 442 insertions(+), 1 deletion(-) create mode 100644 manifests/composite/elastic-stack/elasticsearch-data/kustomization.yaml create mode 100644 manifests/composite/elastic-stack/elasticsearch-data/patches/elasticsearch-data.yaml create mode 100644 manifests/composite/elastic-stack/elasticsearch-data/replacements/kustomization.yaml create mode 100644 manifests/composite/elastic-stack/elasticsearch-data/replacements/versions.yaml create mode 100644 manifests/composite/elastic-stack/elasticsearch-ingest/kustomization.yaml create mode 100644 manifests/composite/elastic-stack/elasticsearch-ingest/patches/elasticsearch-ingest.yaml create mode 100644 manifests/composite/elastic-stack/elasticsearch-ingest/replacements/kustomization.yaml create mode 100644 manifests/composite/elastic-stack/elasticsearch-ingest/replacements/versions.yaml create mode 100644 manifests/composite/elastic-stack/kustomization.yaml create mode 100644 manifests/composite/elastic-stack/patches/kibana.yaml create mode 100644 manifests/composite/elastic-stack/replacements/kustomization.yaml create mode 100644 manifests/function/elasticsearch/helmrelease.yaml create mode 100644 manifests/function/elasticsearch/helmrepository.yaml create mode 100644 manifests/function/elasticsearch/kustomization.yaml create mode 100644 manifests/function/elasticsearch/replacements/kustomization.yaml create mode 100644 manifests/function/elasticsearch/replacements/versions.yaml create mode 100644 manifests/function/kibana/helmrelease.yaml create mode 100644 manifests/function/kibana/helmrepository.yaml create mode 100644 manifests/function/kibana/kustomization.yaml create mode 100644 manifests/function/kibana/replacements/kustomization.yaml create mode 100644 manifests/function/kibana/replacements/versions.yaml diff --git a/manifests/composite/elastic-stack/elasticsearch-data/kustomization.yaml b/manifests/composite/elastic-stack/elasticsearch-data/kustomization.yaml new file mode 100644 index 000000000..d39e8259e --- /dev/null +++ b/manifests/composite/elastic-stack/elasticsearch-data/kustomization.yaml @@ -0,0 +1,20 @@ +resources: + - ../../../function/elasticsearch + +patches: + - patch: |- + - op: replace + path: /metadata/name + value: elasticsearch-data + target: + kind: HelmRelease + name: elasticsearch + - patch: |- + - op: replace + path: /metadata/name + value: elasticsearch-data + target: + kind: HelmRepository + name: elasticsearch + + - path: patches/elasticsearch-data.yaml diff --git a/manifests/composite/elastic-stack/elasticsearch-data/patches/elasticsearch-data.yaml b/manifests/composite/elastic-stack/elasticsearch-data/patches/elasticsearch-data.yaml new file mode 100644 index 000000000..b58775f64 --- /dev/null +++ b/manifests/composite/elastic-stack/elasticsearch-data/patches/elasticsearch-data.yaml @@ -0,0 +1,27 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: elasticsearch-data +spec: + releaseName: elasticsearch-data + chart: + spec: + sourceRef: + name: elasticsearch-data + values: + nodeGroup: "data" + masterService: elasticsearch-ingest + roles: + master: "false" + ingest: "false" + data: "true" + replicas: 1 + resources: + requests: + cpu: "1m" + memory: "0.5Gi" + limits: + cpu: "1000m" + memory: "2Gi" + persistence: + enabled: false diff --git a/manifests/composite/elastic-stack/elasticsearch-data/replacements/kustomization.yaml b/manifests/composite/elastic-stack/elasticsearch-data/replacements/kustomization.yaml new file mode 100644 index 000000000..ee1becf0d --- /dev/null +++ b/manifests/composite/elastic-stack/elasticsearch-data/replacements/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - versions.yaml diff --git a/manifests/composite/elastic-stack/elasticsearch-data/replacements/versions.yaml b/manifests/composite/elastic-stack/elasticsearch-data/replacements/versions.yaml new file mode 100644 index 000000000..873c166fb --- /dev/null +++ b/manifests/composite/elastic-stack/elasticsearch-data/replacements/versions.yaml @@ -0,0 +1,59 @@ +apiVersion: airshipit.org/v1alpha1 +kind: ReplacementTransformer +metadata: + name: elasticsearch-data-versions-replacements + annotations: + config.kubernetes.io/function: |- + container: + image: quay.io/airshipit/replacement-transformer:v2 +replacements: +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.helm_repositories.elastic.url}" + target: + objref: + kind: HelmRepository + name: elasticsearch-data + fieldrefs: ["{.spec.url}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.charts.elasticsearch.chart}" + target: + objref: + kind: HelmRelease + name: elasticsearch-data + fieldrefs: ["{.spec.chart.spec.chart}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.charts.elasticsearch.version}" + target: + objref: + kind: HelmRelease + name: elasticsearch-data + fieldrefs: ["{.spec.chart.version}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.image_components.elasticsearch.elasticsearch.repository}" + target: + objref: + kind: HelmRelease + name: elasticsearch-data + fieldrefs: ["{.spec.values.image}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.image_components.elasticsearch.elasticsearch.tag}" + target: + objref: + kind: HelmRelease + name: elasticsearch-data + fieldrefs: ["{.spec.values.imageTag}"] \ No newline at end of file diff --git a/manifests/composite/elastic-stack/elasticsearch-ingest/kustomization.yaml b/manifests/composite/elastic-stack/elasticsearch-ingest/kustomization.yaml new file mode 100644 index 000000000..9eba86f6a --- /dev/null +++ b/manifests/composite/elastic-stack/elasticsearch-ingest/kustomization.yaml @@ -0,0 +1,19 @@ +resources: + - ../../../function/elasticsearch + +patches: + - patch: |- + - op: replace + path: /metadata/name + value: elasticsearch-ingest + target: + kind: HelmRelease + name: elasticsearch + - patch: |- + - op: replace + path: /metadata/name + value: elasticsearch-ingest + target: + kind: HelmRepository + name: elasticsearch + - path: patches/elasticsearch-ingest.yaml diff --git a/manifests/composite/elastic-stack/elasticsearch-ingest/patches/elasticsearch-ingest.yaml b/manifests/composite/elastic-stack/elasticsearch-ingest/patches/elasticsearch-ingest.yaml new file mode 100644 index 000000000..9d7be3c37 --- /dev/null +++ b/manifests/composite/elastic-stack/elasticsearch-ingest/patches/elasticsearch-ingest.yaml @@ -0,0 +1,26 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: elasticsearch-ingest +spec: + releaseName: elasticsearch-ingest + chart: + spec: + sourceRef: + name: elasticsearch-ingest + values: + nodeGroup: "ingest" + roles: + master: "true" + ingest: "true" + data: "false" + replicas: 1 + resources: + requests: + cpu: "1m" + memory: "0.5Gi" + limits: + cpu: "1000m" + memory: "2Gi" + persistence: + enabled: false diff --git a/manifests/composite/elastic-stack/elasticsearch-ingest/replacements/kustomization.yaml b/manifests/composite/elastic-stack/elasticsearch-ingest/replacements/kustomization.yaml new file mode 100644 index 000000000..ee1becf0d --- /dev/null +++ b/manifests/composite/elastic-stack/elasticsearch-ingest/replacements/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - versions.yaml diff --git a/manifests/composite/elastic-stack/elasticsearch-ingest/replacements/versions.yaml b/manifests/composite/elastic-stack/elasticsearch-ingest/replacements/versions.yaml new file mode 100644 index 000000000..76448f6c8 --- /dev/null +++ b/manifests/composite/elastic-stack/elasticsearch-ingest/replacements/versions.yaml @@ -0,0 +1,59 @@ +apiVersion: airshipit.org/v1alpha1 +kind: ReplacementTransformer +metadata: + name: elasticsearch-ingest-versions-replacements + annotations: + config.kubernetes.io/function: |- + container: + image: quay.io/airshipit/replacement-transformer:v2 +replacements: +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.helm_repositories.elastic.url}" + target: + objref: + kind: HelmRepository + name: elasticsearch-ingest + fieldrefs: ["{.spec.url}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.charts.elasticsearch.chart}" + target: + objref: + kind: HelmRelease + name: elasticsearch-ingest + fieldrefs: ["{.spec.chart.spec.chart}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.charts.elasticsearch.version}" + target: + objref: + kind: HelmRelease + name: elasticsearch-ingest + fieldrefs: ["{.spec.chart.version}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.image_components.elasticsearch.elasticsearch.repository}" + target: + objref: + kind: HelmRelease + name: elasticsearch-ingest + fieldrefs: ["{.spec.values.image}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.image_components.elasticsearch.elasticsearch.tag}" + target: + objref: + kind: HelmRelease + name: elasticsearch-ingest + fieldrefs: ["{.spec.values.imageTag}"] \ No newline at end of file diff --git a/manifests/composite/elastic-stack/kustomization.yaml b/manifests/composite/elastic-stack/kustomization.yaml new file mode 100644 index 000000000..10cffa3fc --- /dev/null +++ b/manifests/composite/elastic-stack/kustomization.yaml @@ -0,0 +1,9 @@ +resources: + - elasticsearch-ingest + - elasticsearch-data + - ../../function/kibana + +namespace: lma + +patches: + - path: patches/kibana.yaml diff --git a/manifests/composite/elastic-stack/patches/kibana.yaml b/manifests/composite/elastic-stack/patches/kibana.yaml new file mode 100644 index 000000000..95cd933e5 --- /dev/null +++ b/manifests/composite/elastic-stack/patches/kibana.yaml @@ -0,0 +1,24 @@ +apiVersion: "helm.toolkit.fluxcd.io/v2beta1" +kind: HelmRelease +metadata: + name: kibana +spec: + values: + elasticsearchHosts: "http://elasticsearch-ingest.lma.svc.cluster.local:9200" + replicas: 1 + # Extra environment variables to append to this nodeGroup + # This will be appended to the current 'env:' key. You can use any of the kubernetes env + # syntax here + extraEnvs: + - name: "NODE_OPTIONS" + value: "--max-old-space-size=1800" + # additionals labels + labels: {} + resources: + requests: + memory: "0.5Gi" + cpu: "1m" + limits: + memory: "1024Mi" + cpu: "1000m" + protocol: http diff --git a/manifests/composite/elastic-stack/replacements/kustomization.yaml b/manifests/composite/elastic-stack/replacements/kustomization.yaml new file mode 100644 index 000000000..7797506eb --- /dev/null +++ b/manifests/composite/elastic-stack/replacements/kustomization.yaml @@ -0,0 +1,4 @@ +resources: + - ../elasticsearch-ingest/replacements + - ../elasticsearch-data/replacements + - ../../../function/kibana/replacements diff --git a/manifests/composite/lma-infra/patches/logging-spec.yaml b/manifests/composite/lma-infra/patches/logging-spec.yaml index 8277f6d1d..2366ef915 100644 --- a/manifests/composite/lma-infra/patches/logging-spec.yaml +++ b/manifests/composite/lma-infra/patches/logging-spec.yaml @@ -36,7 +36,7 @@ spec: - name: elasticsearch spec: elasticsearch: - host: elasticsearch-elasticsearch-cluster.default.svc.cluster.local + host: elasticsearch-ingest.lma-infra.svc.cluster.local port: 9200 scheme: https ssl_verify: false diff --git a/manifests/function/elasticsearch/helmrelease.yaml b/manifests/function/elasticsearch/helmrelease.yaml new file mode 100644 index 000000000..4ca967efc --- /dev/null +++ b/manifests/function/elasticsearch/helmrelease.yaml @@ -0,0 +1,16 @@ +apiVersion: "helm.toolkit.fluxcd.io/v2beta1" +kind: HelmRelease +metadata: + name: elasticsearch +spec: + releaseName: elasticsearch + interval: 5m + chart: + spec: + sourceRef: + kind: HelmRepository + name: elasticsearch + interval: 1m + wait: false + timeout: 5m + values: {} diff --git a/manifests/function/elasticsearch/helmrepository.yaml b/manifests/function/elasticsearch/helmrepository.yaml new file mode 100644 index 000000000..dba0f02d2 --- /dev/null +++ b/manifests/function/elasticsearch/helmrepository.yaml @@ -0,0 +1,6 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: HelmRepository +metadata: + name: elasticsearch +spec: + interval: 10m diff --git a/manifests/function/elasticsearch/kustomization.yaml b/manifests/function/elasticsearch/kustomization.yaml new file mode 100644 index 000000000..25a9bbac8 --- /dev/null +++ b/manifests/function/elasticsearch/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - helmrepository.yaml + - helmrelease.yaml diff --git a/manifests/function/elasticsearch/replacements/kustomization.yaml b/manifests/function/elasticsearch/replacements/kustomization.yaml new file mode 100644 index 000000000..ee1becf0d --- /dev/null +++ b/manifests/function/elasticsearch/replacements/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - versions.yaml diff --git a/manifests/function/elasticsearch/replacements/versions.yaml b/manifests/function/elasticsearch/replacements/versions.yaml new file mode 100644 index 000000000..543124ba4 --- /dev/null +++ b/manifests/function/elasticsearch/replacements/versions.yaml @@ -0,0 +1,59 @@ +apiVersion: airshipit.org/v1alpha1 +kind: ReplacementTransformer +metadata: + name: elasticseach-versions-replacements + annotations: + config.kubernetes.io/function: |- + container: + image: quay.io/airshipit/replacement-transformer:v2 +replacements: +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.helm_repositories.elastic.url}" + target: + objref: + kind: HelmRepository + name: elasticseach + fieldrefs: ["{.spec.url}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.charts.elasticseach.chart}" + target: + objref: + kind: HelmRelease + name: elasticseach + fieldrefs: ["{.spec.chart.spec.chart}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.charts.elasticseach.version}" + target: + objref: + kind: HelmRelease + name: elasticseach + fieldrefs: ["{.spec.chart.version}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.image_components.elasticseach.elasticseach.image}" + target: + objref: + kind: HelmRelease + name: elasticseach + fieldrefs: ["{.spec.values.image}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.image_components.elasticseach.elasticseach.imageTag}" + target: + objref: + kind: HelmRelease + name: elasticseach + fieldrefs: ["{.spec.values.imageTag}"] \ No newline at end of file diff --git a/manifests/function/kibana/helmrelease.yaml b/manifests/function/kibana/helmrelease.yaml new file mode 100644 index 000000000..da36c8e69 --- /dev/null +++ b/manifests/function/kibana/helmrelease.yaml @@ -0,0 +1,16 @@ +apiVersion: "helm.toolkit.fluxcd.io/v2beta1" +kind: HelmRelease +metadata: + name: kibana +spec: + releaseName: kibana + interval: 5m + chart: + spec: + sourceRef: + kind: HelmRepository + name: kibana + interval: 1m + wait: false + timeout: 5m + values: {} diff --git a/manifests/function/kibana/helmrepository.yaml b/manifests/function/kibana/helmrepository.yaml new file mode 100644 index 000000000..8ade95e52 --- /dev/null +++ b/manifests/function/kibana/helmrepository.yaml @@ -0,0 +1,6 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: HelmRepository +metadata: + name: kibana +spec: + interval: 10m diff --git a/manifests/function/kibana/kustomization.yaml b/manifests/function/kibana/kustomization.yaml new file mode 100644 index 000000000..25a9bbac8 --- /dev/null +++ b/manifests/function/kibana/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - helmrepository.yaml + - helmrelease.yaml diff --git a/manifests/function/kibana/replacements/kustomization.yaml b/manifests/function/kibana/replacements/kustomization.yaml new file mode 100644 index 000000000..ee1becf0d --- /dev/null +++ b/manifests/function/kibana/replacements/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - versions.yaml diff --git a/manifests/function/kibana/replacements/versions.yaml b/manifests/function/kibana/replacements/versions.yaml new file mode 100644 index 000000000..46d360d95 --- /dev/null +++ b/manifests/function/kibana/replacements/versions.yaml @@ -0,0 +1,59 @@ +apiVersion: airshipit.org/v1alpha1 +kind: ReplacementTransformer +metadata: + name: kibana-versions-replacements + annotations: + config.kubernetes.io/function: |- + container: + image: quay.io/airshipit/replacement-transformer:v2 +replacements: +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.helm_repositories.elastic.url}" + target: + objref: + kind: HelmRepository + name: kibana + fieldrefs: ["{.spec.url}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.charts.kibana.chart}" + target: + objref: + kind: HelmRelease + name: kibana + fieldrefs: ["{.spec.chart.spec.chart}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.charts.kibana.version}" + target: + objref: + kind: HelmRelease + name: kibana + fieldrefs: ["{.spec.chart.version}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.image_components.kibana.kibana.repository}" + target: + objref: + kind: HelmRelease + name: kibana + fieldrefs: ["{.spec.values.image}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.image_components.kibana.kibana.tag}" + target: + objref: + kind: HelmRelease + name: kibana + fieldrefs: ["{.spec.values.imageTag}"] diff --git a/manifests/function/treasuremap-base-catalogues/versions-treasuremap.yaml b/manifests/function/treasuremap-base-catalogues/versions-treasuremap.yaml index bb2531155..77bb4a950 100644 --- a/manifests/function/treasuremap-base-catalogues/versions-treasuremap.yaml +++ b/manifests/function/treasuremap-base-catalogues/versions-treasuremap.yaml @@ -12,6 +12,8 @@ spec: checksum: 4a6909d1480ac30d676accd7b37ec711 helm_repositories: + elastic: + url: https://helm.elastic.co grafana: url: https://grafana.github.io/helm-charts prometheus_community: @@ -29,9 +31,15 @@ spec: url: http://helm-chart-collator.collator.svc:8080 charts: + elasticsearch: + chart: elasticsearch + version: 7.12.1 grafana: chart: grafana version: 6.9.1 + kibana: + chart: kibana + version: 7.12.1 kube_prometheus_stack: chart: kube-prometheus-stack version: 16.0.1 @@ -133,10 +141,18 @@ spec: # image_components are organized by # : # : + elasticsearch: + elasticsearch: + repository: docker.elastic.co/elasticsearch/elasticsearch + tag: 7.12.1 grafana: grafana: repository: grafana/grafana tag: 7.5.5 + kibana: + kibana: + repository: docker.elastic.co/kibana/kibana + tag: 7.12.1 kube_prometheus_stack: admission_webhooks: repository: jettech/kube-webhook-certgen diff --git a/manifests/site/test-site/target/lma-stack/kustomization.yaml b/manifests/site/test-site/target/lma-stack/kustomization.yaml index f736c402a..f6373d672 100644 --- a/manifests/site/test-site/target/lma-stack/kustomization.yaml +++ b/manifests/site/test-site/target/lma-stack/kustomization.yaml @@ -1,10 +1,12 @@ resources: + - ../../../../composite/elastic-stack - ../../../../composite/monitoring-stack - ../../../../function/minio - ../catalogues - minio-admin-secret.yaml transformers: + - ../../../../composite/elastic-stack/replacements - ../../../../composite/monitoring-stack/replacements - ../../../../function/minio/replacements