Merge "Add ldap support to grafana, update version, add helm tests"

grafana/templates/bin/_datasource.sh.tpl

@@ -1,24 +0,0 @@
-#!/bin/bash
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-set -ex
-
-exec curl "http://${GF_SECURITY_ADMIN_USER}:${GF_SECURITY_ADMIN_PASSWORD}@${GRAFANA_SERVICE}:${GRAFANA_PORT}/api/datasources" \
-            -H "Content-Type: application/json;charset=UTF-8" --data-binary \
-            {{- with .Values.conf.datasource }}
-            "{\"name\":\"{{ .name }}\",\"type\":\"{{ .type }}\",\"url\":\"$PROMETHEUS_URL\",\"database\":\"{{ .database }}\",\"jsonData\":{ {{ .jsonData }} },\"access\":\"{{ .access }}\",\"isDefault\":{{ .isDefault }}}"
-            {{- end }}

grafana/templates/bin/_helm-tests.sh.tpl

@@ -0,0 +1,50 @@
+#!/bin/bash
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+
+set -ex
+
+function check_datasource () {
+  echo "Verifying prometheus datasource configured"
+  datasource_type=$(curl -K- <<< "--user ${GF_SECURITY_ADMIN_USER}:${GF_SECURITY_ADMIN_PASSWORD}" \
+  "${GRAFANA_ENDPOINT}/api/datasources" \
+  | python -c "import sys, json; print json.load(sys.stdin)[0]['type']")
+  if [ "$datasource_type" == "prometheus" ];
+  then
+     echo "PASS: Prometheus datasource found!";
+  else
+     echo "FAIL: Prometheus datasource not found!";
+     exit 1;
+  fi
+}
+
+function check_dashboard_count () {
+  echo "Verifying number of configured dashboards"
+  dashboard_count=$(curl -K- <<< "--user ${GF_SECURITY_ADMIN_USER}:${GF_SECURITY_ADMIN_PASSWORD}" \
+  "${GRAFANA_ENDPOINT}/api/admin/stats" \
+  | python -c "import sys, json; print json.load(sys.stdin)['dashboards']")
+  if [ "$dashboard_count" == "$DASHBOARD_COUNT" ];
+  then
+     echo "PASS: Reported number:$dashboard_count, expected number: $DASHBOARD_COUNT";
+  else
+     echo "FAIL: Reported number:$dashboard_count, expected number: $DASHBOARD_COUNT";
+     exit 1;
+  fi
+}
+
+check_datasource
+check_dashboard_count

grafana/templates/configmap-bin.yaml

@@ -28,8 +28,8 @@ data:
 {{ tuple "bin/_db-session-sync.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   image-repo-sync.sh: |
 {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
-  datasource.sh: |
+  helm-tests.sh: |
-{{ tuple "bin/_datasource.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   grafana.sh: |
 {{ tuple "bin/_grafana.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

grafana/templates/configmap-etc.yaml

@@ -35,8 +35,16 @@ kind: ConfigMap
 metadata:
   name: grafana-etc
 data:
+  datasources.yaml: |
+{{ toYaml .Values.conf.provisioning.datasources | indent 4 }}
+  dashboards.yaml: |
+{{ toYaml .Values.conf.provisioning.dashboards | indent 4 }}
   grafana.ini: |
 {{ include "helm-toolkit.utils.to_ini" .Values.conf.grafana | indent 4 }}
+{{ if not (empty .Values.conf.ldap) }}
+  ldap.toml: |
+{{ .Values.conf.ldap | indent 4 }}
+{{ end }}
 {{ range $key, $value := .Values.conf.dashboards }}
   {{$key}}.json: |
 {{ toJson $value | indent 4 }}

grafana/templates/deployment.yaml

@@ -69,16 +69,29 @@ spec:
                 secretKeyRef:
                   name: grafana-admin-creds
                   key: GRAFANA_ADMIN_PASSWORD
+            - name: PROMETHEUS_URL
+              value: {{ tuple "monitoring" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
           volumeMounts:
           - name: pod-etc-grafana
             mountPath: /etc/grafana
+          - name: pod-provisioning-grafana
+            mountPath: /var/lib/grafana/provisioning
           - name: grafana-bin
             mountPath: /tmp/grafana.sh
             subPath: grafana.sh
             readOnly: true
+          - name: grafana-etc
+            mountPath: /var/lib/grafana/provisioning/dashboards/dashboards.yaml
+            subPath: dashboards.yaml
+          - name: grafana-etc
+            mountPath: /var/lib/grafana/provisioning/datasources/datasources.yaml
+            subPath: datasources.yaml
           - name: grafana-etc
             mountPath: /etc/grafana/grafana.ini
             subPath: grafana.ini
+          - name: grafana-etc
+            mountPath: /etc/grafana/ldap.toml
+            subPath: ldap.toml
           - name: data
             mountPath: /var/lib/grafana/data
           {{- range $key, $value := .Values.conf.dashboards }}
@@ -90,6 +103,8 @@ spec:
       volumes:
         - name: pod-etc-grafana
           emptyDir: {}
+        - name: pod-provisioning-grafana
+          emptyDir: {}
         - name: grafana-bin
           configMap:
             name: grafana-bin

grafana/templates/job-prometheus-datasource.yaml

@@ -1,72 +0,0 @@
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if .Values.manifests.job_datasource }}
-{{- $envAll := . }}
-
-{{- $serviceAccountName := "grafana-register-datasource" }}
-{{ tuple $envAll "register_datasource" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: grafana-register-datasource
-spec:
-  template:
-    metadata:
-      labels:
-{{ tuple $envAll "grafana" "datasource" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
-    spec:
-      serviceAccountName: {{ $serviceAccountName }}
-      restartPolicy: OnFailure
-      nodeSelector:
-        {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
-      initContainers:
-{{ tuple $envAll "register_datasource" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
-      containers:
-        - name: grafana-datasource
-{{ tuple $envAll "datasource" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ tuple $envAll $envAll.Values.pod.resources.jobs.datasource | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-          command:
-            - /tmp/datasource.sh
-          env:
-            - name: GF_SECURITY_ADMIN_USER
-              valueFrom:
-                secretKeyRef:
-                  name: grafana-admin-creds
-                  key: GRAFANA_ADMIN_USERNAME
-            - name: GF_SECURITY_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: grafana-admin-creds
-                  key: GRAFANA_ADMIN_PASSWORD
-            - name: GRAFANA_SERVICE
-              value: {{ tuple "grafana" "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
-            - name: GRAFANA_PORT
-              value: {{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
-            - name: PROMETHEUS_URL
-              value: {{ tuple "monitoring" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
-          volumeMounts:
-            - name: grafana-bin
-              mountPath: /tmp/datasource.sh
-              subPath: datasource.sh
-              readOnly: true
-      volumes:
-        - name: grafana-bin
-          configMap:
-            name: grafana-bin
-            defaultMode: 0555
-{{- end }}

grafana/templates/pod-helm-tests.yaml

@@ -0,0 +1,60 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.helm_tests }}
+{{- $dashboardCount := len .Values.conf.dashboards }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{.Release.Name}}-test"
+  annotations:
+    "helm.sh/hook": test-success
+spec:
+  restartPolicy: Never
+  containers:
+    - name: {{.Release.Name}}-helm-tests
+{{ tuple $envAll "helm_tests" | include "helm-toolkit.snippets.image" | indent 6 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
+      command:
+        - /tmp/helm-tests.sh
+      env:
+        - name: DASHBOARD_COUNT
+          value: {{ $dashboardCount | quote }}
+        - name: GF_SECURITY_ADMIN_USER
+          valueFrom:
+            secretKeyRef:
+              name: grafana-admin-creds
+              key: GRAFANA_ADMIN_USERNAME
+        - name: GF_SECURITY_ADMIN_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: grafana-admin-creds
+              key: GRAFANA_ADMIN_PASSWORD
+        - name: GRAFANA_ENDPOINT
+          value: {{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+      volumeMounts:
+        - name: grafana-bin
+          mountPath: /tmp/helm-tests.sh
+          subPath: helm-tests.sh
+          readOnly: true
+  volumes:
+    - name: grafana-bin
+      configMap:
+        name: grafana-bin
+        defaultMode: 0555
+{{- end }}

grafana/values.yaml

@@ -18,11 +18,11 @@
 
 images:
   tags:
-    grafana: docker.io/grafana/grafana:4.5.2
+    grafana: docker.io/grafana/grafana:5.0.0
-    datasource: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
     db_init: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
     grafana_db_session_sync: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
+    helm_tests: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: IfNotPresent
   local_registry:
@@ -101,6 +101,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      tests:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
     grafana:
       requests:
         memory: "128Mi"
@@ -165,7 +172,7 @@ endpoints:
     auth:
       admin:
         username: admin
-        password: admin
+        password: password
     hosts:
       default: grafana-dashboard
       public: grafana
@@ -195,6 +202,21 @@ endpoints:
       api:
         default: 9090
         public: 80
+  ldap:
+    hosts:
+      default: ldap
+    auth:
+      admin:
+        password: password
+    host_fqdn_override:
+      default: null
+    path:
+      default: "/ou=People,dc=cluster,dc=local"
+    scheme:
+      default: ldap
+    port:
+      ldap:
+        default: 389
 
 dependencies:
   dynamic:
@@ -231,7 +253,7 @@ dependencies:
       services:
         - endpoint: internal
           service: local_image_registry
-    register_datasource:
+    tests:
       services:
         - endpoint: internal
           service: grafana
@@ -263,10 +285,10 @@ manifests:
   configmap_etc: true
   deployment: true
   ingress: true
+  helm_tests: true
   job_db_init: true
   job_db_init_session: true
   job_db_session_sync: true
-  job_datasource: true
   job_image_repo_sync: true
   secret_db: true
   secret_db_session: true
@@ -275,16 +297,67 @@ manifests:
   service_ingress: true
 
 conf:
-  datasource:
+  ldap: |
-    name: prometheus
+    verbose_logging = true
-    type: prometheus
+
-    database:
+    [[servers]]
-    access: proxy
+    host = "ldap.openstack.svc.cluster.local"
-    isDefault: true
+    port = 389
+    use_ssl = false
+    start_tls = false
+    ssl_skip_verify = false
+    bind_dn = "cn=admin,dc=cluster,dc=local"
+    bind_password = 'password'
+    search_filter = "(uid=%s)"
+    search_base_dns = ["dc=cluster,dc=local"]
+    group_search_filter = "(&(objectclass=posixGroup)(memberUID=uid=%s,ou=People,dc=cluster,dc=local))"
+    group_search_base_dns = ["ou=Groups,dc=cluster,dc=local"]
+
+    [servers.attributes]
+    username = "uid"
+    surname = "sn"
+    member_of = "cn"
+    email = "mail"
+
+    [[servers.group_mappings]]
+    group_dn = "cn=admin,dc=cluster,dc=local"
+    org_role = "Admin"
+
+    [[servers.group_mappings]]
+    group_dn = "*"
+    org_role = "Editor"
+  provisioning:
+    dashboards:
+      apiVersion: 1
+      providers:
+      - name: 'osh-infra-dashboards'
+        orgId: 1
+        folder: ''
+        type: file
+        disableDeletion: false
+        editable: false
+        options:
+          path: /var/lib/grafana/dashboards
+    datasources:
+      apiVersion: 1
+      datasources:
+      - name: prometheus
+        type: prometheus
+        access: proxy
+        orgId: 1
+        editable: true
+        url: 'http://prom-metrics.openstack.svc.cluster.local:9090'
   grafana:
+    auth.basic:
+      enabled: true
+    auth.ldap:
+      enabled: true
+      config_file: /etc/grafana/ldap.toml
+      allow_sign_up: true
     paths:
       data: /var/lib/grafana/data
       plugins: /var/lib/grafana/plugins
+      provisioning: /var/lib/grafana/provisioning
     server:
       protocol: http
       http_port: 3000
@@ -306,17 +379,10 @@ conf:
       allow_sign_up: false
       allow_org_create: false
       auto_assign_org: true
-      auto_assign_org_role: Admin
       default_theme: dark
     log:
       mode: console
       level: info
-    log.console:
-      level: info
-      format: console
-    dashboards.json:
-      enabled: true
-      path: /var/lib/grafana/dashboards
     grafana_net:
       url: https://grafana.net
   dashboards:

tools/deployment/multinode/090-grafana.sh

@@ -52,3 +52,6 @@ helm upgrade --install grafana ./grafana \
 
 #NOTE: Validate Deployment info
 helm status grafana
+
+#NOTE: Run helm tests
+helm test grafana