Update alertmanager include snmp_notifier function

Change-Id: I5aedbdcdbba397a9fddde19a0898cb91de08553a

prometheus-alertmanager/templates/bin/_alertmanager.sh.tpl

@@ -20,8 +20,8 @@ COMMAND="${@:-start}"
 function start () {
   exec /bin/alertmanager \
     --config.file=/etc/alertmanager/config.yml \
-    --storage.path={{ .Values.conf.command_flags.storage.path }} \
+    --storage.path={{ .Values.conf.command_flags.alertmanager.storage.path }} \
-    --cluster.listen-address={{ .Values.conf.command_flags.cluster.listen_address }} \
+    --cluster.listen-address={{ .Values.conf.command_flags.alertmanager.cluster.listen_address }} \
     $(generate_peers)
 }
 

prometheus-alertmanager/templates/configmap-etc.yaml

@@ -20,8 +20,7 @@ kind: ConfigMap
 metadata:
   name: alertmanager-etc
 data:
-  config.yml: |
+{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.alertmanager "key" "config.yml") | indent 2 }}
-{{ toYaml .Values.conf.alertmanager | indent 4 }}
   alert-templates.tmpl: |
 {{- if .Values.conf.alert_templates }}
 {{ .Values.conf.alert_templates | indent 4 }}

prometheus-alertmanager/templates/snmp-notifier/snmp-deployment.yaml

@@ -0,0 +1,75 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.snmpnotifier.deployment }}
+{{- $envAll := . }}
+
+{{- $mounts_snmpnotifier := .Values.pod.mounts.snmpnotifier.snmpnotifier }}
+{{- $mounts_snmpnotifier_init := .Values.pod.mounts.snmpnotifier.init_container }}
+
+{{- $serviceAccountName := "snmpnotifier" }}
+{{ tuple $envAll "snmpnotifier" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: snmpnotifier
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+  labels:
+{{ tuple $envAll "snmpnotifier" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  podManagementPolicy: "Parallel"
+  replicas: {{ .Values.pod.replicas.snmpnotifier }}
+  selector:
+    matchLabels:
+{{ tuple $envAll "snmpnotifier" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "snmpnotifier" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+{{ dict "envAll" $envAll "podName" "snmpnotifier" "containerNames" (list "snmpnotifier" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+    spec:
+{{ dict "envAll" $envAll "application" "server" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      serviceAccountName: {{ $serviceAccountName }}
+      affinity:
+{{ tuple $envAll "snmpnotifier" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+      nodeSelector:
+        {{ .Values.labels.snmpnotifier.node_selector_key }}: {{ .Values.labels.snmpnotifier.node_selector_value | quote }}
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.snmpnotifier.timeout | default "30" }}
+      containers:
+        - name: snmpnotifier
+{{ tuple $envAll "snmpnotifier" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.snmpnotifier | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "server" "container" "snmpnotifier" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          args:
+          - --alert.severity-label={{ .Values.conf.command_flags.snmpnotifier.alert_severity_label}}
+          - --alert.default-severity={{ .Values.conf.command_flags.snmpnotifier.alert_default_severity}}
+          - --snmp.version={{ .Values.conf.command_flags.snmpnotifier.snmp_version}}
+          - --snmp.destination={{ .Values.conf.command_flags.snmpnotifier.snmp_desination}}
+          - --snmp.trap-default-oid={{ .Values.conf.command_flags.snmpnotifier.snmp_trap_default_oid}}
+          - --snmp.trap-description-template={{ .Values.conf.command_flags.snmpnotifier.snmp_trap_description_template}}
+          - --snmp.community={{ .Values.conf.command_flags.snmpnotifier.snmp_community}}
+          - --log.level={{ .Values.conf.command_flags.snmpnotifier.log_level}}
+          ports:
+            - name: snmp-api
+              containerPort: {{ tuple "snmpnotifier" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: {{ tuple "snmpnotifier" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            initialDelaySeconds: 30
+            timeoutSeconds: 30
+{{- end }}

prometheus-alertmanager/templates/snmp-notifier/snmp-service.yaml

@@ -0,0 +1,34 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.snmpnotifier.service }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "snmpnotifier" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+spec:
+  ports:
+  - name: snmpnotifier-api
+    {{ if .Values.network.snmpnotifier.node_port.enabled }}
+    nodePort: {{ .Values.network.snmpnotifier.node_port.port }}
+    {{ end }}
+    port: {{ tuple "snmpnotifier" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  selector:
+{{ tuple $envAll "snmpnotifier" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+  {{ if .Values.network.snmpnotifier.node_port.enabled }}
+  type: NodePort
+  {{ end }}
+{{- end }}

prometheus-alertmanager/templates/statefulset.yaml

@@ -132,7 +132,7 @@ spec:
             name: alertmanager-bin
             defaultMode: 0555
 {{ if $mounts_alertmanager.volumes }}{{ toYaml $mounts_alertmanager.volumes | indent 8 }}{{ end }}
-{{- if not .Values.storage.enabled }}
+{{- if not .Values.storage.alertmanager.enabled }}
         - name: alertmanager-data
           emptyDir: {}
 {{- else }}
@@ -140,10 +140,10 @@ spec:
     - metadata:
         name: alertmanager-data
       spec:
-        accessModes: {{ .Values.storage.pvc.access_mode }}
+        accessModes: {{ .Values.storage.alertmanager.pvc.access_mode }}
         resources:
           requests:
-            storage: {{ .Values.storage.requests.storage  }}
+            storage: {{ .Values.storage.alertmanager.requests.storage  }}
-        storageClassName: {{ .Values.storage.storage_class }}
+        storageClassName: {{ .Values.storage.alertmanager.storage_class }}
 {{- end }}
 {{- end }}

prometheus-alertmanager/values.yaml

@@ -19,6 +19,7 @@
 images:
   tags:
     prometheus-alertmanager: docker.io/prom/alertmanager:v0.20.0
+    snmpnotifier: docker.io/maxwo/snmp-notifier:v1.0.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: IfNotPresent
@@ -32,6 +33,9 @@ labels:
   alertmanager:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
+  snmpnotifier:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
   job:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
@@ -60,15 +64,23 @@ pod:
     alertmanager:
       alertmanager:
       init_container: null
+    snmpnotifier:
+      snmpnotifier:
+      init_container: null
   replicas:
     alertmanager: 1
+    snmpnotifier: 1
   lifecycle:
     upgrades:
+      deployment:
+        pod_replacement_strategy: RollingUpdate
       statefulsets:
         pod_replacement_strategy: RollingUpdate
     termination_grace_period:
       alertmanager:
         timeout: 30
+      snmpnotifier:
+        timeout: 30
   resources:
     enabled: false
     alertmanager:
@@ -86,6 +98,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+    snmpnotifier:
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
+      requests:
+        memory: "128Mi"
+        cpu: "500m"
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -127,6 +146,20 @@ endpoints:
         public: 80
       mesh:
         default: 9094
+  snmpnotifier:
+    name: snmpnotifier
+    namespace: null
+    hosts:
+      default: snmp-engine
+    host_fqdn_override:
+      default: null
+    path:
+      default: /alerts
+    scheme:
+      default: 'http'
+    port:
+      api:
+        default: 9464
 
 dependencies:
   dynamic:
@@ -157,6 +190,10 @@ network:
     node_port:
       enabled: false
       port: 30903
+  snmpnotifier:
+    node_port:
+      enabled: false
+      port: 30464
 
 secrets:
   tls:
@@ -165,6 +202,7 @@ secrets:
         public: alerts-tls-public
 
 storage:
+  alertmanager:
     enabled: true
     pvc:
       access_mode: ["ReadWriteOnce"]
@@ -184,6 +222,9 @@ manifests:
   service_discovery: true
   service_ingress: true
   statefulset: true
+  snmpnotifier:
+    service: true
+    deployment: true
 
 network_policy:
   alertmanager:
@@ -194,11 +235,21 @@ network_policy:
 
 conf:
   command_flags:
+    alertmanager:
       storage:
         path: /var/lib/alertmanager/data
       cluster:
         listen_address: "0.0.0.0:9094"
-  alertmanager:
+    snmpnotifier:
+      alert_severity_label: severity
+      alert_default_severity: crititcal
+      snmp_version: V2c
+      snmp_desination: 192.168.89.128:162
+      snmp_trap_default_oid: 1.3.6.1.4.1.98789.0.1
+      snmp_trap_description_template: /etc/snmp_notifier/description-template.tpl
+      snmp_community: public
+      log_level: debug
+  alertmanager: |
     global:
       # The smarthost and SMTP sender used for mail notifications.
       smtp_smarthost: 'localhost:25'
@@ -234,7 +285,8 @@ conf:
       # resend them.
       repeat_interval: 3h
       # A default receiver
-      receiver: team-X-mails
+      # receiver: team-X-mails
+      receiver: snmp_notifier
       # All the above attributes are inherited by all child routes and can
       # overwritten on each.
       # The child route trees.
@@ -291,6 +343,11 @@ conf:
           - cluster
           - service
     receivers:
+      - name: 'snmp_notifier'
+        webhook_configs:
+        - send_resolved: true
+          #url: http://snmp-engine.osh-infra.svc.cluster.local:9464/alerts
+          url: {{ tuple "snmpnotifier" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
       - name: 'team-X-mails'
         email_configs:
           - to: 'team-X+alerts@example.org'
@@ -313,6 +370,6 @@ conf:
           - auth_token: <auth_token>
             room_id: 85
             message_format: html
-            notify: true
+            notify: false
   alert_templates: null
 ...

prometheus/values.yaml

@@ -156,13 +156,13 @@ endpoints:
         default: 9090
       http:
         default: 80
-  alerts:
+  alertmanager:
-    name: alertmanager
+    name: prometheus-alertmanager
     namespace: null
     hosts:
       default: alerts-engine
-      public: alertmanager
+      public: prometheus-alertmanager
-      discovery: alertmanager-discovery
+      discovery: prometheus-alertmanager-discovery
     host_fqdn_override:
       default: null
     path:
@@ -1081,7 +1081,7 @@ conf:
             bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
             relabel_configs:
             - source_labels: [__meta_kubernetes_pod_label_application]
-              regex: alertmanager
+              regex: prometheus-alertmanager
               action: keep
             - source_labels: [__meta_kubernetes_pod_container_port_name]
               regex: alerts-api

tools/deployment/multinode/060-alertmanager.sh

@@ -19,8 +19,7 @@ make prometheus-alertmanager
 
 #NOTE: Deploy command
 helm upgrade --install alertmanager ./prometheus-alertmanager \
-    --namespace=osh-infra \
+    --namespace=osh-infra
-    --set pod.replicas.alertmanager=3
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh osh-infra