Add peer meshing to Alertmanager

Adds additional flags to Alertmanager for the peer meshing. This
also adds a headless discovery service so each instance can
calculate the DNS names of its mesh peers on startup.

Change-Id: I2ba7f4aec88f73e6bc3ff31117973ebb4e85ceba

prometheus-alertmanager/templates/bin/_alertmanager.sh.tpl

@@ -22,7 +22,17 @@ COMMAND="${@:-start}"
 function start () {
   exec /bin/alertmanager \
     -config.file=/etc/config/alertmanager.yml \
-    -storage.path=/var/lib/alertmanager/data
+    -storage.path={{ .Values.conf.command_flags.storage.path }} \
+    -mesh.listen-address={{ .Values.conf.command_flags.mesh.listen_address }} \
+    $(generate_peers)
+}
+
+function generate_peers () {
+  final_pod_suffix=$(( {{ .Values.pod.replicas.alertmanager }}-1 ))
+  for pod_suffix in `seq 0 "$final_pod_suffix"`
+  do
+    echo -mesh.peer={{ .Release.Name }}-$pod_suffix.$DISCOVERY_SVC:6783
+  done
 }
 
 function stop () {

prometheus-alertmanager/templates/service-discovery.yaml

@@ -0,0 +1,32 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.service_discovery }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "alerts" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+  - name: peer-mesh
+    port: {{ .Values.network.alertmanager.mesh_port }}
+  selector:
+{{ tuple $envAll "alertmanager" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- end }}

prometheus-alertmanager/templates/statefulset.yaml

@@ -29,7 +29,7 @@ kind: StatefulSet
 metadata:
   name: alertmanager
 spec:
-  serviceName: {{ tuple "alerts" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  serviceName: {{ tuple "alerts" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
   replicas: {{ .Values.pod.replicas.alertmanager }}
   template:
     metadata:
@@ -60,9 +60,14 @@ spec:
                   - /tmp/alertmanager.sh
                   - stop
 {{ tuple $envAll $envAll.Values.pod.resources.alertmanager | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: DISCOVERY_SVC
+              value: {{ tuple "alerts" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
           ports:
             - name: alerts-api
               containerPort: {{ .Values.network.alertmanager.port }}
+            - name: peer-mesh
+              containerPort: {{ .Values.network.alertmanager.mesh_port }}
           readinessProbe:
             httpGet:
               path: /#/status

prometheus-alertmanager/values.yaml

@@ -92,8 +92,9 @@ endpoints:
     name: alertmanager
     namespace: null
     hosts:
-      default: alerts-api
+      default: alerts-engine
       public: alertmanager
+      discovery: alertmanager-discovery
     host_fqdn_override:
       default: null
     path:
@@ -104,6 +105,8 @@ endpoints:
       api:
         default: 9093
         public: 80
+      mesh:
+        default: 6783
 
 dependencies:
   alertmanager:
@@ -130,6 +133,7 @@ network:
       enabled: false
       port: 30903
     port: 9093
+    mesh_port: 6783
 
 storage:
   enabled: true
@@ -149,11 +153,17 @@ manifests:
   pvc: true
   rbac_entrypoint: true
   service: true
+  service_discovery: true
   service_ingress: true
   serviceaccount: true
   statefulset: true
 
 conf:
+  command_flags:
+    storage:
+      path: /var/lib/alertmanager/data
+    mesh:
+      listen_address: "0.0.0.0:6783"
   alertmanager: |
     global:
       # The smarthost and SMTP sender used for mail notifications.