aic-helm normalization

* As of helm 2.0.0, it no longer seems necessary to
copy a "globals.tpl" into each child chart, defines in
child charts now seem to be reliably rendered in the
parent.  This was not working correctly in 2.0.0rc2
and so after a lot of testing, the globals define
insertion has been removed from the Makefiles. Also,
includes piped to b64encode are working now, meaning
we no longer have to do Makefile magic to base64
encode successfully.  That has been removed.  Once
.Files.Get works properly in a child chart context
we can remove all Make help.

* The openstack-base chart has been renamed to common
to better reflect that it is common to everything in
aic-helm, even non-openstack projects like ceph.  All
charts now include it as a requirement.

* A first pass at a "cleaner" template directory approach
has been applied to rabbitmq as a test chart.  This allows
files such as _start_rabbit to live in their raw form
and organizes files by config vs script in 'etc' or 'bin'
directories.  If accepted, will apply to all other charts
for consistency.

.gitignore

@@ -2,4 +2,4 @@
 **/*.tgz
 .idea/
 **/_partials.tpl
-**/_common.tpl
+**/_globals.tpl

Makefile

@@ -1,14 +1,14 @@
-.PHONY: ceph mariadb keystone memcached rabbitmq openstack-base openstack all clean base64
+.PHONY: ceph mariadb keystone memcached rabbitmq common openstack all clean 
 
-B64_DIRS := openstack-base/secrets
+B64_DIRS := common/secrets
-B64_EXCLUDE := $(wildcard openstack-base/secrets/*.b64)
+B64_EXCLUDE := $(wildcard common/secrets/*.b64)
 
 CHARTS := ceph mariadb rabbitmq memcached keystone openstack
-COMMON_TPL := openstack-base/templates/_common.tpl
+COMMON_TPL := common/templates/_globals.tpl
 
-all: openstack-base ceph mariadb rabbitmq memcached keystone openstack
+all: common ceph mariadb rabbitmq memcached keystone openstack
 
-openstack-base: build-openstack-base
+common: build-common
 
 ceph: build-ceph
 
@@ -23,24 +23,12 @@ memcached: build-memcached
 openstack: build-openstack
 	
 clean:  
-	$(shell rm -rf openstack-base/secrets/*.b64)
+	$(shell rm -rf common/secrets/*.b64)
 	$(shell rm -rf */templates/_partials.tpl)
-	$(shell rm -rf */templates/_common.tpl)
+	$(shell rm -rf */templates/_globals.tpl)
-	echo "Removed all .b64, _partials.tpl, and _common.tpl files"
+	echo "Removed all .b64, _partials.tpl, and _globals.tpl files"
-
-build-openstack-base:
-	# rebuild all base64 values
-	$(eval B64_OBJS = $(foreach dir,$(B64_DIRS),$(shell find $(dir)/* -type f $(foreach e,$(B64_EXCLUDE), -not -path "$(e)"))))
-	$(foreach var,$(B64_OBJS),cat $(var) | base64 | perl -pe 'chomp if eof' > $(var).b64;)
-
-	if [ -f openstack-base/Makefile ]; then make -C openstack-base; fi
-	if [ -f openstack-base/requirements.yaml ]; then helm dep up openstack-base; fi
-	helm lint openstack-base
-	helm package openstack-base
-	$(foreach var,$(CHARTS),$(shell cp $(COMMON_TPL) $(var)/templates))
 
 build-%:
-	if [ ! -f $*/templates/_common.tpl ]; then echo; seq -s= 30|tr -d '[:digit:]'; echo "You need to run 'make openstack-base' first to generate _common.tpl"; seq -s= 30|tr -d '[:digit:]'; exit 1; fi; 
 	if [ -f $*/Makefile ]; then make -C $*; fi
 	if [ -f $*/requirements.yaml ]; then helm dep up $*; fi
 	helm lint $*

README.md

@@ -57,7 +57,7 @@ make
 # generate secrets (ceph, etc.)
 export osd_cluster_network=10.32.0.0/12
 export osd_public_network=10.32.0.0/12
-cd openstack-base/utils/secret-generator
+cd common/utils/secret-generator
 ./generate_secrets.sh all `./generate_secrets.sh fsid`
 cd ../../..
 

ceph/README.md

@@ -84,7 +84,7 @@ kubectl label nodes node-type=storage --all
 You will need to generate ceph keys and configuration.  There is a simple to use utility that can do this quickly.  Please note the generator utility (per ceph-docker) requires the sigil template framework: (https://github.com/gliderlabs/sigil) to be installed and on the current path.
 
 ```
-cd openstack-base/utils/secret-generator
+cd common/utils/secret-generator
 ./generate_secrets.sh all `./generate_secrets.sh fsid`
 cd ../../..
 ```

ceph/requirements.lock

@@ -1,3 +1,6 @@
-dependencies: []
+dependencies:
-digest: sha256:81059fe6210ccee4e3349c0f34c12d180f995150128a913d63b65b7937c6b152
+- name: common
-generated: 2016-11-25T16:24:27.602432249-08:00
+  repository: http://localhost:8879/charts
+  version: 0.1.0
+digest: sha256:c6a7e430c900036912fe3fdc14213e9280c5da0b6607ce4dcf6dc95535d114fc
+generated: 2016-11-29T12:54:37.241999965-08:00

ceph/requirements.yaml

@@ -1 +1,5 @@
-dependencies: []
+dependencies: 
+  - name: common
+    repository: http://localhost:8879/charts
+    version: 0.1.0
+

ceph/templates/secrets.yaml

@@ -1,24 +1,16 @@
 ---
 apiVersion: v1
 kind: Secret
-metadata:
-  name: "ceph-conf-combined-storageclass"
-type: kubernetes.io/rbd
-data:
-  key: {{ include "secrets/ceph-client-key.b64"  . | quote }}
----
-apiVersion: v1
-kind: Secret
 metadata:
   name: "ceph-conf-combined"
 type: Opaque
 data:
   ceph.conf: |
-{{ include "secrets/ceph.conf.b64"  .| indent 4 }}
+{{ include "secrets/ceph.conf" . | b64enc | indent 4 }}
   ceph.client.admin.keyring: |
-{{ include "secrets/ceph.client.admin.keyring.b64" . | indent 4 }}
+{{ include "secrets/ceph.client.admin.keyring" . | b64enc | indent 4 }}
   ceph.mon.keyring: |
-{{ include "secrets/ceph.mon.keyring.b64" . | indent 4 }}
+{{ include "secrets/ceph.mon.keyring" . | b64enc | indent 4 }}
 ---
 apiVersion: v1
 kind: Secret
@@ -27,7 +19,7 @@ metadata:
 type: Opaque
 data:
   ceph.keyring: |
-{{ include "secrets/ceph.rgw.keyring.b64" . | indent 4 }}
+{{ include "secrets/ceph.rgw.keyring" . | b64enc | indent 4 }}
 ---
 apiVersion: v1
 kind: Secret
@@ -36,7 +28,7 @@ metadata:
 type: Opaque
 data:
   ceph.keyring: |
-{{ include "secrets/ceph.mds.keyring.b64"  . | indent 4 }}
+{{ include "secrets/ceph.mds.keyring"  . | b64enc | indent 4 }}
 ---
 apiVersion: v1
 kind: Secret
@@ -45,7 +37,7 @@ metadata:
 type: Opaque
 data:
   ceph.keyring: |
-{{ include "secrets/ceph.osd.keyring.b64"  . | indent 4 }}
+{{ include "secrets/ceph.osd.keyring"  . | b64enc | indent 4 }}
 ---
 apiVersion: v1
 kind: Secret
@@ -53,4 +45,4 @@ metadata:
   name: "ceph-client-key"
 type: Opaque
 data:
-  ceph-client-key: {{ include "secrets/ceph-client-key.b64"  . | quote }}  
+  ceph-client-key: {{ include "secrets/ceph-client-key"  . | b64enc | quote }}  

ceph/templates/storage.yaml

@@ -1,3 +1,5 @@
+# note that these secrets are handled by the common chart, not the ceph
+# chart, as we likely want them "everywhere"
 ---
 apiVersion: storage.k8s.io/v1beta1
 kind: StorageClass
@@ -5,10 +7,10 @@ metadata:
     name: general
 provisioner: kubernetes.io/rbd
 parameters:
-    monitors: ceph-mon.ceph:6789
+    monitors: ceph-mon.{{.Release.Namespace}}:6789
     adminId: admin
-    adminSecretName: ceph-conf-combined-storageclass
+    adminSecretName: pvc-ceph-conf-combined-storageclass
-    adminSecretNamespace: ceph
+    adminSecretNamespace: {{.Release.Namespace}}
     pool: rbd
     userId: admin
-    userSecretName: ceph-client-key
+    userSecretName: pvc-ceph-client-key

common/.gitignore

@@ -0,0 +1,2 @@
+secrets/*
+templates/_secrets.tpl

common/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 description: A base chart for all openstack charts
-name: openstack-base
+name: common
 version: 0.1.0

common/Makefile

@@ -0,0 +1,7 @@
+EXCLUDE := templates/* charts/* Chart.yaml requirement* values.yaml Makefile utils/* common/Chart.yaml
+SECRETS := $(shell find secrets -type f $(foreach e,$(EXCLUDE), -not -path "$(e)") )
+
+templates/_secrets.tpl: Makefile $(SECRETS)
+	echo Generating $(CURDIR)/$@
+	rm -f $@
+	for i in $(SECRETS); do  printf '{{ define "'$$i'" }}' >> $@; cat $$i >> $@; printf "{{ end }}\n" >> $@; done

common/templates/_funcs.tpl

@@ -0,0 +1,9 @@
+{{- define "template" -}}
+{{- $name := index . 0 -}}
+{{- $context := index . 1 -}}
+{{- $v:= $context.Template.Name | split "/" -}}
+{{- $n := len $v -}}
+{{- $last := sub $n 1 | printf "_%d" | index $v -}}
+{{- $wtf := $context.Template.Name | replace $last $name -}}
+{{ include $wtf $context }}
+{{- end -}}

common/templates/secrets.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "pvc-ceph-conf-combined-storageclass"
+type: kubernetes.io/rbd
+data:
+  key: {{ include "secrets/ceph-client-key"  . | quote }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "pvc-ceph-client-key"
+type: kubernetes.io/rbd
+data:
+  key: {{ include "secrets/ceph-client-key"  . | quote }}

keystone/requirements.yaml

@@ -1,5 +1,5 @@
 dependencies:
-  - name: openstack-base
+  - name: common
     repository: http://localhost:8879/charts
     version: 0.1.0
 dependencies:

mariadb/requirements.lock

@@ -1,6 +1,6 @@
 dependencies:
-- name: openstack-base
+- name: common
   repository: http://localhost:8879/charts
   version: 0.1.0
-digest: sha256:178ec6c44cf73c5d3c6d47eccee30b1b7f7e373c57af4e37fe7ea85f3549660e
+digest: sha256:c6a7e430c900036912fe3fdc14213e9280c5da0b6607ce4dcf6dc95535d114fc
-generated: 2016-11-25T16:24:28.454876345-08:00
+generated: 2016-11-29T11:31:25.670403608-08:00

mariadb/requirements.yaml

@@ -1,4 +1,4 @@
 dependencies:
-  - name: openstack-base
+  - name: common
     repository: http://localhost:8879/charts
     version: 0.1.0

memcached/requirements.lock

@@ -1,6 +1,6 @@
 dependencies:
-- name: openstack-base
+- name: common
   repository: http://localhost:8879/charts
   version: 0.1.0
-digest: sha256:178ec6c44cf73c5d3c6d47eccee30b1b7f7e373c57af4e37fe7ea85f3549660e
+digest: sha256:c6a7e430c900036912fe3fdc14213e9280c5da0b6607ce4dcf6dc95535d114fc
-generated: 2016-11-25T16:24:49.79575501-08:00
+generated: 2016-11-29T11:31:27.05384225-08:00

memcached/requirements.yaml

@@ -1,4 +1,4 @@
 dependencies:
-  - name: openstack-base
+  - name: common
     repository: http://localhost:8879/charts
     version: 0.1.0

openstack-base/.gitignore

@@ -1 +0,0 @@
-secrets/*

openstack-base/Makefile

@@ -1,7 +0,0 @@
-EXCLUDE := templates/* charts/* Chart.yaml requirement* values.yaml Makefile utils/* openstack-base/Chart.yaml
-FILES := $(shell find * -type f $(foreach e,$(EXCLUDE), -not -path "$(e)") )
-
-templates/_common.tpl: Makefile $(FILES)
-	echo Generating $(CURDIR)/$@
-	rm -f $@
-	for i in $(FILES); do  printf '{{ define "'$$i'" }}' >> $@; cat $$i >> $@; printf "{{ end }}\n" >> $@; done

openstack/Makefile

@@ -1,7 +0,0 @@
-EXCLUDE := templates/* charts/* Chart.yaml requirement* values.yaml Makefile openstack-base/Chart.yaml
-FILES := $(shell find * -type f $(foreach e,$(EXCLUDE), -not -path "$(e)") )
-
-templates/_partials.tpl: Makefile $(FILES)
-	echo Generating $(CURDIR)/$@
-	rm -f $@
-	for i in $(FILES); do  printf '{{ define "'$$i'" }}' >> $@; cat $$i >> $@; printf "{{ end }}\n" >> $@; done

openstack/requirements.lock

@@ -1,5 +1,5 @@
 dependencies:
-- name: openstack-base
+- name: common
   repository: http://localhost:8879/charts
   version: 0.1.0
 - name: memcached
@@ -14,5 +14,5 @@ dependencies:
 - name: keystone
   repository: http://localhost:8879/charts
   version: 0.1.0
-digest: sha256:cbe0cda88c1f8f7daf37f58e8d38c913478630687b22f8e622950c08cb2fdc6d
+digest: sha256:e92d6b6811d65492a95e4db258d516bfd7dd540108bb3d0e92e7dabc13ae2bbf
-generated: 2016-11-25T15:34:47.932395616-08:00
+generated: 2016-11-29T11:31:33.541933538-08:00

openstack/requirements.yaml

@@ -1,5 +1,5 @@
 dependencies:
-  - name: openstack-base
+  - name: common
     version: 0.1.0
     repository: http://localhost:8879/charts
   - name: memcached

openstack/templates/secrets.yaml

@@ -1,9 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
-  namespace: {{.Release.Namespace}}
-  name: "ceph-client-key"
-type: kubernetes.io/rbd
-data:
-  key: {{ include "secrets/ceph-client-key.b64"  . | quote }}

rabbitmq/requirements.lock

@@ -1,6 +1,6 @@
 dependencies:
-- name: openstack-base
+- name: common
   repository: http://localhost:8879/charts
   version: 0.1.0
-digest: sha256:178ec6c44cf73c5d3c6d47eccee30b1b7f7e373c57af4e37fe7ea85f3549660e
+digest: sha256:c6a7e430c900036912fe3fdc14213e9280c5da0b6607ce4dcf6dc95535d114fc
-generated: 2016-11-25T16:24:29.266938266-08:00
+generated: 2016-11-29T11:31:26.360492977-08:00

rabbitmq/requirements.yaml

@@ -1,4 +1,4 @@
 dependencies:
-  - name: openstack-base
+  - name: common
     repository: http://localhost:8879/charts
     version: 0.1.0

rabbitmq/templates/bin-configmap.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rabbitmq-bin
+  labels:
+    system: openstack
+    type: configuration
+    component: messaging
+data:
+  start_rabbitmq.sh: |
+{{ tuple "bin/_start_rabbit.tpl" . | include "template" | indent 4 }}

rabbitmq/templates/bin/_start_rabbit.tpl

@@ -0,0 +1,18 @@
+chown -R rabbitmq:rabbitmq /var/lib/rabbitmq
+
+/etc/init.d/rabbitmq-server start
+
+rabbitmq-plugins enable rabbitmq_tracing
+rabbitmqctl trace_on
+
+rabbitmqctl add_user {{ .Values.auth.default_user }} {{ .Values.auth.default_pass }} || true
+rabbitmqctl set_permissions {{ .Values.auth.default_user }} ".*" ".*" ".*" || true
+
+rabbitmqctl add_user {{ .Values.auth.admin_user }} {{ .Values.auth.admin_pass }}|| true
+rabbitmqctl set_permissions {{ .Values.auth.admin_user }} ".*" ".*" ".*" || true
+rabbitmqctl set_user_tags {{ .Values.auth.admin_user }} administrator || true
+
+rabbitmqctl change_password guest  {{ .Values.auth.default_pass }} || true
+rabbitmqctl set_user_tags guest monitoring || true
+/etc/init.d/rabbitmq-server stop
+exec rabbitmq-server

rabbitmq/templates/start_rabbitmq.yaml

@@ -1,31 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: start-rabbitmq
-  labels:
-    system: openstack
-    type: configuration
-    component: messaging
-
-data:
-  start_rabbitmq.sh: |
-
-    chown -R rabbitmq:rabbitmq /var/lib/rabbitmq
-
-    /etc/init.d/rabbitmq-server start
-
-    rabbitmq-plugins enable rabbitmq_tracing
-    rabbitmqctl trace_on
-
-    rabbitmqctl add_user {{ .Values.auth.default_user }} {{ .Values.auth.default_pass }} || true
-    rabbitmqctl set_permissions {{ .Values.auth.default_user }} ".*" ".*" ".*" || true
-
-    rabbitmqctl add_user {{ .Values.auth.admin_user }} {{ .Values.auth.admin_pass }}|| true
-    rabbitmqctl set_permissions {{ .Values.auth.admin_user }} ".*" ".*" ".*" || true
-    rabbitmqctl set_user_tags {{ .Values.auth.admin_user }} administrator || true
-
-    rabbitmqctl change_password guest  {{ .Values.auth.default_pass }} || true
-    rabbitmqctl set_user_tags guest monitoring || true
-    /etc/init.d/rabbitmq-server stop
-    exec rabbitmq-server
-