MariaDB: simplify db init process

This PS simplifys the db start and init script, increasing
robustness of the mariadb chart on low end hardware. It also
removes some unrequired annotations and moves to more explicit
qualification of the clustering variables.

Change-Id: I6cff5fb6ccfdcc6c82203b0e5eff445ee98bc298

mariadb/templates/bin/_readiness.sh.tpl

@@ -48,4 +48,8 @@ fi
 
 echo "${POD_NAME} ready." 1>&2
 
+if [ -e ${BOOTSTRAP_FILE} ]; then
+  rm -f ${BOOTSTRAP_FILE}
+fi
+
 exit 0

mariadb/templates/bin/_start.sh.tpl

@@ -25,11 +25,13 @@ if [ ! -d /var/lib/mysql/mysql ]; then
         if [ "x${FORCE_BOOTSTRAP}" = "xtrue" ]; then
             echo 'force_bootstrap set, so will force-initialize node 0.'
             CLUSTER_INIT_ARGS=--wsrep-new-cluster
+            CLUSTER_BOOTSTRAP=true
         elif ! mysql --defaults-file=/etc/mysql/admin_user.cnf \
                      --connect-timeout 2 \
                      -e 'select 1'; then
             echo 'No other nodes found, so will initialize cluster.'
             CLUSTER_INIT_ARGS=--wsrep-new-cluster
+            CLUSTER_BOOTSTRAP=true
         else
             echo 'Found other live nodes, will attempt to join them.'
             mkdir /var/lib/mysql/mysql
@@ -38,6 +40,7 @@ if [ ! -d /var/lib/mysql/mysql ]; then
         echo 'Not pod 0, so will avoid upstream database initialization.'
         mkdir /var/lib/mysql/mysql
     fi
+    chown -R mysql:mysql /var/lib/mysql
 fi
 
 # Construct cluster config
@@ -50,8 +53,8 @@ for i in $(seq 1 ${MARIADB_REPLICAS}); do
     else
       NUM="$(expr $i - 1)"
     fi
-    CANDIDATE_POD="${SERVICE_NAME}-$NUM.${DISCOVERY_SERVICE_NAME}"
-    if [ "x${CANDIDATE_POD}" != "x${POD_NAME}.${DISCOVERY_SERVICE_NAME}" ]; then
+    CANDIDATE_POD="${SERVICE_NAME}-$NUM.$(hostname -d)"
+    if [ "x${CANDIDATE_POD}" != "x${POD_NAME}.$(hostname -d)" ]; then
         if [ -n "${MEMBERS}" ]; then
             MEMBERS+=,
         fi
@@ -60,13 +63,25 @@ for i in $(seq 1 ${MARIADB_REPLICAS}); do
 done
 
 echo "Writing cluster config for ${POD_NAME} to ${CLUSTER_CONFIG_PATH}"
-cat >> ${CLUSTER_CONFIG_PATH} <<EOF
+cat >> ${CLUSTER_CONFIG_PATH} << EOF
 [mysqld]
 wsrep_cluster_address="gcomm://${MEMBERS}"
 wsrep_node_address=${POD_IP}
-wsrep_node_name=${POD_NAME}.${DISCOVERY_SERVICE_NAME}
+wsrep_node_name=${POD_NAME}.$(hostname -d)
 EOF
 
-echo 'Executing upstream docker-entrypoint.'
-set +e
-exec /usr/local/bin/docker-entrypoint.sh mysqld ${CLUSTER_INIT_ARGS}
+if [ "x${CLUSTER_BOOTSTRAP}" = "xtrue" ]; then
+  mysql_install_db --user=mysql --datadir=/var/lib/mysql
+
+  cat > "${BOOTSTRAP_FILE}" << EOF
+DELETE FROM mysql.user ;
+CREATE OR REPLACE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ;
+GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ;
+DROP DATABASE IF EXISTS test ;
+FLUSH PRIVILEGES ;
+EOF
+
+  CLUSTER_INIT_ARGS="${CLUSTER_INIT_ARGS} --init-file=${BOOTSTRAP_FILE}"
+fi
+
+exec mysqld ${CLUSTER_INIT_ARGS}

mariadb/templates/etc/_00-base.cnf.tpl

@@ -90,7 +90,7 @@ binlog_format=ROW
 default-storage-engine=InnoDB
 innodb_autoinc_lock_mode=2
 innodb_flush_log_at_trx_commit=2
-wsrep_cluster_name={{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+wsrep_cluster_name={{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | replace "." "_" }}
 wsrep_on=1
 wsrep_provider=/usr/lib/galera/libgalera_smm.so
 wsrep_provider_options="gmcast.listen_addr=tcp://0.0.0.0:{{ tuple "oslo_db" "internal" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}"

mariadb/templates/service-discovery.yaml

@@ -14,8 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-# This service could be used for cluster pod discovery, though instead it's
-# primarily here to allow DNS lookups of cluster pods.
 {{- if .Values.manifests.service_discovery }}
 {{- $envAll := . }}
 ---

mariadb/templates/service.yaml

@@ -21,14 +21,6 @@ apiVersion: v1
 kind: Service
 metadata:
   name: {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
-  annotations:
-    # This is needed to make the peer-finder work properly and to help avoid
-    # edge cases where instance 0 comes up after losing its data and needs to
-    # decide whether it should create a new cluster or try to join an existing
-    # one. If it creates a new cluster when it should have joined an existing
-    # one, we'd end up with two separate clusters listening at the same service
-    # endpoint, which would be very bad.
-    service.alpha.kubernetes.io/tolerate-unready-endpoints: "false"
 spec:
   ports:
     - name: db

mariadb/templates/statefulset.yaml

@@ -26,7 +26,7 @@ kind: StatefulSet
 metadata:
   name: mariadb
 spec:
-  serviceName: "{{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}-discovery"
+  serviceName: "{{ tuple "oslo_db" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}"
   replicas: {{ .Values.pod.replicas.server }}
   template:
     metadata:
@@ -58,21 +58,21 @@ spec:
                   fieldPath: metadata.name
             - name: FORCE_BOOTSTRAP
               value: {{ .Values.force_bootstrap | quote }}
+            - name: BOOTSTRAP_FILE
+              value: {{ printf "/tmp/%s.sql" (randAlphaNum 8) }}
             - name: MARIADB_REPLICAS
               value: {{ .Values.pod.replicas.server | quote }}
             - name: WSREP_PORT
               value: {{ tuple "oslo_db" "internal" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: SERVICE_NAME
               value: {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
-            - name: DISCOVERY_SERVICE_NAME
-              value: {{ tuple "oslo_db" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup"}}
             - name: MYSQL_ROOT_PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: mariadb-db-root-password
                   key: MYSQL_ROOT_PASSWORD
           readinessProbe:
-            initialDelaySeconds: 60
+            initialDelaySeconds: 30
             periodSeconds: 30
             timeoutSeconds: 3
             exec: