diff --git a/barbican/values.yaml b/barbican/values.yaml index 2b8625b003..3b39ce0df2 100644 --- a/barbican/values.yaml +++ b/barbican/values.yaml @@ -63,8 +63,6 @@ keystone: dependencies: db_init: - jobs: - - mariadb-seed service: - mariadb db_sync: diff --git a/cinder/values.yaml b/cinder/values.yaml index e6e28871af..6f9f9695df 100644 --- a/cinder/values.yaml +++ b/cinder/values.yaml @@ -107,8 +107,6 @@ misc: dependencies: db_init: - jobs: - - mariadb-seed service: - mariadb db_sync: diff --git a/doc/source/install/developer/minikube.rst b/doc/source/install/developer/minikube.rst index 2cf3dbc0df..cf68f885ee 100644 --- a/doc/source/install/developer/minikube.rst +++ b/doc/source/install/developer/minikube.rst @@ -84,7 +84,7 @@ use the following commands: kubectl label nodes openstack-control-plane=enabled --all --namespace=openstack # Deploy each chart: - helm install --name mariadb local/mariadb --namespace=openstack --set development.enabled=true + helm install --name mariadb local/mariadb --namespace=openstack --set volume.enabled=false helm install --name=memcached local/memcached --namespace=openstack helm install --name=etcd-rabbitmq local/etcd --namespace=openstack helm install --name=rabbitmq local/rabbitmq --namespace=openstack @@ -263,7 +263,7 @@ child charts. :: - helm install --name mariadb --set development.enabled=true local/mariadb --namespace=openstack + helm install --name mariadb --set volume.enabled=false local/mariadb --namespace=openstack .. note:: MariaDB seeding tasks run for quite a while. This is expected diff --git a/glance/values.yaml b/glance/values.yaml index 5b8ece2377..b222c3a1c2 100644 --- a/glance/values.yaml +++ b/glance/values.yaml @@ -210,8 +210,6 @@ resources: dependencies: db_init: - jobs: - - mariadb-seed service: - mariadb db_sync: diff --git a/heat/values.yaml b/heat/values.yaml index ef6edba798..d028679674 100644 --- a/heat/values.yaml +++ b/heat/values.yaml @@ -173,8 +173,6 @@ network: dependencies: db_init: - jobs: - - mariadb-seed service: - mariadb db_sync: diff --git a/keystone/values.yaml b/keystone/values.yaml index 9e7c32d74b..e78563efac 100644 --- a/keystone/values.yaml +++ b/keystone/values.yaml @@ -66,19 +66,15 @@ network: dependencies: api: jobs: - - mariadb-seed - keystone-db-sync service: - mariadb db_sync: jobs: - keystone-db-init - - mariadb-seed service: - mariadb init: - jobs: - - mariadb-seed service: - mariadb diff --git a/magnum/values.yaml b/magnum/values.yaml index 5ec6fcdde0..1b49dfc178 100644 --- a/magnum/values.yaml +++ b/magnum/values.yaml @@ -98,8 +98,6 @@ network: dependencies: db_init: - jobs: - - mariadb-seed service: - mariadb db_sync: diff --git a/mariadb/Chart.yaml b/mariadb/Chart.yaml index ec0b033b83..33d211dcd7 100644 --- a/mariadb/Chart.yaml +++ b/mariadb/Chart.yaml @@ -15,4 +15,4 @@ apiVersion: v1 description: OpenStack-Helm MariaDB name: mariadb -version: 0.1.0 +version: 0.5.0 diff --git a/mariadb/templates/bin/_bootstrap-db.sh.tpl b/mariadb/templates/bin/_bootstrap-db.sh.tpl deleted file mode 100644 index afadf67d34..0000000000 --- a/mariadb/templates/bin/_bootstrap-db.sh.tpl +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -SLEEP_TIMEOUT=5 - -# Initialize system .Values.database. -mysql_install_db --datadir=/var/lib/mysql - -# Start mariadb and wait for it to be ready. -# -# note that we bind to 127.0.0.1 here because we want -# to interact with the database but we dont want to expose it -# yet for other cluster members to accidently connect yet -mysqld_safe --defaults-file=/etc/my.cnf \ - --console \ - --wsrep-new-cluster \ - --wsrep_cluster_address='gcomm://' \ - --bind-address='127.0.0.1' \ - --wsrep_node_address="127.0.0.1:{{ .Values.network.port.wsrep }}" \ - --wsrep_provider_options="gmcast.listen_addr=tcp://127.0.0.1:{{ .Values.network.port.wsrep }}" & - -TIMEOUT=120 -while [[ ! -f /var/lib/mysql/mariadb.pid ]]; do -if [[ ${TIMEOUT} -gt 0 ]]; then - let TIMEOUT-=1 - sleep 1 -else - exit 1 -fi -done - -# Reset permissions. -# kolla_security_reset requires to be run from home directory -cd /var/lib/mysql ; DB_ROOT_PASSWORD="{{ .Values.database.root_password }}" kolla_security_reset - -mysql -u root --password="{{ .Values.database.root_password }}" --port="{{ .Values.network.port.mariadb }}" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY '{{ .Values.database.root_password }}' WITH GRANT OPTION;" -mysql -u root --password="{{ .Values.database.root_password }}" --port="{{ .Values.network.port.mariadb }}" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '{{ .Values.database.root_password }}' WITH GRANT OPTION;" - -# Restart .Values.database. -mysqladmin -uroot -p"{{ .Values.database.root_password }}" --port="{{ .Values.network.port.mariadb }}" shutdown - -# Wait for the mariadb server to shut down -SHUTDOWN_TIMEOUT=60 -while [[ -f /var/lib/mysql/mariadb.pid ]]; do - if [[ ${SHUTDOWN_TIMEOUT} -gt 0 ]]; then - let SHUTDOWN_TIMEOUT-=1 - sleep 1 - else - echo "MariaDB instance couldn't be properly shut down" - exit 1 - fi -done diff --git a/mariadb/templates/bin/_peer-finder.py.tpl b/mariadb/templates/bin/_peer-finder.py.tpl deleted file mode 100644 index bceab182a5..0000000000 --- a/mariadb/templates/bin/_peer-finder.py.tpl +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import urllib2 -import ssl -import socket -import sys -import time - -URL = ('https://kubernetes.default.svc.cluster.local/api/v1/namespaces/{namespace}' - '/endpoints/{service_name}') -TOKEN_FILE = '/var/run/secrets/kubernetes.io/serviceaccount/token' - - -def get_service_endpoints(service_name): - url = URL.format(namespace=os.environ['NAMESPACE'], service_name=service_name) - try: - token = file (TOKEN_FILE, 'r').read() - except KeyError: - exit("Unable to open a file with token.") - header = {'Authorization': " Bearer {}".format(token)} - req = urllib2.Request(url=url, headers=header) - - ctx = create_ctx() - connection = urllib2.urlopen(req, context=ctx) - data = connection.read() - - # parse to dict - json_acceptable_string = data.replace("'", "\"") - output = json.loads(json_acceptable_string) - - return output - - -def get_ip_addresses(output, force_only_members=False): - subsets = output['subsets'][0] - if not 'addresses' in subsets: - return [] - - # where we are seeding, the only cluster member is the seed job - if not force_only_members: - for subset in subsets['addresses']: - if subset.has_key('name'): - if 'seed' in subset['name']: - return [subset['ip']] - - # otherwise use the other cluster members - ip_addresses = [x['ip'] for x in subsets['addresses']] - my_ip = get_my_ip_address() - if my_ip in ip_addresses: - ip_addresses.remove(my_ip) - return ip_addresses - - -def get_my_ip_address(): - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect(('kubernetes.default.svc.cluster.local', 0)) - return s.getsockname()[0] - - -def create_ctx(): - ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - return ctx - - -def print_galera_cluster_address(service_name, force_only_members): - while True: - output = get_service_endpoints(service_name) - ips = get_ip_addresses(output, force_only_members) - #print "=== OUTPUT: %s" % output - #print "=== IPS: %s" % ips - if len(ips): - wsrep_cluster_address = '--wsrep_cluster_address=gcomm://{}'.format( - ','.join(get_ip_addresses(output))) - print wsrep_cluster_address - break - time.sleep(5) - - -def main(): - if len(sys.argv) != 3: - exit('peer-finder: You need to pass argument <1|0 for force cluster members>') - service_name = sys.argv[1] - force_only_members = int(sys.argv[2]) - print_galera_cluster_address(service_name, force_only_members) - -if __name__ == '__main__': - main() diff --git a/mariadb/templates/bin/_readiness.py.tpl b/mariadb/templates/bin/_readiness.py.tpl deleted file mode 100644 index 5261b87490..0000000000 --- a/mariadb/templates/bin/_readiness.py.tpl +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import time -import pymysql - -DB_HOST = "127.0.0.1" -DB_PORT = int(os.environ.get('MARIADB_SERVICE_PORT', '3306')) - -while True: - try: - pymysql.connections.Connection(host=DB_HOST, port=DB_PORT, - connect_timeout=1) - sys.exit(0) - except pymysql.err.OperationalError as e: - code, message = e.args - - if code == 2003 and 'time out' in message: - print('Connection timeout, sleeping') - time.sleep(1) - continue - if code == 1045: - print('Mysql ready to use. Exiting') - sys.exit(0) - - # other error - raise diff --git a/mariadb/templates/bin/_readiness.sh.tpl b/mariadb/templates/bin/_readiness.sh.tpl new file mode 100644 index 0000000000..88c3044e57 --- /dev/null +++ b/mariadb/templates/bin/_readiness.sh.tpl @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o pipefail + +PASSWORD={{ .Values.database.root_password | quote }} + +MYSQL="mysql -u root --password=${PASSWORD}" + +if [ ! $($MYSQL -e 'select 1') ]; then + echo "Could not SELECT 1" 1>&2 + exit 1 +fi + +# Set this late, so that we can give a nicer error message above. +set -o errexit + +CLUSTER_STATUS=$($MYSQL -e "show status like 'wsrep_cluster_status'" | tail -n 1 | cut -f 2) +if [ "x${CLUSTER_STATUS}" != "xPrimary" ]; then + echo "Not in primary cluster: '${CLUSTER_STATUS}'" 1>&2 + exit 1 +fi + +WSREP_READY=$($MYSQL -e "show status like 'wsrep_ready'" | tail -n 1 | cut -f 2) +if [ "x${WSREP_READY}" != "xON" ]; then + echo "WSREP not ready: '${WSREP_READY}'" 1>&2 + exit 1 +fi + +WSREP_STATE=$($MYSQL -e "show status like 'wsrep_local_state_comment'" | tail -n 1 | cut -f 2) +if [ "x${WSREP_STATE}" != "xSynced" ]; then + echo "WSREP not synced: '${WSREP_STATE}'" 1>&2 + exit 1 +fi + +echo "${POD_NAME} ready." 1>&2 + +exit 0 diff --git a/mariadb/templates/bin/_seed.sh.tpl b/mariadb/templates/bin/_seed.sh.tpl deleted file mode 100644 index 94a35fdec2..0000000000 --- a/mariadb/templates/bin/_seed.sh.tpl +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/sh -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -SLEEP_TIMEOUT=5 - -function wait_for_cluster { - - # Wait for the mariadb server to be "Ready" before starting the security reset with a max timeout - TIMEOUT=600 - while [[ ! -f /var/lib/mysql/mariadb.pid ]]; do - if [[ ${TIMEOUT} -gt 0 ]]; then - let TIMEOUT-=1 - sleep 1 - else - exit 1 - fi - done - - REPLICAS={{ .Values.replicas }} - # We need to count seed instance here. - MINIMUM_CLUSTER_SIZE=$(( $REPLICAS + 1 )) - - # wait until we have at least two more members in a cluster. - while true ; do - CLUSTER_SIZE=`mysql -uroot -h ${POD_IP} -p"{{ .Values.database.root_password }}" --port="{{ .Values.network.port.mariadb }}" -e'show status' | grep wsrep_cluster_size | awk ' { if($2 ~ /[0-9]/){ print $2 } else { print 0 } } '` - if [ "${CLUSTER_SIZE}" -lt ${MINIMUM_CLUSTER_SIZE} ] ; then - echo "Cluster seed not finished, waiting." - sleep ${SLEEP_TIMEOUT} - continue - fi - CLUSTER_STATUS=`mysql -uroot -h ${POD_IP} -p"{{ .Values.database.root_password }}" --port="{{ .Values.network.port.mariadb }}" -e'show status' | grep wsrep_local_state_comment | awk ' { print $2 } '` - if [ "${CLUSTER_STATUS}" != "Synced" ] ; then - echo "Cluster not synced, waiting." - sleep ${SLEEP_TIMEOUT} - continue - fi - # Count number of endpoint separators. - ENDPOINTS_CNT=`python /tmp/peer-finder.py mariadb 1 | grep -o ',' | wc -l` - # TODO(tomasz.paszkowski): Fix a corner case when only one endpoint is on the list. - # Add +1 for seed node and +1 as first item does not have a separator - ENDPOINTS_CNT=$(($ENDPOINTS_CNT+2)) - if [ "${ENDPOINTS_CNT}" != "${CLUSTER_SIZE}" ] ; then - echo "Cluster not synced, waiting." - sleep ${SLEEP_TIMEOUT} - continue - fi - echo "Cluster ready, exiting seed." - kill -- -$$ - break - done -} - -# With the DaemonSet implementation, there may be a difference -# in the number of replicas and actual number of nodes matching -# mariadb node selector label. Problem will be solved when -# the implementation will be switched to Deployment -# (using anti-affinity feature). - -{{- if .Values.development.enabled }} -REPLICAS=1 -{{- else }} -REPLICAS={{ .Values.replicas }} -{{- end }} - -if [ "$REPLICAS" -eq 1 ] ; then - echo "Requested to build one-instance MariaDB cluster. There is no need to run seed. Exiting." - exit 0 -elif [ "$REPLICAS" -eq 2 ] ; then - echo "2-instance cluster is not a valid MariaDB configuration." - exit 1 -fi - -. /tmp/bootstrap-db.sh -mysqld_safe --defaults-file=/etc/my.cnf \ - --console \ - --wsrep-new-cluster \ - --wsrep_cluster_address='gcomm://' \ - --bind-address="0.0.0.0" \ - --wsrep_node_address="${POD_IP}:{{ .Values.network.port.wsrep }}" \ - --wsrep_provider_options="gmcast.listen_addr=tcp://${POD_IP}:{{ .Values.network.port.wsrep }}" & -wait_for_cluster -exit 0 diff --git a/mariadb/templates/bin/_start.sh.tpl b/mariadb/templates/bin/_start.sh.tpl index 4db3c3f438..c9f76120d3 100644 --- a/mariadb/templates/bin/_start.sh.tpl +++ b/mariadb/templates/bin/_start.sh.tpl @@ -13,54 +13,65 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -ex -trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT +export MYSQL_ROOT_PASSWORD={{ .Values.database.root_password | quote }} -sudo chown mysql: /var/lib/mysql -rm -rf /var/lib/mysql/lost+found +# +# Bootstrap database +# +CLUSTER_INIT_ARGS= -{{- if .Values.development.enabled }} -REPLICAS=1 -{{- else }} -REPLICAS={{ .Values.replicas }} -{{- end }} -PETSET_NAME={{ printf "%s" .Values.service_name }} -INIT_MARKER="/var/lib/mysql/init_done" - -function join_by { local IFS="$1"; shift; echo "$*"; } - -# Remove mariadb.pid if exists -if [[ -f /var/lib/mysql/mariadb.pid ]]; then - if [[ `pgrep -c $(cat /var/lib/mysql/mariadb.pid)` -eq 0 ]]; then - rm -vf /var/lib/mysql/mariadb.pid +if [ ! -d /var/lib/mysql/mysql ]; then + if [ "x${POD_NAME}" = "x{{ .Values.service_name }}-0" ]; then + echo No data found for pod 0 + if [ "xtrue" = "x{{ .Values.force_bootstrap }}" ]; then + echo force_bootstrap set, so will force-initialize node 0. + CLUSTER_INIT_ARGS=--wsrep-new-cluster + elif ! mysql -h {{ .Values.service_name }} -u root --password=${MYSQL_ROOT_PASSWORD} -e 'select 1'; then + echo No other nodes found, so will initialize cluster. + CLUSTER_INIT_ARGS=--wsrep-new-cluster + else + echo Found other live nodes, will attempt to join them. + mkdir /var/lib/mysql/mysql + fi + else + echo Not pod 0, so will avoid upstream database initialization. + mkdir /var/lib/mysql/mysql fi fi -if [ "$REPLICAS" -eq 1 ] ; then - if [[ ! -f ${INIT_MARKER} ]]; then - cd /var/lib/mysql - echo "Creating one-instance MariaDB." - bash /tmp/bootstrap-db.sh - touch ${INIT_MARKER} +# +# Construct cluster config +# +CLUSTER_CONFIG_PATH=/etc/mysql/conf.d/10-cluster-config.cnf + +MEMBERS= +for i in $(seq 1 {{ .Values.replicas }}); do + NUM=$(expr $i - 1) + CANDIDATE_POD="{{ .Values.service_name }}-$NUM.{{ .Values.service_name }}-discovery" + if [ "x${CANDIDATE_POD}" != "x${POD_NAME}.{{ .Values.service_name }}-discovery" ]; then + if [ -n "${MEMBERS}" ]; then + MEMBERS+=, + fi + MEMBERS+="${CANDIDATE_POD}:{{ .Values.network.port.wsrep }}" fi - exec mysqld_safe --defaults-file=/etc/my.cnf \ - --console \ - --wsrep-new-cluster \ - --wsrep_cluster_address='gcomm://' -else +done - # give the seed more of a chance to be ready by the time - # we start the first pet so we succeed on the first pass - # a little hacky, but prevents restarts as we aren't waiting - # for job completion here so I'm not sure what else - # to look for - sleep 30 +echo +echo Writing cluster config for ${POD_NAME} to ${CLUSTER_CONFIG_PATH} +echo vvv - export WSREP_OPTIONS=`python /tmp/peer-finder.py mariadb 0` - exec mysqld --defaults-file=/etc/my.cnf \ - --console \ - --bind-address="0.0.0.0" \ - --wsrep_node_address="${POD_IP}:{{ .Values.network.port.wsrep }}" \ - --wsrep_provider_options="gmcast.listen_addr=tcp://${POD_IP}:{{ .Values.network.port.wsrep }}" \ - $WSREP_OPTIONS -fi +cat <