openstack/kolla-ansible
Code Issues Proposed changes

Merge "Refactor mariadb to support shards"

Browse Source
This commit is contained in:
Zuul 2021-04-09 14:23:54 +00:00 committed by Gerrit Code Review
commit d4db69c079
15 changed files with 115 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ansible/group_vars/all.yml
View File

@ -362,6 +362,10 @@ mariadb_wsrep_port: "4567"
mariadb_ist_port: "4568"
mariadb_sst_port: "4444"
mariadb_clustercheck_port: "4569"
mariadb_monitor_user: "haproxy"
mariadb_default_database_shard_id: 0
mariadb_default_database_shard_hosts: "{% set default_shard = [] %}{% for host in groups['mariadb'] %}{% if hostvars[host]['mariadb_shard_id'] is not defined or hostvars[host]['mariadb_shard_id'] == mariadb_default_database_shard_id %}{{ default_shard.append(host) }}{% endif %}{% endfor %}{{ default_shard }}"
mariadb_loadbalancer: "haproxy"
masakari_api_port: "15868"

24
ansible/roles/mariadb/defaults/main.yml
View File

@ -4,7 +4,7 @@ project_name: "mariadb"
mariadb_services:
mariadb:
container_name: mariadb
group: mariadb
group: "{{ mariadb_shard_group }}"
enabled: true
image: "{{ mariadb_image_full }}"
volumes: "{{ mariadb_default_volumes + mariadb_extra_volumes }}"
@ -37,13 +37,13 @@ mariadb_services:
custom_member_list: "{{ external_haproxy_members.split(';') }}"
mariadb-clustercheck:
container_name: mariadb_clustercheck
group: mariadb
group: "{{ mariadb_shard_group }}"
enabled: "{{ enable_mariadb_clustercheck | bool }}"
image: "{{ mariadb_clustercheck_image_full }}"
volumes: "{{ mariadb_clustercheck_default_volumes + mariadb_clustercheck_extra_volumes }}"
dimensions: "{{ mariadb_clustercheck_dimensions }}"
environment:
MYSQL_USERNAME: "haproxy"
MYSQL_USERNAME: "{{ mariadb_monitor_user }}"
MYSQL_PASSWORD: ""
MYSQL_HOST: "{{ api_interface_address }}"
AVAILABLE_WHEN_DONOR: "1"
@ -57,8 +57,8 @@ database_max_timeout: 120
####################
# HAProxy
####################
internal_haproxy_members: "{% for host in groups['mariadb'] %}server {{ hostvars[host]['ansible_hostname'] }} {{ 'api' | kolla_address(host) }}:{{ mariadb_port }} check {% if enable_mariadb_clustercheck | bool %}port {{ mariadb_clustercheck_port }} {% endif %} inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}"
external_haproxy_members: "{% for host in groups['mariadb'] %}server {{ host }} {{ host }}:{{ mariadb_port }} check {% if enable_mariadb_clustercheck | bool %}port {{ mariadb_clustercheck_port}} {% endif %} inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}"
internal_haproxy_members: "{% for host in mariadb_default_database_shard_hosts %} server {{ hostvars[host]['ansible_hostname'] }} {{ 'api' | kolla_address(host) }}:{{ mariadb_port }} check {% if enable_mariadb_clustercheck | bool %}port {{ mariadb_clustercheck_port }}{% endif %} inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}"
external_haproxy_members: "{% for host in mariadb_default_database_shard_hosts %} server {{ host }} {{ host }}:{{ mariadb_port }} check {% if enable_mariadb_clustercheck | bool %}port {{ mariadb_clustercheck_port }}{% endif %} inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}"
####################
# Docker
@ -95,6 +95,7 @@ mariadb_clustercheck_extra_volumes: "{{ default_extra_volumes }}"
# Vars used within recover_cluster.yml
########################################
mariadb_service: "{{ mariadb_services['mariadb'] }}"
mariadb_recover_tmp_file_path: "/tmp/kolla_mariadb_recover_inventory_name_{{ mariadb_shard_name }}"
###############
# WSREP options
@ -108,12 +109,21 @@ mariabackup_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ doc
mariabackup_tag: "{{ openstack_tag }}"
mariabackup_image_full: "{{ mariabackup_image }}:{{ mariabackup_tag }}"
mariadb_backup_host: "{{ groups['mariadb'][0] }}"
mariadb_backup_host: "{{ groups[mariadb_shard_group][0] }}"
mariadb_backup_database_schema: "PERCONA_SCHEMA"
mariadb_backup_database_user: "backup"
mariadb_backup_database_user: "{% if mariadb_loadbalancer == 'haproxy' %}backup{% else %}backup_{{ mariadb_shard_name }}{% endif %}"
mariadb_backup_type: "full"
mariadb_backup: "{{ mariadb_loadbalancer != 'haproxy' or inventory_hostname in mariadb_default_database_shard_hosts }}"
####################
# Clustercheck
####################
enable_mariadb_clustercheck: "yes"
####################
# Sharding
####################
mariadb_shard_id: "{{ mariadb_default_database_shard_id }}"
mariadb_shard_name: "shard_{{ mariadb_shard_id }}"
mariadb_shard_group: "mariadb_{{ mariadb_shard_name }}"
mariadb_shard_database_user: "{% if mariadb_loadbalancer == 'haproxy' %}{{ database_user }}{% else %}root_{{ mariadb_shard_name }}{% endif %}"

12
ansible/roles/mariadb/handlers/main.yml
View File

@ -57,7 +57,7 @@
login_port: "{{ mariadb_port }}"
login_user: "{{ database_user }}"
login_password: "{{ database_password }}"
name: "haproxy"
name: "{{ mariadb_monitor_user }}"
password: ""
host: "%"
priv: "*.*:USAGE"
@ -66,9 +66,9 @@
- name: Restart MariaDB on existing cluster members
include_tasks: 'restart_services.yml'
when:
- groups.mariadb_port_alive_True is defined
- inventory_hostname in groups.mariadb_port_alive_True
- groups.mariadb_port_alive_True.index(inventory_hostname) % 4 == item
- groups[mariadb_shard_group + '_port_alive_True'] is defined
- inventory_hostname in groups[mariadb_shard_group + '_port_alive_True']
- groups[mariadb_shard_group + '_port_alive_True'].index(inventory_hostname) % 4 == item
- kolla_action != "config"
listen: restart mariadb
loop:
@ -81,8 +81,8 @@
include_tasks: 'restart_services.yml'
when:
- bootstrap_host is not defined or bootstrap_host != inventory_hostname
- groups.mariadb_port_alive_False is defined
- inventory_hostname in groups.mariadb_port_alive_False
- groups[mariadb_shard_group + '_port_alive_False'] is defined
- inventory_hostname in groups[mariadb_shard_group + '_port_alive_False']
- kolla_action != "config"
listen: restart mariadb

1
ansible/roles/mariadb/tasks/backup.yml
View File

@ -20,3 +20,4 @@
- "kolla_logs:/var/log/kolla/"
when:
- inventory_hostname == mariadb_backup_host
- mariadb_backup | bool

2
ansible/roles/mariadb/tasks/bootstrap.yml
View File

@ -4,7 +4,7 @@
- include_tasks: bootstrap_cluster.yml
when:
- not mariadb_cluster_exists
- inventory_hostname == groups['mariadb'][0]
- inventory_hostname == groups[mariadb_shard_group][0]
- include_tasks: recover_cluster.yml
when: mariadb_recover | default(False)

11
ansible/roles/mariadb/tasks/check.yml
View File

@ -1,9 +1,16 @@
---
- name: Waiting for MariaDB service to be ready through VIP
# Explicitly wait for the database to be accessible via the load balancer.
# Sometimes it can reject connections even when all database services are up,
# due to the health check polling in HAProxy.
- name: Wait for MariaDB service to be ready through VIP
become: true
command: "docker exec {{ mariadb_service.container_name }} mysql -h {{ database_address }} -P {{ database_port }} -u haproxy -e 'show databases;'"
command: >
docker exec {{ mariadb_service.container_name }}
mysql -h {{ database_address }} -P {{ database_port }}
-u {{ mariadb_shard_database_user }} -p{{ database_password }} -e 'show databases;'
register: result
until: result is success
changed_when: False
retries: 6
delay: 10
when: mariadb_shard_id == mariadb_default_database_shard_id

1
ansible/roles/mariadb/tasks/deploy.yml
View File

@ -10,5 +10,4 @@
- import_tasks: register.yml
# Test haproxy user through VIP
- import_tasks: check.yml

23
ansible/roles/mariadb/tasks/lookup_cluster.yml
View File

@ -9,12 +9,12 @@
- name: Divide hosts by their MariaDB volume availability
group_by:
key: mariadb_had_volume_{{ mariadb_volume is not changed }}
key: "{{ mariadb_shard_group }}_had_volume_{{ mariadb_volume is not changed }}"
changed_when: false
- name: Establish whether the cluster has already existed
set_fact:
mariadb_cluster_exists: "{{ groups.mariadb_had_volume_True is defined }}"
mariadb_cluster_exists: "{{ groups[mariadb_shard_group + '_had_volume_True'] is defined }}"
- block:
- name: Check MariaDB service port liveness
@ -29,7 +29,7 @@
- name: Divide hosts by their MariaDB service port liveness
group_by:
key: mariadb_port_alive_{{ check_mariadb_port_liveness is success }}
key: "{{ mariadb_shard_group }}_port_alive_{{ check_mariadb_port_liveness is success }}"
changed_when: false
- name: Fail on existing but stopped cluster
@ -37,9 +37,9 @@
msg: MariaDB cluster exists but is stopped. Please start it using kolla-ansible mariadb_recovery
when:
# NOTE(yoctozepto): we allow single-node cluster to start
- groups['mariadb'] | length > 1
- groups[mariadb_shard_group] | length > 1
- mariadb_cluster_exists
- groups.mariadb_port_alive_True is not defined
- groups[mariadb_shard_group + '_port_alive_True'] is not defined
- block:
- name: Check MariaDB service WSREP sync status
@ -60,19 +60,20 @@
set_fact:
mariadb_sync_status: "{{ check_mariadb_sync_status.stdout.split('\t')[1] }}"
when:
- groups.mariadb_port_alive_True is defined
- inventory_hostname in groups.mariadb_port_alive_True
- groups[mariadb_shard_group + '_port_alive_True'] is defined
- inventory_hostname in groups[mariadb_shard_group + '_port_alive_True']
- name: Divide hosts by their MariaDB service WSREP sync status
group_by:
key: mariadb_sync_status_{{ mariadb_sync_status | default('NA') }}
key: "{{ mariadb_shard_group }}_sync_status_{{ mariadb_sync_status | default('NA') }}"
changed_when: false
- name: Fail when MariaDB services are not synced across the whole cluster
fail:
msg: MariaDB cluster is not synced. Please wait for WSREP sync before proceeding.
when:
- groups.mariadb_port_alive_True is defined
- groups.mariadb_sync_status_Synced is not defined or
groups.mariadb_port_alive_True | sort != groups.mariadb_sync_status_Synced | sort
- groups[mariadb_shard_group + '_port_alive_True'] is defined
- groups[mariadb_shard_group + '_sync_status_Synced'] is not defined or
groups[mariadb_shard_group + '_port_alive_True'] | sort != groups[mariadb_shard_group + '_sync_status_Synced'] | sort
when: not mariadb_recover | default(False)

5
ansible/roles/mariadb/tasks/main.yml
View File

@ -1,2 +1,7 @@
---
- name: Group MariaDB hosts based on shards
group_by:
key: "{{ mariadb_shard_group }}"
changed_when: false
- include_tasks: "{{ kolla_action }}.yml"

18
ansible/roles/mariadb/tasks/recover_cluster.yml
View File

@ -4,21 +4,15 @@
msg: "MariaDB cluster was not found. Is your inventory correct?"
when: not mariadb_cluster_exists
- name: Cleaning up temp file on mariadb hosts
file:
path: /tmp/kolla_mariadb_grastate.dat
state: absent
changed_when: false
check_mode: no
- name: Cleaning up temp file on localhost
file:
path: /tmp/kolla_mariadb_recover_inventory_name
path: "{{ item }}"
state: absent
delegate_to: localhost
changed_when: false
check_mode: no
run_once: true
with_fileglob: "/tmp/kolla_mariadb_recover_inventory_name_*"
- block:
- name: Stop MariaDB containers
@ -76,7 +70,7 @@
if [[ ! -z {{ hostvars[inventory_hostname]['seqno'] }} && ! -z {{ hostvars[item]['seqno'] }} &&
{{ hostvars[inventory_hostname]['seqno'] }} =~ ^-?[0-9]+$ && {{ hostvars[item]['seqno'] }} =~ ^-?[0-9]+$ &&
{{ hostvars[inventory_hostname]['seqno'] }} -lt {{ hostvars[item]['seqno'] }} ]]; then echo {{ hostvars[item]['seqno'] }}; fi
with_items: "{{ groups['mariadb'] }}"
with_items: "{{ groups[mariadb_shard_group] }}"
register: seqno_compare
args:
executable: /bin/bash
@ -85,7 +79,7 @@
- name: Writing hostname of host with the largest seqno to temp file
copy:
content: "{{ inventory_hostname }}"
dest: /tmp/kolla_mariadb_recover_inventory_name
dest: "{{ mariadb_recover_tmp_file_path }}"
mode: 0644
delegate_to: localhost
changed_when: false
@ -93,7 +87,7 @@
- name: Registering mariadb_recover_inventory_name from temp file
set_fact:
mariadb_recover_inventory_name: "{{ lookup('file', '/tmp/kolla_mariadb_recover_inventory_name') }}"
mariadb_recover_inventory_name: "{{ lookup('file', mariadb_recover_tmp_file_path) }}"
when:
- mariadb_recover_inventory_name is not defined
@ -230,4 +224,4 @@
- bootstrap_host is defined
- bootstrap_host == inventory_hostname
- import_tasks: wait_for_loadbalancer.yml
- import_tasks: check.yml

56
ansible/roles/mariadb/tasks/register.yml
View File

@ -1,52 +1,82 @@
---
- import_tasks: wait_for_loadbalancer.yml
- name: Creating shard root mysql user
become: true
kolla_toolbox:
module_name: mysql_user
module_args:
login_host: "{{ api_interface_address }}"
login_port: "{{ mariadb_port }}"
login_user: "{{ database_user }}"
login_password: "{{ database_password }}"
name: "{{ mariadb_shard_database_user }}"
password: "{{ database_password }}"
host: "%"
priv: "*.*:ALL,GRANT"
when:
- inventory_hostname == groups[mariadb_shard_group][0]
- name: Creating mysql monitor user
become: true
kolla_toolbox:
module_name: mysql_user
module_args:
login_host: "{{ api_interface_address }}"
login_port: "{{ mariadb_port }}"
login_user: "{{ database_user }}"
login_password: "{{ database_password }}"
name: "{{ mariadb_monitor_user }}"
password: ""
host: "%"
priv: "*.*:USAGE"
when:
- inventory_hostname == groups[mariadb_shard_group][0]
- name: Creating the Mariabackup database
become: true
kolla_toolbox:
module_name: mysql_db
module_args:
login_host: "{{ database_address }}"
login_port: "{{ database_port }}"
login_user: "{{ database_user }}"
login_host: "{{ api_interface_address }}"
login_port: "{{ mariadb_port }}"
login_user: "{{ mariadb_shard_database_user }}"
login_password: "{{ database_password }}"
name: "{{ mariadb_backup_database_schema }}"
run_once: True
when:
- enable_mariabackup | bool
- inventory_hostname == mariadb_backup_host
- name: Creating database backup user and setting permissions
become: true
kolla_toolbox:
module_name: mysql_user
module_args:
login_host: "{{ database_address }}"
login_port: "{{ database_port }}"
login_user: "{{ database_user }}"
login_host: "{{ api_interface_address }}"
login_port: "{{ mariadb_port }}"
login_user: "{{ mariadb_shard_database_user }}"
login_password: "{{ database_password }}"
name: "{{ mariadb_backup_database_user }}"
password: "{{ mariadb_backup_database_password }}"
host: "%"
priv: "*.*:CREATE TABLESPACE,RELOAD,PROCESS,SUPER,LOCK TABLES,REPLICATION CLIENT"
append_privs: True
run_once: True
when:
- enable_mariabackup | bool
- inventory_hostname == mariadb_backup_host
- name: Granting permissions on Mariabackup database to backup user
become: true
kolla_toolbox:
module_name: mysql_user
module_args:
login_host: "{{ database_address }}"
login_port: "{{ database_port }}"
login_user: "{{ database_user }}"
login_host: "{{ api_interface_address }}"
login_port: "{{ mariadb_port }}"
login_user: "{{ mariadb_shard_database_user }}"
login_password: "{{ database_password }}"
name: "{{ mariadb_backup_database_user }}"
password: "{{ mariadb_backup_database_password }}"
host: "%"
priv: "{{ mariadb_backup_database_schema }}.*:CREATE,INSERT,SELECT"
append_privs: True
run_once: True
when:
- enable_mariabackup | bool
- inventory_hostname == mariadb_backup_host

4
ansible/roles/mariadb/tasks/restart_services.yml
View File

@ -42,5 +42,5 @@
# NOTE(yoctozepto): we don't want to wait for new nodes to fully sync
# with an existing cluster as this could take time
- not mariadb_cluster_exists or
(groups.mariadb_port_alive_True is defined and
inventory_hostname in groups.mariadb_port_alive_True)
(groups[mariadb_shard_group + '_port_alive_True'] is defined and
inventory_hostname in groups[mariadb_shard_group + '_port_alive_True'])

16
ansible/roles/mariadb/tasks/wait_for_loadbalancer.yml
View File

@ -1,16 +0,0 @@
---
# Explicitly wait for the database to be accessible via the load balancer.
# Sometimes it can reject connections even when all database services are up,
# due to the health check polling in HAProxy.
- name: wait for MariaDB to be available via HAProxy
wait_for:
host: "{{ database_address }}"
port: "{{ database_port }}"
connect_timeout: 1
timeout: 60
search_regex: "MariaDB"
register: check_mariadb_port
until: check_mariadb_port is success
retries: 10
delay: 6
run_once: True

2
ansible/roles/mariadb/templates/galera.cnf.j2
View File

@ -26,7 +26,7 @@ character-set-server = utf8
datadir=/var/lib/mysql/
wsrep_cluster_address=gcomm://{% if (groups['mariadb'] | length) > 1 %}{% for host in groups['mariadb'] %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ mariadb_wsrep_port }}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
wsrep_cluster_address=gcomm://{% if (groups[mariadb_shard_group] | length) > 1 %}{% for host in groups[mariadb_shard_group] %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ mariadb_wsrep_port }}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
wsrep_provider_options=gmcast.listen_addr=tcp://{{ api_interface_address | put_address_in_context('url') }}:{{ mariadb_wsrep_port }};ist.recv_addr={{ api_interface_address | put_address_in_context('url') }}:{{ mariadb_ist_port }};{% for option in mariadb_wsrep_extra_provider_options %}{{ option }}{% if not loop.last %};{% endif %}{% endfor %}

8
releasenotes/notes/mariadb-role-install-several-clusters-at-once-110057a091600d2c.yaml Normal file
View File

@ -0,0 +1,8 @@
---
features:
- |
The Mariadb role now allows the creation of multiple clusters.
This provides a benefit to operators as they are able to install
and maintain several clusters at once using kolla-ansible.
This is useful when deploying db clusters for cells or db clusters
for services that have large demands on the database.