diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 951eb66a09..0d7c8362ac 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -336,6 +336,7 @@ mariadb_port: "{{ database_port }}" mariadb_wsrep_port: "4567" mariadb_ist_port: "4568" mariadb_sst_port: "4444" +mariadb_clustercheck_port: "4569" masakari_api_port: "15868" diff --git a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 index 98797b09b8..44d9484b27 100644 --- a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 +++ b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 @@ -10,3 +10,17 @@ format1 /^(?.*)/ enable_watch_timer false + + @type tail + path /var/log/kolla/mariadb/xinetd.log + pos_file /var/run/{{ fluentd_binary }}/mariadb-xinetd.pos + tag infra.mariadb.xinetd + format multiline + format_firstline /^\d{2}/\d{1,2}/\d{1,2}@\d{1,2}:\d{1,2}:\d{1,2}\: (START|EXIT)\: / + format1 /^(?\S+) (?.*)?$/ + time_key Timestamp + keep_time_key true + time_format %y/%-m/%-d@%T + ignore_repeated_permission_error true + enable_watch_timer false + diff --git a/ansible/roles/mariadb/defaults/main.yml b/ansible/roles/mariadb/defaults/main.yml index dcab1dbe10..84b8605851 100644 --- a/ansible/roles/mariadb/defaults/main.yml +++ b/ansible/roles/mariadb/defaults/main.yml @@ -21,7 +21,7 @@ mariadb_services: backend_tcp_extra: - "option srvtcpka" - "timeout server 3600s" - - "option mysql-check user haproxy post-41" + - "option {% if enable_mariadb_clustercheck | bool %}httpchk{% else %}mysql-check user haproxy post-41{% endif %}" custom_member_list: "{{ internal_haproxy_members.split(';') }}" mariadb_external_lb: enabled: "{{ enable_external_mariadb_load_balancer|bool }}" @@ -35,6 +35,18 @@ mariadb_services: - "option srvtcpka" - "timeout server 3600s" custom_member_list: "{{ external_haproxy_members.split(';') }}" + mariadb-clustercheck: + container_name: mariadb_clustercheck + group: mariadb + enabled: "{{ enable_mariadb_clustercheck | bool }}" + image: "{{ mariadb_clustercheck_image_full }}" + volumes: "{{ mariadb_clustercheck_default_volumes + mariadb_clustercheck_extra_volumes }}" + dimensions: "{{ mariadb_clustercheck_dimensions }}" + environment: + MYSQL_USERNAME: "haproxy" + MYSQL_PASSWORD: "" + MYSQL_HOST: "{{ api_interface_address }}" + AVAILABLE_WHEN_DONOR: "1" #################### # Database @@ -45,8 +57,8 @@ database_max_timeout: 120 #################### # HAProxy #################### -internal_haproxy_members: "{% for host in groups['mariadb'] %}server {{ hostvars[host]['ansible_hostname'] }} {{ 'api' | kolla_address(host) }}:{{ mariadb_port }} check inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}" -external_haproxy_members: "{% for host in groups['mariadb'] %}server {{ host }} {{ host }}:{{ mariadb_port }} check inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}" +internal_haproxy_members: "{% for host in groups['mariadb'] %}server {{ hostvars[host]['ansible_hostname'] }} {{ 'api' | kolla_address(host) }}:{{ mariadb_port }} check {% if enable_mariadb_clustercheck | bool %}port {{ mariadb_clustercheck_port }} {% endif %} inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}" +external_haproxy_members: "{% for host in groups['mariadb'] %}server {{ host }} {{ host }}:{{ mariadb_port }} check {% if enable_mariadb_clustercheck | bool %}port {{ mariadb_clustercheck_port}} {% endif %} inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}" #################### # Docker @@ -55,7 +67,14 @@ mariadb_install_type: "{{ kolla_install_type }}" mariadb_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ mariadb_install_type }}-mariadb" mariadb_tag: "{{ openstack_tag }}" mariadb_image_full: "{{ mariadb_image }}:{{ mariadb_tag }}" + +mariadb_clustercheck_install_type: "{{ kolla_install_type }}" +mariadb_clustercheck_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ mariadb_clustercheck_install_type }}-mariadb-clustercheck" +mariadb_clustercheck_tag: "{{ openstack_tag }}" +mariadb_clustercheck_image_full: "{{ mariadb_clustercheck_image }}:{{ mariadb_clustercheck_tag }}" + mariadb_dimensions: "{{ default_container_dimensions }}" +mariadb_clustercheck_dimensions: "{{ default_container_dimensions }}" mariadb_default_volumes: - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro" @@ -63,7 +82,14 @@ mariadb_default_volumes: - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}" - "mariadb:/var/lib/mysql" - "kolla_logs:/var/log/kolla/" +mariadb_clustercheck_default_volumes: + - "{{ node_config_directory }}/mariadb-clustercheck/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}" + - "kolla_logs:/var/log/kolla/" + mariadb_extra_volumes: "{{ default_extra_volumes }}" +mariadb_clustercheck_extra_volumes: "{{ default_extra_volumes }}" ######################################## # Vars used within recover_cluster.yml @@ -87,3 +113,8 @@ mariadb_backup_database_schema: "PERCONA_SCHEMA" mariadb_backup_database_user: "backup" mariadb_backup_database_address: "{{ database_address }}" mariadb_backup_type: "full" + +#################### +# Clustercheck +#################### +enable_mariadb_clustercheck: "yes" diff --git a/ansible/roles/mariadb/handlers/main.yml b/ansible/roles/mariadb/handlers/main.yml index 79e52d9084..9c3cb057de 100644 --- a/ansible/roles/mariadb/handlers/main.yml +++ b/ansible/roles/mariadb/handlers/main.yml @@ -89,3 +89,19 @@ - name: Ensure MariaDB is running normally on bootstrap host include_tasks: 'restart_services.yml' listen: Bootstrap MariaDB cluster + +- name: Restart mariadb-clustercheck container + vars: + service_name: "mariadb-clustercheck" + service: "{{ mariadb_services[service_name] }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + image: "{{ service.image }}" + name: "{{ service.container_name }}" + volumes: "{{ service.volumes }}" + dimensions: "{{ service.dimensions }}" + environment: "{{ service.environment }}" + listen: + - restart mariadb-clustercheck diff --git a/ansible/roles/mariadb/tasks/config.yml b/ansible/roles/mariadb/tasks/config.yml index e8eca2db25..273d19303f 100644 --- a/ansible/roles/mariadb/tasks/config.yml +++ b/ansible/roles/mariadb/tasks/config.yml @@ -40,19 +40,17 @@ - inventory_hostname == mariadb_backup_host - name: Copying over config.json files for services - vars: - service_name: "mariadb" - service: "{{ mariadb_services[service_name] }}" template: - src: "{{ service_name }}.json.j2" - dest: "{{ node_config_directory }}/{{ service_name }}/config.json" + src: "{{ item.key }}.json.j2" + dest: "{{ node_config_directory }}/{{ item.key }}/config.json" mode: "0660" become: true when: - - inventory_hostname in groups[service.group] - - service.enabled | bool + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ mariadb_services }}" notify: - - restart mariadb + - "restart {{ item.key }}" - name: Copying over config.json files for mariabackup vars: @@ -97,5 +95,20 @@ notify: - restart mariadb +- name: Copying over xinetd clustercheck.conf + vars: + service_name: "mariadb-clustercheck" + service: "{{ mariadb_services[service_name] }}" + template: + src: "{{ role_path }}/templates/xinetd-clustercheck.j2" + dest: "{{ node_config_directory }}/{{ service_name }}/clustercheck" + mode: "0660" + become: true + when: + - inventory_hostname in groups[service.group] + - service.enabled | bool + notify: + - restart mariadb-clustercheck + - include_tasks: check-containers.yml when: kolla_action != "config" diff --git a/ansible/roles/mariadb/tasks/recover_cluster.yml b/ansible/roles/mariadb/tasks/recover_cluster.yml index 2147d98192..15ec23a017 100644 --- a/ansible/roles/mariadb/tasks/recover_cluster.yml +++ b/ansible/roles/mariadb/tasks/recover_cluster.yml @@ -185,15 +185,17 @@ - bootstrap_host is defined - bootstrap_host == inventory_hostname -- name: Restart slave MariaDB container +- name: Restart slave MariaDB container(s) become: true kolla_docker: action: "start_container" common_options: "{{ docker_common_options }}" - name: "{{ mariadb_service.container_name }}" - image: "{{ mariadb_service.image }}" - volumes: "{{ mariadb_service.volumes }}" - dimensions: "{{ mariadb_service.dimensions }}" + name: "{{ item.value.container_name }}" + image: "{{ item.value.image }}" + volumes: "{{ item.value.volumes }}" + dimensions: "{{ item.value.dimensions }}" + environment: "{{ item.value.environment | default({}) }}" + with_dict: "{{ mariadb_services }}" when: - bootstrap_host is defined - bootstrap_host != inventory_hostname @@ -213,15 +215,17 @@ - bootstrap_host is defined - bootstrap_host != inventory_hostname -- name: Restart master MariaDB container +- name: Restart master MariaDB container(s) become: true kolla_docker: action: "recreate_or_restart_container" common_options: "{{ docker_common_options }}" - name: "{{ mariadb_service.container_name }}" - image: "{{ mariadb_service.image }}" - volumes: "{{ mariadb_service.volumes }}" - dimensions: "{{ mariadb_service.dimensions }}" + name: "{{ item.value.container_name }}" + image: "{{ item.value.image }}" + volumes: "{{ item.value.volumes }}" + dimensions: "{{ item.value.dimensions }}" + environment: "{{ item.value.environment | default({}) }}" + with_dict: "{{ mariadb_services }}" when: - bootstrap_host is defined - bootstrap_host == inventory_hostname diff --git a/ansible/roles/mariadb/templates/mariadb-clustercheck.json.j2 b/ansible/roles/mariadb/templates/mariadb-clustercheck.json.j2 new file mode 100644 index 0000000000..8083ec31f4 --- /dev/null +++ b/ansible/roles/mariadb/templates/mariadb-clustercheck.json.j2 @@ -0,0 +1,18 @@ +{ + "command": "/usr/sbin/xinetd -dontfork -filelog /var/log/kolla/mariadb/xinetd.log", + "config_files": [ + { + "source": "{{ container_config_directory }}/clustercheck", + "dest": "/etc/xinetd.d/clustercheck", + "owner": "root", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/mariadb", + "owner": "mysql:mysql", + "recurse": true + } + ] +} diff --git a/ansible/roles/mariadb/templates/xinetd-clustercheck.j2 b/ansible/roles/mariadb/templates/xinetd-clustercheck.j2 new file mode 100644 index 0000000000..1d719ae1e0 --- /dev/null +++ b/ansible/roles/mariadb/templates/xinetd-clustercheck.j2 @@ -0,0 +1,16 @@ +# default: on +# description: mysqlchk +service mysqlchk +{ + disable = no + flags = REUSE + socket_type = stream + type = UNLISTED + port = {{ mariadb_clustercheck_port }} + wait = no + user = mysql + server = /usr/bin/clustercheck + log_type = FILE /var/log/kolla/mariadb/xinetd.log + log_on_failure = HOST ATTEMPT + per_source = UNLIMITED +} diff --git a/releasenotes/notes/mariadb-clustercheck-39b5837c09868f1e.yaml b/releasenotes/notes/mariadb-clustercheck-39b5837c09868f1e.yaml new file mode 100644 index 0000000000..d4641b2a0a --- /dev/null +++ b/releasenotes/notes/mariadb-clustercheck-39b5837c09868f1e.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Enable Galera node state checking by using ``clustercheck`` script that is + used by HAProxy to define node up/down state.