Add commands for database backup and recovery

Uses the support [1] for database backups added to Kolla Ansible in the
Stein release.

Adds the following new commands:

kayobe overcloud database backup [--incremental]
kayobe overcloud database recover [--force-recovery-host <host>]

The recovery command is for recovering a clustered database that has
lost quorum.

[1] https://docs.openstack.org/kolla-ansible/latest/admin/mariadb-backup-and-restore.html

Change-Id: Ie16354cd01ea7dd83cd3d4058dd8451b8387600b
Story: 2005015
Task: 29493

ansible/group_vars/all/kolla

@@ -369,6 +369,7 @@ kolla_enable_skydive: "no"
 kolla_enable_storm: "{{ 'yes' if kolla_enable_monasca | bool else 'no' }}"
 kolla_enable_swift: "no"
 kolla_enable_telegraf: "no"
+kolla_enable_xtrabackup: "no"
 kolla_enable_zookeeper: "{{ 'yes' if kolla_enable_kafka | bool or kolla_enable_storm | bool else 'no' }}"
 
 ###############################################################################

ansible/kolla-openstack.yml

@@ -119,6 +119,7 @@
             - { name: nova, file: nova.conf }
             - { name: octavia, file: octavia.conf }
             - { name: sahara, file: sahara.conf }
+            - { name: xtrabackup, file: backup.my.cnf }
             - { name: zookeeper, file: zookeeper.cfg }
 
         - name: Initialise a fact containing extra configuration
@@ -223,5 +224,6 @@
       kolla_extra_nova: "{{ kolla_extra_config.nova | default }}"
       kolla_extra_octavia: "{{ kolla_extra_config.octavia | default }}"
       kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}"
+      kolla_extra_xtrabackup: "{{ kolla_extra_config.xtrabackup | default }}"
       kolla_extra_zookeeper: "{{ kolla_extra_config.zookeeper | default }}"
       kolla_extra_config_path: "{{ kayobe_config_path }}/kolla/config"

ansible/roles/kolla-ansible/vars/main.yml

@@ -152,5 +152,6 @@ kolla_feature_flags:
   - vitrage
   - vmtp
   - watcher
+  - xtrabackup
   - zookeeper
   - zun

ansible/roles/kolla-openstack/defaults/main.yml

@@ -404,6 +404,15 @@ kolla_enable_storm:
 # Whether to enable swift.
 kolla_enable_swift:
 
+###############################################################################
+# Xtrabackup configuration.
+
+# Whether to enable Xtrabackup.
+kolla_enable_xtrabackup:
+
+# Free form extra configuration to append to backup.my.cnf.
+kolla_extra_xtrabackup:
+
 ###############################################################################
 # Zookeeper configuration.
 

ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml

@@ -96,6 +96,10 @@ provisioner:
           foo=bar
         kolla_enable_swift: true
         kolla_enable_storm: true
+        kolla_enable_xtrabackup: true
+        kolla_extra_xtrabackup: |
+          [extra-backup.my.cnf]
+          foo=bar
         kolla_enable_zookeeper: true
         kolla_extra_zookeeper: |
           [extra-zookeeper.cfg]

ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py

@@ -78,6 +78,7 @@ def test_service_config_directory(host, path):
      'nova.conf',
      'octavia.conf',
      'sahara.conf',
+     'backup.my.cnf',
      'zookeeper.cfg'])
 def test_service_ini_file(host, path):
     # TODO(mgoddard): Check more of config file contents.

ansible/roles/kolla-openstack/tasks/config.yml

@@ -34,6 +34,7 @@
     - { src: pxelinux.default.j2, dest: ironic/pxelinux.default, enabled: "{{ kolla_enable_ironic }}" }
     - { src: inspector.ipxe.j2, dest: ironic/inspector.ipxe, enabled: "{{ kolla_enable_ironic_ipxe }}" }
     - { src: sahara.conf.j2, dest: sahara.conf, enabled: "{{ kolla_enable_sahara }}" }
+    - { src: backup.my.cnf.j2, dest: backup.my.cnf, enabled: "{{ kolla_enable_xtrabackup }}" }
     - { src: zookeeper.cfg.j2, dest: zookeeper.cfg, enabled: "{{ kolla_enable_zookeeper }}" }
   when: item.enabled | bool
 

ansible/roles/kolla-openstack/templates/backup.my.cnf.j2

@@ -0,0 +1,9 @@
+# {{ ansible_managed }}
+
+{% if kolla_extra_xtrabackup %}
+#######################
+# Extra configuration
+#######################
+
+{{ kolla_extra_xtrabackup }}
+{% endif %}

ansible/roles/kolla-openstack/vars/main.yml

@@ -166,6 +166,11 @@ kolla_openstack_custom_config:
       - container.ring.gz
       - object.builder
       - object.ring.gz
+  # Xtrabackup.
+  - src: "{{ kolla_extra_config_path }}/xtrabackup"
+    dest: "{{ kolla_node_custom_config_path }}/xtrabackup"
+    patterns: "*"
+    enabled: "{{ kolla_enable_xtrabackup }}"
   # Zookeeper.
   - src: "{{ kolla_extra_config_path }}/zookeeper"
     dest: "{{ kolla_node_custom_config_path }}/zookeeper"

doc/source/administration/overcloud.rst

@@ -35,6 +35,8 @@ For example::
 To execute the command with root privileges, add the ``--become`` argument.
 Adding the ``--verbose`` argument allows the output of the command to be seen.
 
+.. _overcloud-administration-reconfigure:
+
 Reconfiguring Containerised Services
 ====================================
 
@@ -129,3 +131,35 @@ The configuration will be generated remotely on the overcloud hosts in the
 specified directory, with one subdirectory per container. This command may be
 followed by ``kayobe ovecloud service configuration save`` to gather the
 generated configuration to the Ansible control host.
+
+Performing Database Backups
+===========================
+
+Database backups can be performed using the underlying support in Kolla
+Ansible.
+
+In order to enable backups, enable Xtrabackup in
+``${KAYOBE_CONFIG_PATH}/kolla.yml``:
+
+.. code-block:: console
+
+   kolla_enable_xtrabackup: true
+
+To apply this change, use the :ref:`kayobe overcloud service reconfigure
+<overcloud-administration-reconfigure>` command.
+
+To perform a full backup, run the following command:
+
+.. code-block:: console
+
+   kayobe overcloud database backup
+
+Or to perform an incremental backup, run the following command:
+
+.. code-block:: console
+
+   kayobe overcloud database backup --incremental
+
+Further information on backing up and restoring the database is available in
+the `Kolla Ansible documentation
+<https://docs.openstack.org/kolla-ansible/latest/admin/mariadb-backup-and-restore.html>`__.

kayobe/cli/commands.py

@@ -970,6 +970,49 @@ class OvercloudHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command):
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
 
 
+class OvercloudDatabaseBackup(KollaAnsibleMixin, VaultMixin, Command):
+    """Backup the overcloud database."""
+
+    def get_parser(self, prog_name):
+        parser = super(OvercloudDatabaseBackup, self).get_parser(prog_name)
+        group = parser.add_argument_group("Overcloud Database Backup")
+        group.add_argument("--incremental", action='store_true',
+                           help="Whether to perform an incremental database "
+                                "backup. Default is false.")
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Performing overcloud database backup")
+        extra_args = []
+        if parsed_args.incremental:
+            extra_args.append('--incremental')
+        self.run_kolla_ansible_overcloud(parsed_args, "mariadb_backup",
+                                         extra_args=extra_args)
+
+
+class OvercloudDatabaseRecover(KollaAnsibleMixin, VaultMixin, Command):
+    """Recover the overcloud database."""
+
+    def get_parser(self, prog_name):
+        parser = super(OvercloudDatabaseRecover, self).get_parser(prog_name)
+        group = parser.add_argument_group("Overcloud Database Recovery")
+        group.add_argument("--force-recovery-host",
+                           help="Name of a host to use to perform the "
+                                "recovery. By default kolla-ansible will "
+                                "automatically determine which host to use, "
+                                "and this option should not be used.")
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Performing overcloud database recovery")
+        extra_vars = {}
+        if parsed_args.force_recovery_host:
+            extra_vars['mariadb_recover_inventory_name'] = (
+                parsed_args.force_recovery_host)
+        self.run_kolla_ansible_overcloud(parsed_args, "mariadb_recovery",
+                                         extra_vars=extra_vars)
+
+
 class OvercloudServiceConfigurationGenerate(KayobeAnsibleMixin,
                                             KollaAnsibleMixin, VaultMixin,
                                             Command):

kayobe/tests/unit/cli/test_commands.py

@@ -1254,6 +1254,76 @@ class TestCase(unittest.TestCase):
         ]
         self.assertEqual(expected_calls, mock_run.call_args_list)
 
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_overcloud")
+    def test_overcloud_database_backup(self, mock_run):
+        command = commands.OvercloudDatabaseBackup(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "mariadb_backup",
+                extra_args=[]
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_overcloud")
+    def test_overcloud_database_backup_incremental(self, mock_run):
+        command = commands.OvercloudDatabaseBackup(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args(["--incremental"])
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "mariadb_backup",
+                extra_args=["--incremental"]
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_overcloud")
+    def test_overcloud_database_recover(self, mock_run):
+        command = commands.OvercloudDatabaseRecover(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "mariadb_recovery",
+                extra_vars={}
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_overcloud")
+    def test_overcloud_database_recover_force_host(self, mock_run):
+        command = commands.OvercloudDatabaseRecover(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args(["--force-recovery-host", "foo"])
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "mariadb_recovery",
+                extra_vars={
+                    "mariadb_recover_inventory_name": "foo"
+                }
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
     def test_overcloud_service_configuration_save(self, mock_run):

releasenotes/notes/db-backup-recovery-d4f1ced2ebd7dac4.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Adds commands to make use of the database backup and recovery features in
+    Kolla Ansible.
+
+    ``kayobe overcloud database backup [--incremental]`` can be used to take a
+    full or incremental backup of the database using Xtrabackup.
+
+    ``kayobe overcloud database recover [--force-recovery-host <host>]`` can be
+    used to recover a database cluster that has lost Quorum.

setup.cfg

@@ -54,6 +54,8 @@ kayobe.cli=
     overcloud_bios_raid_configure = kayobe.cli.commands:OvercloudBIOSRAIDConfigure
     overcloud_container_image_build = kayobe.cli.commands:OvercloudContainerImageBuild
     overcloud_container_image_pull = kayobe.cli.commands:OvercloudContainerImagePull
+    overcloud_database_backup = kayobe.cli.commands:OvercloudDatabaseBackup
+    overcloud_database_recover = kayobe.cli.commands:OvercloudDatabaseRecover
     overcloud_deployment_image_build = kayobe.cli.commands:OvercloudDeploymentImageBuild
     overcloud_deprovision = kayobe.cli.commands:OvercloudDeprovision
     overcloud_hardware_inspect = kayobe.cli.commands:OvercloudHardwareInspect