Refactor backup content

This commit aims to refactor the way we select folders for backup in a single place, giving the option to remove folders/subfolders from the backup archive using paths/globs. This will give the opportunity to provide a more detailed input of what we want add to and exclude from the backup and reduce the backup archive size to allow for a quicker backup & restore process. TEST PLAN -- CentOS -- PASS backup and restore on SX PASS add new deployments before taking backup, check if they are present and working well after the restore PASS backup system with stx-openstack installed, restored and checked if stx-openstack was working -- Debian -- PASS backup and restore on SX (local and remote play) PASS backup and restore on DX PASS backup system with storage backend configured as Ceph, restore without issues PASS backup system with no storage backend configured, restore with no issues PASS add new deployments before taking backup, check if they are present and working well after the restore Story: 2010117 Task: 45980 Signed-off-by: Thiago Brito <thiago.brito@windriver.com> Change-Id: I750be41aaaf8e136ecfaa5442a6d0d304ac9490f
2022-07-28 15:58:41 -03:00 · 2022-07-28 15:58:41 -03:00 · c53535bffc
commit c53535bffc
parent 9d59383e08
5 changed files with 100 additions and 77 deletions
--- a/playbookconfig/src/playbooks/roles/backup/backup-system/tasks/main.yml
+++ b/playbookconfig/src/playbooks/roles/backup/backup-system/tasks/main.yml
@ -197,9 +197,9 @@
    # overrides are already created when we uploaded the application.
    - name: Create Helm overrides temp dir
      file:
-        path: "{{ tempdir.path }}/helm_overrides_dir"
+        path: "{{ tempdir.path }}/helm_overrides_sqldump_dir"
        state: directory
-      register: helm_overrides_dir
+      register: helm_overrides_sqldump_dir

    - name: Get the openstack Helm overrides from the from the database
      shell: >-
@ -219,34 +219,16 @@

    - name: Backup Helm overrides
      copy:
-        dest: "{{ helm_overrides_dir.path }}/helm_overrides_dump.sql"
+        dest: "{{ helm_overrides_sqldump_dir.path }}/helm_overrides_dump.sql"
        mode: 0755
        content: '{{ updates_list | default("") }}'

    # Now Postgres data and MariaDB data are stored in staging dir, we can estimate
    # the disk size requirement for the backup archive.
    - name: Check the size (in KiB) of directories that will be backed up for platform
-      shell: "du -sh -k  {{ item }} | awk '{print $1}'"
-      with_items:
-        - /etc
-        - "{{ homedir }} "
-        - "{{ config_permdir }}"
-        - "{{ sysinv_permdir }}"
-        - "{{ puppet_permdir }}/hieradata"
-        - "{{ keyring_permdir }}"
-        - "{{ extra_permdir }}"
-        - "{{ patching_permdir }}"
-        - "{{ patching_repo_permdir }}"
-        - "{{ extension_permdir }}"
-        - "{{ deploy_permdir }}"
-        - "{{ postgres_dir.path }}"
-        - "{{ armada_permdir }}"
-        - "{{ fluxcd_permdir }}"
-        - "{{ helm_charts_permdir }}"
-        - "{{ helm_overrides_permdir }}"
-        - "{{ helm_overrides_dir.path }}"
-        - "{{ fernet_keys_permdir }}"
+      shell: "du -sh -k  {{ item }} --exclude {{ backup.exclude|join(' --exclude ') }} | awk '{print $1}'"
      register: size_output_platform
+      with_items: "{{ backup.targets }}"

    # Estimate the backup size. We add 128M overhead for things like ceph crushmap,
    # ldap data, etc. that will be generated and stored in the staging dir later on.
@ -294,7 +276,7 @@
        shell: "du -sh -k  {{ item }} | awk '{print $1}'"
        with_items:
          - "{{ mariadb_dir.path }}"
-          - "{{ helm_overrides_dir.path }}"
+          - "{{ helm_overrides_sqldump_dir.path }}"
          - "{{ armada_permdir }}/{{ openstack_app_name }}"
          - "{{ helm_charts_permdir }}/starlingx"
        register: size_output_openstack
@ -497,37 +479,40 @@
        msg: "Failed to run post-backup action [{{ post_backup_notification_result.rc }}]"
      when: post_backup_notification_result.rc != 0

+    # NOTE: Backup contents are defined on roles/backup/backup-system/vars/main.yml
+    # Some additional files generated during this playbook's execution are appended to the backup targets here
+    - name: Set fact for backup targets with extra files
+      set_fact:
+        final_backup_targets: "{{ (backup.targets + [
+          etcd_snapshot_file,
+          helm_overrides_sqldump_dir.path,
+          ldap_db_backup,
+          override_backup_file,
+          postgres_dir.path,
+          crushmap_file_,
+          ceph_conf_ctrl_0_
+        ]) | reject('equalto', '') | list }}"  # rejecting vars that might be empty
+      vars:
+        crushmap_file_: "{{ crushmap_file | default(\"\") }}"
+        ceph_conf_ctrl_0_: "{{ ceph_conf_ctrl_0 | default(\"\") }}"

-    # Archive module has a known bug that doesn't handle empty symbolic links
-    # well. Restore to tar command. Can add -P option to keep the leading
-    # '/'s in file names in the tar file, so that the tasks that strip leading
-    # '/' from the directory names before untar won't be required.
+    # This is nasty to understand, but check the -vvv output to see what is going on
+    # When we update the community.general collection from 1.3.6 to >=5.2.0, we can use "archive"
+    # with exclusion_patterns
    - name: Create a tgz archive for platform backup
-      shell: "tar -czf {{ platform_backup_file_path }} $(ls -d \
-           {{ override_backup_file }} \
-           /etc \
-           {{ homedir }} \
-           {{ config_permdir }} \
-           {{ sysinv_permdir }} \
-           {{ puppet_permdir }}/hieradata \
-           {{ keyring_permdir }} \
-           {{ extra_permdir }} \
-           {{ patching_permdir }} \
-           {{ patching_repo_permdir }} \
-           {{ extension_permdir }} \
-           {{ deploy_permdir }} \
-           {{ crushmap_file | default(\"\") }} \
-           {{ ceph_conf_ctrl_0 | default(\"\") }} \
-           {{ etcd_snapshot_file }} \
-           {{ ldap_db_backup }} \
-           {{ postgres_dir.path }} \
-           {{ armada_permdir }} \
-           {{ fluxcd_permdir }} \
-           {{ helm_overrides_permdir }} \
-           {{ helm_charts_permdir }} \
-           {{ fernet_keys_permdir }} 2>/dev/null)"
+      shell: >-
+        tar
+        --exclude {{ backup.exclude | map('regex_replace', '^/', '') | list | join(' --exclude ') }}
+        -czf {{ platform_backup_file_path }}
+        $(ls -d
+        {{ final_backup_targets | join(' ') }}
+        2> /dev/null)
      args:
        warn: false
+      # Changing the failed_when behavior to prevent the backup to fail on "file changed as we read it", which
+      # makes tar return 1
+      register: tar_cmd
+      failed_when: tar_cmd.rc >= 2

    - name: Create a tgz archive for dc-vault backup
      shell: "tar -czf {{ dc_vault_backup_file_path }} $(ls -d \
@ -541,7 +526,7 @@
           {{ armada_permdir }}/{{ openstack_app_name }} \
           {{ helm_charts_permdir }}/starlingx \
           {{ mariadb_dir.path }} \
-           {{ helm_overrides_dir.path }} 2>/dev/null)"
+           {{ helm_overrides_sqldump_dir.path }} 2>/dev/null)"
      args:
        warn: false
      when: check_mariadb_pod.rc == 0 or openstack_status.stdout == "uploaded"
--- a/playbookconfig/src/playbooks/roles/backup/backup-system/vars/main.yml
+++ b/playbookconfig/src/playbooks/roles/backup/backup-system/vars/main.yml
@ -1,7 +1,14 @@
 ---
 kube_config_dir: /etc/kubernetes/admin.conf

-skip_os_dbs: ['Database', 'information_schema', 'performance_schema', 'mysql', 'horizon', 'panko', 'gnocchi']
+skip_os_dbs:
+  - 'Database'
+  - 'information_schema'
+  - 'performance_schema'
+  - 'mysql'
+  - 'horizon'
+  - 'panko'
+  - 'gnocchi'

 mariadb_pod: mariadb-server-0

@ -10,3 +17,51 @@ backup_include_apps: []

 # List of apps on DC system controller whose images will be included in the backup
 backup_include_dc_controller_apps: ["platform-integ-apps"]
+
+homedir: "{{ '/home' if os_release == 'centos' else '/var/home' }}"
+armada_permdir: "{{ platform_path }}/armada/{{ software_version }}"
+config_permdir: "{{ platform_path }}/config/{{ software_version }}"
+deploy_permdir: "{{ platform_path }}/deploy/{{ software_version }}"
+extension_permdir: "/opt/extension"
+extra_permdir: "{{ platform_path }}/extra"
+fernet_keys_permdir: "{{ platform_path }}/keystone/fernet-keys"
+fluxcd_permdir: "{{ platform_path }}/fluxcd/{{ software_version }}"
+helm_overrides_permdir: "{{ platform_path }}/helm/{{ software_version }}"
+helm_charts_permdir: "{{ platform_path }}/helm_charts"
+keyring_permdir: "{{ platform_path }}/.keyring/{{ software_version }}"
+patching_permdir: "/opt/patching"  # TODO(outbrito): Remove when CentOS deprecates
+patching_repo_permdir: "/var/www/pages/updates"  # TODO(outbrito): Remove when CentOS deprecates
+puppet_permdir: "{{ platform_path }}/puppet/{{ software_version }}"
+sysinv_permdir: "{{ platform_path }}/sysinv/{{ software_version }}"
+
+# Use plain paths or glob
+backup:
+  targets:
+    - "/etc"
+    - "{{ armada_permdir }}"
+    - "{{ config_permdir }}"
+    - "{{ deploy_permdir }}"
+    - "{{ extension_permdir }}"
+    - "{{ extra_permdir }}"
+    - "{{ fernet_keys_permdir}}"
+    - "{{ fluxcd_permdir }}"
+    - "{{ helm_charts_permdir }}"
+    - "{{ helm_overrides_permdir }}"
+    - "{{ homedir }}"
+    - "{{ keyring_permdir }}"
+    - "{{ patching_permdir }}"  # TODO(outbrito): Remove when CentOS deprecates
+    - "{{ patching_repo_permdir }}"  # TODO(outbrito): Remove when CentOS deprecates
+    - "{{ puppet_permdir }}/hieradata"
+    - "{{ postgres_dir.path }}"
+    - "{{ sysinv_permdir }}"
+  exclude:
+    - "/etc/selinux/**"
+    - "/etc/selinux"
+    - "/etc/udev/**"
+    - "/etc/udev"
+    - "{{ homedir }}/*/ansible.log"
+    - "{{ homedir }}/*/.ansibl**"
+    - "{{ homedir }}/*/.cache/**"
+    - "{{ homedir }}/*/.cache"
+    - "{{ homedir }}/*/.kube/**"
+    - "{{ homedir }}/*/.kube"
--- a/playbookconfig/src/playbooks/roles/backup/prepare-env/tasks/main.yml
+++ b/playbookconfig/src/playbooks/roles/backup/prepare-env/tasks/main.yml
@ -1,6 +1,6 @@
 ---
 #
-# Copyright (c) 2019 Wind River Systems, Inc.
+# Copyright (c) 2019-2022 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@ -29,10 +29,6 @@
    msg: "Backup can only be done on the active controller."
  when: active_ctlr.rc != 0

- name: Declare homedir fact
-  set_fact:
-    homedir: "{{ '/home' if os_release == 'centos' else '/var/home' }}"
-
 - name: Check disk usage of /home directory
  shell: "du -sh -m  {{ homedir }}  | awk '{print $1}'"
  register: home_dir_usage
@ -57,23 +53,9 @@
  set_fact:
    system_mode: "{{ check_system_mode.stdout.split('=')[1] }}"

- name: Set config path facts
+- name: Set dc-vault config path fact
  set_fact:
-    keyring_permdir: "{{ platform_path }}/.keyring/{{ software_version }}"
-    config_permdir: "{{ platform_path }}/config/{{ software_version }}"
-    sysinv_permdir: "{{ platform_path }}/sysinv/{{ software_version }}"
-    puppet_permdir: "{{ platform_path }}/puppet/{{ software_version }}"
-    armada_permdir: "{{ platform_path }}/armada/{{ software_version }}"
-    fluxcd_permdir: "{{ platform_path }}/fluxcd/{{ software_version }}"
-    deploy_permdir: "{{ platform_path }}/deploy/{{ software_version }}"
-    helm_overrides_permdir: "{{ platform_path }}/helm/{{ software_version }}"
-    helm_charts_permdir: "{{ platform_path }}/helm_charts"
-    patching_permdir: "/opt/patching"
-    patching_repo_permdir: "/var/www/pages/updates"
-    extension_permdir: "/opt/extension"
    dc_vault_permdir: "/opt/dc-vault"
-    extra_permdir: "{{ platform_path }}/extra"
-    fernet_keys_permdir: "{{ platform_path }}/keystone/fernet-keys"

 - name: Check if ceph is configured
  stat:
--- a/playbookconfig/src/playbooks/roles/restore-openstack/restore/tasks/main.yml
+++ b/playbookconfig/src/playbooks/roles/restore-openstack/restore/tasks/main.yml
@ -83,24 +83,24 @@
  - name: Set Helm Overrides backup directory
    tempfile:
      state: directory
-    register: helm_overrides_dir
+    register: helm_overrides_sqldump_dir
    become_user: postgres

  - name: Restore Helm Overrides dump file
    command: >-
-      tar -C {{ helm_overrides_dir.path }} -xpf {{ target_backup_dir }}/{{ backup_filename }}
+      tar -C {{ helm_overrides_sqldump_dir.path }} -xpf {{ target_backup_dir }}/{{ backup_filename }}
      --wildcards --transform='s,.*/,,' '*/helm_overrides*.sql'
    args:
      warn: false

  - name: Apply Helm Overrides dump file
    shell: >-
-      psql -f {{ helm_overrides_dir.path }}/helm_overrides_dump.sql sysinv
+      psql -f {{ helm_overrides_sqldump_dir.path }}/helm_overrides_dump.sql sysinv
    become_user: postgres

  - name: Remove Helm Overrides backup directory
    file:
-      path: "{{ helm_overrides_dir.path }}"
+      path: "{{ helm_overrides_sqldump_dir.path }}"
      state: absent

  # This override is required by the tidy_storage helper script when we do a full restore.
--- a/tox.ini
+++ b/tox.ini
@ -2,6 +2,7 @@
 envlist = linters,pep8
 minversion = 2.3
 skipsdist = True
+toxworkdir = /tmp/{env:USER}_ansible-playbookstox

 [testenv]
 install_command = pip install -U {opts} {packages}