Use glusterfs to synchronise repo server contents

The /var/lib/glusterd directory is bind mounted into
/openstack/glusterd/<...> to preserve the peer state, specifically
the peer UUID.

Depends-On: https://review.opendev.org/c/openstack/ansible-role-systemd_mount/+/837750
Depends-On: https://review.opendev.org/c/openstack/openstack-ansible-repo_server/+/839411
Depends-On: https://review.opendev.org/c/openstack/openstack-ansible-plugins/+/837582
Change-Id: I5c01e0ffd4bc7a5340309d00a1061e6d2f300bd5

ansible-collection-requirements.yml

@@ -25,6 +25,9 @@ collections:
 - name: https://github.com/ansible-collections/ansible.posix
   version: 1.3.0
   type: git
+- name: https://github.com/gluster/gluster-ansible-collection
+  version: 1.0.2
+  type: git
 # NOTE(noonedeadpunk): needs version in galaxy.yml to pull from git
 - name: ansible.netcommon
   version: 2.6.1

doc/source/admin/upgrades/distribution-upgrades.rst

@@ -94,10 +94,6 @@ Pre-Requisites
       and can visit https://admin:password@external_lb_vip_address:1936/ and read
       'Statistics Report for pid # on infrastructure_host'
 
-   *  repo_container
-
-      Check all your repo_containers and look for /etc/lsyncd/lsyncd.conf.lua
-
 Warnings
 ========
 

inventory/group_vars/repo_all.yml

@@ -15,3 +15,21 @@
 
 # Ensure that the package state matches the global setting
 repo_server_package_state: "{{ package_state }}"
+
+# Transition to shared filesystem, disable lsyncd deployment
+repo_server_enable_sync_manager: False
+
+# Repo container default list of bind mounts
+repo_container_bind_mounts:
+  - bind_dir_path: "/var/lib/glusterd"
+    mount_path: "/openstack/glusterd/{{inventory_hostname }}"
+
+# Enable glusterfs mount
+repo_server_systemd_mounts:
+  - what: "{{ ansible_hostname }}:gfs-repo"
+    where: "/var/www/repo"
+    type: glusterfs
+    state: 'started'
+    enabled: true
+    
+openstack_repo_server_enable_glusterfs: True

playbooks/repo-install.yml

@@ -32,14 +32,34 @@
 
     - include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       vars:
-        list_of_bind_mounts:
-          - mount_path: "/openstack/{{ inventory_hostname }}"
-            bind_dir_path: "/var/www"
+        list_of_bind_mounts: "{{ repo_container_bind_mounts }}"
       when: not is_metal
 
     - include_tasks: common-tasks/unbound-clients.yml
       when:
         - hostvars['localhost']['resolvconf_enabled'] | bool
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+- name: Setup repo server gluster cluster
+  hosts: repo_all
+  gather_facts: false
+  user: root
+  tasks:
+    - name: Create gluster cluster
+      include_role:
+        name: openstack.osa.glusterfs
+      vars:
+        glusterfs_server_group_name: "repo_all"
+        glusterfs_server_volume_name: "gfs-repo"
+      when:
+        - openstack_repo_server_enable_glusterfs | default(False)
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+- name: Setup repo servers
+  hosts: repo_all
+  gather_facts: false
+  serial: "{{ repo_serial | default(['1','100%']) }}"
+  user: root
   roles:
     - role: "repo_server"
 
@@ -51,13 +71,8 @@
         rsyslog_client_log_files: "{{ rsyslog_var.log_files | default([]) }}"
         rsyslog_client_config_name: "{{ rsyslog_var.config_name }}"
       with_items:
-        - log_rotate_file: lsyncd_log_rotate
-          log_dir: "/var/log/lsyncd"
-          config_name: "99-lsyncd-rsyslog-client.conf"
         - log_rotate_file: repo_nginx_log_rotate
           log_dir: "/var/log/nginx"
-          log_files:
-            - /var/log/rsyncd.log
           config_name: "99-repo-nginx-rsyslog-client.conf"
       loop_control:
         loop_var: rsyslog_var

releasenotes/notes/repo-server-glusterfs-181d0ab5c62e13d6.yaml

@@ -0,0 +1,26 @@
+---
+features:
+  - |
+    The mechanism used previously to syncronise repo server contents between
+    highly available sets of repo servers in a multinode deployment (lsyncd
+    and rsync over ssh) is removed and replaced with a shared filesystem
+    mount. This permits much easier support for multi operating system and
+    multi processor architectures in the deployment when building and serving
+    python wheels using the repo server. The default deployment will run a
+    glusterfs server in each repo server host, and mount the glusterfs
+    fileystem at /var/www/repo using the system_mount ansible role. If a
+    deployment wishes to use an alternative external shared filesystem, the
+    new variable `openstack_repo_server_enable_glusterfs` can be set to `false`
+    and alternative mounts created by overriding the new `repo_server_systemd_mounts`
+    variable. It is mandatory to use some type of shared filesystem for the
+    repo server in all deployments.
+upgrade:
+  - |
+    The repo server hosts will stop and uninstall existing lsyncd and rsync
+    services from the repo server hosts. This functionality will be replaced
+    by default with a glusterfs shared filesystem. If a deployment uses a
+    firewall on the control plane, the rules should be updated to allow the
+    glusterfs traffic between the repo server hosts. Alternative external shared
+    filesystems (eg NFS, cephfs, others) may be used if required and the new
+    variables `repo_server_systemd_mounts` and `openstack_repo_server_enable_glusterfs`
+    allow a deployment to override the default use of glusterfs.