From f1bcb6a58692786c5c0a1abde1b26d06c6cfbd29 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Wed, 15 Sep 2021 16:08:10 -0700
Subject: [PATCH] Set a gerrit replication timeout of 15 minutes

We are seeing that replication tasks occasionally sit around forever and
have had to take manual intervention. One theory is that this is related
to networking between the gerrit server and the gitea servers. We don't
set maxRetries which means replication should be retried infinitely
which means if we hit the timeout we should try again. 15 minutes was
sort of arbitrarily chosen as ~twice the time it takes to clone a large
repo like nova.

Change-Id: Iec2536ad149a2e625a1f0107b9fcee3079493607
---
 inventory/service/host_vars/review02.opendev.org.yaml  | 8 ++++++++
 playbooks/roles/gerrit/templates/replication.config.j2 | 3 +++
 2 files changed, 11 insertions(+)

diff --git a/inventory/service/host_vars/review02.opendev.org.yaml b/inventory/service/host_vars/review02.opendev.org.yaml
index 2cd9dca1f4..d4ed6987b0 100644
--- a/inventory/service/host_vars/review02.opendev.org.yaml
+++ b/inventory/service/host_vars/review02.opendev.org.yaml
@@ -6,6 +6,7 @@ gerrit_replication:
     replicatePermissions: false
     mirror: true
     threads: '4'
+    timeout: '900'
   - name: 'gitea02'
     url: 'ssh://git@gitea02.opendev.org:222/'
     authGroup: 'Anonymous Users'
@@ -13,6 +14,7 @@ gerrit_replication:
     replicatePermissions: false
     mirror: true
     threads: '4'
+    timeout: '900'
   - name: 'gitea03'
     url: 'ssh://git@gitea03.opendev.org:222/'
     authGroup: 'Anonymous Users'
@@ -20,6 +22,7 @@ gerrit_replication:
     replicatePermissions: false
     mirror: true
     threads: '4'
+    timeout: '900'
   - name: 'gitea04'
     url: 'ssh://git@gitea04.opendev.org:222/'
     authGroup: 'Anonymous Users'
@@ -27,6 +30,7 @@ gerrit_replication:
     replicatePermissions: false
     mirror: true
     threads: '4'
+    timeout: '900'
   - name: 'gitea05'
     url: 'ssh://git@gitea05.opendev.org:222/'
     authGroup: 'Anonymous Users'
@@ -34,6 +38,7 @@ gerrit_replication:
     replicatePermissions: false
     mirror: true
     threads: '4'
+    timeout: '900'
   - name: 'gitea06'
     url: 'ssh://git@gitea06.opendev.org:222/'
     authGroup: 'Anonymous Users'
@@ -41,6 +46,7 @@ gerrit_replication:
     replicatePermissions: false
     mirror: true
     threads: '4'
+    timeout: '900'
   - name: 'gitea07'
     url: 'ssh://git@gitea07.opendev.org:222/'
     authGroup: 'Anonymous Users'
@@ -48,6 +54,7 @@ gerrit_replication:
     replicatePermissions: false
     mirror: true
     threads: '4'
+    timeout: '900'
   - name: 'gitea08'
     url: 'ssh://git@gitea08.opendev.org:222/'
     authGroup: 'Anonymous Users'
@@ -55,6 +62,7 @@ gerrit_replication:
     replicatePermissions: false
     mirror: true
     threads: '4'
+    timeout: '900'
 gerrit_storyboard_url: https://storyboard.openstack.org
 gerrit_vhost_name: review.opendev.org
 gerrit_serverid: 4a232e18-c5a9-48ee-94c0-e04e7cca6543
diff --git a/playbooks/roles/gerrit/templates/replication.config.j2 b/playbooks/roles/gerrit/templates/replication.config.j2
index 4834d238d3..e553c292bc 100644
--- a/playbooks/roles/gerrit/templates/replication.config.j2
+++ b/playbooks/roles/gerrit/templates/replication.config.j2
@@ -12,6 +12,9 @@ replicateOnStartup = false
 {% if 'replicationDelay' in replication %}
 	replicationDelay = {{ replication.replicationDelay }}
 {% endif %}
+{% if 'timeout' in replication %}
+	timeout = {{ replication.timeout }}
+{% endif %}
 {% if 'threads' in replication %}
 	threads = {{ replication.threads }}
 {% endif %}