Keepalives for gerrit connections

It is off by default (keepalive=0), which means connections may hang forever
when media is not reliable.
This change applies the same change as proposed for Zuul:
https://review.openstack.org/238988

Support in puppets going with dependent review

Change-Id: Ifaf6aa28ad65305592839c1bd2ca90157a32ab3c

monitoring/scoreboard/ci-scoreboard.conf

@@ -3,5 +3,6 @@ GERRIT_USER: some-ci-user
 GERRIT_KEY: /home/ubuntu/.ssh/gerrit_key
 GERRIT_HOSTNAME: review.openstack.org
 GERRIT_PORT: 29418
+GERRIT_KEEPALIVE: 0
 DB_URI: mongodb://localhost:27017
 LOG_FILE_LOCATION: scoreboard.log

monitoring/scoreboard/config.py

@@ -37,6 +37,13 @@ class Config:
     def gerrit_port(self):
         return self._int_value('GERRIT_PORT')
 
+    def gerrit_keepalive(self):
+        keepalive = self._int_value('GERRIT_KEEPALIVE')
+        # 0 is the safe default, meaning no keepalives
+        if keepalive is None:
+            keepalive = 0
+        return keepalive
+
     def db_uri(self):
         return self._value('DB_URI')
 

monitoring/scoreboard/infra/gerrit.py

@@ -26,13 +26,15 @@ import pprint
 class GerritWatcher(threading.Thread):
     log = logging.getLogger("gerrit.GerritWatcher")
 
-    def __init__(self, gerrit, username, hostname, port=29418, keyfile=None):
+    def __init__(self, gerrit, username, hostname, port=29418, keyfile=None,
+                 keepalive=0):
         threading.Thread.__init__(self)
         self.username = username
         self.keyfile = keyfile
         self.hostname = hostname
         self.port = port
         self.gerrit = gerrit
+        self.keepalive = keepalive
 
     def _read(self, fd):
         l = fd.readline()
@@ -54,6 +56,8 @@ class GerritWatcher(threading.Thread):
                            username=self.username,
                            port=self.port,
                            key_filename=self.keyfile)
+            transport = client.get_transport()
+            transport.set_keepalive(self.keepalive)
 
             stdin, stdout, stderr = client.exec_command("gerrit stream-events")
 
@@ -76,7 +80,7 @@ class GerritWatcher(threading.Thread):
 class Gerrit(object):
     log = logging.getLogger("gerrit.Gerrit")
 
-    def __init__(self, hostname, username, port=29418, keyfile=None):
+    def __init__(self, hostname, username, port=29418, keyfile=None, keepalive=0):
         self.username = username
         self.hostname = hostname
         self.port = port
@@ -84,6 +88,7 @@ class Gerrit(object):
         self.watcher_thread = None
         self.event_queue = None
         self.client = None
+        self.keepalive = keepalive
 
     def startWatching(self):
         self.event_queue = Queue.Queue()
@@ -92,7 +97,8 @@ class Gerrit(object):
             self.username,
             self.hostname,
             self.port,
-            keyfile=self.keyfile)
+            keyfile=self.keyfile,
+            keepalive=self.keepalive)
         self.watcher_thread.daemon = True
         self.watcher_thread.start()
 
@@ -186,6 +192,8 @@ class Gerrit(object):
                        username=self.username,
                        port=self.port,
                        key_filename=self.keyfile)
+        transport = client.get_transport()
+        transport.set_keepalive(self.keepalive)
         self.client = client
 
     def _ssh(self, command):

monitoring/scoreboard/scoreboard_gerrit_listener.py

@@ -128,8 +128,10 @@ class GerritCIListener():
         username = self.cfg.gerrit_user()
         port = self.cfg.gerrit_port()
         keyfile = self.cfg.gerrit_key()
+        keepalive = self.cfg.gerrit_keepalive()
 
-        self.g = gerrit.Gerrit(hostname, username, port=port, keyfile=keyfile)
+        self.g = gerrit.Gerrit(hostname, username, port=port,
+                               keyfile=keyfile, keepalive=keepalive)
         self.g.startWatching()
 
         self.periodic_query_users()
@@ -144,4 +146,4 @@ class GerritCIListener():
 
 if __name__ == '__main__':
     listener = GerritCIListener()
-    listener.run()
+    listener.run()