Self-resetting PrivContext

When nova-compute, an oslo.service, receives SIGHUP, running PrivContext
clients terminate, resulting in the shutdown of their corresponding
ClientChannel threads. Subsequent attempts to execute privileged methods
fail with EPIPE as a result.

The PrivContext._wrap'per (the meat of the entrypoint decorator) already
had a check to lazily start() the ClientChannel if that hasn't already
happened.

This commit makes ClientChannel store state indicating whether it's
still running; and adds logic to PrivContext._wrap to check that state
and reset (stop() and re-start()) the ClientChannel if it was previously
created but terminated.

Change-Id: I8096fc7fd014e6dd299fae8ab073336c7cae362a
Closes-Bug: #1715374

oslo_privsep/comm.py

@@ -113,6 +113,7 @@ class Future(object):
 
 class ClientChannel(object):
     def __init__(self, sock):
+        self.running = False
         self.writer = Serializer(sock)
         self.lock = threading.Lock()
         self.reader_thread = threading.Thread(
@@ -127,6 +128,8 @@ class ClientChannel(object):
 
     def _reader_main(self, reader):
         """This thread owns and demuxes the read channel"""
+        with self.lock:
+            self.running = True
         for msg in reader:
             msgid, data = msg
             if msgid is None:
@@ -148,6 +151,7 @@ class ClientChannel(object):
         with self.lock:
             for mbox in self.outstanding_msgs.values():
                 mbox.set_exception(exc)
+            self.running = False
 
     def out_of_band(self, msg):
         """Received OOB message. Subclasses might want to override this."""

oslo_privsep/priv_context.py

@@ -237,6 +237,9 @@ class PrivContext(object):
     def _wrap(self, func, *args, **kwargs):
         if self.client_mode:
             name = '%s.%s' % (func.__module__, func.__name__)
+            if self.channel is not None and not self.channel.running:
+                LOG.warning("RESTARTING PrivContext for %s", name)
+                self.stop()
             if self.channel is None:
                 self.start()
             return self.channel.remote_call(name, args, kwargs)