372bc4947e
The concurrent.futures module is one of the ways that async activities can be done in the future, so we should try to work on getting to that future by using more futures. To enable this (as well as to enable getting off eventlet), add a thread pool based executor which will process incoming messages using the pool. Also begins adding according docs as well for the different types of executors that are available. Change-Id: I1482fd70abbf69f4e2994597c5e95d91fecb815e
132 lines
4.8 KiB
Python
132 lines
4.8 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import collections
|
|
import functools
|
|
import sys
|
|
import threading
|
|
|
|
from concurrent import futures
|
|
import six
|
|
|
|
from oslo.messaging._executors import base
|
|
from oslo.utils import excutils
|
|
|
|
|
|
class ThreadExecutor(base.PooledExecutorBase):
|
|
"""A message executor which integrates with threads.
|
|
|
|
A message process that polls for messages from a dispatching thread and
|
|
on reception of an incoming message places the message to be processed in
|
|
a thread pool to be executed at a later time.
|
|
"""
|
|
|
|
# NOTE(harlowja): if eventlet is being used and the thread module is monkey
|
|
# patched this should/is supposed to work the same as the eventlet based
|
|
# executor.
|
|
|
|
# NOTE(harlowja): Make it somewhat easy to change this via
|
|
# inheritance (since there does exist other executor types that could be
|
|
# used/tried here).
|
|
_executor_cls = futures.ThreadPoolExecutor
|
|
|
|
def __init__(self, conf, listener, dispatcher):
|
|
super(ThreadExecutor, self).__init__(conf, listener, dispatcher)
|
|
self._poller = None
|
|
self._executor = None
|
|
self._tombstone = threading.Event()
|
|
self._incomplete = collections.deque()
|
|
self._mutator = threading.Lock()
|
|
|
|
def _completer(self, exit_method, fut):
|
|
"""Completes futures."""
|
|
try:
|
|
exc = fut.exception()
|
|
if exc is not None:
|
|
exc_type = type(exc)
|
|
# Not available on < 3.x due to this being an added feature
|
|
# of pep-3134 (exception chaining and embedded tracebacks).
|
|
if six.PY3:
|
|
exc_tb = exc.__traceback__
|
|
else:
|
|
exc_tb = None
|
|
if not exit_method(exc_type, exc, exc_tb):
|
|
six.reraise(exc_type, exc, tb=exc_tb)
|
|
else:
|
|
exit_method(None, None, None)
|
|
finally:
|
|
with self._mutator:
|
|
try:
|
|
self._incomplete.remove(fut)
|
|
except ValueError:
|
|
pass
|
|
|
|
@excutils.forever_retry_uncaught_exceptions
|
|
def _runner(self):
|
|
while not self._tombstone.is_set():
|
|
incoming = self.listener.poll()
|
|
if incoming is None:
|
|
continue
|
|
# This is hacky, needs to be fixed....
|
|
context = self.dispatcher(incoming)
|
|
enter_method = context.__enter__()
|
|
exit_method = context.__exit__
|
|
try:
|
|
fut = self._executor.submit(enter_method)
|
|
except RuntimeError:
|
|
# This is triggered when the executor has been shutdown...
|
|
#
|
|
# TODO(harlowja): should we put whatever we pulled off back
|
|
# since when this is thrown it means the executor has been
|
|
# shutdown already??
|
|
exit_method(*sys.exc_info())
|
|
return
|
|
else:
|
|
with self._mutator:
|
|
self._incomplete.append(fut)
|
|
# Run the other half (__exit__) when done...
|
|
fut.add_done_callback(functools.partial(self._completer,
|
|
exit_method))
|
|
|
|
def start(self):
|
|
if self._executor is None:
|
|
self._executor = self._executor_cls(self.conf.rpc_thread_pool_size)
|
|
self._tombstone.clear()
|
|
if self._poller is None or not self._poller.is_alive():
|
|
self._poller = threading.Thread(target=self._runner)
|
|
self._poller.daemon = True
|
|
self._poller.start()
|
|
|
|
def stop(self):
|
|
if self._executor is not None:
|
|
self._executor.shutdown(wait=False)
|
|
self._tombstone.set()
|
|
self.listener.stop()
|
|
|
|
def wait(self):
|
|
# TODO(harlowja): this method really needs a timeout.
|
|
if self._poller is not None:
|
|
self._tombstone.wait()
|
|
self._poller.join()
|
|
self._poller = None
|
|
if self._executor is not None:
|
|
with self._mutator:
|
|
incomplete_fs = list(self._incomplete)
|
|
self._incomplete.clear()
|
|
if incomplete_fs:
|
|
futures.wait(incomplete_fs, return_when=futures.ALL_COMPLETED)
|
|
self._executor = None
|