openstack/oslo.utils
Code Issues Proposed changes

Add a eventlet utils helper module

Browse Source 
When greenthreads are being used (and/or eventlet is available and
some modules have been monkey-patched) emit warnings to the users
that certain modules should be monkey-patched and if they are not
then spurious or unexpected lock-ups and/or hangs may result due to
this type of mixed usage which doesn't typically end well.

This commit adds that functionality to a new eventlet utils module
that can be used by calling code when that code really knows it will
not work without modules being patched or partially being patched.

Change-Id: I9a856fc0a6502b438c8da9b2f589154a6fa89a9f
This commit is contained in:
Joshua Harlow 2015-01-23 10:55:19 -08:00
parent ff05ecc7ea
commit ed4e2d775a
3 changed files with 216 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/source/api/eventletutils.rst Normal file
View File

@ -0,0 +1,6 @@
===============
eventletutils
===============
.. automodule:: oslo_utils.eventletutils
:members:

98
oslo_utils/eventletutils.py Normal file
View File

@ -0,0 +1,98 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import warnings
from oslo_utils import importutils
# These may or may not exist; so carefully import them if we can...
_eventlet = importutils.try_import('eventlet')
_patcher = importutils.try_import('eventlet.patcher')
# Attribute that can be used by others to see if eventlet is even currently
# useable (can be used in unittests to skip test cases or test classes that
# require eventlet to work).
EVENTLET_AVAILABLE = all((_eventlet, _patcher))
# Taken from eventlet.py (v0.16.1) patcher code (it's not a accessible set
# for some reason...)
_ALL_PATCH = frozenset(['__builtin__', 'MySQLdb', 'os',
'psycopg', 'select', 'socket', 'thread', 'time'])
def warn_eventlet_not_patched(expected_patched_modules=None,
what='this library'):
"""Warns if eventlet is being used without patching provided modules.
:param expected_patched_modules: list of modules to check to ensure that
they are patched (and to warn if they
are not); these names should correspond
to the names passed into the eventlet
monkey_patch() routine. If not provided
then *all* the modules that could be
patched are checked. The currently valid
selection is one or multiple of
['MySQLdb', '__builtin__', 'all', 'os',
'psycopg', 'select', 'socket', 'thread',
'time'] (where 'all' has an inherent
special meaning).
:type expected_patched_modules: list/tuple/iterable
:param what: string to merge into the warnings message to identify
what is being checked (used in forming the emitted warnings
message).
:type what: string
"""
if not expected_patched_modules:
expanded_patched_modules = _ALL_PATCH.copy()
else:
expanded_patched_modules = set()
for m in expected_patched_modules:
if m == 'all':
expanded_patched_modules.update(_ALL_PATCH)
else:
if m not in _ALL_PATCH:
raise ValueError("Unknown module '%s' requested to check"
" if patched" % m)
else:
expanded_patched_modules.add(m)
if EVENTLET_AVAILABLE:
try:
# The patcher code stores a dictionary here of all modules
# names -> whether it was patched...
#
# Example:
#
# >>> _patcher.monkey_patch(os=True)
# >>> print(_patcher.already_patched)
# {'os': True}
maybe_patched = bool(_patcher.already_patched)
except AttributeError:
# Assume it is patched (the attribute used here doesn't appear
# to be a public documented API so we will assume that everything
# is patched when that attribute isn't there to be safe...)
maybe_patched = True
if maybe_patched:
not_patched = []
for m in sorted(expanded_patched_modules):
if not _patcher.is_monkey_patched(m):
not_patched.append(m)
if not_patched:
warnings.warn("It is highly recommended that when eventlet"
" is used that the %s modules are monkey"
" patched when using %s (to avoid"
" spurious or unexpected lock-ups"
" and/or hangs)" % (not_patched, what),
RuntimeWarning, stacklevel=3)

112
oslo_utils/tests/test_eventletutils.py Normal file
View File

@ -0,0 +1,112 @@
# Copyright 2012, Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import warnings
import mock
from oslotest import base as test_base
import six
from oslo_utils import eventletutils
class EventletUtilsTest(test_base.BaseTestCase):
def setUp(self):
super(EventletUtilsTest, self).setUp()
self._old_avail = eventletutils.EVENTLET_AVAILABLE
eventletutils.EVENTLET_AVAILABLE = True
def tearDown(self):
super(EventletUtilsTest, self).tearDown()
eventletutils.EVENTLET_AVAILABLE = self._old_avail
@mock.patch("oslo_utils.eventletutils._patcher")
def test_warning_not_patched(self, mock_patcher):
mock_patcher.already_patched = True
mock_patcher.is_monkey_patched.return_value = False
with warnings.catch_warnings(record=True) as capture:
warnings.simplefilter("always")
eventletutils.warn_eventlet_not_patched(['os'])
self.assertEqual(1, len(capture))
w = capture[0]
self.assertEqual(RuntimeWarning, w.category)
self.assertIn('os', six.text_type(w.message))
@mock.patch("oslo_utils.eventletutils._patcher")
def test_warning_not_patched_none_provided(self, mock_patcher):
mock_patcher.already_patched = True
mock_patcher.is_monkey_patched.return_value = False
with warnings.catch_warnings(record=True) as capture:
warnings.simplefilter("always")
eventletutils.warn_eventlet_not_patched()
self.assertEqual(1, len(capture))
w = capture[0]
self.assertEqual(RuntimeWarning, w.category)
for m in eventletutils._ALL_PATCH:
self.assertIn(m, six.text_type(w.message))
@mock.patch("oslo_utils.eventletutils._patcher")
def test_warning_not_patched_all(self, mock_patcher):
mock_patcher.already_patched = True
mock_patcher.is_monkey_patched.return_value = False
with warnings.catch_warnings(record=True) as capture:
warnings.simplefilter("always")
eventletutils.warn_eventlet_not_patched(['all'])
self.assertEqual(1, len(capture))
w = capture[0]
self.assertEqual(RuntimeWarning, w.category)
for m in eventletutils._ALL_PATCH:
self.assertIn(m, six.text_type(w.message))
@mock.patch("oslo_utils.eventletutils._patcher")
def test_no_warning(self, mock_patcher):
mock_patcher.already_patched = True
mock_patcher.is_monkey_patched.return_value = True
with warnings.catch_warnings(record=True) as capture:
warnings.simplefilter("always")
eventletutils.warn_eventlet_not_patched(['os'])
self.assertEqual(0, len(capture))
@mock.patch("oslo_utils.eventletutils._patcher")
def test_partially_patched_warning(self, mock_patcher):
is_patched = set()
mock_patcher.already_patched = True
mock_patcher.is_monkey_patched.side_effect = lambda m: m in is_patched
with warnings.catch_warnings(record=True) as capture:
warnings.simplefilter("always")
eventletutils.warn_eventlet_not_patched(['os'])
self.assertEqual(1, len(capture))
is_patched.add('os')
with warnings.catch_warnings(record=True) as capture:
warnings.simplefilter("always")
eventletutils.warn_eventlet_not_patched(['os'])
self.assertEqual(0, len(capture))
is_patched.add('thread')
with warnings.catch_warnings(record=True) as capture:
warnings.simplefilter("always")
eventletutils.warn_eventlet_not_patched(['os', 'thread'])
self.assertEqual(0, len(capture))
with warnings.catch_warnings(record=True) as capture:
warnings.simplefilter("always")
eventletutils.warn_eventlet_not_patched(['all'])
self.assertEqual(1, len(capture))
w = capture[0]
self.assertEqual(RuntimeWarning, w.category)
for m in ['os', 'thread']:
self.assertNotIn(m, six.text_type(w.message))
def test_invalid_patch_check(self):
self.assertRaises(ValueError,
eventletutils.warn_eventlet_not_patched,
['blah.blah'])