From 4af5dfa3309931b036ce86e4d0dfd3d71d7626f8 Mon Sep 17 00:00:00 2001 From: Derek Higgins Date: Tue, 28 Oct 2014 09:42:09 +0000 Subject: [PATCH] Switch to oslo.concurrency Nova has removed nova/openstack/common/lockutils.py and switched to oslo.concurrency so we can no longer import lockutils from the nova tree. Make the same switch in the ironic tree. Closes-Bug: #1386631 Change-Id: I8db99d61dbe6c50c9edae37077242e2696bc5671 --- ironic/common/dhcp_factory.py | 2 +- ironic/common/disk_partitioner.py | 2 +- ironic/common/driver_factory.py | 2 +- ironic/common/images.py | 2 +- ironic/common/utils.py | 2 +- ironic/conductor/manager.py | 2 +- ironic/drivers/modules/console_utils.py | 2 +- ironic/drivers/modules/deploy_utils.py | 2 +- ironic/drivers/modules/image_cache.py | 2 +- ironic/drivers/modules/ipmitool.py | 2 +- ironic/drivers/modules/ssh.py | 2 +- ironic/nova/compute/manager.py | 2 +- ironic/openstack/common/lockutils.py | 379 --------------------- ironic/openstack/common/processutils.py | 283 --------------- ironic/tests/drivers/test_console_utils.py | 2 +- ironic/tests/drivers/test_deploy_utils.py | 2 +- ironic/tests/drivers/test_ipmitool.py | 2 +- ironic/tests/drivers/test_ssh.py | 2 +- ironic/tests/test_images.py | 2 +- ironic/tests/test_utils.py | 2 +- openstack-common.conf | 2 - requirements.txt | 1 + 22 files changed, 19 insertions(+), 682 deletions(-) delete mode 100644 ironic/openstack/common/lockutils.py delete mode 100644 ironic/openstack/common/processutils.py diff --git a/ironic/common/dhcp_factory.py b/ironic/common/dhcp_factory.py index 84958ee814..c6848c6d01 100644 --- a/ironic/common/dhcp_factory.py +++ b/ironic/common/dhcp_factory.py @@ -12,11 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.concurrency import lockutils from oslo.config import cfg import stevedore from ironic.common import exception -from ironic.openstack.common import lockutils dhcp_provider_opts = [ diff --git a/ironic/common/disk_partitioner.py b/ironic/common/disk_partitioner.py index 11c2d0da3f..fd09ffc3dd 100644 --- a/ironic/common/disk_partitioner.py +++ b/ironic/common/disk_partitioner.py @@ -15,6 +15,7 @@ import re +from oslo.concurrency import processutils from oslo.config import cfg from ironic.common import exception @@ -23,7 +24,6 @@ from ironic.common.i18n import _LW from ironic.common import utils from ironic.openstack.common import log as logging from ironic.openstack.common import loopingcall -from ironic.openstack.common import processutils opts = [ cfg.IntOpt('check_device_interval', diff --git a/ironic/common/driver_factory.py b/ironic/common/driver_factory.py index 69b2e781a9..2d69ffd19f 100644 --- a/ironic/common/driver_factory.py +++ b/ironic/common/driver_factory.py @@ -13,12 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.concurrency import lockutils from oslo.config import cfg from stevedore import dispatch from ironic.common import exception from ironic.common.i18n import _LI -from ironic.openstack.common import lockutils from ironic.openstack.common import log diff --git a/ironic/common/images.py b/ironic/common/images.py index 15597a1618..ccff1b6d09 100644 --- a/ironic/common/images.py +++ b/ironic/common/images.py @@ -23,6 +23,7 @@ import os import shutil import jinja2 +from oslo.concurrency import processutils from oslo.config import cfg from ironic.common import exception @@ -34,7 +35,6 @@ from ironic.common import utils from ironic.openstack.common import fileutils from ironic.openstack.common import imageutils from ironic.openstack.common import log as logging -from ironic.openstack.common import processutils LOG = logging.getLogger(__name__) diff --git a/ironic/common/utils.py b/ironic/common/utils.py index 49018ad694..89e58c5633 100644 --- a/ironic/common/utils.py +++ b/ironic/common/utils.py @@ -29,6 +29,7 @@ import tempfile import uuid import netaddr +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils import paramiko @@ -39,7 +40,6 @@ from ironic.common.i18n import _ from ironic.common.i18n import _LE from ironic.common.i18n import _LW from ironic.openstack.common import log as logging -from ironic.openstack.common import processutils utils_opts = [ cfg.StrOpt('rootwrap_config', diff --git a/ironic/conductor/manager.py b/ironic/conductor/manager.py index 6de809fe32..b765b2e3a1 100644 --- a/ironic/conductor/manager.py +++ b/ironic/conductor/manager.py @@ -47,6 +47,7 @@ import threading import eventlet from eventlet import greenpool +from oslo.concurrency import lockutils from oslo.config import cfg from oslo.db import exception as db_exception from oslo import messaging @@ -70,7 +71,6 @@ from ironic.conductor import utils from ironic.db import api as dbapi from ironic import objects from ironic.openstack.common import context as ironic_context -from ironic.openstack.common import lockutils from ironic.openstack.common import log from ironic.openstack.common import periodic_task diff --git a/ironic/drivers/modules/console_utils.py b/ironic/drivers/modules/console_utils.py index 5a58d84141..fcc10d3490 100644 --- a/ironic/drivers/modules/console_utils.py +++ b/ironic/drivers/modules/console_utils.py @@ -24,6 +24,7 @@ import subprocess import tempfile import time +from oslo.concurrency import processutils from oslo.config import cfg from ironic.common import exception @@ -32,7 +33,6 @@ from ironic.common.i18n import _LW from ironic.common import utils from ironic.openstack.common import log as logging from ironic.openstack.common import loopingcall -from ironic.openstack.common import processutils opts = [ diff --git a/ironic/drivers/modules/deploy_utils.py b/ironic/drivers/modules/deploy_utils.py index fd12c49990..3d43ac8920 100644 --- a/ironic/drivers/modules/deploy_utils.py +++ b/ironic/drivers/modules/deploy_utils.py @@ -20,6 +20,7 @@ import socket import stat import time +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils @@ -30,7 +31,6 @@ from ironic.common.i18n import _LE from ironic.common import utils from ironic.drivers.modules import image_cache from ironic.openstack.common import log as logging -from ironic.openstack.common import processutils LOG = logging.getLogger(__name__) diff --git a/ironic/drivers/modules/image_cache.py b/ironic/drivers/modules/image_cache.py index 97634de712..72ae5248cd 100644 --- a/ironic/drivers/modules/image_cache.py +++ b/ironic/drivers/modules/image_cache.py @@ -22,6 +22,7 @@ import os import tempfile import time +from oslo.concurrency import lockutils from oslo.config import cfg from ironic.common import exception @@ -31,7 +32,6 @@ from ironic.common.i18n import _LW from ironic.common import images from ironic.common import utils from ironic.openstack.common import fileutils -from ironic.openstack.common import lockutils from ironic.openstack.common import log as logging diff --git a/ironic/drivers/modules/ipmitool.py b/ironic/drivers/modules/ipmitool.py index b0b9395cb5..154bf25516 100644 --- a/ironic/drivers/modules/ipmitool.py +++ b/ironic/drivers/modules/ipmitool.py @@ -36,6 +36,7 @@ import stat import tempfile import time +from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils @@ -51,7 +52,6 @@ from ironic.drivers import base from ironic.drivers.modules import console_utils from ironic.openstack.common import log as logging from ironic.openstack.common import loopingcall -from ironic.openstack.common import processutils CONF = cfg.CONF diff --git a/ironic/drivers/modules/ssh.py b/ironic/drivers/modules/ssh.py index e49a7df961..cfa4f2f322 100644 --- a/ironic/drivers/modules/ssh.py +++ b/ironic/drivers/modules/ssh.py @@ -28,6 +28,7 @@ Currently supported environments are: import os +from oslo.concurrency import processutils from oslo.config import cfg from ironic.common import boot_devices @@ -41,7 +42,6 @@ from ironic.conductor import task_manager from ironic.drivers import base from ironic.drivers import utils as driver_utils from ironic.openstack.common import log as logging -from ironic.openstack.common import processutils libvirt_opts = [ cfg.StrOpt('libvirt_uri', diff --git a/ironic/nova/compute/manager.py b/ironic/nova/compute/manager.py index 1e2d04f50a..90b51dde34 100644 --- a/ironic/nova/compute/manager.py +++ b/ironic/nova/compute/manager.py @@ -23,7 +23,7 @@ work. The goal here is to generalise the areas where n-c talking to a clustered hypervisor has issues, and long term fold them into the main ComputeManager. """ -from nova.openstack.common import lockutils +from oslo.concurrency import lockutils from nova.compute import manager import nova.context diff --git a/ironic/openstack/common/lockutils.py b/ironic/openstack/common/lockutils.py deleted file mode 100644 index 6a39b8e14a..0000000000 --- a/ironic/openstack/common/lockutils.py +++ /dev/null @@ -1,379 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import contextlib -import errno -import functools -import logging -import os -import shutil -import subprocess -import sys -import tempfile -import threading -import time -import weakref - -from oslo.config import cfg - -from ironic.openstack.common import fileutils -from ironic.openstack.common.gettextutils import _, _LE, _LI - - -LOG = logging.getLogger(__name__) - - -util_opts = [ - cfg.BoolOpt('disable_process_locking', default=False, - help='Enables or disables inter-process locks.'), - cfg.StrOpt('lock_path', - default=os.environ.get("IRONIC_LOCK_PATH"), - help='Directory to use for lock files.') -] - - -CONF = cfg.CONF -CONF.register_opts(util_opts) - - -def set_defaults(lock_path): - cfg.set_defaults(util_opts, lock_path=lock_path) - - -class _FileLock(object): - """Lock implementation which allows multiple locks, working around - issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does - not require any cleanup. Since the lock is always held on a file - descriptor rather than outside of the process, the lock gets dropped - automatically if the process crashes, even if __exit__ is not executed. - - There are no guarantees regarding usage by multiple green threads in a - single process here. This lock works only between processes. Exclusive - access between local threads should be achieved using the semaphores - in the @synchronized decorator. - - Note these locks are released when the descriptor is closed, so it's not - safe to close the file descriptor while another green thread holds the - lock. Just opening and closing the lock file can break synchronisation, - so lock files must be accessed only using this abstraction. - """ - - def __init__(self, name): - self.lockfile = None - self.fname = name - - def acquire(self): - basedir = os.path.dirname(self.fname) - - if not os.path.exists(basedir): - fileutils.ensure_tree(basedir) - LOG.info(_LI('Created lock path: %s'), basedir) - - self.lockfile = open(self.fname, 'w') - - while True: - try: - # Using non-blocking locks since green threads are not - # patched to deal with blocking locking calls. - # Also upon reading the MSDN docs for locking(), it seems - # to have a laughable 10 attempts "blocking" mechanism. - self.trylock() - LOG.debug('Got file lock "%s"', self.fname) - return True - except IOError as e: - if e.errno in (errno.EACCES, errno.EAGAIN): - # external locks synchronise things like iptables - # updates - give it some time to prevent busy spinning - time.sleep(0.01) - else: - raise threading.ThreadError(_("Unable to acquire lock on" - " `%(filename)s` due to" - " %(exception)s") % - { - 'filename': self.fname, - 'exception': e, - }) - - def __enter__(self): - self.acquire() - return self - - def release(self): - try: - self.unlock() - self.lockfile.close() - LOG.debug('Released file lock "%s"', self.fname) - except IOError: - LOG.exception(_LE("Could not release the acquired lock `%s`"), - self.fname) - - def __exit__(self, exc_type, exc_val, exc_tb): - self.release() - - def exists(self): - return os.path.exists(self.fname) - - def trylock(self): - raise NotImplementedError() - - def unlock(self): - raise NotImplementedError() - - -class _WindowsLock(_FileLock): - def trylock(self): - msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1) - - def unlock(self): - msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1) - - -class _FcntlLock(_FileLock): - def trylock(self): - fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) - - def unlock(self): - fcntl.lockf(self.lockfile, fcntl.LOCK_UN) - - -class _PosixLock(object): - def __init__(self, name): - # Hash the name because it's not valid to have POSIX semaphore - # names with things like / in them. Then use base64 to encode - # the digest() instead taking the hexdigest() because the - # result is shorter and most systems can't have shm sempahore - # names longer than 31 characters. - h = hashlib.sha1() - h.update(name.encode('ascii')) - self.name = str((b'/' + base64.urlsafe_b64encode( - h.digest())).decode('ascii')) - - def acquire(self, timeout=None): - self.semaphore = posix_ipc.Semaphore(self.name, - flags=posix_ipc.O_CREAT, - initial_value=1) - self.semaphore.acquire(timeout) - return self - - def __enter__(self): - self.acquire() - return self - - def release(self): - self.semaphore.release() - self.semaphore.close() - - def __exit__(self, exc_type, exc_val, exc_tb): - self.release() - - def exists(self): - try: - semaphore = posix_ipc.Semaphore(self.name) - except posix_ipc.ExistentialError: - return False - else: - semaphore.close() - return True - - -if os.name == 'nt': - import msvcrt - InterProcessLock = _WindowsLock - FileLock = _WindowsLock -else: - import base64 - import fcntl - import hashlib - - import posix_ipc - InterProcessLock = _PosixLock - FileLock = _FcntlLock - -_semaphores = weakref.WeakValueDictionary() -_semaphores_lock = threading.Lock() - - -def _get_lock_path(name, lock_file_prefix, lock_path=None): - # NOTE(mikal): the lock name cannot contain directory - # separators - name = name.replace(os.sep, '_') - if lock_file_prefix: - sep = '' if lock_file_prefix.endswith('-') else '-' - name = '%s%s%s' % (lock_file_prefix, sep, name) - - local_lock_path = lock_path or CONF.lock_path - - if not local_lock_path: - # NOTE(bnemec): Create a fake lock path for posix locks so we don't - # unnecessarily raise the RequiredOptError below. - if InterProcessLock is not _PosixLock: - raise cfg.RequiredOptError('lock_path') - local_lock_path = 'posixlock:/' - - return os.path.join(local_lock_path, name) - - -def external_lock(name, lock_file_prefix=None, lock_path=None): - LOG.debug('Attempting to grab external lock "%(lock)s"', - {'lock': name}) - - lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path) - - # NOTE(bnemec): If an explicit lock_path was passed to us then it - # means the caller is relying on file-based locking behavior, so - # we can't use posix locks for those calls. - if lock_path: - return FileLock(lock_file_path) - return InterProcessLock(lock_file_path) - - -def remove_external_lock_file(name, lock_file_prefix=None): - """Remove an external lock file when it's not used anymore - This will be helpful when we have a lot of lock files - """ - with internal_lock(name): - lock_file_path = _get_lock_path(name, lock_file_prefix) - try: - os.remove(lock_file_path) - except OSError: - LOG.info(_LI('Failed to remove file %(file)s'), - {'file': lock_file_path}) - - -def internal_lock(name): - with _semaphores_lock: - try: - sem = _semaphores[name] - except KeyError: - sem = threading.Semaphore() - _semaphores[name] = sem - - LOG.debug('Got semaphore "%(lock)s"', {'lock': name}) - return sem - - -@contextlib.contextmanager -def lock(name, lock_file_prefix=None, external=False, lock_path=None): - """Context based lock - - This function yields a `threading.Semaphore` instance (if we don't use - eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is - True, in which case, it'll yield an InterProcessLock instance. - - :param lock_file_prefix: The lock_file_prefix argument is used to provide - lock files on disk with a meaningful prefix. - - :param external: The external keyword argument denotes whether this lock - should work across multiple processes. This means that if two different - workers both run a method decorated with @synchronized('mylock', - external=True), only one of them will execute at a time. - """ - int_lock = internal_lock(name) - with int_lock: - if external and not CONF.disable_process_locking: - ext_lock = external_lock(name, lock_file_prefix, lock_path) - with ext_lock: - yield ext_lock - else: - yield int_lock - LOG.debug('Released semaphore "%(lock)s"', {'lock': name}) - - -def synchronized(name, lock_file_prefix=None, external=False, lock_path=None): - """Synchronization decorator. - - Decorating a method like so:: - - @synchronized('mylock') - def foo(self, *args): - ... - - ensures that only one thread will execute the foo method at a time. - - Different methods can share the same lock:: - - @synchronized('mylock') - def foo(self, *args): - ... - - @synchronized('mylock') - def bar(self, *args): - ... - - This way only one of either foo or bar can be executing at a time. - """ - - def wrap(f): - @functools.wraps(f) - def inner(*args, **kwargs): - try: - with lock(name, lock_file_prefix, external, lock_path): - LOG.debug('Got semaphore / lock "%(function)s"', - {'function': f.__name__}) - return f(*args, **kwargs) - finally: - LOG.debug('Semaphore / lock released "%(function)s"', - {'function': f.__name__}) - return inner - return wrap - - -def synchronized_with_prefix(lock_file_prefix): - """Partial object generator for the synchronization decorator. - - Redefine @synchronized in each project like so:: - - (in nova/utils.py) - from nova.openstack.common import lockutils - - synchronized = lockutils.synchronized_with_prefix('nova-') - - - (in nova/foo.py) - from nova import utils - - @utils.synchronized('mylock') - def bar(self, *args): - ... - - The lock_file_prefix argument is used to provide lock files on disk with a - meaningful prefix. - """ - - return functools.partial(synchronized, lock_file_prefix=lock_file_prefix) - - -def main(argv): - """Create a dir for locks and pass it to command from arguments - - If you run this: - python -m openstack.common.lockutils python setup.py testr - - a temporary directory will be created for all your locks and passed to all - your tests in an environment variable. The temporary dir will be deleted - afterwards and the return value will be preserved. - """ - - lock_dir = tempfile.mkdtemp() - os.environ["IRONIC_LOCK_PATH"] = lock_dir - try: - ret_val = subprocess.call(argv[1:]) - finally: - shutil.rmtree(lock_dir, ignore_errors=True) - return ret_val - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/ironic/openstack/common/processutils.py b/ironic/openstack/common/processutils.py deleted file mode 100644 index 5280b1142d..0000000000 --- a/ironic/openstack/common/processutils.py +++ /dev/null @@ -1,283 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -System-level utilities and helper functions. -""" - -import errno -import logging -import multiprocessing -import os -import random -import shlex -import signal - -from eventlet.green import subprocess -from eventlet import greenthread -import six - -from ironic.openstack.common.gettextutils import _ -from ironic.openstack.common import strutils - - -LOG = logging.getLogger(__name__) - - -class InvalidArgumentError(Exception): - def __init__(self, message=None): - super(InvalidArgumentError, self).__init__(message) - - -class UnknownArgumentError(Exception): - def __init__(self, message=None): - super(UnknownArgumentError, self).__init__(message) - - -class ProcessExecutionError(Exception): - def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, - description=None): - self.exit_code = exit_code - self.stderr = stderr - self.stdout = stdout - self.cmd = cmd - self.description = description - - if description is None: - description = _("Unexpected error while running command.") - if exit_code is None: - exit_code = '-' - message = _('%(description)s\n' - 'Command: %(cmd)s\n' - 'Exit code: %(exit_code)s\n' - 'Stdout: %(stdout)r\n' - 'Stderr: %(stderr)r') % {'description': description, - 'cmd': cmd, - 'exit_code': exit_code, - 'stdout': stdout, - 'stderr': stderr} - super(ProcessExecutionError, self).__init__(message) - - -class NoRootWrapSpecified(Exception): - def __init__(self, message=None): - super(NoRootWrapSpecified, self).__init__(message) - - -def _subprocess_setup(): - # Python installs a SIGPIPE handler by default. This is usually not what - # non-Python subprocesses expect. - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - - -def execute(*cmd, **kwargs): - """Helper method to shell out and execute a command through subprocess. - - Allows optional retry. - - :param cmd: Passed to subprocess.Popen. - :type cmd: string - :param process_input: Send to opened process. - :type process_input: string - :param env_variables: Environment variables and their values that - will be set for the process. - :type env_variables: dict - :param check_exit_code: Single bool, int, or list of allowed exit - codes. Defaults to [0]. Raise - :class:`ProcessExecutionError` unless - program exits with one of these code. - :type check_exit_code: boolean, int, or [int] - :param delay_on_retry: True | False. Defaults to True. If set to True, - wait a short amount of time before retrying. - :type delay_on_retry: boolean - :param attempts: How many times to retry cmd. - :type attempts: int - :param run_as_root: True | False. Defaults to False. If set to True, - the command is prefixed by the command specified - in the root_helper kwarg. - :type run_as_root: boolean - :param root_helper: command to prefix to commands called with - run_as_root=True - :type root_helper: string - :param shell: whether or not there should be a shell used to - execute this command. Defaults to false. - :type shell: boolean - :param loglevel: log level for execute commands. - :type loglevel: int. (Should be logging.DEBUG or logging.INFO) - :returns: (stdout, stderr) from process execution - :raises: :class:`UnknownArgumentError` on - receiving unknown arguments - :raises: :class:`ProcessExecutionError` - """ - - process_input = kwargs.pop('process_input', None) - env_variables = kwargs.pop('env_variables', None) - check_exit_code = kwargs.pop('check_exit_code', [0]) - ignore_exit_code = False - delay_on_retry = kwargs.pop('delay_on_retry', True) - attempts = kwargs.pop('attempts', 1) - run_as_root = kwargs.pop('run_as_root', False) - root_helper = kwargs.pop('root_helper', '') - shell = kwargs.pop('shell', False) - loglevel = kwargs.pop('loglevel', logging.DEBUG) - - if isinstance(check_exit_code, bool): - ignore_exit_code = not check_exit_code - check_exit_code = [0] - elif isinstance(check_exit_code, int): - check_exit_code = [check_exit_code] - - if kwargs: - raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs) - - if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0: - if not root_helper: - raise NoRootWrapSpecified( - message=_('Command requested root, but did not ' - 'specify a root helper.')) - cmd = shlex.split(root_helper) + list(cmd) - - cmd = map(str, cmd) - - while attempts > 0: - attempts -= 1 - try: - LOG.log(loglevel, 'Running cmd (subprocess): %s', - strutils.mask_password(' '.join(cmd))) - _PIPE = subprocess.PIPE # pylint: disable=E1101 - - if os.name == 'nt': - preexec_fn = None - close_fds = False - else: - preexec_fn = _subprocess_setup - close_fds = True - - obj = subprocess.Popen(cmd, - stdin=_PIPE, - stdout=_PIPE, - stderr=_PIPE, - close_fds=close_fds, - preexec_fn=preexec_fn, - shell=shell, - env=env_variables) - result = None - for _i in six.moves.range(20): - # NOTE(russellb) 20 is an arbitrary number of retries to - # prevent any chance of looping forever here. - try: - if process_input is not None: - result = obj.communicate(process_input) - else: - result = obj.communicate() - except OSError as e: - if e.errno in (errno.EAGAIN, errno.EINTR): - continue - raise - break - obj.stdin.close() # pylint: disable=E1101 - _returncode = obj.returncode # pylint: disable=E1101 - LOG.log(loglevel, 'Result was %s' % _returncode) - if not ignore_exit_code and _returncode not in check_exit_code: - (stdout, stderr) = result - raise ProcessExecutionError(exit_code=_returncode, - stdout=stdout, - stderr=stderr, - cmd=' '.join(cmd)) - return result - except ProcessExecutionError: - if not attempts: - raise - else: - LOG.log(loglevel, '%r failed. Retrying.', cmd) - if delay_on_retry: - greenthread.sleep(random.randint(20, 200) / 100.0) - finally: - # NOTE(termie): this appears to be necessary to let the subprocess - # call clean something up in between calls, without - # it two execute calls in a row hangs the second one - greenthread.sleep(0) - - -def trycmd(*args, **kwargs): - """A wrapper around execute() to more easily handle warnings and errors. - - Returns an (out, err) tuple of strings containing the output of - the command's stdout and stderr. If 'err' is not empty then the - command can be considered to have failed. - - :discard_warnings True | False. Defaults to False. If set to True, - then for succeeding commands, stderr is cleared - - """ - discard_warnings = kwargs.pop('discard_warnings', False) - - try: - out, err = execute(*args, **kwargs) - failed = False - except ProcessExecutionError as exn: - out, err = '', six.text_type(exn) - failed = True - - if not failed and discard_warnings and err: - # Handle commands that output to stderr but otherwise succeed - err = '' - - return out, err - - -def ssh_execute(ssh, cmd, process_input=None, - addl_env=None, check_exit_code=True): - LOG.debug('Running cmd (SSH): %s', cmd) - if addl_env: - raise InvalidArgumentError(_('Environment not supported over SSH')) - - if process_input: - # This is (probably) fixable if we need it... - raise InvalidArgumentError(_('process_input not supported over SSH')) - - stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd) - channel = stdout_stream.channel - - # NOTE(justinsb): This seems suspicious... - # ...other SSH clients have buffering issues with this approach - stdout = stdout_stream.read() - stderr = stderr_stream.read() - stdin_stream.close() - - exit_status = channel.recv_exit_status() - - # exit_status == -1 if no exit code was returned - if exit_status != -1: - LOG.debug('Result was %s' % exit_status) - if check_exit_code and exit_status != 0: - raise ProcessExecutionError(exit_code=exit_status, - stdout=stdout, - stderr=stderr, - cmd=cmd) - - return (stdout, stderr) - - -def get_worker_count(): - """Utility to get the default worker count. - - @return: The number of CPUs if that can be determined, else a default - worker count of 1 is returned. - """ - try: - return multiprocessing.cpu_count() - except NotImplementedError: - return 1 diff --git a/ironic/tests/drivers/test_console_utils.py b/ironic/tests/drivers/test_console_utils.py index 95de1529e9..99ffee95f5 100644 --- a/ironic/tests/drivers/test_console_utils.py +++ b/ironic/tests/drivers/test_console_utils.py @@ -24,13 +24,13 @@ import subprocess import tempfile import mock +from oslo.concurrency import processutils from oslo.config import cfg from ironic.common import exception from ironic.common import utils from ironic.drivers.modules import console_utils from ironic.drivers.modules import ipmitool as ipmi -from ironic.openstack.common import processutils from ironic.tests.db import base as db_base from ironic.tests.db import utils as db_utils from ironic.tests.objects import utils as obj_utils diff --git a/ironic/tests/drivers/test_deploy_utils.py b/ironic/tests/drivers/test_deploy_utils.py index 2918d6a7a5..d991b93f43 100644 --- a/ironic/tests/drivers/test_deploy_utils.py +++ b/ironic/tests/drivers/test_deploy_utils.py @@ -19,6 +19,7 @@ import tempfile import fixtures import mock +from oslo.concurrency import processutils from oslo.config import cfg from ironic.common import disk_partitioner @@ -26,7 +27,6 @@ from ironic.common import exception from ironic.common import utils as common_utils from ironic.drivers.modules import deploy_utils as utils from ironic.drivers.modules import image_cache -from ironic.openstack.common import processutils from ironic.tests import base as tests_base _PXECONF_DEPLOY = """ diff --git a/ironic/tests/drivers/test_ipmitool.py b/ironic/tests/drivers/test_ipmitool.py index 4c6dad1fb8..4b163288d5 100644 --- a/ironic/tests/drivers/test_ipmitool.py +++ b/ironic/tests/drivers/test_ipmitool.py @@ -25,6 +25,7 @@ import tempfile import time import mock +from oslo.concurrency import processutils from oslo.config import cfg from ironic.common import boot_devices @@ -35,7 +36,6 @@ from ironic.common import utils from ironic.conductor import task_manager from ironic.drivers.modules import console_utils from ironic.drivers.modules import ipmitool as ipmi -from ironic.openstack.common import processutils from ironic.tests import base from ironic.tests.conductor import utils as mgr_utils from ironic.tests.db import base as db_base diff --git a/ironic/tests/drivers/test_ssh.py b/ironic/tests/drivers/test_ssh.py index 22285fcb5d..020e406494 100644 --- a/ironic/tests/drivers/test_ssh.py +++ b/ironic/tests/drivers/test_ssh.py @@ -27,12 +27,12 @@ from ironic.common import utils from ironic.conductor import task_manager from ironic.drivers.modules import ssh from ironic.drivers import utils as driver_utils -from ironic.openstack.common import processutils from ironic.tests.conductor import utils as mgr_utils from ironic.tests.db import base as db_base from ironic.tests.db import utils as db_utils from ironic.tests.objects import utils as obj_utils +from oslo.concurrency import processutils from oslo.config import cfg CONF = cfg.CONF diff --git a/ironic/tests/test_images.py b/ironic/tests/test_images.py index 5a84233ce6..7b73ab6945 100644 --- a/ironic/tests/test_images.py +++ b/ironic/tests/test_images.py @@ -20,6 +20,7 @@ import os import shutil import mock +from oslo.concurrency import processutils from oslo.config import cfg import six.moves.builtins as __builtin__ @@ -28,7 +29,6 @@ from ironic.common import image_service from ironic.common import images from ironic.common import utils from ironic.openstack.common import imageutils -from ironic.openstack.common import processutils from ironic.tests import base CONF = cfg.CONF diff --git a/ironic/tests/test_utils.py b/ironic/tests/test_utils.py index 73e9fb2b0d..44deee9cbd 100644 --- a/ironic/tests/test_utils.py +++ b/ironic/tests/test_utils.py @@ -23,13 +23,13 @@ import uuid import mock import netaddr +from oslo.concurrency import processutils from oslo.config import cfg import six import six.moves.builtins as __builtin__ from ironic.common import exception from ironic.common import utils -from ironic.openstack.common import processutils from ironic.tests import base CONF = cfg.CONF diff --git a/openstack-common.conf b/openstack-common.conf index c6c6d49b81..87cc25c642 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -6,12 +6,10 @@ module=context module=fileutils module=gettextutils module=imageutils -module=lockutils module=log module=loopingcall module=periodic_task module=policy -module=processutils module=service module=versionutils diff --git a/requirements.txt b/requirements.txt index 9b1ca4c983..c387d91bef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ python-swiftclient>=2.2.0 stevedore>=1.0.0 # Apache-2.0 pysendfile==2.0.0 websockify>=0.6.0,<0.7 +oslo.concurrency>=0.1.0 # Apache-2.0 oslo.config>=1.4.0 # Apache-2.0 oslo.db>=1.0.0 # Apache-2.0 oslo.rootwrap>=1.3.0