diff --git a/etc/ironic/ironic.conf.sample b/etc/ironic/ironic.conf.sample index b40ccee95f..06ea5a2eb0 100644 --- a/etc/ironic/ironic.conf.sample +++ b/etc/ironic/ironic.conf.sample @@ -4,123 +4,6 @@ # Options defined in oslo.messaging # -# Use durable queues in AMQP. (boolean value) -# Deprecated group/name - [DEFAULT]/rabbit_durable_queues -#amqp_durable_queues=false - -# Auto-delete queues in AMQP. (boolean value) -#amqp_auto_delete=false - -# Size of RPC connection pool. (integer value) -#rpc_conn_pool_size=30 - -# Qpid broker hostname. (string value) -#qpid_hostname=localhost - -# Qpid broker port. (integer value) -#qpid_port=5672 - -# Qpid HA cluster host:port pairs. (list value) -#qpid_hosts=$qpid_hostname:$qpid_port - -# Username for Qpid connection. (string value) -#qpid_username= - -# Password for Qpid connection. (string value) -#qpid_password= - -# Space separated list of SASL mechanisms to use for auth. -# (string value) -#qpid_sasl_mechanisms= - -# Seconds between connection keepalive heartbeats. (integer -# value) -#qpid_heartbeat=60 - -# Transport to use, either 'tcp' or 'ssl'. (string value) -#qpid_protocol=tcp - -# Whether to disable the Nagle algorithm. (boolean value) -#qpid_tcp_nodelay=true - -# The number of prefetched messages held by receiver. (integer -# value) -#qpid_receiver_capacity=1 - -# The qpid topology version to use. Version 1 is what was -# originally used by impl_qpid. Version 2 includes some -# backwards-incompatible changes that allow broker federation -# to work. Users should update to version 2 when they are -# able to take everything down, as it requires a clean break. -# (integer value) -#qpid_topology_version=1 - -# SSL version to use (valid only if SSL enabled). Valid values -# are TLSv1 and SSLv23. SSLv2, SSLv3, TLSv1_1, and TLSv1_2 may -# be available on some distributions. (string value) -#kombu_ssl_version= - -# SSL key file (valid only if SSL enabled). (string value) -#kombu_ssl_keyfile= - -# SSL cert file (valid only if SSL enabled). (string value) -#kombu_ssl_certfile= - -# SSL certification authority file (valid only if SSL -# enabled). (string value) -#kombu_ssl_ca_certs= - -# How long to wait before reconnecting in response to an AMQP -# consumer cancel notification. (floating point value) -#kombu_reconnect_delay=1.0 - -# The RabbitMQ broker address where a single node is used. -# (string value) -#rabbit_host=localhost - -# The RabbitMQ broker port where a single node is used. -# (integer value) -#rabbit_port=5672 - -# RabbitMQ HA cluster host:port pairs. (list value) -#rabbit_hosts=$rabbit_host:$rabbit_port - -# Connect over SSL for RabbitMQ. (boolean value) -#rabbit_use_ssl=false - -# The RabbitMQ userid. (string value) -#rabbit_userid=guest - -# The RabbitMQ password. (string value) -#rabbit_password=guest - -# The RabbitMQ login method. (string value) -#rabbit_login_method=AMQPLAIN - -# The RabbitMQ virtual host. (string value) -#rabbit_virtual_host=/ - -# How frequently to retry connecting with RabbitMQ. (integer -# value) -#rabbit_retry_interval=1 - -# How long to backoff for between retries when connecting to -# RabbitMQ. (integer value) -#rabbit_retry_backoff=2 - -# Maximum number of RabbitMQ connection retries. Default is 0 -# (infinite retry count). (integer value) -#rabbit_max_retries=0 - -# Use HA queues in RabbitMQ (x-ha-policy: all). If you change -# this option, you must wipe the RabbitMQ database. (boolean -# value) -#rabbit_ha_queues=false - -# Deprecated, use rpc_backend=kombu+memory or rpc_backend=fake -# (boolean value) -#fake_rabbit=false - # ZeroMQ bind address. Should be a wildcard (*), an ethernet # interface, or IP. The "host" option should point or resolve # to this address. (string value) @@ -444,22 +327,12 @@ # -# Options defined in ironic.openstack.common.policy +# Options defined in ironic.openstack.common.versionutils # -# The JSON file that defines policies. (string value) -#policy_file=policy.json - -# Default rule. Enforced when a requested rule is not found. -# (string value) -#policy_default_rule=default - -# Directories where policy configuration files are stored. -# They can be relative to any directory in the search path -# defined by the config_dir option, or absolute paths. The -# file defined by policy_file must exist for these directories -# to be searched. (multi valued) -#policy_dirs=policy.d +# Enables or disables fatal status of deprecations. (boolean +# value) +#fatal_deprecations=false [agent] @@ -752,20 +625,22 @@ # connection lost. (boolean value) #use_db_reconnect=false -# Seconds between database connection retries. (integer value) +# Seconds between retries of a database transaction. (integer +# value) #db_retry_interval=1 -# If True, increases the interval between database connection -# retries up to db_max_retry_interval. (boolean value) +# If True, increases the interval between retries of a +# database operation up to db_max_retry_interval. (boolean +# value) #db_inc_retry_interval=true # If db_inc_retry_interval is set, the maximum seconds between -# database connection retries. (integer value) +# retries of a database operation. (integer value) #db_max_retry_interval=10 -# Maximum database connection retries before error is raised. -# Set to -1 to specify an infinite retry count. (integer -# value) +# Maximum retries in case of connection error or deadlock +# error before error is raised. Set to -1 to specify an +# infinite retry count. (integer value) #db_max_retries=20 @@ -1269,6 +1144,147 @@ #allow_insecure_clients=false +[oslo_messaging_qpid] + +# +# Options defined in oslo.messaging +# + +# Use durable queues in AMQP. (boolean value) +# Deprecated group/name - [DEFAULT]/rabbit_durable_queues +#amqp_durable_queues=false + +# Auto-delete queues in AMQP. (boolean value) +#amqp_auto_delete=false + +# Size of RPC connection pool. (integer value) +#rpc_conn_pool_size=30 + +# Qpid broker hostname. (string value) +#qpid_hostname=ironic + +# Qpid broker port. (integer value) +#qpid_port=5672 + +# Qpid HA cluster host:port pairs. (list value) +#qpid_hosts=$qpid_hostname:$qpid_port + +# Username for Qpid connection. (string value) +#qpid_username= + +# Password for Qpid connection. (string value) +#qpid_password= + +# Space separated list of SASL mechanisms to use for auth. +# (string value) +#qpid_sasl_mechanisms= + +# Seconds between connection keepalive heartbeats. (integer +# value) +#qpid_heartbeat=60 + +# Transport to use, either 'tcp' or 'ssl'. (string value) +#qpid_protocol=tcp + +# Whether to disable the Nagle algorithm. (boolean value) +#qpid_tcp_nodelay=true + +# The number of prefetched messages held by receiver. (integer +# value) +#qpid_receiver_capacity=1 + +# The qpid topology version to use. Version 1 is what was +# originally used by impl_qpid. Version 2 includes some +# backwards-incompatible changes that allow broker federation +# to work. Users should update to version 2 when they are +# able to take everything down, as it requires a clean break. +# (integer value) +#qpid_topology_version=1 + + +[oslo_messaging_rabbit] + +# +# Options defined in oslo.messaging +# + +# Use durable queues in AMQP. (boolean value) +# Deprecated group/name - [DEFAULT]/rabbit_durable_queues +#amqp_durable_queues=false + +# Auto-delete queues in AMQP. (boolean value) +#amqp_auto_delete=false + +# Size of RPC connection pool. (integer value) +#rpc_conn_pool_size=30 + +# SSL version to use (valid only if SSL enabled). Valid values +# are TLSv1 and SSLv23. SSLv2, SSLv3, TLSv1_1, and TLSv1_2 may +# be available on some distributions. (string value) +#kombu_ssl_version= + +# SSL key file (valid only if SSL enabled). (string value) +#kombu_ssl_keyfile= + +# SSL cert file (valid only if SSL enabled). (string value) +#kombu_ssl_certfile= + +# SSL certification authority file (valid only if SSL +# enabled). (string value) +#kombu_ssl_ca_certs= + +# How long to wait before reconnecting in response to an AMQP +# consumer cancel notification. (floating point value) +#kombu_reconnect_delay=1.0 + +# The RabbitMQ broker address where a single node is used. +# (string value) +#rabbit_host=ironic + +# The RabbitMQ broker port where a single node is used. +# (integer value) +#rabbit_port=5672 + +# RabbitMQ HA cluster host:port pairs. (list value) +#rabbit_hosts=$rabbit_host:$rabbit_port + +# Connect over SSL for RabbitMQ. (boolean value) +#rabbit_use_ssl=false + +# The RabbitMQ userid. (string value) +#rabbit_userid=guest + +# The RabbitMQ password. (string value) +#rabbit_password=guest + +# The RabbitMQ login method. (string value) +#rabbit_login_method=AMQPLAIN + +# The RabbitMQ virtual host. (string value) +#rabbit_virtual_host=/ + +# How frequently to retry connecting with RabbitMQ. (integer +# value) +#rabbit_retry_interval=1 + +# How long to backoff for between retries when connecting to +# RabbitMQ. (integer value) +#rabbit_retry_backoff=2 + +# Maximum number of RabbitMQ connection retries. Default is 0 +# (infinite retry count). (integer value) +#rabbit_max_retries=0 + +# Use HA queues in RabbitMQ (x-ha-policy: all). If you change +# this option, you must wipe the RabbitMQ database. (boolean +# value) +#rabbit_ha_queues=false + +# Deprecated, use rpc_backend=kombu+memory or rpc_backend=fake +# (boolean value) +#fake_rabbit=false + + [pxe] # diff --git a/ironic/openstack/common/_i18n.py b/ironic/openstack/common/_i18n.py index 21dcf4c89b..0e235c78a6 100644 --- a/ironic/openstack/common/_i18n.py +++ b/ironic/openstack/common/_i18n.py @@ -17,14 +17,14 @@ See http://docs.openstack.org/developer/oslo.i18n/usage.html """ try: - import oslo.i18n + import oslo_i18n # NOTE(dhellmann): This reference to o-s-l-o will be replaced by the # application name when this module is synced into the separate # repository. It is OK to have more than one translation function # using the same domain, since there will still only be one message # catalog. - _translators = oslo.i18n.TranslatorFactory(domain='ironic') + _translators = oslo_i18n.TranslatorFactory(domain='ironic') # The primary translation function using the well-known name "_" _ = _translators.primary diff --git a/ironic/openstack/common/eventlet_backdoor.py b/ironic/openstack/common/eventlet_backdoor.py index 2415e076bc..e31104f97c 100644 --- a/ironic/openstack/common/eventlet_backdoor.py +++ b/ironic/openstack/common/eventlet_backdoor.py @@ -16,21 +16,21 @@ from __future__ import print_function +import copy import errno import gc +import logging import os import pprint import socket import sys import traceback -import eventlet import eventlet.backdoor import greenlet -from oslo.config import cfg +from oslo_config import cfg -from ironic.openstack.common.gettextutils import _LI -from ironic.openstack.common import log as logging +from ironic.openstack.common._i18n import _LI help_for_backdoor_port = ( "Acceptable values are 0, , and :, where 0 results " @@ -49,6 +49,12 @@ CONF.register_opts(eventlet_backdoor_opts) LOG = logging.getLogger(__name__) +def list_opts(): + """Entry point for oslo-config-generator. + """ + return [(None, copy.deepcopy(eventlet_backdoor_opts))] + + class EventletBackdoorConfigValueError(Exception): def __init__(self, port_range, help_msg, ex): msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. ' diff --git a/ironic/openstack/common/fileutils.py b/ironic/openstack/common/fileutils.py index ec26eaf9cd..9097c35d45 100644 --- a/ironic/openstack/common/fileutils.py +++ b/ironic/openstack/common/fileutils.py @@ -17,22 +17,25 @@ import contextlib import errno import logging import os +import stat import tempfile -from oslo.utils import excutils +from oslo_utils import excutils LOG = logging.getLogger(__name__) _FILE_CACHE = {} +DEFAULT_MODE = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO -def ensure_tree(path): +def ensure_tree(path, mode=DEFAULT_MODE): """Create a directory (and any ancestor directories required) :param path: Directory to create + :param mode: Directory creation permissions """ try: - os.makedirs(path) + os.makedirs(path, mode) except OSError as exc: if exc.errno == errno.EEXIST: if not os.path.isdir(path): diff --git a/ironic/openstack/common/imageutils.py b/ironic/openstack/common/imageutils.py index e157eef007..66c1d1207e 100644 --- a/ironic/openstack/common/imageutils.py +++ b/ironic/openstack/common/imageutils.py @@ -21,8 +21,9 @@ Helper methods to deal with images. import re -from ironic.openstack.common.gettextutils import _ -from ironic.openstack.common import strutils +from oslo_utils import strutils + +from ironic.openstack.common._i18n import _ class QemuImgInfo(object): @@ -100,10 +101,9 @@ class QemuImgInfo(object): real_details = real_details.strip().lower() elif root_cmd == 'snapshot_list': # Next line should be a header, starting with 'ID' - if not lines_after or not lines_after[0].startswith("ID"): + if not lines_after or not lines_after.pop(0).startswith("ID"): msg = _("Snapshot list encountered but no header found!") raise ValueError(msg) - del lines_after[0] real_details = [] # This is the sprintf pattern we will try to match # "%-10s%-20s%7s%20s%15s" @@ -118,6 +118,7 @@ class QemuImgInfo(object): date_pieces = line_pieces[5].split(":") if len(date_pieces) != 3: break + lines_after.pop(0) real_details.append({ 'id': line_pieces[0], 'tag': line_pieces[1], @@ -125,7 +126,6 @@ class QemuImgInfo(object): 'date': line_pieces[3], 'vm_clock': line_pieces[4] + " " + line_pieces[5], }) - del lines_after[0] return real_details def _parse(self, cmd_output): diff --git a/ironic/openstack/common/loopingcall.py b/ironic/openstack/common/loopingcall.py index f0fe645615..f0bfe40ffc 100644 --- a/ironic/openstack/common/loopingcall.py +++ b/ironic/openstack/common/loopingcall.py @@ -15,14 +15,14 @@ # License for the specific language governing permissions and limitations # under the License. +import logging import sys import time from eventlet import event from eventlet import greenthread -from ironic.openstack.common.gettextutils import _LE, _LW -from ironic.openstack.common import log as logging +from ironic.openstack.common._i18n import _LE, _LW LOG = logging.getLogger(__name__) @@ -84,9 +84,9 @@ class FixedIntervalLoopingCall(LoopingCallBase): break delay = end - start - interval if delay > 0: - LOG.warn(_LW('task %(func_name)s run outlasted ' + LOG.warn(_LW('task %(func_name)r run outlasted ' 'interval by %(delay).2f sec'), - {'func_name': repr(self.f), 'delay': delay}) + {'func_name': self.f, 'delay': delay}) greenthread.sleep(-delay if delay < 0 else 0) except LoopingCallDone as e: self.stop() @@ -127,9 +127,9 @@ class DynamicLoopingCall(LoopingCallBase): if periodic_interval_max is not None: idle = min(idle, periodic_interval_max) - LOG.debug('Dynamic looping call %(func_name)s sleeping ' + LOG.debug('Dynamic looping call %(func_name)r sleeping ' 'for %(idle).02f seconds', - {'func_name': repr(self.f), 'idle': idle}) + {'func_name': self.f, 'idle': idle}) greenthread.sleep(idle) except LoopingCallDone as e: self.stop() diff --git a/ironic/openstack/common/network_utils.py b/ironic/openstack/common/network_utils.py deleted file mode 100644 index 00bd286534..0000000000 --- a/ironic/openstack/common/network_utils.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Network-related utilities and helper functions. -""" - -import logging -import socket - -from six.moves.urllib import parse - -from ironic.openstack.common.gettextutils import _LW - -LOG = logging.getLogger(__name__) - - -def parse_host_port(address, default_port=None): - """Interpret a string as a host:port pair. - - An IPv6 address MUST be escaped if accompanied by a port, - because otherwise ambiguity ensues: 2001:db8:85a3::8a2e:370:7334 - means both [2001:db8:85a3::8a2e:370:7334] and - [2001:db8:85a3::8a2e:370]:7334. - - >>> parse_host_port('server01:80') - ('server01', 80) - >>> parse_host_port('server01') - ('server01', None) - >>> parse_host_port('server01', default_port=1234) - ('server01', 1234) - >>> parse_host_port('[::1]:80') - ('::1', 80) - >>> parse_host_port('[::1]') - ('::1', None) - >>> parse_host_port('[::1]', default_port=1234) - ('::1', 1234) - >>> parse_host_port('2001:db8:85a3::8a2e:370:7334', default_port=1234) - ('2001:db8:85a3::8a2e:370:7334', 1234) - >>> parse_host_port(None) - (None, None) - """ - if not address: - return (None, None) - - if address[0] == '[': - # Escaped ipv6 - _host, _port = address[1:].split(']') - host = _host - if ':' in _port: - port = _port.split(':')[1] - else: - port = default_port - else: - if address.count(':') == 1: - host, port = address.split(':') - else: - # 0 means ipv4, >1 means ipv6. - # We prohibit unescaped ipv6 addresses with port. - host = address - port = default_port - - return (host, None if port is None else int(port)) - - -class ModifiedSplitResult(parse.SplitResult): - """Split results class for urlsplit.""" - - # NOTE(dims): The functions below are needed for Python 2.6.x. - # We can remove these when we drop support for 2.6.x. - @property - def hostname(self): - netloc = self.netloc.split('@', 1)[-1] - host, port = parse_host_port(netloc) - return host - - @property - def port(self): - netloc = self.netloc.split('@', 1)[-1] - host, port = parse_host_port(netloc) - return port - - -def urlsplit(url, scheme='', allow_fragments=True): - """Parse a URL using urlparse.urlsplit(), splitting query and fragments. - This function papers over Python issue9374 when needed. - - The parameters are the same as urlparse.urlsplit. - """ - scheme, netloc, path, query, fragment = parse.urlsplit( - url, scheme, allow_fragments) - if allow_fragments and '#' in path: - path, fragment = path.split('#', 1) - if '?' in path: - path, query = path.split('?', 1) - return ModifiedSplitResult(scheme, netloc, - path, query, fragment) - - -def set_tcp_keepalive(sock, tcp_keepalive=True, - tcp_keepidle=None, - tcp_keepalive_interval=None, - tcp_keepalive_count=None): - """Set values for tcp keepalive parameters - - This function configures tcp keepalive parameters if users wish to do - so. - - :param tcp_keepalive: Boolean, turn on or off tcp_keepalive. If users are - not sure, this should be True, and default values will be used. - - :param tcp_keepidle: time to wait before starting to send keepalive probes - :param tcp_keepalive_interval: time between successive probes, once the - initial wait time is over - :param tcp_keepalive_count: number of probes to send before the connection - is killed - """ - - # NOTE(praneshp): Despite keepalive being a tcp concept, the level is - # still SOL_SOCKET. This is a quirk. - if isinstance(tcp_keepalive, bool): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, tcp_keepalive) - else: - raise TypeError("tcp_keepalive must be a boolean") - - if not tcp_keepalive: - return - - # These options aren't available in the OS X version of eventlet, - # Idle + Count * Interval effectively gives you the total timeout. - if tcp_keepidle is not None: - if hasattr(socket, 'TCP_KEEPIDLE'): - sock.setsockopt(socket.IPPROTO_TCP, - socket.TCP_KEEPIDLE, - tcp_keepidle) - else: - LOG.warning(_LW('tcp_keepidle not available on your system')) - if tcp_keepalive_interval is not None: - if hasattr(socket, 'TCP_KEEPINTVL'): - sock.setsockopt(socket.IPPROTO_TCP, - socket.TCP_KEEPINTVL, - tcp_keepalive_interval) - else: - LOG.warning(_LW('tcp_keepintvl not available on your system')) - if tcp_keepalive_count is not None: - if hasattr(socket, 'TCP_KEEPCNT'): - sock.setsockopt(socket.IPPROTO_TCP, - socket.TCP_KEEPCNT, - tcp_keepalive_count) - else: - LOG.warning(_LW('tcp_keepknt not available on your system')) diff --git a/ironic/openstack/common/periodic_task.py b/ironic/openstack/common/periodic_task.py index 147f3d1083..f3c99e5ced 100644 --- a/ironic/openstack/common/periodic_task.py +++ b/ironic/openstack/common/periodic_task.py @@ -12,14 +12,14 @@ # under the License. import copy +import logging import random import time -from oslo.config import cfg +from oslo_config import cfg import six from ironic.openstack.common._i18n import _, _LE, _LI -from ironic.openstack.common import log as logging periodic_opts = [ diff --git a/ironic/openstack/common/service.py b/ironic/openstack/common/service.py index 9de7302eec..e4eed8a2e3 100644 --- a/ironic/openstack/common/service.py +++ b/ironic/openstack/common/service.py @@ -18,7 +18,7 @@ """Generic Node base class for all workers that run on hosts.""" import errno -import logging as std_logging +import logging import os import random import signal @@ -35,11 +35,10 @@ except ImportError: import eventlet from eventlet import event -from oslo.config import cfg +from oslo_config import cfg from ironic.openstack.common import eventlet_backdoor from ironic.openstack.common._i18n import _LE, _LI, _LW -from ironic.openstack.common import log as logging from ironic.openstack.common import systemd from ironic.openstack.common import threadgroup @@ -163,7 +162,7 @@ class ServiceLauncher(Launcher): signo = 0 LOG.debug('Full set of CONF:') - CONF.log_opt_values(LOG, std_logging.DEBUG) + CONF.log_opt_values(LOG, logging.DEBUG) try: if ready_callback: @@ -200,16 +199,12 @@ class ServiceWrapper(object): class ProcessLauncher(object): - def __init__(self, wait_interval=0.01): - """Constructor. + def __init__(self): + """Constructor.""" - :param wait_interval: The interval to sleep for between checks - of child process exit. - """ self.children = {} self.sigcaught = None self.running = True - self.wait_interval = wait_interval rfd, self.writepipe = os.pipe() self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r') self.handle_signal() @@ -334,8 +329,8 @@ class ProcessLauncher(object): def _wait_child(self): try: - # Don't block if no child processes have exited - pid, status = os.waitpid(0, os.WNOHANG) + # Block while any of child processes have exited + pid, status = os.waitpid(0, 0) if not pid: return None except OSError as exc: @@ -364,10 +359,6 @@ class ProcessLauncher(object): while self.running: wrap = self._wait_child() if not wrap: - # Yield to other threads if no children have exited - # Sleep for a short time to avoid excessive CPU usage - # (see bug #1095346) - eventlet.greenthread.sleep(self.wait_interval) continue while self.running and len(wrap.children) < wrap.workers: self._start_child(wrap) @@ -377,7 +368,7 @@ class ProcessLauncher(object): systemd.notify_once() LOG.debug('Full set of CONF:') - CONF.log_opt_values(LOG, std_logging.DEBUG) + CONF.log_opt_values(LOG, logging.DEBUG) try: while True: diff --git a/ironic/openstack/common/strutils.py b/ironic/openstack/common/strutils.py deleted file mode 100644 index 3f509b222e..0000000000 --- a/ironic/openstack/common/strutils.py +++ /dev/null @@ -1,295 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -System-level utilities and helper functions. -""" - -import math -import re -import sys -import unicodedata - -import six - -from ironic.openstack.common.gettextutils import _ - - -UNIT_PREFIX_EXPONENT = { - 'k': 1, - 'K': 1, - 'Ki': 1, - 'M': 2, - 'Mi': 2, - 'G': 3, - 'Gi': 3, - 'T': 4, - 'Ti': 4, -} -UNIT_SYSTEM_INFO = { - 'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')), - 'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')), -} - -TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes') -FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no') - -SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]") -SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+") - - -# NOTE(flaper87): The following 3 globals are used by `mask_password` -_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] - -# NOTE(ldbragst): Let's build a list of regex objects using the list of -# _SANITIZE_KEYS we already have. This way, we only have to add the new key -# to the list of _SANITIZE_KEYS and we can generate regular expressions -# for XML and JSON automatically. -_SANITIZE_PATTERNS = [] -_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', - r'(<%(key)s>).*?()', - r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', - r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])', - r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])' - '.*?([\'"])', - r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)'] - -for key in _SANITIZE_KEYS: - for pattern in _FORMAT_PATTERNS: - reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) - _SANITIZE_PATTERNS.append(reg_ex) - - -def int_from_bool_as_string(subject): - """Interpret a string as a boolean and return either 1 or 0. - - Any string value in: - - ('True', 'true', 'On', 'on', '1') - - is interpreted as a boolean True. - - Useful for JSON-decoded stuff and config file parsing - """ - return bool_from_string(subject) and 1 or 0 - - -def bool_from_string(subject, strict=False, default=False): - """Interpret a string as a boolean. - - A case-insensitive match is performed such that strings matching 't', - 'true', 'on', 'y', 'yes', or '1' are considered True and, when - `strict=False`, anything else returns the value specified by 'default'. - - Useful for JSON-decoded stuff and config file parsing. - - If `strict=True`, unrecognized values, including None, will raise a - ValueError which is useful when parsing values passed in from an API call. - Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. - """ - if not isinstance(subject, six.string_types): - subject = six.text_type(subject) - - lowered = subject.strip().lower() - - if lowered in TRUE_STRINGS: - return True - elif lowered in FALSE_STRINGS: - return False - elif strict: - acceptable = ', '.join( - "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS)) - msg = _("Unrecognized value '%(val)s', acceptable values are:" - " %(acceptable)s") % {'val': subject, - 'acceptable': acceptable} - raise ValueError(msg) - else: - return default - - -def safe_decode(text, incoming=None, errors='strict'): - """Decodes incoming text/bytes string using `incoming` if they're not - already unicode. - - :param incoming: Text's current encoding - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: text or a unicode `incoming` encoded - representation of it. - :raises TypeError: If text is not an instance of str - """ - if not isinstance(text, (six.string_types, six.binary_type)): - raise TypeError("%s can't be decoded" % type(text)) - - if isinstance(text, six.text_type): - return text - - if not incoming: - incoming = (sys.stdin.encoding or - sys.getdefaultencoding()) - - try: - return text.decode(incoming, errors) - except UnicodeDecodeError: - # Note(flaper87) If we get here, it means that - # sys.stdin.encoding / sys.getdefaultencoding - # didn't return a suitable encoding to decode - # text. This happens mostly when global LANG - # var is not set correctly and there's no - # default encoding. In this case, most likely - # python will use ASCII or ANSI encoders as - # default encodings but they won't be capable - # of decoding non-ASCII characters. - # - # Also, UTF-8 is being used since it's an ASCII - # extension. - return text.decode('utf-8', errors) - - -def safe_encode(text, incoming=None, - encoding='utf-8', errors='strict'): - """Encodes incoming text/bytes string using `encoding`. - - If incoming is not specified, text is expected to be encoded with - current python's default encoding. (`sys.getdefaultencoding`) - - :param incoming: Text's current encoding - :param encoding: Expected encoding for text (Default UTF-8) - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: text or a bytestring `encoding` encoded - representation of it. - :raises TypeError: If text is not an instance of str - """ - if not isinstance(text, (six.string_types, six.binary_type)): - raise TypeError("%s can't be encoded" % type(text)) - - if not incoming: - incoming = (sys.stdin.encoding or - sys.getdefaultencoding()) - - if isinstance(text, six.text_type): - return text.encode(encoding, errors) - elif text and encoding != incoming: - # Decode text before encoding it with `encoding` - text = safe_decode(text, incoming, errors) - return text.encode(encoding, errors) - else: - return text - - -def string_to_bytes(text, unit_system='IEC', return_int=False): - """Converts a string into an float representation of bytes. - - The units supported for IEC :: - - Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it) - KB, KiB, MB, MiB, GB, GiB, TB, TiB - - The units supported for SI :: - - kb(it), Mb(it), Gb(it), Tb(it) - kB, MB, GB, TB - - Note that the SI unit system does not support capital letter 'K' - - :param text: String input for bytes size conversion. - :param unit_system: Unit system for byte size conversion. - :param return_int: If True, returns integer representation of text - in bytes. (default: decimal) - :returns: Numerical representation of text in bytes. - :raises ValueError: If text has an invalid value. - - """ - try: - base, reg_ex = UNIT_SYSTEM_INFO[unit_system] - except KeyError: - msg = _('Invalid unit system: "%s"') % unit_system - raise ValueError(msg) - match = reg_ex.match(text) - if match: - magnitude = float(match.group(1)) - unit_prefix = match.group(2) - if match.group(3) in ['b', 'bit']: - magnitude /= 8 - else: - msg = _('Invalid string format: %s') % text - raise ValueError(msg) - if not unit_prefix: - res = magnitude - else: - res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix]) - if return_int: - return int(math.ceil(res)) - return res - - -def to_slug(value, incoming=None, errors="strict"): - """Normalize string. - - Convert to lowercase, remove non-word characters, and convert spaces - to hyphens. - - Inspired by Django's `slugify` filter. - - :param value: Text to slugify - :param incoming: Text's current encoding - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: slugified unicode representation of `value` - :raises TypeError: If text is not an instance of str - """ - value = safe_decode(value, incoming, errors) - # NOTE(aababilov): no need to use safe_(encode|decode) here: - # encodings are always "ascii", error handling is always "ignore" - # and types are always known (first: unicode; second: str) - value = unicodedata.normalize("NFKD", value).encode( - "ascii", "ignore").decode("ascii") - value = SLUGIFY_STRIP_RE.sub("", value).strip().lower() - return SLUGIFY_HYPHENATE_RE.sub("-", value) - - -def mask_password(message, secret="***"): - """Replace password with 'secret' in message. - - :param message: The string which includes security information. - :param secret: value with which to replace passwords. - :returns: The unicode value of message with the password fields masked. - - For example: - - >>> mask_password("'adminPass' : 'aaaaa'") - "'adminPass' : '***'" - >>> mask_password("'admin_pass' : 'aaaaa'") - "'admin_pass' : '***'" - >>> mask_password('"password" : "aaaaa"') - '"password" : "***"' - >>> mask_password("'original_password' : 'aaaaa'") - "'original_password' : '***'" - >>> mask_password("u'original_password' : u'aaaaa'") - "u'original_password' : u'***'" - """ - message = six.text_type(message) - - # NOTE(ldbragst): Check to see if anything in message contains any key - # specified in _SANITIZE_KEYS, if not then just return the message since - # we don't have to mask any passwords. - if not any(key in message for key in _SANITIZE_KEYS): - return message - - secret = r'\g<1>' + secret + r'\g<2>' - for pattern in _SANITIZE_PATTERNS: - message = re.sub(pattern, secret, message) - return message diff --git a/ironic/openstack/common/systemd.py b/ironic/openstack/common/systemd.py index 8a432b5ed6..36243b342a 100644 --- a/ironic/openstack/common/systemd.py +++ b/ironic/openstack/common/systemd.py @@ -16,12 +16,11 @@ Helper module for systemd service readiness notification. """ +import logging import os import socket import sys -from ironic.openstack.common import log as logging - LOG = logging.getLogger(__name__) diff --git a/ironic/openstack/common/threadgroup.py b/ironic/openstack/common/threadgroup.py index 46755d7ea2..95378faa8d 100644 --- a/ironic/openstack/common/threadgroup.py +++ b/ironic/openstack/common/threadgroup.py @@ -11,12 +11,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import logging import threading import eventlet from eventlet import greenpool -from ironic.openstack.common import log as logging from ironic.openstack.common import loopingcall @@ -96,6 +96,8 @@ class ThreadGroup(object): continue try: x.stop() + except eventlet.greenlet.GreenletExit: + pass except Exception as ex: LOG.exception(ex) diff --git a/ironic/openstack/common/versionutils.py b/ironic/openstack/common/versionutils.py index 68f60bc134..dce07a82e0 100644 --- a/ironic/openstack/common/versionutils.py +++ b/ironic/openstack/common/versionutils.py @@ -18,14 +18,25 @@ Helpers for comparing version strings. """ import functools +import inspect +import logging +from oslo_config import cfg import pkg_resources +import six -from ironic.openstack.common.gettextutils import _ -from ironic.openstack.common import log as logging +from ironic.openstack.common._i18n import _ LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +opts = [ + cfg.BoolOpt('fatal_deprecations', + default=False, + help='Enables or disables fatal status of deprecations.'), +] class deprecated(object): @@ -72,6 +83,7 @@ class deprecated(object): ICEHOUSE = 'I' JUNO = 'J' KILO = 'K' + LIBERTY = 'L' _RELEASES = { # NOTE(morganfainberg): Bexar is used for unit test purposes, it is @@ -83,6 +95,7 @@ class deprecated(object): 'I': 'Icehouse', 'J': 'Juno', 'K': 'Kilo', + 'L': 'Liberty', } _deprecated_msg_with_alternative = _( @@ -116,16 +129,34 @@ class deprecated(object): self.remove_in = remove_in self.what = what - def __call__(self, func): + def __call__(self, func_or_cls): if not self.what: - self.what = func.__name__ + '()' + self.what = func_or_cls.__name__ + '()' + msg, details = self._build_message() - @functools.wraps(func) - def wrapped(*args, **kwargs): - msg, details = self._build_message() - LOG.deprecated(msg, details) - return func(*args, **kwargs) - return wrapped + if inspect.isfunction(func_or_cls): + + @six.wraps(func_or_cls) + def wrapped(*args, **kwargs): + report_deprecated_feature(LOG, msg, details) + return func_or_cls(*args, **kwargs) + return wrapped + elif inspect.isclass(func_or_cls): + orig_init = func_or_cls.__init__ + + # TODO(tsufiev): change `functools` module to `six` as + # soon as six 1.7.4 (with fix for passing `assigned` + # argument to underlying `functools.wraps`) is released + # and added to the oslo-incubator requrements + @functools.wraps(orig_init, assigned=('__name__', '__doc__')) + def new_init(self, *args, **kwargs): + report_deprecated_feature(LOG, msg, details) + orig_init(self, *args, **kwargs) + func_or_cls.__init__ = new_init + return func_or_cls + else: + raise TypeError('deprecated can be used only with functions or ' + 'classes') def _get_safe_to_remove_release(self, release): # TODO(dstanek): this method will have to be reimplemented once @@ -181,3 +212,44 @@ def is_compatible(requested_version, current_version, same_major=True): return False return current_parts >= requested_parts + + +# Track the messages we have sent already. See +# report_deprecated_feature(). +_deprecated_messages_sent = {} + + +def report_deprecated_feature(logger, msg, *args, **kwargs): + """Call this function when a deprecated feature is used. + + If the system is configured for fatal deprecations then the message + is logged at the 'critical' level and :class:`DeprecatedConfig` will + be raised. + + Otherwise, the message will be logged (once) at the 'warn' level. + + :raises: :class:`DeprecatedConfig` if the system is configured for + fatal deprecations. + """ + stdmsg = _("Deprecated: %s") % msg + CONF.register_opts(opts) + if CONF.fatal_deprecations: + logger.critical(stdmsg, *args, **kwargs) + raise DeprecatedConfig(msg=stdmsg) + + # Using a list because a tuple with dict can't be stored in a set. + sent_args = _deprecated_messages_sent.setdefault(msg, list()) + + if args in sent_args: + # Already logged this message, so don't log it again. + return + + sent_args.append(args) + logger.warn(stdmsg, *args, **kwargs) + + +class DeprecatedConfig(Exception): + message = _("Fatal call to deprecated config: %(msg)s") + + def __init__(self, msg): + super(Exception, self).__init__(self.message % dict(msg=msg)) diff --git a/openstack-common.conf b/openstack-common.conf index 35c32281ce..b3da6b5fc0 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,9 +1,7 @@ [DEFAULT] # The list of modules to copy from oslo-incubator -module=config.generator module=fileutils -module=gettextutils module=imageutils module=log module=loopingcall @@ -13,8 +11,8 @@ module=versionutils # Tools script=tools/install_venv_common.py -script=tools/config/generate_sample.sh -script=tools/config/check_uptodate.sh +#script=tools/config/generate_sample.sh +#script=tools/config/check_uptodate.sh # The base module to hold the copy of openstack.common base=ironic