Remove dependency on python-quantumclient

* quantum previously depended on python-quantumclient for the
   functionality in quantum/common.  This change migrates
   the required modules to the quantum repo and removes that
   dependency.
 * Addresses bug 977711

Change-Id: Ia17a486859eb1aa993900a5f3b582b258c6a6d78

quantum/__init__.py

@@ -14,6 +14,3 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)

quantum/common/__init__.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
-
+# Copyright 2011 Nicira Networks, Inc.
-# Copyright 2011 OpenStack LLC
+# All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -13,3 +13,4 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+# @author: Somik Behera, Nicira Networks, Inc.

quantum/common/config.py

@@ -0,0 +1,345 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 Nicira Networks, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Routines for configuring Quantum
+"""
+
+import ConfigParser
+import logging
+import logging.config
+import logging.handlers
+import optparse
+import os
+import re
+import sys
+import socket
+
+from paste import deploy
+
+from quantum.common import flags
+from quantum.common import exceptions as exception
+
+DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s"
+DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+FLAGS = flags.FLAGS
+LOG = logging.getLogger('quantum.wsgi')
+
+
+def parse_options(parser, cli_args=None):
+    """
+    Returns the parsed CLI options, command to run and its arguments, merged
+    with any same-named options found in a configuration file.
+
+    The function returns a tuple of (options, args), where options is a
+    mapping of option key/str(value) pairs, and args is the set of arguments
+    (not options) supplied on the command-line.
+
+    The reason that the option values are returned as strings only is that
+    ConfigParser and paste.deploy only accept string values...
+
+    :param parser: The option parser
+    :param cli_args: (Optional) Set of arguments to process. If not present,
+                     sys.argv[1:] is used.
+    :retval tuple of (options, args)
+    """
+
+    (options, args) = parser.parse_args(cli_args)
+
+    return (vars(options), args)
+
+
+def add_common_options(parser):
+    """
+    Given a supplied optparse.OptionParser, adds an OptionGroup that
+    represents all common configuration options.
+
+    :param parser: optparse.OptionParser
+    """
+    help_text = "The following configuration options are common to "\
+                "all quantum programs."
+
+    group = optparse.OptionGroup(parser, "Common Options", help_text)
+    group.add_option('-v', '--verbose', default=False, dest="verbose",
+                     action="store_true",
+                     help="Print more verbose output")
+    group.add_option('-d', '--debug', default=False, dest="debug",
+                     action="store_true",
+                     help="Print debugging output")
+    group.add_option('--config-file', default=None, metavar="PATH",
+                     help="Path to the config file to use. When not specified "
+                          "(the default), we generally look at the first "
+                          "argument specified to be a config file, and if "
+                          "that is also missing, we search standard "
+                          "directories for a config file.")
+    parser.add_option_group(group)
+
+
+def add_log_options(parser):
+    """
+    Given a supplied optparse.OptionParser, adds an OptionGroup that
+    represents all the configuration options around logging.
+
+    :param parser: optparse.OptionParser
+    """
+    help_text = "The following configuration options are specific to logging "\
+                "functionality for this program."
+
+    group = optparse.OptionGroup(parser, "Logging Options", help_text)
+    group.add_option('--log-config', default=None, metavar="PATH",
+                     help="If this option is specified, the logging "
+                          "configuration file specified is used and overrides "
+                          "any other logging options specified. Please see "
+                          "the Python logging module documentation for "
+                          "details on logging configuration files.")
+    group.add_option('--log-date-format', metavar="FORMAT",
+                      default=DEFAULT_LOG_DATE_FORMAT,
+                      help="Format string for %(asctime)s in log records. "
+                           "Default: %default")
+    group.add_option('--use-syslog', default=False,
+                      action="store_true",
+                      help="Output logs to syslog.")
+    group.add_option('--log-file', default=None, metavar="PATH",
+                      help="(Optional) Name of log file to output to. "
+                           "If not set, logging will go to stdout.")
+    group.add_option("--log-dir", default=None,
+                      help="(Optional) The directory to keep log files in "
+                           "(will be prepended to --logfile)")
+    parser.add_option_group(group)
+
+
+def setup_logging(options, conf):
+    """
+    Sets up the logging options for a log with supplied name
+
+    :param options: Mapping of typed option key/values
+    :param conf: Mapping of untyped key/values from config file
+    """
+
+    if options.get('log_config', None):
+        # Use a logging configuration file for all settings...
+        if os.path.exists(options['log_config']):
+            logging.config.fileConfig(options['log_config'])
+            return
+        else:
+            raise RuntimeError("Unable to locate specified logging "
+                               "config file: %s" % options['log_config'])
+
+    # If either the CLI option or the conf value
+    # is True, we set to True
+    debug = options.get('debug') or \
+            get_option(conf, 'debug', type='bool', default=False)
+    verbose = options.get('verbose') or \
+            get_option(conf, 'verbose', type='bool', default=False)
+    root_logger = logging.root
+    if debug:
+        root_logger.setLevel(logging.DEBUG)
+    elif verbose:
+        root_logger.setLevel(logging.INFO)
+    else:
+        root_logger.setLevel(logging.WARNING)
+
+    # Set log configuration from options...
+    # Note that we use a hard-coded log format in the options
+    # because of Paste.Deploy bug #379
+    # http://trac.pythonpaste.org/pythonpaste/ticket/379
+    log_format = options.get('log_format', DEFAULT_LOG_FORMAT)
+    log_date_format = options.get('log_date_format', DEFAULT_LOG_DATE_FORMAT)
+    formatter = logging.Formatter(log_format, log_date_format)
+
+    syslog = options.get('use_syslog')
+    if not syslog:
+        syslog = conf.get('use_syslog')
+
+    if syslog:
+        SysLogHandler = logging.handlers.SysLogHandler
+        try:
+            handler = SysLogHandler(address='/dev/log',
+                                    facility=SysLogHandler.LOG_SYSLOG)
+        except socket.error:
+            handler = SysLogHandler(address='/var/run/syslog',
+                                    facility=SysLogHandler.LOG_SYSLOG)
+        handler.setFormatter(formatter)
+        root_logger.addHandler(handler)
+
+    logfile = options.get('log_file')
+    if not logfile:
+        logfile = conf.get('log_file')
+
+    if logfile:
+        logdir = options.get('log_dir')
+        if not logdir:
+            logdir = conf.get('log_dir')
+        if logdir:
+            logfile = os.path.join(logdir, logfile)
+        logfile = logging.FileHandler(logfile)
+        logfile.setFormatter(formatter)
+        logfile.setFormatter(formatter)
+        root_logger.addHandler(logfile)
+    else:
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setFormatter(formatter)
+        root_logger.addHandler(handler)
+
+
+def find_config_file(options, args, config_file='quantum.conf'):
+    """
+    Return the first config file found.
+
+    We search for the paste config file in the following order:
+    * If --config-file option is used, use that
+    * If args[0] is a file, use that
+    * Search for the configuration file in standard directories:
+        * .
+        * ~.quantum/
+        * ~
+        * $FLAGS.state_path/etc/quantum
+        * $FLAGS.state_path/etc
+
+    :retval Full path to config file, or None if no config file found
+    """
+
+    fix_path = lambda p: os.path.abspath(os.path.expanduser(p))
+    if options.get('config_file'):
+        if os.path.exists(options['config_file']):
+            return fix_path(options['config_file'])
+    elif args:
+        if os.path.exists(args[0]):
+            return fix_path(args[0])
+
+    dir_to_common = os.path.dirname(os.path.abspath(__file__))
+    root = os.path.join(dir_to_common, '..', '..', '..', '..')
+    # Handle standard directory search for the config file
+    config_file_dirs = [fix_path(os.path.join(os.getcwd(), 'etc')),
+                        fix_path(os.path.join('~', '.quantum-venv', 'etc',
+                                 'quantum')),
+                        fix_path('~'),
+                        os.path.join(FLAGS.state_path, 'etc'),
+                        os.path.join(FLAGS.state_path, 'etc', 'quantum'),
+                        fix_path(os.path.join('~', '.local',
+                                              'etc', 'quantum')),
+                        '/usr/etc/quantum',
+                        '/usr/local/etc/quantum',
+                        '/etc/quantum/',
+                        '/etc']
+
+    if 'plugin' in options:
+        config_file_dirs = [os.path.join(x, 'quantum', 'plugins',
+                                             options['plugin'])
+                            for x in config_file_dirs]
+
+    if os.path.exists(os.path.join(root, 'plugins')):
+        plugins = [fix_path(os.path.join(root, 'plugins', p, 'etc'))
+                  for p in os.listdir(os.path.join(root, 'plugins'))]
+        plugins = [p for p in plugins if os.path.isdir(p)]
+        config_file_dirs.extend(plugins)
+
+    for cfg_dir in config_file_dirs:
+        cfg_file = os.path.join(cfg_dir, config_file)
+        if os.path.exists(cfg_file):
+            return cfg_file
+
+
+def load_paste_config(app_name, options, args):
+    """
+    Looks for a config file to use for an app and returns the
+    config file path and a configuration mapping from a paste config file.
+
+    We search for the paste config file in the following order:
+    * If --config-file option is used, use that
+    * If args[0] is a file, use that
+    * Search for quantum.conf in standard directories:
+        * .
+        * ~.quantum/
+        * ~
+        * /etc/quantum
+        * /etc
+
+    :param app_name: Name of the application to load config for, or None.
+                     None signifies to only load the [DEFAULT] section of
+                     the config file.
+    :param options: Set of typed options returned from parse_options()
+    :param args: Command line arguments from argv[1:]
+    :retval Tuple of (conf_file, conf)
+
+    :raises RuntimeError when config file cannot be located or there was a
+            problem loading the configuration file.
+    """
+    conf_file = find_config_file(options, args)
+    if not conf_file:
+        raise RuntimeError("Unable to locate any configuration file. "
+                            "Cannot load application %s" % app_name)
+    try:
+        conf = deploy.appconfig("config:%s" % conf_file, name=app_name)
+        return conf_file, conf
+    except Exception, e:
+        raise RuntimeError("Error trying to load config %s: %s"
+                           % (conf_file, e))
+
+
+def load_paste_app(app_name, options, args):
+    """
+    Builds and returns a WSGI app from a paste config file.
+
+    We search for the paste config file in the following order:
+    * If --config-file option is used, use that
+    * If args[0] is a file, use that
+    * Search for quantum.conf in standard directories:
+        * .
+        * ~.quantum/
+        * ~
+        * /etc/quantum
+        * /etc
+
+    :param app_name: Name of the application to load
+    :param options: Set of typed options returned from parse_options()
+    :param args: Command line arguments from argv[1:]
+
+    :raises RuntimeError when config file cannot be located or application
+            cannot be loaded from config file
+    """
+    conf_file, conf = load_paste_config(app_name, options, args)
+
+    try:
+        app = deploy.loadapp("config:%s" % conf_file, name=app_name)
+    except (LookupError, ImportError), e:
+        raise RuntimeError("Unable to load %(app_name)s from "
+                           "configuration file %(conf_file)s."
+                           "\nGot: %(e)r" % locals())
+    return conf, app
+
+
+def get_option(options, option, **kwargs):
+    if option in options:
+        value = options[option]
+        type_ = kwargs.get('type', 'str')
+        if type_ == 'bool':
+            if hasattr(value, 'lower'):
+                return value.lower() == 'true'
+            else:
+                return value
+        elif type_ == 'int':
+            return int(value)
+        elif type_ == 'float':
+            return float(value)
+        else:
+            return value
+    elif 'default' in kwargs:
+        return kwargs['default']
+    else:
+        raise KeyError("option '%s' not found" % option)

quantum/common/exceptions.py

@@ -0,0 +1,114 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 Nicira Networks, Inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Quantum base exception handling.
+"""
+
+
+class QuantumException(Exception):
+    """Base Quantum Exception
+
+    Taken from nova.exception.NovaException
+    To correctly use this class, inherit from it and define
+    a 'message' property. That message will get printf'd
+    with the keyword arguments provided to the constructor.
+
+    """
+    message = _("An unknown exception occurred.")
+
+    def __init__(self, **kwargs):
+        try:
+            self._error_string = self.message % kwargs
+
+        except Exception:
+            # at least get the core message out if something happened
+            self._error_string = self.message
+
+    def __str__(self):
+        return self._error_string
+
+
+class NotFound(QuantumException):
+    pass
+
+
+class ClassNotFound(NotFound):
+    message = _("Class %(class_name)s could not be found")
+
+
+class NetworkNotFound(NotFound):
+    message = _("Network %(net_id)s could not be found")
+
+
+class PortNotFound(NotFound):
+    message = _("Port %(port_id)s could not be found " \
+                "on network %(net_id)s")
+
+
+class StateInvalid(QuantumException):
+    message = _("Unsupported port state: %(port_state)s")
+
+
+class NetworkInUse(QuantumException):
+    message = _("Unable to complete operation on network %(net_id)s. " \
+                "There is one or more attachments plugged into its ports.")
+
+
+class PortInUse(QuantumException):
+    message = _("Unable to complete operation on port %(port_id)s " \
+                "for network %(net_id)s. The attachment '%(att_id)s" \
+                "is plugged into the logical port.")
+
+
+class AlreadyAttached(QuantumException):
+    message = _("Unable to plug the attachment %(att_id)s into port " \
+                "%(port_id)s for network %(net_id)s. The attachment is " \
+                "already plugged into port %(att_port_id)s")
+
+
+class ProcessExecutionError(IOError):
+    def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
+                 description=None):
+        if description is None:
+            description = "Unexpected error while running command."
+        if exit_code is None:
+            exit_code = '-'
+        message = "%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % (
+                  description, cmd, exit_code, stdout, stderr)
+        IOError.__init__(self, message)
+
+
+class Error(Exception):
+    def __init__(self, message=None):
+        super(Error, self).__init__(message)
+
+
+class MalformedRequestBody(QuantumException):
+    message = _("Malformed request body: %(reason)s")
+
+
+class Invalid(Error):
+    pass
+
+
+class InvalidContentType(Invalid):
+    message = _("Invalid content type %(content_type)s.")
+
+
+class NotImplementedError(Error):
+    pass

quantum/common/flags.py

@@ -0,0 +1,249 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 Citrix Systems, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Command-line flag library.
+
+Wraps gflags.
+Global flags should be defined here, the rest are defined where they're used.
+
+"""
+import getopt
+import gflags
+import os
+import string
+import sys
+
+
+class FlagValues(gflags.FlagValues):
+    """Extension of gflags.FlagValues that allows undefined and runtime flags.
+
+    Unknown flags will be ignored when parsing the command line, but the
+    command line will be kept so that it can be replayed if new flags are
+    defined after the initial parsing.
+
+    """
+
+    def __init__(self, extra_context=None):
+        gflags.FlagValues.__init__(self)
+        self.__dict__['__dirty'] = []
+        self.__dict__['__was_already_parsed'] = False
+        self.__dict__['__stored_argv'] = []
+        self.__dict__['__extra_context'] = extra_context
+
+    def __call__(self, argv):
+        # We're doing some hacky stuff here so that we don't have to copy
+        # out all the code of the original verbatim and then tweak a few lines.
+        # We're hijacking the output of getopt so we can still return the
+        # leftover args at the end
+        sneaky_unparsed_args = {"value": None}
+        original_argv = list(argv)
+
+        if self.IsGnuGetOpt():
+            orig_getopt = getattr(getopt, 'gnu_getopt')
+            orig_name = 'gnu_getopt'
+        else:
+            orig_getopt = getattr(getopt, 'getopt')
+            orig_name = 'getopt'
+
+        def _sneaky(*args, **kw):
+            optlist, unparsed_args = orig_getopt(*args, **kw)
+            sneaky_unparsed_args['value'] = unparsed_args
+            return optlist, unparsed_args
+
+        try:
+            setattr(getopt, orig_name, _sneaky)
+            args = gflags.FlagValues.__call__(self, argv)
+        except gflags.UnrecognizedFlagError:
+            # Undefined args were found, for now we don't care so just
+            # act like everything went well
+            # (these three lines are copied pretty much verbatim from the end
+            # of the __call__ function we are wrapping)
+            unparsed_args = sneaky_unparsed_args['value']
+            if unparsed_args:
+                if self.IsGnuGetOpt():
+                    args = argv[:1] + unparsed_args
+                else:
+                    args = argv[:1] + original_argv[-len(unparsed_args):]
+            else:
+                args = argv[:1]
+        finally:
+            setattr(getopt, orig_name, orig_getopt)
+
+        # Store the arguments for later, we'll need them for new flags
+        # added at runtime
+        self.__dict__['__stored_argv'] = original_argv
+        self.__dict__['__was_already_parsed'] = True
+        self.ClearDirty()
+        return args
+
+    def Reset(self):
+        gflags.FlagValues.Reset(self)
+        self.__dict__['__dirty'] = []
+        self.__dict__['__was_already_parsed'] = False
+        self.__dict__['__stored_argv'] = []
+
+    def SetDirty(self, name):
+        """Mark a flag as dirty so that accessing it will case a reparse."""
+        self.__dict__['__dirty'].append(name)
+
+    def IsDirty(self, name):
+        return name in self.__dict__['__dirty']
+
+    def ClearDirty(self):
+        self.__dict__['__is_dirty'] = []
+
+    def WasAlreadyParsed(self):
+        return self.__dict__['__was_already_parsed']
+
+    def ParseNewFlags(self):
+        if '__stored_argv' not in self.__dict__:
+            return
+        new_flags = FlagValues(self)
+        for k in self.__dict__['__dirty']:
+            new_flags[k] = gflags.FlagValues.__getitem__(self, k)
+
+        new_flags(self.__dict__['__stored_argv'])
+        for k in self.__dict__['__dirty']:
+            setattr(self, k, getattr(new_flags, k))
+        self.ClearDirty()
+
+    def __setitem__(self, name, flag):
+        gflags.FlagValues.__setitem__(self, name, flag)
+        if self.WasAlreadyParsed():
+            self.SetDirty(name)
+
+    def __getitem__(self, name):
+        if self.IsDirty(name):
+            self.ParseNewFlags()
+        return gflags.FlagValues.__getitem__(self, name)
+
+    def __getattr__(self, name):
+        if self.IsDirty(name):
+            self.ParseNewFlags()
+        val = gflags.FlagValues.__getattr__(self, name)
+        if type(val) is str:
+            tmpl = string.Template(val)
+            context = [self, self.__dict__['__extra_context']]
+            return tmpl.substitute(StrWrapper(context))
+        return val
+
+
+class StrWrapper(object):
+    """Wrapper around FlagValues objects.
+
+    Wraps FlagValues objects for string.Template so that we're
+    sure to return strings.
+
+    """
+    def __init__(self, context_objs):
+        self.context_objs = context_objs
+
+    def __getitem__(self, name):
+        for context in self.context_objs:
+            val = getattr(context, name, False)
+            if val:
+                return str(val)
+        raise KeyError(name)
+
+
+# Copied from gflags with small mods to get the naming correct.
+# Originally gflags checks for the first module that is not gflags that is
+# in the call chain, we want to check for the first module that is not gflags
+# and not this module.
+def _GetCallingModule():
+    """Returns the name of the module that's calling into this module.
+
+    We generally use this function to get the name of the module calling a
+    DEFINE_foo... function.
+
+    """
+    # Walk down the stack to find the first globals dict that's not ours.
+    for depth in range(1, sys.getrecursionlimit()):
+        if not sys._getframe(depth).f_globals is globals():
+            module_name = __GetModuleName(sys._getframe(depth).f_globals)
+            if module_name == 'gflags':
+                continue
+            if module_name is not None:
+                return module_name
+    raise AssertionError("No module was found")
+
+
+# Copied from gflags because it is a private function
+def __GetModuleName(globals_dict):
+    """Given a globals dict, returns the name of the module that defines it.
+
+    Args:
+    globals_dict: A dictionary that should correspond to an environment
+      providing the values of the globals.
+
+    Returns:
+    A string (the name of the module) or None (if the module could not
+    be identified.
+
+    """
+    for name, module in sys.modules.iteritems():
+        if getattr(module, '__dict__', None) is globals_dict:
+            if name == '__main__':
+                return sys.argv[0]
+            return name
+    return None
+
+
+def _wrapper(func):
+    def _wrapped(*args, **kw):
+        kw.setdefault('flag_values', FLAGS)
+        func(*args, **kw)
+    _wrapped.func_name = func.func_name
+    return _wrapped
+
+
+FLAGS = FlagValues()
+gflags.FLAGS = FLAGS
+gflags._GetCallingModule = _GetCallingModule
+
+
+DEFINE = _wrapper(gflags.DEFINE)
+DEFINE_string = _wrapper(gflags.DEFINE_string)
+DEFINE_integer = _wrapper(gflags.DEFINE_integer)
+DEFINE_bool = _wrapper(gflags.DEFINE_bool)
+DEFINE_boolean = _wrapper(gflags.DEFINE_boolean)
+DEFINE_float = _wrapper(gflags.DEFINE_float)
+DEFINE_enum = _wrapper(gflags.DEFINE_enum)
+DEFINE_list = _wrapper(gflags.DEFINE_list)
+DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist)
+DEFINE_multistring = _wrapper(gflags.DEFINE_multistring)
+DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int)
+DEFINE_flag = _wrapper(gflags.DEFINE_flag)
+HelpFlag = gflags.HelpFlag
+HelpshortFlag = gflags.HelpshortFlag
+HelpXMLFlag = gflags.HelpXMLFlag
+
+
+def DECLARE(name, module_string, flag_values=FLAGS):
+    if module_string not in sys.modules:
+        __import__(module_string, globals(), locals())
+    if name not in flag_values:
+        raise gflags.UnrecognizedFlag(
+                "%s not defined by %s" % (name, module_string))
+
+
+# __GLOBAL FLAGS ONLY__
+# Define any app-specific flags in their own files, docs at:
+# http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#a9
+
+DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../../'),
+              "Top-level directory for maintaining quantum's state")

quantum/common/test_lib.py

@@ -0,0 +1,291 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 OpenStack, LLC
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+# Colorizer Code is borrowed from Twisted:
+# Copyright (c) 2001-2010 Twisted Matrix Laboratories.
+#
+#    Permission is hereby granted, free of charge, to any person obtaining
+#    a copy of this software and associated documentation files (the
+#    "Software"), to deal in the Software without restriction, including
+#    without limitation the rights to use, copy, modify, merge, publish,
+#    distribute, sublicense, and/or sell copies of the Software, and to
+#    permit persons to whom the Software is furnished to do so, subject to
+#    the following conditions:
+#
+#    The above copyright notice and this permission notice shall be
+#    included in all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+#    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+#    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+#    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import gettext
+import os
+import unittest
+import sys
+import logging
+
+from nose import result
+from nose import core
+from nose import config
+
+
+class _AnsiColorizer(object):
+    """
+    A colorizer is an object that loosely wraps around a stream, allowing
+    callers to write text to the stream in a particular color.
+
+    Colorizer classes must implement C{supported()} and C{write(text, color)}.
+    """
+    _colors = dict(black=30, red=31, green=32, yellow=33,
+                   blue=34, magenta=35, cyan=36, white=37)
+
+    def __init__(self, stream):
+        self.stream = stream
+
+    def supported(cls, stream=sys.stdout):
+        """
+        A class method that returns True if the current platform supports
+        coloring terminal output using this method. Returns False otherwise.
+        """
+        if not stream.isatty():
+            return False  # auto color only on TTYs
+        try:
+            import curses
+        except ImportError:
+            return False
+        else:
+            try:
+                try:
+                    return curses.tigetnum("colors") > 2
+                except curses.error:
+                    curses.setupterm()
+                    return curses.tigetnum("colors") > 2
+            except:
+                raise
+                # guess false in case of error
+                return False
+    supported = classmethod(supported)
+
+    def write(self, text, color):
+        """
+        Write the given text to the stream in the given color.
+
+        @param text: Text to be written to the stream.
+
+        @param color: A string label for a color. e.g. 'red', 'white'.
+        """
+        color = self._colors[color]
+        self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text))
+
+
+class _Win32Colorizer(object):
+    """
+    See _AnsiColorizer docstring.
+    """
+    def __init__(self, stream):
+        from win32console import GetStdHandle, STD_OUT_HANDLE, \
+             FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \
+             FOREGROUND_INTENSITY
+        red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN,
+                                  FOREGROUND_BLUE, FOREGROUND_INTENSITY)
+        self.stream = stream
+        self.screenBuffer = GetStdHandle(STD_OUT_HANDLE)
+        self._colors = {
+            'normal': red | green | blue,
+            'red': red | bold,
+            'green': green | bold,
+            'blue': blue | bold,
+            'yellow': red | green | bold,
+            'magenta': red | blue | bold,
+            'cyan': green | blue | bold,
+            'white': red | green | blue | bold}
+
+    def supported(cls, stream=sys.stdout):
+        try:
+            import win32console
+            screenBuffer = win32console.GetStdHandle(
+                win32console.STD_OUT_HANDLE)
+        except ImportError:
+            return False
+        import pywintypes
+        try:
+            screenBuffer.SetConsoleTextAttribute(
+                win32console.FOREGROUND_RED |
+                win32console.FOREGROUND_GREEN |
+                win32console.FOREGROUND_BLUE)
+        except pywintypes.error:
+            return False
+        else:
+            return True
+    supported = classmethod(supported)
+
+    def write(self, text, color):
+        color = self._colors[color]
+        self.screenBuffer.SetConsoleTextAttribute(color)
+        self.stream.write(text)
+        self.screenBuffer.SetConsoleTextAttribute(self._colors['normal'])
+
+
+class _NullColorizer(object):
+    """
+    See _AnsiColorizer docstring.
+    """
+    def __init__(self, stream):
+        self.stream = stream
+
+    def supported(cls, stream=sys.stdout):
+        return True
+    supported = classmethod(supported)
+
+    def write(self, text, color):
+        self.stream.write(text)
+
+
+class QuantumTestResult(result.TextTestResult):
+    def __init__(self, *args, **kw):
+        result.TextTestResult.__init__(self, *args, **kw)
+        self._last_case = None
+        self.colorizer = None
+        # NOTE(vish, tfukushima): reset stdout for the terminal check
+        stdout = sys.__stdout__
+        sys.stdout = sys.__stdout__
+        for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]:
+            if colorizer.supported():
+                self.colorizer = colorizer(self.stream)
+                break
+        sys.stdout = stdout
+
+    def getDescription(self, test):
+        return str(test)
+
+    # NOTE(vish, tfukushima): copied from unittest with edit to add color
+    def addSuccess(self, test):
+        unittest.TestResult.addSuccess(self, test)
+        if self.showAll:
+            self.colorizer.write("OK", 'green')
+            self.stream.writeln()
+        elif self.dots:
+            self.stream.write('.')
+            self.stream.flush()
+
+    # NOTE(vish, tfukushima): copied from unittest with edit to add color
+    def addFailure(self, test, err):
+        unittest.TestResult.addFailure(self, test, err)
+        if self.showAll:
+            self.colorizer.write("FAIL", 'red')
+            self.stream.writeln()
+        elif self.dots:
+            self.stream.write('F')
+            self.stream.flush()
+
+    # NOTE(vish, tfukushima): copied from unittest with edit to add color
+    def addError(self, test, err):
+        """Overrides normal addError to add support for errorClasses.
+        If the exception is a registered class, the error will be added
+        to the list for that class, not errors.
+        """
+        stream = getattr(self, 'stream', None)
+        ec, ev, tb = err
+        try:
+            exc_info = self._exc_info_to_string(err, test)
+        except TypeError:
+            # This is for compatibility with Python 2.3.
+            exc_info = self._exc_info_to_string(err)
+        for cls, (storage, label, isfail) in self.errorClasses.items():
+            if result.isclass(ec) and issubclass(ec, cls):
+                if isfail:
+                    test.passwd = False
+                storage.append((test, exc_info))
+                # Might get patched into a streamless result
+                if stream is not None:
+                    if self.showAll:
+                        message = [label]
+                        detail = result._exception_details(err[1])
+                        if detail:
+                            message.append(detail)
+                        stream.writeln(": ".join(message))
+                    elif self.dots:
+                        stream.write(label[:1])
+                return
+        self.errors.append((test, exc_info))
+        test.passed = False
+        if stream is not None:
+            if self.showAll:
+                self.colorizer.write("ERROR", 'red')
+                self.stream.writeln()
+            elif self.dots:
+                stream.write('E')
+
+    def startTest(self, test):
+        unittest.TestResult.startTest(self, test)
+        current_case = test.test.__class__.__name__
+
+        if self.showAll:
+            if current_case != self._last_case:
+                self.stream.writeln(current_case)
+                self._last_case = current_case
+            #NOTE(salvatore-orlando):
+            #slightly changed in order to print test case class
+            #together with unit test name
+            self.stream.write(
+                '    %s' % str(test.test).ljust(60))
+            self.stream.flush()
+
+
+class QuantumTestRunner(core.TextTestRunner):
+    def _makeResult(self):
+        return QuantumTestResult(self.stream,
+                              self.descriptions,
+                              self.verbosity,
+                              self.config)
+
+
+def run_tests(c=None):
+    logger = logging.getLogger()
+    hdlr = logging.StreamHandler()
+    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+    hdlr.setFormatter(formatter)
+    logger.addHandler(hdlr)
+    logger.setLevel(logging.DEBUG)
+
+    # NOTE(bgh): I'm not entirely sure why but nose gets confused here when
+    # calling run_tests from a plugin directory run_tests.py (instead of the
+    # main run_tests.py).  It will call run_tests with no arguments and the
+    # testing of run_tests will fail (though the plugin tests will pass).  For
+    # now we just return True to let the run_tests test pass.
+    if not c:
+        return True
+
+    runner = QuantumTestRunner(stream=c.stream,
+                            verbosity=c.verbosity,
+                            config=c)
+    return not core.run(config=c, testRunner=runner)
+
+# describes parameters used by different unit/functional tests
+# a plugin-specific testing mechanism should import this dictionary
+# and override the values in it if needed (e.g., run_tests.py in
+# quantum/plugins/openvswitch/ )
+test_config = {
+    "plugin_name": "quantum.plugins.sample.SamplePlugin.FakePlugin",
+    "default_net_op_status": "UP",
+    "default_port_op_status": "UP",
+}

quantum/common/utils.py

@@ -0,0 +1,267 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011, Nicira Networks, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# Borrowed from nova code base, more utilities will be added/borrowed as and
+# when needed.
+# @author: Somik Behera, Nicira Networks, Inc.
+
+"""Utilities and helper functions."""
+
+import ConfigParser
+import datetime
+import inspect
+import logging
+import os
+import random
+import subprocess
+import socket
+import sys
+import base64
+import functools
+import json
+import re
+import string
+import struct
+import time
+import types
+
+from quantum.common import flags
+from quantum.common import exceptions as exception
+from quantum.common.exceptions import ProcessExecutionError
+
+
+def import_class(import_str):
+    """Returns a class from a string including module and class."""
+    mod_str, _sep, class_str = import_str.rpartition('.')
+    try:
+        __import__(mod_str)
+        return getattr(sys.modules[mod_str], class_str)
+    except (ImportError, ValueError, AttributeError), exc:
+        print(('Inner Exception: %s'), exc)
+        raise exception.ClassNotFound(class_name=class_str)
+
+
+def import_object(import_str):
+    """Returns an object including a module or module and class."""
+    try:
+        __import__(import_str)
+        return sys.modules[import_str]
+    except ImportError:
+        cls = import_class(import_str)
+        return cls()
+
+
+def to_primitive(value):
+    if type(value) is type([]) or type(value) is type((None,)):
+        o = []
+        for v in value:
+            o.append(to_primitive(v))
+        return o
+    elif type(value) is type({}):
+        o = {}
+        for k, v in value.iteritems():
+            o[k] = to_primitive(v)
+        return o
+    elif isinstance(value, datetime.datetime):
+        return str(value)
+    elif hasattr(value, 'iteritems'):
+        return to_primitive(dict(value.iteritems()))
+    elif hasattr(value, '__iter__'):
+        return to_primitive(list(value))
+    else:
+        return value
+
+
+def dumps(value):
+    try:
+        return json.dumps(value)
+    except TypeError:
+        pass
+    return json.dumps(to_primitive(value))
+
+
+def loads(s):
+    return json.loads(s)
+
+TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+FLAGS = flags.FLAGS
+
+
+def int_from_bool_as_string(subject):
+    """
+    Interpret a string as a boolean and return either 1 or 0.
+
+    Any string value in:
+        ('True', 'true', 'On', 'on', '1')
+    is interpreted as a boolean True.
+
+    Useful for JSON-decoded stuff and config file parsing
+    """
+    return bool_from_string(subject) and 1 or 0
+
+
+def bool_from_string(subject):
+    """
+    Interpret a string as a boolean.
+
+    Any string value in:
+        ('True', 'true', 'On', 'on', '1')
+    is interpreted as a boolean True.
+
+    Useful for JSON-decoded stuff and config file parsing
+    """
+    if type(subject) == type(bool):
+        return subject
+    if hasattr(subject, 'startswith'):  # str or unicode...
+        if subject.strip().lower() in ('true', 'on', '1'):
+            return True
+    return False
+
+
+def fetchfile(url, target):
+    logging.debug("Fetching %s" % url)
+#    c = pycurl.Curl()
+#    fp = open(target, "wb")
+#    c.setopt(c.URL, url)
+#    c.setopt(c.WRITEDATA, fp)
+#    c.perform()
+#    c.close()
+#    fp.close()
+    execute("curl --fail %s -o %s" % (url, target))
+
+
+def execute(cmd, process_input=None, addl_env=None, check_exit_code=True):
+    logging.debug("Running cmd: %s", cmd)
+    env = os.environ.copy()
+    if addl_env:
+        env.update(addl_env)
+    obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+    result = None
+    if process_input != None:
+        result = obj.communicate(process_input)
+    else:
+        result = obj.communicate()
+    obj.stdin.close()
+    if obj.returncode:
+        logging.debug("Result was %s" % (obj.returncode))
+        if check_exit_code and obj.returncode != 0:
+            (stdout, stderr) = result
+            raise ProcessExecutionError(exit_code=obj.returncode,
+                                        stdout=stdout,
+                                        stderr=stderr,
+                                        cmd=cmd)
+    return result
+
+
+def abspath(s):
+    return os.path.join(os.path.dirname(__file__), s)
+
+
+# TODO(sirp): when/if utils is extracted to common library, we should remove
+# the argument's default.
+#def default_flagfile(filename='nova.conf'):
+def default_flagfile(filename='quantum.conf'):
+    for arg in sys.argv:
+        if arg.find('flagfile') != -1:
+            break
+    else:
+        if not os.path.isabs(filename):
+            # turn relative filename into an absolute path
+            script_dir = os.path.dirname(inspect.stack()[-1][1])
+            filename = os.path.abspath(os.path.join(script_dir, filename))
+        if os.path.exists(filename):
+            sys.argv = \
+                sys.argv[:1] + ['--flagfile=%s' % filename] + sys.argv[1:]
+
+
+def debug(arg):
+    logging.debug('debug in callback: %s', arg)
+    return arg
+
+
+def runthis(prompt, cmd, check_exit_code=True):
+    logging.debug("Running %s" % (cmd))
+    exit_code = subprocess.call(cmd.split(" "))
+    logging.debug(prompt % (exit_code))
+    if check_exit_code and exit_code != 0:
+        raise ProcessExecutionError(exit_code=exit_code,
+                                    stdout=None,
+                                    stderr=None,
+                                    cmd=cmd)
+
+
+def generate_uid(topic, size=8):
+    return '%s-%s' % (topic, ''.join(
+        [random.choice('01234567890abcdefghijklmnopqrstuvwxyz')
+         for x in xrange(size)]))
+
+
+def generate_mac():
+    mac = [0x02, 0x16, 0x3e, random.randint(0x00, 0x7f),
+           random.randint(0x00, 0xff), random.randint(0x00, 0xff)]
+    return ':'.join(map(lambda x: "%02x" % x, mac))
+
+
+def last_octet(address):
+    return int(address.split(".")[-1])
+
+
+def isotime(at=None):
+    if not at:
+        at = datetime.datetime.utcnow()
+    return at.strftime(TIME_FORMAT)
+
+
+def parse_isotime(timestr):
+    return datetime.datetime.strptime(timestr, TIME_FORMAT)
+
+
+def get_plugin_from_config(file="config.ini"):
+        Config = ConfigParser.ConfigParser()
+        Config.read(file)
+        return Config.get("PLUGIN", "provider")
+
+
+class LazyPluggable(object):
+    """A pluggable backend loaded lazily based on some value."""
+
+    def __init__(self, pivot, **backends):
+        self.__backends = backends
+        self.__pivot = pivot
+        self.__backend = None
+
+    def __get_backend(self):
+        if not self.__backend:
+            backend_name = self.__pivot.value
+            if backend_name not in self.__backends:
+                raise exception.Error('Invalid backend: %s' % backend_name)
+
+            backend = self.__backends[backend_name]
+            if type(backend) == type(tuple()):
+                name = backend[0]
+                fromlist = backend[1]
+            else:
+                name = backend
+                fromlist = backend
+
+            self.__backend = __import__(name, None, None, fromlist)
+            logging.info('backend %s', self.__backend)
+        return self.__backend
+
+    def __getattr__(self, key):
+        backend = self.__get_backend()
+        return getattr(backend, key)

quantum/plugins/cisco/client/cli.py

@@ -36,20 +36,13 @@ import subprocess
 
 from optparse import OptionParser
 
-sys.path.append(os.getcwd())
+from quantum.plugins.cisco.common import cisco_constants as const
-import quantum.client.cli as qcli
+from quantumclient import Client
+import quantumclient.cli as qcli
 
-POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
-                                   os.pardir,
-                                   os.pardir))
-if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'quantum', '__init__.py')):
-    sys.path.insert(0, POSSIBLE_TOPDIR)
 
 gettext.install('quantum', unicode=1)
 
-from quantum.client import Client
-
-from quantum.plugins.cisco.common import cisco_constants as const
 
 LOG = logging.getLogger('quantum')
 FORMAT = 'json'

quantum/server/__init__.py

@@ -24,13 +24,6 @@ import optparse
 import os
 import sys
 
-
-possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
-                                   os.pardir,
-                                   os.pardir))
-if os.path.exists(os.path.join(possible_topdir, 'quantum', '__init__.py')):
-    sys.path.insert(0, possible_topdir)
-
 gettext.install('quantum', unicode=1)
 
 from quantum import service

quantum/tests/unit/client_tools/stubs.py

@@ -1,65 +0,0 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright 2011 OpenStack LLC
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-""" Stubs for client tools unit tests """
-
-
-from quantum import api as server
-from quantum.tests.unit import testlib_api
-
-
-class FakeStdout:
-
-    def __init__(self):
-        self.content = []
-
-    def write(self, text):
-        self.content.append(text)
-
-    def make_string(self):
-        result = ''
-        for line in self.content:
-            result = result + line
-        return result
-
-
-class FakeHTTPConnection:
-    """ stub HTTP connection class for CLI testing """
-    def __init__(self, _1, _2):
-        # Ignore host and port parameters
-        self._req = None
-        plugin = 'quantum.plugins.sample.SamplePlugin.FakePlugin'
-        options = dict(plugin_provider=plugin)
-        self._api = server.APIRouterV11(options)
-
-    def request(self, method, action, body, headers):
-        # TODO: remove version prefix from action!
-        parts = action.split('/', 2)
-        path = '/' + parts[2]
-        self._req = testlib_api.create_request(path, body, "application/json",
-                                               method)
-
-    def getresponse(self):
-        res = self._req.get_response(self._api)
-
-        def _fake_read():
-            """ Trick for making a webob.Response look like a
-                httplib.Response
-
-            """
-            return res.body
-
-        setattr(res, 'read', _fake_read)
-        return res

quantum/tests/unit/testlib_api.py

@@ -1,5 +1,5 @@
 from quantum import wsgi
-from quantum.common.serializer import Serializer
+from quantum.wsgi import Serializer
 
 
 def create_request(path, body, content_type, method='GET', query_string=None):

tools/pip-requires

@@ -8,5 +8,3 @@ python-gflags==1.3
 simplejson
 sqlalchemy
 webob==1.0.8
-
--e git+https://review.openstack.org/p/openstack/python-quantumclient#egg=python-quantumclient