Sync charm-helpers

Change-Id: I01088b681a59a1343ec2f996c87d995a8ee9c13e

hooks/charmhelpers/contrib/network/ip.py

@@ -27,6 +27,7 @@ from charmhelpers.core.hookenv import (
     network_get_primary_address,
     unit_get,
     WARNING,
+    NoNetworkBinding,
 )
 
 from charmhelpers.core.host import (
@@ -109,7 +110,12 @@ def get_address_in_network(network, fallback=None, fatal=False):
         _validate_cidr(network)
         network = netaddr.IPNetwork(network)
         for iface in netifaces.interfaces():
+            try:
                 addresses = netifaces.ifaddresses(iface)
+            except ValueError:
+                # If an instance was deleted between
+                # netifaces.interfaces() run and now, its interfaces are gone
+                continue
             if network.version == 4 and netifaces.AF_INET in addresses:
                 for addr in addresses[netifaces.AF_INET]:
                     cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
@@ -578,6 +584,9 @@ def get_relation_ip(interface, cidr_network=None):
     except NotImplementedError:
         # If network-get is not available
         address = get_host_ip(unit_get('private-address'))
+    except NoNetworkBinding:
+        log("No network binding for {}".format(interface), WARNING)
+        address = get_host_ip(unit_get('private-address'))
 
     if config('prefer-ipv6'):
         # Currently IPv6 has priority, eventually we want IPv6 to just be

hooks/charmhelpers/contrib/openstack/amulet/utils.py

@@ -92,7 +92,7 @@ class OpenStackAmuletUtils(AmuletUtils):
             return 'endpoint not found'
 
     def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port,
-                                  public_port, expected):
+                                  public_port, expected, expected_num_eps=3):
         """Validate keystone v3 endpoint data.
 
         Validate the v3 endpoint data which has changed from v2.  The
@@ -138,7 +138,7 @@ class OpenStackAmuletUtils(AmuletUtils):
                 if ret:
                     return 'unexpected endpoint data - {}'.format(ret)
 
-        if len(found) != 3:
+        if len(found) != expected_num_eps:
             return 'Unexpected number of endpoints found'
 
     def validate_svc_catalog_endpoint_data(self, expected, actual):

hooks/charmhelpers/contrib/openstack/context.py

@@ -617,7 +617,9 @@ class HAProxyContext(OSContextGenerator):
     """
     interfaces = ['cluster']
 
-    def __init__(self, singlenode_mode=False):
+    def __init__(self, singlenode_mode=False,
+                 address_types=ADDRESS_TYPES):
+        self.address_types = address_types
         self.singlenode_mode = singlenode_mode
 
     def __call__(self):
@@ -631,7 +633,7 @@ class HAProxyContext(OSContextGenerator):
 
         # NOTE(jamespage): build out map of configured network endpoints
         # and associated backends
-        for addr_type in ADDRESS_TYPES:
+        for addr_type in self.address_types:
             cfg_opt = 'os-{}-network'.format(addr_type)
             # NOTE(thedac) For some reason the ADDRESS_MAP uses 'int' rather
             # than 'internal'
@@ -1635,18 +1637,84 @@ class InternalEndpointContext(OSContextGenerator):
     endpoints by default so this allows admins to optionally use internal
     endpoints.
     """
-    def __init__(self, ost_rel_check_pkg_name):
+    def __call__(self):
-        self.ost_rel_check_pkg_name = ost_rel_check_pkg_name
+        return {'use_internal_endpoints': config('use-internal-endpoints')}
+
+
+class VolumeAPIContext(InternalEndpointContext):
+    """Volume API context.
+
+    This context provides information regarding the volume endpoint to use
+    when communicating between services. It determines which version of the
+    API is appropriate for use.
+
+    This value will be determined in the resulting context dictionary
+    returned from calling the VolumeAPIContext object. Information provided
+    by this context is as follows:
+
+        volume_api_version: the volume api version to use, currently
+            'v2' or 'v3'
+        volume_catalog_info: the information to use for a cinder client
+            configuration that consumes API endpoints from the keystone
+            catalog. This is defined as the type:name:endpoint_type string.
+    """
+    # FIXME(wolsen) This implementation is based on the provider being able
+    # to specify the package version to check but does not guarantee that the
+    # volume service api version selected is available. In practice, it is
+    # quite likely the volume service *is* providing the v3 volume service.
+    # This should be resolved when the service-discovery spec is implemented.
+    def __init__(self, pkg):
+        """
+        Creates a new VolumeAPIContext for use in determining which version
+        of the Volume API should be used for communication. A package codename
+        should be supplied for determining the currently installed OpenStack
+        version.
+
+        :param pkg: the package codename to use in order to determine the
+            component version (e.g. nova-common). See
+            charmhelpers.contrib.openstack.utils.PACKAGE_CODENAMES for more.
+        """
+        super(VolumeAPIContext, self).__init__()
+        self._ctxt = None
+        if not pkg:
+            raise ValueError('package name must be provided in order to '
+                             'determine current OpenStack version.')
+        self.pkg = pkg
+
+    @property
+    def ctxt(self):
+        if self._ctxt is not None:
+            return self._ctxt
+        self._ctxt = self._determine_ctxt()
+        return self._ctxt
+
+    def _determine_ctxt(self):
+        """Determines the Volume API endpoint information.
+
+        Determines the appropriate version of the API that should be used
+        as well as the catalog_info string that would be supplied. Returns
+        a dict containing the volume_api_version and the volume_catalog_info.
+        """
+        rel = os_release(self.pkg, base='icehouse')
+        version = '2'
+        if CompareOpenStackReleases(rel) >= 'pike':
+            version = '3'
+
+        service_type = 'volumev{version}'.format(version=version)
+        service_name = 'cinderv{version}'.format(version=version)
+        endpoint_type = 'publicURL'
+        if config('use-internal-endpoints'):
+            endpoint_type = 'internalURL'
+        catalog_info = '{type}:{name}:{endpoint}'.format(
+            type=service_type, name=service_name, endpoint=endpoint_type)
+
+        return {
+            'volume_api_version': version,
+            'volume_catalog_info': catalog_info,
+        }
 
     def __call__(self):
-        ctxt = {'use_internal_endpoints': config('use-internal-endpoints')}
+        return self.ctxt
-        rel = os_release(self.ost_rel_check_pkg_name, base='icehouse')
-        if CompareOpenStackReleases(rel) >= 'pike':
-            ctxt['volume_api_version'] = '3'
-        else:
-            ctxt['volume_api_version'] = '2'
-
-        return ctxt
 
 
 class AppArmorContext(OSContextGenerator):
@@ -1784,3 +1852,30 @@ class MemcacheContext(OSContextGenerator):
                     ctxt['memcache_server_formatted'],
                     ctxt['memcache_port'])
         return ctxt
+
+
+class EnsureDirContext(OSContextGenerator):
+    '''
+    Serves as a generic context to create a directory as a side-effect.
+
+    Useful for software that supports drop-in files (.d) in conjunction
+    with config option-based templates. Examples include:
+        * OpenStack oslo.policy drop-in files;
+        * systemd drop-in config files;
+        * other software that supports overriding defaults with .d files
+
+    Another use-case is when a subordinate generates a configuration for
+    primary to render in a separate directory.
+
+    Some software requires a user to create a target directory to be
+    scanned for drop-in files with a specific format. This is why this
+    context is needed to do that before rendering a template.
+   '''
+
+    def __init__(self, dirname):
+        '''Used merely to ensure that a given directory exists.'''
+        self.dirname = dirname
+
+    def __call__(self):
+        mkdir(self.dirname)
+        return {}

hooks/charmhelpers/contrib/openstack/templating.py

@@ -93,7 +93,8 @@ class OSConfigTemplate(object):
     Associates a config file template with a list of context generators.
     Responsible for constructing a template context based on those generators.
     """
-    def __init__(self, config_file, contexts):
+
+    def __init__(self, config_file, contexts, config_template=None):
         self.config_file = config_file
 
         if hasattr(contexts, '__call__'):
@@ -103,6 +104,8 @@ class OSConfigTemplate(object):
 
         self._complete_contexts = []
 
+        self.config_template = config_template
+
     def context(self):
         ctxt = {}
         for context in self.contexts:
@@ -124,6 +127,11 @@ class OSConfigTemplate(object):
         self.context()
         return self._complete_contexts
 
+    @property
+    def is_string_template(self):
+        """:returns: Boolean if this instance is a template initialised with a string"""
+        return self.config_template is not None
+
 
 class OSConfigRenderer(object):
     """
@@ -148,6 +156,10 @@ class OSConfigRenderer(object):
                          contexts=[context.IdentityServiceContext()])
         configs.register(config_file='/etc/haproxy/haproxy.conf',
                          contexts=[context.HAProxyContext()])
+        configs.register(config_file='/etc/keystone/policy.d/extra.cfg',
+                         contexts=[context.ExtraPolicyContext()
+                                   context.KeystoneContext()],
+                         config_template=hookenv.config('extra-policy'))
         # write out a single config
         configs.write('/etc/nova/nova.conf')
         # write out all registered configs
@@ -218,14 +230,23 @@ class OSConfigRenderer(object):
             else:
                 apt_install('python3-jinja2')
 
-    def register(self, config_file, contexts):
+    def register(self, config_file, contexts, config_template=None):
         """
         Register a config file with a list of context generators to be called
         during rendering.
+        config_template can be used to load a template from a string instead of
+        using template loaders and template files.
+        :param config_file (str): a path where a config file will be rendered
+        :param contexts (list): a list of context dictionaries with kv pairs
+        :param config_template (str): an optional template string to use
         """
-        self.templates[config_file] = OSConfigTemplate(config_file=config_file,
+        self.templates[config_file] = OSConfigTemplate(
-                                                       contexts=contexts)
+            config_file=config_file,
-        log('Registered config file: %s' % config_file, level=INFO)
+            contexts=contexts,
+            config_template=config_template
+        )
+        log('Registered config file: {}'.format(config_file),
+            level=INFO)
 
     def _get_tmpl_env(self):
         if not self._tmpl_env:
@@ -235,32 +256,58 @@ class OSConfigRenderer(object):
     def _get_template(self, template):
         self._get_tmpl_env()
         template = self._tmpl_env.get_template(template)
-        log('Loaded template from %s' % template.filename, level=INFO)
+        log('Loaded template from {}'.format(template.filename),
+            level=INFO)
+        return template
+
+    def _get_template_from_string(self, ostmpl):
+        '''
+        Get a jinja2 template object from a string.
+        :param ostmpl: OSConfigTemplate to use as a data source.
+        '''
+        self._get_tmpl_env()
+        template = self._tmpl_env.from_string(ostmpl.config_template)
+        log('Loaded a template from a string for {}'.format(
+            ostmpl.config_file),
+            level=INFO)
         return template
 
     def render(self, config_file):
         if config_file not in self.templates:
-            log('Config not registered: %s' % config_file, level=ERROR)
+            log('Config not registered: {}'.format(config_file), level=ERROR)
             raise OSConfigException
-        ctxt = self.templates[config_file].context()
 
+        ostmpl = self.templates[config_file]
+        ctxt = ostmpl.context()
+
+        if ostmpl.is_string_template:
+            template = self._get_template_from_string(ostmpl)
+            log('Rendering from a string template: '
+                '{}'.format(config_file),
+                level=INFO)
+        else:
             _tmpl = os.path.basename(config_file)
             try:
                 template = self._get_template(_tmpl)
             except exceptions.TemplateNotFound:
-            # if no template is found with basename, try looking for it
+                # if no template is found with basename, try looking
-            # using a munged full path, eg:
+                # for it using a munged full path, eg:
                 # /etc/apache2/apache2.conf -> etc_apache2_apache2.conf
                 _tmpl = '_'.join(config_file.split('/')[1:])
                 try:
                     template = self._get_template(_tmpl)
                 except exceptions.TemplateNotFound as e:
-                log('Could not load template from %s by %s or %s.' %
+                    log('Could not load template from {} by {} or {}.'
-                    (self.templates_dir, os.path.basename(config_file), _tmpl),
+                        ''.format(
+                            self.templates_dir,
+                            os.path.basename(config_file),
+                            _tmpl
+                        ),
                         level=ERROR)
                     raise e
 
-        log('Rendering from template: %s' % _tmpl, level=INFO)
+            log('Rendering from template: {}'.format(config_file),
+                level=INFO)
         return template.render(ctxt)
 
     def write(self, config_file):

hooks/charmhelpers/core/hookenv.py

@@ -820,6 +820,10 @@ class Hooks(object):
         return wrapper
 
 
+class NoNetworkBinding(Exception):
+    pass
+
+
 def charm_dir():
     """Return the root directory of the current charm"""
     d = os.environ.get('JUJU_CHARM_DIR')
@@ -1106,7 +1110,17 @@ def network_get_primary_address(binding):
     :raise: NotImplementedError if run on Juju < 2.0
     '''
     cmd = ['network-get', '--primary-address', binding]
-    return subprocess.check_output(cmd).decode('UTF-8').strip()
+    try:
+        response = subprocess.check_output(
+            cmd,
+            stderr=subprocess.STDOUT).decode('UTF-8').strip()
+    except CalledProcessError as e:
+        if 'no network config found for binding' in e.output.decode('UTF-8'):
+            raise NoNetworkBinding("No network binding for {}"
+                                   .format(binding))
+        else:
+            raise
+    return response
 
 
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)

hooks/charmhelpers/core/templating.py

@@ -20,7 +20,8 @@ from charmhelpers.core import hookenv
 
 
 def render(source, target, context, owner='root', group='root',
-           perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
+           perms=0o444, templates_dir=None, encoding='UTF-8',
+           template_loader=None, config_template=None):
     """
     Render a template.
 
@@ -32,6 +33,9 @@ def render(source, target, context, owner='root', group='root',
     The context should be a dict containing the values to be replaced in the
     template.
 
+    config_template may be provided to render from a provided template instead
+    of loading from a file.
+
     The `owner`, `group`, and `perms` options will be passed to `write_file`.
 
     If omitted, `templates_dir` defaults to the `templates` folder in the charm.
@@ -65,6 +69,11 @@ def render(source, target, context, owner='root', group='root',
         if templates_dir is None:
             templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
         template_env = Environment(loader=FileSystemLoader(templates_dir))
+
+    # load from a string if provided explicitly
+    if config_template is not None:
+        template = template_env.from_string(config_template)
+    else:
         try:
             source = source
             template = template_env.get_template(source)

tests/charmhelpers/contrib/network/ip.py

@@ -27,6 +27,7 @@ from charmhelpers.core.hookenv import (
     network_get_primary_address,
     unit_get,
     WARNING,
+    NoNetworkBinding,
 )
 
 from charmhelpers.core.host import (
@@ -109,7 +110,12 @@ def get_address_in_network(network, fallback=None, fatal=False):
         _validate_cidr(network)
         network = netaddr.IPNetwork(network)
         for iface in netifaces.interfaces():
+            try:
                 addresses = netifaces.ifaddresses(iface)
+            except ValueError:
+                # If an instance was deleted between
+                # netifaces.interfaces() run and now, its interfaces are gone
+                continue
             if network.version == 4 and netifaces.AF_INET in addresses:
                 for addr in addresses[netifaces.AF_INET]:
                     cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
@@ -578,6 +584,9 @@ def get_relation_ip(interface, cidr_network=None):
     except NotImplementedError:
         # If network-get is not available
         address = get_host_ip(unit_get('private-address'))
+    except NoNetworkBinding:
+        log("No network binding for {}".format(interface), WARNING)
+        address = get_host_ip(unit_get('private-address'))
 
     if config('prefer-ipv6'):
         # Currently IPv6 has priority, eventually we want IPv6 to just be

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -92,7 +92,7 @@ class OpenStackAmuletUtils(AmuletUtils):
             return 'endpoint not found'
 
     def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port,
-                                  public_port, expected):
+                                  public_port, expected, expected_num_eps=3):
         """Validate keystone v3 endpoint data.
 
         Validate the v3 endpoint data which has changed from v2.  The
@@ -138,7 +138,7 @@ class OpenStackAmuletUtils(AmuletUtils):
                 if ret:
                     return 'unexpected endpoint data - {}'.format(ret)
 
-        if len(found) != 3:
+        if len(found) != expected_num_eps:
             return 'Unexpected number of endpoints found'
 
     def validate_svc_catalog_endpoint_data(self, expected, actual):

tests/charmhelpers/core/hookenv.py

@@ -820,6 +820,10 @@ class Hooks(object):
         return wrapper
 
 
+class NoNetworkBinding(Exception):
+    pass
+
+
 def charm_dir():
     """Return the root directory of the current charm"""
     d = os.environ.get('JUJU_CHARM_DIR')
@@ -1106,7 +1110,17 @@ def network_get_primary_address(binding):
     :raise: NotImplementedError if run on Juju < 2.0
     '''
     cmd = ['network-get', '--primary-address', binding]
-    return subprocess.check_output(cmd).decode('UTF-8').strip()
+    try:
+        response = subprocess.check_output(
+            cmd,
+            stderr=subprocess.STDOUT).decode('UTF-8').strip()
+    except CalledProcessError as e:
+        if 'no network config found for binding' in e.output.decode('UTF-8'):
+            raise NoNetworkBinding("No network binding for {}"
+                                   .format(binding))
+        else:
+            raise
+    return response
 
 
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)

tests/charmhelpers/core/templating.py

@@ -20,7 +20,8 @@ from charmhelpers.core import hookenv
 
 
 def render(source, target, context, owner='root', group='root',
-           perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
+           perms=0o444, templates_dir=None, encoding='UTF-8',
+           template_loader=None, config_template=None):
     """
     Render a template.
 
@@ -32,6 +33,9 @@ def render(source, target, context, owner='root', group='root',
     The context should be a dict containing the values to be replaced in the
     template.
 
+    config_template may be provided to render from a provided template instead
+    of loading from a file.
+
     The `owner`, `group`, and `perms` options will be passed to `write_file`.
 
     If omitted, `templates_dir` defaults to the `templates` folder in the charm.
@@ -65,6 +69,11 @@ def render(source, target, context, owner='root', group='root',
         if templates_dir is None:
             templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
         template_env = Environment(loader=FileSystemLoader(templates_dir))
+
+    # load from a string if provided explicitly
+    if config_template is not None:
+        template = template_env.from_string(config_template)
+    else:
         try:
             source = source
             template = template_env.get_template(source)

tox.ini

@@ -9,7 +9,7 @@ skipsdist = True
 setenv = VIRTUAL_ENV={envdir}
          PYTHONHASHSEED=0
          CHARM_DIR={envdir}
-         AMULET_SETUP_TIMEOUT=2700
+         AMULET_SETUP_TIMEOUT=5400
 install_command =
   pip install --allow-unverified python-apt {opts} {packages}
 commands = ostestr {posargs}