Resync charmhelpers

And enable Python 3.8 tox target.

Uncap flake8, tidy any essential lint.

Change-Id: I5f0c57dbf0e11a7d2746f289f60cbf8cd1df44e6

hooks/charmhelpers/contrib/openstack/policyd.py

@@ -17,7 +17,6 @@ import contextlib
 import os
 import six
 import shutil
-import sys
 import yaml
 import zipfile
 
@@ -531,7 +530,7 @@ def clean_policyd_dir_for(service, keep_paths=None, user=None, group=None):
     hookenv.log("Cleaning path: {}".format(path), level=hookenv.DEBUG)
     if not os.path.exists(path):
         ch_host.mkdir(path, owner=_user, group=_group, perms=0o775)
-    _scanner = os.scandir if sys.version_info > (3, 4) else _py2_scandir
+    _scanner = os.scandir if hasattr(os, 'scandir') else _fallback_scandir
     for direntry in _scanner(path):
         # see if the path should be kept.
         if direntry.path in keep_paths:
@@ -560,23 +559,25 @@ def maybe_create_directory_for(path, user, group):
 
 
 @contextlib.contextmanager
-def _py2_scandir(path):
+def _fallback_scandir(path):
-    """provide a py2 implementation of os.scandir if this module ever gets used
+    """Fallback os.scandir implementation.
-    in a py2 charm (unlikely).  uses os.listdir() to get the names in the path,
+
-    and then mocks the is_dir() function using os.path.isdir() to check for a
+    provide a fallback implementation of os.scandir if this module ever gets
+    used in a py2 or py34 charm. Uses os.listdir() to get the names in the path,
+    and then mocks the is_dir() function using os.path.isdir() to check for
     directory.
 
     :param path: the path to list the directories for
     :type path: str
-    :returns: Generator that provides _P27Direntry objects
+    :returns: Generator that provides _FBDirectory objects
-    :rtype: ContextManager[_P27Direntry]
+    :rtype: ContextManager[_FBDirectory]
     """
     for f in os.listdir(path):
-        yield _P27Direntry(f)
+        yield _FBDirectory(f)
 
 
-class _P27Direntry(object):
+class _FBDirectory(object):
-    """Mock a scandir Direntry object with enough to use in
+    """Mock a scandir Directory object with enough to use in
     clean_policyd_dir_for
     """
 

hooks/charmhelpers/contrib/openstack/utils.py

@@ -278,7 +278,7 @@ PACKAGE_CODENAMES = {
         ('14', 'rocky'),
         ('15', 'stein'),
         ('16', 'train'),
-        ('17', 'ussuri'),
+        ('18', 'ussuri'),
     ]),
     'ceilometer-common': OrderedDict([
         ('5', 'liberty'),
@@ -326,7 +326,7 @@ PACKAGE_CODENAMES = {
         ('14', 'rocky'),
         ('15', 'stein'),
         ('16', 'train'),
-        ('17', 'ussuri'),
+        ('18', 'ussuri'),
     ]),
 }
 
@@ -555,9 +555,8 @@ def reset_os_release():
     _os_rel = None
 
 
-def os_release(package, base=None, reset_cache=False):
+def os_release(package, base=None, reset_cache=False, source_key=None):
-    '''
+    """Returns OpenStack release codename from a cached global.
-    Returns OpenStack release codename from a cached global.
 
     If reset_cache then unset the cached os_release version and return the
     freshly determined version.
@@ -565,7 +564,20 @@ def os_release(package, base=None, reset_cache=False):
     If the codename can not be determined from either an installed package or
     the installation source, the earliest release supported by the charm should
     be returned.
-    '''
+
+    :param package: Name of package to determine release from
+    :type package: str
+    :param base: Fallback codename if endavours to determine from package fail
+    :type base: Optional[str]
+    :param reset_cache: Reset any cached codename value
+    :type reset_cache: bool
+    :param source_key: Name of source configuration option
+                       (default: 'openstack-origin')
+    :type source_key: Optional[str]
+    :returns: OpenStack release codename
+    :rtype: str
+    """
+    source_key = source_key or 'openstack-origin'
     if not base:
         base = UBUNTU_OPENSTACK_RELEASE[lsb_release()['DISTRIB_CODENAME']]
     global _os_rel
@@ -575,7 +587,7 @@ def os_release(package, base=None, reset_cache=False):
         return _os_rel
     _os_rel = (
         get_os_codename_package(package, fatal=False) or
-        get_os_codename_install_source(config('openstack-origin')) or
+        get_os_codename_install_source(config(source_key)) or
         base)
     return _os_rel
 
@@ -658,6 +670,93 @@ def config_value_changed(option):
         return current != saved
 
 
+def get_endpoint_key(service_name, relation_id, unit_name):
+    """Return the key used to refer to an ep changed notification from a unit.
+
+    :param service_name: Service name eg nova, neutron, placement etc
+    :type service_name: str
+    :param relation_id: The id of the relation the unit is on.
+    :type relation_id: str
+    :param unit_name: The name of the unit publishing the notification.
+    :type unit_name: str
+    :returns: The key used to refer to an ep changed notification from a unit
+    :rtype: str
+    """
+    return '{}-{}-{}'.format(
+        service_name,
+        relation_id.replace(':', '_'),
+        unit_name.replace('/', '_'))
+
+
+def get_endpoint_notifications(service_names, rel_name='identity-service'):
+    """Return all notifications for the given services.
+
+    :param service_names: List of service name.
+    :type service_name: List
+    :param rel_name: Name of the relation to query
+    :type rel_name: str
+    :returns: A dict containing the source of the notification and its nonce.
+    :rtype: Dict[str, str]
+    """
+    notifications = {}
+    for rid in relation_ids(rel_name):
+        for unit in related_units(relid=rid):
+            ep_changed_json = relation_get(
+                rid=rid,
+                unit=unit,
+                attribute='ep_changed')
+            if ep_changed_json:
+                ep_changed = json.loads(ep_changed_json)
+                for service in service_names:
+                    if ep_changed.get(service):
+                        key = get_endpoint_key(service, rid, unit)
+                        notifications[key] = ep_changed[service]
+    return notifications
+
+
+def endpoint_changed(service_name, rel_name='identity-service'):
+    """Whether a new notification has been recieved for an endpoint.
+
+    :param service_name: Service name eg nova, neutron, placement etc
+    :type service_name: str
+    :param rel_name: Name of the relation to query
+    :type rel_name: str
+    :returns: Whether endpoint has changed
+    :rtype: bool
+    """
+    changed = False
+    with unitdata.HookData()() as t:
+        db = t[0]
+        notifications = get_endpoint_notifications(
+            [service_name],
+            rel_name=rel_name)
+        for key, nonce in notifications.items():
+            if db.get(key) != nonce:
+                juju_log(('New endpoint change notification found: '
+                          '{}={}').format(key, nonce),
+                         'INFO')
+                changed = True
+                break
+    return changed
+
+
+def save_endpoint_changed_triggers(service_names, rel_name='identity-service'):
+    """Save the enpoint triggers in  db so it can be tracked if they changed.
+
+    :param service_names: List of service name.
+    :type service_name: List
+    :param rel_name: Name of the relation to query
+    :type rel_name: str
+    """
+    with unitdata.HookData()() as t:
+        db = t[0]
+        notifications = get_endpoint_notifications(
+            service_names,
+            rel_name=rel_name)
+        for key, nonce in notifications.items():
+            db.set(key, nonce)
+
+
 def save_script_rc(script_path="scripts/scriptrc", **env_vars):
     """
     Write an rc file in the charm-delivered directory containing

hooks/charmhelpers/contrib/openstack/vaultlocker.py

@@ -37,7 +37,19 @@ class VaultKVContext(context.OSContextGenerator):
         )
 
     def __call__(self):
-        import hvac
+        try:
+            import hvac
+        except ImportError:
+            # BUG: #1862085 - if the relation is made to vault, but the
+            # 'encrypt' option is not made, then the charm errors with an
+            # import warning.  This catches that, logs a warning, and returns
+            # with an empty context.
+            hookenv.log("VaultKVContext: trying to use hvac pythong module "
+                        "but it's not available.  Is secrets-stroage relation "
+                        "made, but encrypt option not set?",
+                        level=hookenv.WARNING)
+            # return an emptry context on hvac import error
+            return {}
         ctxt = {}
         # NOTE(hopem): see https://bugs.launchpad.net/charm-helpers/+bug/1849323
         db = unitdata.kv()

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -1042,7 +1042,7 @@ def filesystem_mounted(fs):
 def make_filesystem(blk_device, fstype='ext4', timeout=10):
     """Make a new filesystem on the specified block device."""
     count = 0
-    e_noent = os.errno.ENOENT
+    e_noent = errno.ENOENT
     while not os.path.exists(blk_device):
         if count >= timeout:
             log('Gave up waiting on block device %s' % blk_device,

hooks/charmhelpers/core/host_factory/ubuntu.py

@@ -25,6 +25,7 @@ UBUNTU_RELEASES = (
     'cosmic',
     'disco',
     'eoan',
+    'focal'
 )
 
 

hooks/charmhelpers/osplatform.py

@@ -1,4 +1,5 @@
 import platform
+import os
 
 
 def get_platform():
@@ -9,9 +10,13 @@ def get_platform():
     This string is used to decide which platform module should be imported.
     """
     # linux_distribution is deprecated and will be removed in Python 3.7
-    # Warings *not* disabled, as we certainly need to fix this.
+    # Warnings *not* disabled, as we certainly need to fix this.
-    tuple_platform = platform.linux_distribution()
+    if hasattr(platform, 'linux_distribution'):
-    current_platform = tuple_platform[0]
+        tuple_platform = platform.linux_distribution()
+        current_platform = tuple_platform[0]
+    else:
+        current_platform = _get_platform_from_fs()
+
     if "Ubuntu" in current_platform:
         return "ubuntu"
     elif "CentOS" in current_platform:
@@ -26,3 +31,16 @@ def get_platform():
     else:
         raise RuntimeError("This module is not supported on {}."
                            .format(current_platform))
+
+
+def _get_platform_from_fs():
+    """Get Platform from /etc/os-release."""
+    with open(os.path.join(os.sep, 'etc', 'os-release')) as fin:
+        content = dict(
+            line.split('=', 1)
+            for line in fin.read().splitlines()
+            if '=' in line
+        )
+    for k, v in content.items():
+        content[k] = v.strip('"')
+    return content["NAME"]

test-requirements.txt

@@ -4,7 +4,7 @@
 charm-tools>=2.4.4
 coverage>=3.6
 mock>=1.2
-flake8>=2.2.4,<=2.4.1
+flake8>=2.2.4
 stestr
 requests>=2.18.4
 pyudev  # for ceph-* charm unit tests (not mocked?)

tox.ini

@@ -41,6 +41,11 @@ basepython = python3.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 
+[testenv:py38]
+basepython = python3.8
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+
 [testenv:pep8]
 basepython = python3
 deps = -r{toxinidir}/requirements.txt
@@ -123,5 +128,5 @@ commands =
     functest-run-suite --keep-model --bundle {posargs}
 
 [flake8]
-ignore = E402,E226
+ignore = E402,E226,W504
 exclude = */charmhelpers

unit_tests/test_utils.py

@@ -97,9 +97,9 @@ class TestConfig(object):
         return self.config
 
     def set(self, attr, value):
-            if attr not in self.config:
+        if attr not in self.config:
-                raise KeyError
+            raise KeyError
-            self.config[attr] = value
+        self.config[attr] = value
 
 
 class TestRelation(object):