Resync charm-helpers

Avoid use of 'service --status-all' which is currently
broken on trusty for upstart managed daemons; the change
moves to detecting how the daemon is managed, and then
using upstart status XXX or the return code of service XXX
status to determine whether a process is running.

Fixes for IPv6 network address detection under Ubuntu
16.04 which changes the output format of the ip commands
slightly.

Update the version map to include 8.1.x as a Neutron
version for Mitaka.

Change-Id: I023dbb4b3d7e16d7ec50e5b78636e5c7688ff9cc
Closes-Bug: 1581171
Closes-Bug: 1581598
Closes-Bug: 1580674

hooks/charmhelpers/contrib/network/ip.py

@@ -214,7 +214,16 @@ def format_ipv6_addr(address):
 
 def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
                    fatal=True, exc_list=None):
-    """Return the assigned IP address for a given interface, if any."""
+    """Return the assigned IP address for a given interface, if any.
+
+    :param iface: network interface on which address(es) are expected to
+                  be found.
+    :param inet_type: inet address family
+    :param inc_aliases: include alias interfaces in search
+    :param fatal: if True, raise exception if address not found
+    :param exc_list: list of addresses to ignore
+    :return: list of ip addresses
+    """
     # Extract nic if passed /dev/ethX
     if '/' in iface:
         iface = iface.split('/')[-1]
@@ -315,6 +324,14 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None,
     We currently only support scope global IPv6 addresses i.e. non-temporary
     addresses. If no global IPv6 address is found, return the first one found
     in the ipv6 address list.
+
+    :param iface: network interface on which ipv6 address(es) are expected to
+                  be found.
+    :param inc_aliases: include alias interfaces in search
+    :param fatal: if True, raise exception if address not found
+    :param exc_list: list of addresses to ignore
+    :param dynamic_only: only recognise dynamic addresses
+    :return: list of ipv6 addresses
     """
     addresses = get_iface_addr(iface=iface, inet_type='AF_INET6',
                                inc_aliases=inc_aliases, fatal=fatal,
@@ -336,7 +353,7 @@ def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None,
             cmd = ['ip', 'addr', 'show', iface]
             out = subprocess.check_output(cmd).decode('UTF-8')
             if dynamic_only:
-                key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*")
+                key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*")
             else:
                 key = re.compile("inet6 (.+)/[0-9]+ scope global.*")
 

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka

@@ -0,0 +1,12 @@
+{% if auth_host -%}
+[keystone_authtoken]
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
+auth_type = password
+project_domain_name = default
+user_domain_name = default
+project_name = {{ admin_tenant_name }}
+username = {{ admin_user }}
+password = {{ admin_password }}
+signing_dir = {{ signing_dir }}
+{% endif -%}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -149,6 +149,7 @@ PACKAGE_CODENAMES = {
     'neutron-common': OrderedDict([
         ('7.0', 'liberty'),
         ('8.0', 'mitaka'),
+        ('8.1', 'mitaka'),
     ]),
     'cinder-common': OrderedDict([
         ('7.0', 'liberty'),

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -166,12 +166,19 @@ class Pool(object):
         """
         # read-only is easy, writeback is much harder
         mode = get_cache_mode(self.service, cache_pool)
+        version = ceph_version()
         if mode == 'readonly':
             check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'none'])
             check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
 
         elif mode == 'writeback':
-            check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'forward'])
+            pool_forward_cmd = ['ceph', '--id', self.service, 'osd', 'tier',
+                                'cache-mode', cache_pool, 'forward']
+            if version >= '10.1':
+                # Jewel added a mandatory flag
+                pool_forward_cmd.append('--yes-i-really-mean-it')
+
+            check_call(pool_forward_cmd)
             # Flush the cache and wait for it to return
             check_call(['rados', '--id', self.service, '-p', cache_pool, 'cache-flush-evict-all'])
             check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove-overlay', self.name])
@@ -221,6 +228,10 @@ class ReplicatedPool(Pool):
                    self.name, str(self.pg_num)]
             try:
                 check_call(cmd)
+                # Set the pool replica size
+                update_pool(client=self.service,
+                            pool=self.name,
+                            settings={'size': str(self.replicas)})
             except CalledProcessError:
                 raise
 
@@ -604,7 +615,7 @@ def pool_exists(service, name):
     except CalledProcessError:
         return False
 
-    return name in out
+    return name in out.split()
 
 
 def get_osds(service):

hooks/charmhelpers/contrib/storage/linux/utils.py

@@ -64,8 +64,8 @@ def is_device_mounted(device):
     :returns: boolean: True if the path represents a mounted device, False if
         it doesn't.
     '''
-    is_partition = bool(re.search(r".*[0-9]+\b", device))
+    try:
-    out = check_output(['mount']).decode('UTF-8')
+        out = check_output(['lsblk', '-P', device]).decode('UTF-8')
-    if is_partition:
+    except:
-        return bool(re.search(device + r"\b", out))
+        return False
-    return bool(re.search(device + r"[0-9]*\b", out))
+    return bool(re.search(r'MOUNTPOINT=".+"', out))

hooks/charmhelpers/core/host.py

@@ -128,11 +128,8 @@ def service(action, service_name):
     return subprocess.call(cmd) == 0
 
 
-def systemv_services_running():
+_UPSTART_CONF = "/etc/init/{}.conf"
-    output = subprocess.check_output(
+_INIT_D_CONF = "/etc/init.d/{}"
-        ['service', '--status-all'],
-        stderr=subprocess.STDOUT).decode('UTF-8')
-    return [row.split()[-1] for row in output.split('\n') if '[ + ]' in row]
 
 
 def service_running(service_name):
@@ -140,21 +137,21 @@ def service_running(service_name):
     if init_is_systemd():
         return service('is-active', service_name)
     else:
+        if os.path.exists(_UPSTART_CONF.format(service_name)):
             try:
                 output = subprocess.check_output(
-                ['service', service_name, 'status'],
+                    ['status', service_name],
                     stderr=subprocess.STDOUT).decode('UTF-8')
             except subprocess.CalledProcessError:
                 return False
             else:
                 # This works for upstart scripts where the 'service' command
                 # returns a consistent string to represent running 'start/running'
-            if ("start/running" in output or "is running" in output or
+                if "start/running" in output:
-                    "up and running" in output):
                     return True
+        elif os.path.exists(_INIT_D_CONF.format(service_name)):
             # Check System V scripts init script return codes
-            if service_name in systemv_services_running():
+            return service('status', service_name)
-                return True
         return False