charmhelpers sync to fix py3 contexts

Change-Id: Ibf6c49aef5127eb824ea671089cb56cf7789dc32

hooks/charmhelpers/contrib/hahelpers/apache.py

@@ -90,6 +90,6 @@ def install_ca_cert(ca_cert):
             log("CA cert is the same as installed version", level=INFO)
         else:
             log("Installing new CA cert", level=INFO)
-            with open(cert_file, 'w') as crt:
+            with open(cert_file, 'wb') as crt:
                 crt.write(ca_cert)
             subprocess.check_call(['update-ca-certificates', '--fresh'])

hooks/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -250,7 +250,14 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         self.log.debug('Waiting up to {}s for extended status on services: '
                        '{}'.format(timeout, services))
         service_messages = {service: message for service in services}
+
+        # Check for idleness
+        self.d.sentry.wait()
+        # Check for error states and bail early
+        self.d.sentry.wait_for_status(self.d.juju_env, services)
+        # Check for ready messages
         self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+
         self.log.info('OK')
 
     def _get_openstack_release(self):
@@ -263,7 +270,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
          self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
          self.yakkety_newton, self.xenial_ocata, self.zesty_ocata,
-         self.xenial_pike, self.artful_pike) = range(11)
+         self.xenial_pike, self.artful_pike, self.xenial_queens,
+         self.bionic_queens,) = range(13)
 
         releases = {
             ('trusty', None): self.trusty_icehouse,
@@ -274,9 +282,11 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
             ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
             ('xenial', 'cloud:xenial-pike'): self.xenial_pike,
+            ('xenial', 'cloud:xenial-queens'): self.xenial_queens,
             ('yakkety', None): self.yakkety_newton,
             ('zesty', None): self.zesty_ocata,
             ('artful', None): self.artful_pike,
+            ('bionic', None): self.bionic_queens,
         }
         return releases[(self.series, self.openstack)]
 
@@ -291,6 +301,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('yakkety', 'newton'),
             ('zesty', 'ocata'),
             ('artful', 'pike'),
+            ('bionic', 'queens'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]
@@ -303,8 +314,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         test scenario, based on OpenStack release and whether ceph radosgw
         is flagged as present or not."""
 
-        if self._get_openstack_release() <= self.trusty_juno:
+        if self._get_openstack_release() == self.trusty_icehouse:
-            # Juno or earlier
+            # Icehouse
             pools = [
                 'data',
                 'metadata',
@@ -312,7 +323,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                 'cinder-ceph',
                 'glance'
             ]
-        elif (self.trust_kilo <= self._get_openstack_release() <=
+        elif (self.trusty_kilo <= self._get_openstack_release() <=
               self.zesty_ocata):
             # Kilo through Ocata
             pools = [

hooks/charmhelpers/contrib/openstack/amulet/utils.py

@@ -43,7 +43,6 @@ import swiftclient
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
-from charmhelpers.core.decorators import retry_on_exception
 from charmhelpers.core.host import CompareHostReleases
 
 DEBUG = logging.DEBUG
@@ -311,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
-    @retry_on_exception(5, base_delay=10)
     def keystone_wait_for_propagation(self, sentry_relation_pairs,
                                       api_version):
         """Iterate over list of sentry and relation tuples and verify that
@@ -327,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils):
             rel = sentry.relation('identity-service',
                                   relation_name)
             self.log.debug('keystone relation data: {}'.format(rel))
-            if rel['api_version'] != str(api_version):
+            if rel.get('api_version') != str(api_version):
                 raise Exception("api_version not propagated through relation"
                                 " data yet ('{}' != '{}')."
                                 "".format(rel['api_version'], api_version))
@@ -349,6 +347,7 @@ class OpenStackAmuletUtils(AmuletUtils):
 
         config = {'preferred-api-version': api_version}
         deployment.d.configure('keystone', config)
+        deployment._auto_wait_for_status()
         self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
 
     def authenticate_cinder_admin(self, keystone_sentry, username,

hooks/charmhelpers/contrib/openstack/context.py

@@ -293,7 +293,7 @@ class PostgresqlDBContext(OSContextGenerator):
 def db_ssl(rdata, ctxt, ssl_dir):
     if 'ssl_ca' in rdata and ssl_dir:
         ca_path = os.path.join(ssl_dir, 'db-client.ca')
-        with open(ca_path, 'w') as fh:
+        with open(ca_path, 'wb') as fh:
             fh.write(b64decode(rdata['ssl_ca']))
 
         ctxt['database_ssl_ca'] = ca_path
@@ -308,12 +308,12 @@ def db_ssl(rdata, ctxt, ssl_dir):
             log("Waiting 1m for ssl client cert validity", level=INFO)
             time.sleep(60)
 
-        with open(cert_path, 'w') as fh:
+        with open(cert_path, 'wb') as fh:
             fh.write(b64decode(rdata['ssl_cert']))
 
         ctxt['database_ssl_cert'] = cert_path
         key_path = os.path.join(ssl_dir, 'db-client.key')
-        with open(key_path, 'w') as fh:
+        with open(key_path, 'wb') as fh:
             fh.write(b64decode(rdata['ssl_key']))
 
         ctxt['database_ssl_key'] = key_path
@@ -459,7 +459,7 @@ class AMQPContext(OSContextGenerator):
 
                         ca_path = os.path.join(
                             self.ssl_dir, 'rabbit-client-ca.pem')
-                        with open(ca_path, 'w') as fh:
+                        with open(ca_path, 'wb') as fh:
                             fh.write(b64decode(ctxt['rabbit_ssl_ca']))
                             ctxt['rabbit_ssl_ca'] = ca_path
 

hooks/charmhelpers/contrib/openstack/templates/ceph.conf

@@ -18,7 +18,7 @@ rbd default features = {{ rbd_features }}
 
 [client]
 {% if rbd_client_cache_settings -%}
-{% for key, value in rbd_client_cache_settings.iteritems() -%}
+{% for key, value in rbd_client_cache_settings.items() -%}
 {{ key }} = {{ value }}
 {% endfor -%}
-{%- endif %}
+{%- endif %}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -140,6 +140,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('yakkety', 'newton'),
     ('zesty', 'ocata'),
     ('artful', 'pike'),
+    ('bionic', 'queens'),
 ])
 
 
@@ -157,6 +158,7 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2016.2', 'newton'),
     ('2017.1', 'ocata'),
     ('2017.2', 'pike'),
+    ('2018.1', 'queens'),
 ])
 
 # The ugly duckling - must list releases oldest to newest
@@ -187,6 +189,8 @@ SWIFT_CODENAMES = OrderedDict([
         ['2.11.0', '2.12.0', '2.13.0']),
     ('pike',
         ['2.13.0', '2.15.0']),
+    ('queens',
+        ['2.16.0']),
 ])
 
 # >= Liberty version->codename mapping
@@ -412,6 +416,8 @@ def get_os_codename_package(package, fatal=True):
         cmd = ['snap', 'list', package]
         try:
             out = subprocess.check_output(cmd)
+            if six.PY3:
+                out = out.decode('UTF-8')
         except subprocess.CalledProcessError as e:
             return None
         lines = out.split('\n')

hooks/charmhelpers/core/strutils.py

@@ -61,13 +61,19 @@ def bytes_from_string(value):
     if isinstance(value, six.string_types):
         value = six.text_type(value)
     else:
-        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+        msg = "Unable to interpret non-string value '%s' as bytes" % (value)
         raise ValueError(msg)
     matches = re.match("([0-9]+)([a-zA-Z]+)", value)
-    if not matches:
+    if matches:
-        msg = "Unable to interpret string value '%s' as bytes" % (value)
+        size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
-        raise ValueError(msg)
+    else:
-    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+        # Assume that value passed in is bytes
+        try:
+            size = int(value)
+        except ValueError:
+            msg = "Unable to interpret string value '%s' as bytes" % (value)
+            raise ValueError(msg)
+    return size
 
 
 class BasicStringComparator(object):

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -250,7 +250,14 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         self.log.debug('Waiting up to {}s for extended status on services: '
                        '{}'.format(timeout, services))
         service_messages = {service: message for service in services}
+
+        # Check for idleness
+        self.d.sentry.wait()
+        # Check for error states and bail early
+        self.d.sentry.wait_for_status(self.d.juju_env, services)
+        # Check for ready messages
         self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+
         self.log.info('OK')
 
     def _get_openstack_release(self):
@@ -263,7 +270,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
          self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
          self.yakkety_newton, self.xenial_ocata, self.zesty_ocata,
-         self.xenial_pike, self.artful_pike) = range(11)
+         self.xenial_pike, self.artful_pike, self.xenial_queens,
+         self.bionic_queens,) = range(13)
 
         releases = {
             ('trusty', None): self.trusty_icehouse,
@@ -274,9 +282,11 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
             ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
             ('xenial', 'cloud:xenial-pike'): self.xenial_pike,
+            ('xenial', 'cloud:xenial-queens'): self.xenial_queens,
             ('yakkety', None): self.yakkety_newton,
             ('zesty', None): self.zesty_ocata,
             ('artful', None): self.artful_pike,
+            ('bionic', None): self.bionic_queens,
         }
         return releases[(self.series, self.openstack)]
 
@@ -291,6 +301,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('yakkety', 'newton'),
             ('zesty', 'ocata'),
             ('artful', 'pike'),
+            ('bionic', 'queens'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]
@@ -303,8 +314,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         test scenario, based on OpenStack release and whether ceph radosgw
         is flagged as present or not."""
 
-        if self._get_openstack_release() <= self.trusty_juno:
+        if self._get_openstack_release() == self.trusty_icehouse:
-            # Juno or earlier
+            # Icehouse
             pools = [
                 'data',
                 'metadata',
@@ -312,7 +323,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                 'cinder-ceph',
                 'glance'
             ]
-        elif (self.trust_kilo <= self._get_openstack_release() <=
+        elif (self.trusty_kilo <= self._get_openstack_release() <=
               self.zesty_ocata):
             # Kilo through Ocata
             pools = [

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -43,7 +43,6 @@ import swiftclient
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
-from charmhelpers.core.decorators import retry_on_exception
 from charmhelpers.core.host import CompareHostReleases
 
 DEBUG = logging.DEBUG
@@ -311,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
-    @retry_on_exception(5, base_delay=10)
     def keystone_wait_for_propagation(self, sentry_relation_pairs,
                                       api_version):
         """Iterate over list of sentry and relation tuples and verify that
@@ -327,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils):
             rel = sentry.relation('identity-service',
                                   relation_name)
             self.log.debug('keystone relation data: {}'.format(rel))
-            if rel['api_version'] != str(api_version):
+            if rel.get('api_version') != str(api_version):
                 raise Exception("api_version not propagated through relation"
                                 " data yet ('{}' != '{}')."
                                 "".format(rel['api_version'], api_version))
@@ -349,6 +347,7 @@ class OpenStackAmuletUtils(AmuletUtils):
 
         config = {'preferred-api-version': api_version}
         deployment.d.configure('keystone', config)
+        deployment._auto_wait_for_status()
         self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
 
     def authenticate_cinder_admin(self, keystone_sentry, username,

tests/charmhelpers/contrib/openstack/utils.py

@@ -140,6 +140,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('yakkety', 'newton'),
     ('zesty', 'ocata'),
     ('artful', 'pike'),
+    ('bionic', 'queens'),
 ])
 
 
@@ -157,6 +158,7 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2016.2', 'newton'),
     ('2017.1', 'ocata'),
     ('2017.2', 'pike'),
+    ('2018.1', 'queens'),
 ])
 
 # The ugly duckling - must list releases oldest to newest
@@ -187,6 +189,8 @@ SWIFT_CODENAMES = OrderedDict([
         ['2.11.0', '2.12.0', '2.13.0']),
     ('pike',
         ['2.13.0', '2.15.0']),
+    ('queens',
+        ['2.16.0']),
 ])
 
 # >= Liberty version->codename mapping
@@ -412,6 +416,8 @@ def get_os_codename_package(package, fatal=True):
         cmd = ['snap', 'list', package]
         try:
             out = subprocess.check_output(cmd)
+            if six.PY3:
+                out = out.decode('UTF-8')
         except subprocess.CalledProcessError as e:
             return None
         lines = out.split('\n')

tests/charmhelpers/core/strutils.py

@@ -61,13 +61,19 @@ def bytes_from_string(value):
     if isinstance(value, six.string_types):
         value = six.text_type(value)
     else:
-        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+        msg = "Unable to interpret non-string value '%s' as bytes" % (value)
         raise ValueError(msg)
     matches = re.match("([0-9]+)([a-zA-Z]+)", value)
-    if not matches:
+    if matches:
-        msg = "Unable to interpret string value '%s' as bytes" % (value)
+        size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
-        raise ValueError(msg)
+    else:
-    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+        # Assume that value passed in is bytes
+        try:
+            size = int(value)
+        except ValueError:
+            msg = "Unable to interpret string value '%s' as bytes" % (value)
+            raise ValueError(msg)
+    return size
 
 
 class BasicStringComparator(object):