From ec4a06057f6fa29c19270a424df15fe3b9ae2388 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 21 Nov 2017 11:49:46 +0100 Subject: [PATCH] charmhelpers sync to fix py3 contexts Change-Id: Ibf6c49aef5127eb824ea671089cb56cf7789dc32 --- .../charmhelpers/contrib/hahelpers/apache.py | 2 +- .../contrib/openstack/amulet/deployment.py | 19 +++++++++++++++---- .../contrib/openstack/amulet/utils.py | 5 ++--- .../charmhelpers/contrib/openstack/context.py | 8 ++++---- .../contrib/openstack/templates/ceph.conf | 4 ++-- hooks/charmhelpers/contrib/openstack/utils.py | 6 ++++++ hooks/charmhelpers/core/strutils.py | 16 +++++++++++----- .../contrib/openstack/amulet/deployment.py | 19 +++++++++++++++---- .../contrib/openstack/amulet/utils.py | 5 ++--- tests/charmhelpers/contrib/openstack/utils.py | 6 ++++++ tests/charmhelpers/core/strutils.py | 16 +++++++++++----- 11 files changed, 75 insertions(+), 31 deletions(-) diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index d0c69942..22acb683 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -90,6 +90,6 @@ def install_ca_cert(ca_cert): log("CA cert is the same as installed version", level=INFO) else: log("Installing new CA cert", level=INFO) - with open(cert_file, 'w') as crt: + with open(cert_file, 'wb') as crt: crt.write(ca_cert) subprocess.check_call(['update-ca-certificates', '--fresh']) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index aacab1f5..e37f2834 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -250,7 +250,14 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.log.debug('Waiting up to {}s for extended status on services: ' '{}'.format(timeout, services)) service_messages = {service: message for service in services} + + # Check for idleness + self.d.sentry.wait() + # Check for error states and bail early + self.d.sentry.wait_for_status(self.d.juju_env, services) + # Check for ready messages self.d.sentry.wait_for_messages(service_messages, timeout=timeout) + self.log.info('OK') def _get_openstack_release(self): @@ -263,7 +270,8 @@ class OpenStackAmuletDeployment(AmuletDeployment): (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty, self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton, self.yakkety_newton, self.xenial_ocata, self.zesty_ocata, - self.xenial_pike, self.artful_pike) = range(11) + self.xenial_pike, self.artful_pike, self.xenial_queens, + self.bionic_queens,) = range(13) releases = { ('trusty', None): self.trusty_icehouse, @@ -274,9 +282,11 @@ class OpenStackAmuletDeployment(AmuletDeployment): ('xenial', 'cloud:xenial-newton'): self.xenial_newton, ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata, ('xenial', 'cloud:xenial-pike'): self.xenial_pike, + ('xenial', 'cloud:xenial-queens'): self.xenial_queens, ('yakkety', None): self.yakkety_newton, ('zesty', None): self.zesty_ocata, ('artful', None): self.artful_pike, + ('bionic', None): self.bionic_queens, } return releases[(self.series, self.openstack)] @@ -291,6 +301,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): ('yakkety', 'newton'), ('zesty', 'ocata'), ('artful', 'pike'), + ('bionic', 'queens'), ]) if self.openstack: os_origin = self.openstack.split(':')[1] @@ -303,8 +314,8 @@ class OpenStackAmuletDeployment(AmuletDeployment): test scenario, based on OpenStack release and whether ceph radosgw is flagged as present or not.""" - if self._get_openstack_release() <= self.trusty_juno: - # Juno or earlier + if self._get_openstack_release() == self.trusty_icehouse: + # Icehouse pools = [ 'data', 'metadata', @@ -312,7 +323,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): 'cinder-ceph', 'glance' ] - elif (self.trust_kilo <= self._get_openstack_release() <= + elif (self.trusty_kilo <= self._get_openstack_release() <= self.zesty_ocata): # Kilo through Ocata pools = [ diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index c24be8f1..b71b2b19 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -43,7 +43,6 @@ import swiftclient from charmhelpers.contrib.amulet.utils import ( AmuletUtils ) -from charmhelpers.core.decorators import retry_on_exception from charmhelpers.core.host import CompareHostReleases DEBUG = logging.DEBUG @@ -311,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Checking if tenant exists ({})...'.format(tenant)) return tenant in [t.name for t in keystone.tenants.list()] - @retry_on_exception(5, base_delay=10) def keystone_wait_for_propagation(self, sentry_relation_pairs, api_version): """Iterate over list of sentry and relation tuples and verify that @@ -327,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils): rel = sentry.relation('identity-service', relation_name) self.log.debug('keystone relation data: {}'.format(rel)) - if rel['api_version'] != str(api_version): + if rel.get('api_version') != str(api_version): raise Exception("api_version not propagated through relation" " data yet ('{}' != '{}')." "".format(rel['api_version'], api_version)) @@ -349,6 +347,7 @@ class OpenStackAmuletUtils(AmuletUtils): config = {'preferred-api-version': api_version} deployment.d.configure('keystone', config) + deployment._auto_wait_for_status() self.keystone_wait_for_propagation(sentry_relation_pairs, api_version) def authenticate_cinder_admin(self, keystone_sentry, username, diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index ece75df8..e6c0e9fe 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -293,7 +293,7 @@ class PostgresqlDBContext(OSContextGenerator): def db_ssl(rdata, ctxt, ssl_dir): if 'ssl_ca' in rdata and ssl_dir: ca_path = os.path.join(ssl_dir, 'db-client.ca') - with open(ca_path, 'w') as fh: + with open(ca_path, 'wb') as fh: fh.write(b64decode(rdata['ssl_ca'])) ctxt['database_ssl_ca'] = ca_path @@ -308,12 +308,12 @@ def db_ssl(rdata, ctxt, ssl_dir): log("Waiting 1m for ssl client cert validity", level=INFO) time.sleep(60) - with open(cert_path, 'w') as fh: + with open(cert_path, 'wb') as fh: fh.write(b64decode(rdata['ssl_cert'])) ctxt['database_ssl_cert'] = cert_path key_path = os.path.join(ssl_dir, 'db-client.key') - with open(key_path, 'w') as fh: + with open(key_path, 'wb') as fh: fh.write(b64decode(rdata['ssl_key'])) ctxt['database_ssl_key'] = key_path @@ -459,7 +459,7 @@ class AMQPContext(OSContextGenerator): ca_path = os.path.join( self.ssl_dir, 'rabbit-client-ca.pem') - with open(ca_path, 'w') as fh: + with open(ca_path, 'wb') as fh: fh.write(b64decode(ctxt['rabbit_ssl_ca'])) ctxt['rabbit_ssl_ca'] = ca_path diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index ed5c4f10..a11ce8ab 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -18,7 +18,7 @@ rbd default features = {{ rbd_features }} [client] {% if rbd_client_cache_settings -%} -{% for key, value in rbd_client_cache_settings.iteritems() -%} +{% for key, value in rbd_client_cache_settings.items() -%} {{ key }} = {{ value }} {% endfor -%} -{%- endif %} \ No newline at end of file +{%- endif %} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index b073c77b..8a541d40 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -140,6 +140,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('yakkety', 'newton'), ('zesty', 'ocata'), ('artful', 'pike'), + ('bionic', 'queens'), ]) @@ -157,6 +158,7 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2016.2', 'newton'), ('2017.1', 'ocata'), ('2017.2', 'pike'), + ('2018.1', 'queens'), ]) # The ugly duckling - must list releases oldest to newest @@ -187,6 +189,8 @@ SWIFT_CODENAMES = OrderedDict([ ['2.11.0', '2.12.0', '2.13.0']), ('pike', ['2.13.0', '2.15.0']), + ('queens', + ['2.16.0']), ]) # >= Liberty version->codename mapping @@ -412,6 +416,8 @@ def get_os_codename_package(package, fatal=True): cmd = ['snap', 'list', package] try: out = subprocess.check_output(cmd) + if six.PY3: + out = out.decode('UTF-8') except subprocess.CalledProcessError as e: return None lines = out.split('\n') diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py index 685dabde..e8df0452 100644 --- a/hooks/charmhelpers/core/strutils.py +++ b/hooks/charmhelpers/core/strutils.py @@ -61,13 +61,19 @@ def bytes_from_string(value): if isinstance(value, six.string_types): value = six.text_type(value) else: - msg = "Unable to interpret non-string value '%s' as boolean" % (value) + msg = "Unable to interpret non-string value '%s' as bytes" % (value) raise ValueError(msg) matches = re.match("([0-9]+)([a-zA-Z]+)", value) - if not matches: - msg = "Unable to interpret string value '%s' as bytes" % (value) - raise ValueError(msg) - return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) + if matches: + size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) + else: + # Assume that value passed in is bytes + try: + size = int(value) + except ValueError: + msg = "Unable to interpret string value '%s' as bytes" % (value) + raise ValueError(msg) + return size class BasicStringComparator(object): diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index aacab1f5..e37f2834 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -250,7 +250,14 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.log.debug('Waiting up to {}s for extended status on services: ' '{}'.format(timeout, services)) service_messages = {service: message for service in services} + + # Check for idleness + self.d.sentry.wait() + # Check for error states and bail early + self.d.sentry.wait_for_status(self.d.juju_env, services) + # Check for ready messages self.d.sentry.wait_for_messages(service_messages, timeout=timeout) + self.log.info('OK') def _get_openstack_release(self): @@ -263,7 +270,8 @@ class OpenStackAmuletDeployment(AmuletDeployment): (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty, self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton, self.yakkety_newton, self.xenial_ocata, self.zesty_ocata, - self.xenial_pike, self.artful_pike) = range(11) + self.xenial_pike, self.artful_pike, self.xenial_queens, + self.bionic_queens,) = range(13) releases = { ('trusty', None): self.trusty_icehouse, @@ -274,9 +282,11 @@ class OpenStackAmuletDeployment(AmuletDeployment): ('xenial', 'cloud:xenial-newton'): self.xenial_newton, ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata, ('xenial', 'cloud:xenial-pike'): self.xenial_pike, + ('xenial', 'cloud:xenial-queens'): self.xenial_queens, ('yakkety', None): self.yakkety_newton, ('zesty', None): self.zesty_ocata, ('artful', None): self.artful_pike, + ('bionic', None): self.bionic_queens, } return releases[(self.series, self.openstack)] @@ -291,6 +301,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): ('yakkety', 'newton'), ('zesty', 'ocata'), ('artful', 'pike'), + ('bionic', 'queens'), ]) if self.openstack: os_origin = self.openstack.split(':')[1] @@ -303,8 +314,8 @@ class OpenStackAmuletDeployment(AmuletDeployment): test scenario, based on OpenStack release and whether ceph radosgw is flagged as present or not.""" - if self._get_openstack_release() <= self.trusty_juno: - # Juno or earlier + if self._get_openstack_release() == self.trusty_icehouse: + # Icehouse pools = [ 'data', 'metadata', @@ -312,7 +323,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): 'cinder-ceph', 'glance' ] - elif (self.trust_kilo <= self._get_openstack_release() <= + elif (self.trusty_kilo <= self._get_openstack_release() <= self.zesty_ocata): # Kilo through Ocata pools = [ diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index c24be8f1..b71b2b19 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -43,7 +43,6 @@ import swiftclient from charmhelpers.contrib.amulet.utils import ( AmuletUtils ) -from charmhelpers.core.decorators import retry_on_exception from charmhelpers.core.host import CompareHostReleases DEBUG = logging.DEBUG @@ -311,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('Checking if tenant exists ({})...'.format(tenant)) return tenant in [t.name for t in keystone.tenants.list()] - @retry_on_exception(5, base_delay=10) def keystone_wait_for_propagation(self, sentry_relation_pairs, api_version): """Iterate over list of sentry and relation tuples and verify that @@ -327,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils): rel = sentry.relation('identity-service', relation_name) self.log.debug('keystone relation data: {}'.format(rel)) - if rel['api_version'] != str(api_version): + if rel.get('api_version') != str(api_version): raise Exception("api_version not propagated through relation" " data yet ('{}' != '{}')." "".format(rel['api_version'], api_version)) @@ -349,6 +347,7 @@ class OpenStackAmuletUtils(AmuletUtils): config = {'preferred-api-version': api_version} deployment.d.configure('keystone', config) + deployment._auto_wait_for_status() self.keystone_wait_for_propagation(sentry_relation_pairs, api_version) def authenticate_cinder_admin(self, keystone_sentry, username, diff --git a/tests/charmhelpers/contrib/openstack/utils.py b/tests/charmhelpers/contrib/openstack/utils.py index b073c77b..8a541d40 100644 --- a/tests/charmhelpers/contrib/openstack/utils.py +++ b/tests/charmhelpers/contrib/openstack/utils.py @@ -140,6 +140,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('yakkety', 'newton'), ('zesty', 'ocata'), ('artful', 'pike'), + ('bionic', 'queens'), ]) @@ -157,6 +158,7 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2016.2', 'newton'), ('2017.1', 'ocata'), ('2017.2', 'pike'), + ('2018.1', 'queens'), ]) # The ugly duckling - must list releases oldest to newest @@ -187,6 +189,8 @@ SWIFT_CODENAMES = OrderedDict([ ['2.11.0', '2.12.0', '2.13.0']), ('pike', ['2.13.0', '2.15.0']), + ('queens', + ['2.16.0']), ]) # >= Liberty version->codename mapping @@ -412,6 +416,8 @@ def get_os_codename_package(package, fatal=True): cmd = ['snap', 'list', package] try: out = subprocess.check_output(cmd) + if six.PY3: + out = out.decode('UTF-8') except subprocess.CalledProcessError as e: return None lines = out.split('\n') diff --git a/tests/charmhelpers/core/strutils.py b/tests/charmhelpers/core/strutils.py index 685dabde..e8df0452 100644 --- a/tests/charmhelpers/core/strutils.py +++ b/tests/charmhelpers/core/strutils.py @@ -61,13 +61,19 @@ def bytes_from_string(value): if isinstance(value, six.string_types): value = six.text_type(value) else: - msg = "Unable to interpret non-string value '%s' as boolean" % (value) + msg = "Unable to interpret non-string value '%s' as bytes" % (value) raise ValueError(msg) matches = re.match("([0-9]+)([a-zA-Z]+)", value) - if not matches: - msg = "Unable to interpret string value '%s' as bytes" % (value) - raise ValueError(msg) - return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) + if matches: + size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) + else: + # Assume that value passed in is bytes + try: + size = int(value) + except ValueError: + msg = "Unable to interpret string value '%s' as bytes" % (value) + raise ValueError(msg) + return size class BasicStringComparator(object):