Enable Bionic as a gate test

Change bionic test from dev to gate for 18.05.

Change-Id: Id71a674e9c12efc1c37bf8b616d5ed84d2d2be7c

hooks/charmhelpers/contrib/network/ovs/__init__.py

@@ -175,3 +175,22 @@ def full_restart():
         service('start', 'openvswitch-force-reload-kmod')
     else:
         service('force-reload-kmod', 'openvswitch-switch')
+
+
+def enable_ipfix(bridge, target):
+    '''Enable IPfix on bridge to target.
+    :param bridge: Bridge to monitor
+    :param target: IPfix remote endpoint
+    '''
+    cmd = ['ovs-vsctl', 'set', 'Bridge', bridge, 'ipfix=@i', '--',
+           '--id=@i', 'create', 'IPFIX', 'targets="{}"'.format(target)]
+    log('Enabling IPfix on {}.'.format(bridge))
+    subprocess.check_call(cmd)
+
+
+def disable_ipfix(bridge):
+    '''Diable IPfix on target bridge.
+    :param bridge: Bridge to modify
+    '''
+    cmd = ['ovs-vsctl', 'clear', 'Bridge', bridge, 'ipfix']
+    subprocess.check_call(cmd)

hooks/charmhelpers/contrib/network/ufw.py

@@ -151,6 +151,29 @@ def enable(soft_fail=False):
         return True
 
 
+def reload():
+    """
+    Reload ufw
+
+    :returns: True if ufw is successfully enabled
+    """
+    output = subprocess.check_output(['ufw', 'reload'],
+                                     universal_newlines=True,
+                                     env={'LANG': 'en_US',
+                                          'PATH': os.environ['PATH']})
+
+    m = re.findall('^Firewall reloaded\n',
+                   output, re.M)
+    hookenv.log(output, level='DEBUG')
+
+    if len(m) == 0:
+        hookenv.log("ufw couldn't be reloaded", level='WARN')
+        return False
+    else:
+        hookenv.log("ufw reloaded", level='INFO')
+        return True
+
+
 def disable():
     """
     Disable ufw

hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications

@@ -5,4 +5,7 @@ transport_url = {{ transport_url }}
 {% if notification_topics -%}
 topics = {{ notification_topics }}
 {% endif -%}
+{% if notification_format -%}
+notification_format = {{ notification_format }}
+{% endif -%}
 {% endif -%}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -306,7 +306,7 @@ def get_os_codename_install_source(src):
 
     if src.startswith('cloud:'):
         ca_rel = src.split(':')[1]
-        ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0]
+        ca_rel = ca_rel.split('-')[1].split('/')[0]
         return ca_rel
 
     # Best guess match based on deb string provided

hooks/charmhelpers/contrib/openstack/vaultlocker.py

@@ -21,6 +21,7 @@ import charmhelpers.contrib.openstack.context as context
 import charmhelpers.core.hookenv as hookenv
 import charmhelpers.core.host as host
 import charmhelpers.core.templating as templating
+import charmhelpers.core.unitdata as unitdata
 
 VAULTLOCKER_BACKEND = 'charm-vaultlocker'
 
@@ -36,30 +37,39 @@ class VaultKVContext(context.OSContextGenerator):
         )
 
     def __call__(self):
+        db = unitdata.kv()
+        last_token = db.get('last-token')
+        secret_id = db.get('secret-id')
         for relation_id in hookenv.relation_ids(self.interfaces[0]):
             for unit in hookenv.related_units(relation_id):
-                vault_url = hookenv.relation_get(
-                    'vault_url',
-                    unit=unit,
-                    rid=relation_id
-                )
-                role_id = hookenv.relation_get(
-                    '{}_role_id'.format(hookenv.local_unit()),
-                    unit=unit,
-                    rid=relation_id
-                )
+                data = hookenv.relation_get(unit=unit,
+                                            rid=relation_id)
+                vault_url = data.get('vault_url')
+                role_id = data.get('{}_role_id'.format(hookenv.local_unit()))
+                token = data.get('{}_token'.format(hookenv.local_unit()))
+
+                if all([vault_url, role_id, token]):
+                    token = json.loads(token)
+                    vault_url = json.loads(vault_url)
+
+                    # Tokens may change when secret_id's are being
+                    # reissued - if so use token to get new secret_id
+                    if token != last_token:
+                        secret_id = retrieve_secret_id(
+                            url=vault_url,
+                            token=token
+                        )
+                        db.set('secret-id', secret_id)
+                        db.set('last-token', token)
+                        db.flush()
 
-                if vault_url and role_id:
                     ctxt = {
-                        'vault_url': json.loads(vault_url),
+                        'vault_url': vault_url,
                         'role_id': json.loads(role_id),
+                        'secret_id': secret_id,
                         'secret_backend': self.secret_backend,
                     }
-                    vault_ca = hookenv.relation_get(
-                        'vault_ca',
-                        unit=unit,
-                        rid=relation_id
-                    )
+                    vault_ca = data.get('vault_ca')
                     if vault_ca:
                         ctxt['vault_ca'] = json.loads(vault_ca)
                     self.complete = True
@@ -96,3 +106,21 @@ def vault_relation_complete(backend=None):
     vault_kv = VaultKVContext(secret_backend=backend or VAULTLOCKER_BACKEND)
     vault_kv()
     return vault_kv.complete
+
+
+# TODO: contrib a high level unwrap method to hvac that works
+def retrieve_secret_id(url, token):
+    """Retrieve a response-wrapped secret_id from Vault
+
+    :param url: URL to Vault Server
+    :ptype url: str
+    :param token: One shot Token to use
+    :ptype token: str
+    :returns: secret_id to use for Vault Access
+    :rtype: str"""
+    import hvac
+    client = hvac.Client(url=url, token=token)
+    response = client._post('/v1/sys/wrapping/unwrap')
+    if response.status_code == 200:
+        data = response.json()
+        return data['data']['secret_id']

hooks/charmhelpers/core/hookenv.py

@@ -290,7 +290,7 @@ class Config(dict):
         self.implicit_save = True
         self._prev_dict = None
         self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
-        if os.path.exists(self.path):
+        if os.path.exists(self.path) and os.stat(self.path).st_size:
             self.load_previous()
         atexit(self._implicit_save)
 
@@ -310,7 +310,11 @@ class Config(dict):
         """
         self.path = path or self.path
         with open(self.path) as f:
-            self._prev_dict = json.load(f)
+            try:
+                self._prev_dict = json.load(f)
+            except ValueError as e:
+                log('Unable to parse previous config data - {}'.format(str(e)),
+                    level=ERROR)
         for k, v in copy.deepcopy(self._prev_dict).items():
             if k not in self:
                 self[k] = v

hooks/charmhelpers/core/sysctl.py

@@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
 def create(sysctl_dict, sysctl_file):
     """Creates a sysctl.conf file from a YAML associative array
 
-    :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
+    :param sysctl_dict: a dict or YAML-formatted string of sysctl
+                        options eg "{ 'kernel.max_pid': 1337 }"
     :type sysctl_dict: str
     :param sysctl_file: path to the sysctl file to be saved
     :type sysctl_file: str or unicode
     :returns: None
     """
-    try:
-        sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
-    except yaml.YAMLError:
-        log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
-            level=ERROR)
-        return
+    if type(sysctl_dict) is not dict:
+        try:
+            sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
+        except yaml.YAMLError:
+            log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
+                level=ERROR)
+            return
+    else:
+        sysctl_dict_parsed = sysctl_dict
 
     with open(sysctl_file, "w") as fd:
         for key, value in sysctl_dict_parsed.items():

tests/charmhelpers/contrib/openstack/utils.py

@@ -306,7 +306,7 @@ def get_os_codename_install_source(src):
 
     if src.startswith('cloud:'):
         ca_rel = src.split(':')[1]
-        ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0]
+        ca_rel = ca_rel.split('-')[1].split('/')[0]
         return ca_rel
 
     # Best guess match based on deb string provided

tests/charmhelpers/core/hookenv.py

@@ -290,7 +290,7 @@ class Config(dict):
         self.implicit_save = True
         self._prev_dict = None
         self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
-        if os.path.exists(self.path):
+        if os.path.exists(self.path) and os.stat(self.path).st_size:
             self.load_previous()
         atexit(self._implicit_save)
 
@@ -310,7 +310,11 @@ class Config(dict):
         """
         self.path = path or self.path
         with open(self.path) as f:
-            self._prev_dict = json.load(f)
+            try:
+                self._prev_dict = json.load(f)
+            except ValueError as e:
+                log('Unable to parse previous config data - {}'.format(str(e)),
+                    level=ERROR)
         for k, v in copy.deepcopy(self._prev_dict).items():
             if k not in self:
                 self[k] = v

tests/charmhelpers/core/sysctl.py

@@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
 def create(sysctl_dict, sysctl_file):
     """Creates a sysctl.conf file from a YAML associative array
 
-    :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
+    :param sysctl_dict: a dict or YAML-formatted string of sysctl
+                        options eg "{ 'kernel.max_pid': 1337 }"
     :type sysctl_dict: str
     :param sysctl_file: path to the sysctl file to be saved
     :type sysctl_file: str or unicode
     :returns: None
     """
-    try:
-        sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
-    except yaml.YAMLError:
-        log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
-            level=ERROR)
-        return
+    if type(sysctl_dict) is not dict:
+        try:
+            sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
+        except yaml.YAMLError:
+            log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
+                level=ERROR)
+            return
+    else:
+        sysctl_dict_parsed = sysctl_dict
 
     with open(sysctl_file, "w") as fd:
         for key, value in sysctl_dict_parsed.items():

tox.ini

@@ -66,7 +66,7 @@ basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
-    bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-pike --no-destroy
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy
 
 [testenv:func27-dfs]
 # Charm Functional Test