py3: Switch to Python 3 for OpenStack Rocky

Switch to using python3-{nova,neutron} at OpenStack Rocky; purge
previously installed Python 2 modules on upgrade.

Switch default test target to bionic-rocky.

Change-Id: I0a2b48bfc8c950efea3e83c74ec3eb6dd43796c4

hooks/charmhelpers/contrib/hahelpers/apache.py

@@ -23,8 +23,8 @@
 #
 
 import os
-import subprocess
 
+from charmhelpers.core import host
 from charmhelpers.core.hookenv import (
     config as config_get,
     relation_get,
@@ -83,14 +83,4 @@ def retrieve_ca_cert(cert_file):
 
 
 def install_ca_cert(ca_cert):
-    if ca_cert:
+    host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')
-        cert_file = ('/usr/local/share/ca-certificates/'
-                     'keystone_juju_ca_cert.crt')
-        old_cert = retrieve_ca_cert(cert_file)
-        if old_cert and old_cert == ca_cert:
-            log("CA cert is the same as installed version", level=INFO)
-        else:
-            log("Installing new CA cert", level=INFO)
-            with open(cert_file, 'wb') as crt:
-                crt.write(ca_cert)
-            subprocess.check_call(['update-ca-certificates', '--fresh'])

hooks/charmhelpers/contrib/openstack/amulet/utils.py

@@ -680,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils):
             nova.flavors.create(name, ram, vcpus, disk, flavorid,
                                 ephemeral, swap, rxtx_factor, is_public)
 
-    def create_cirros_image(self, glance, image_name):
+    def glance_create_image(self, glance, image_name, image_url,
-        """Download the latest cirros image and upload it to glance,
+                            download_dir='tests',
-        validate and return a resource pointer.
+                            hypervisor_type='qemu',
+                            disk_format='qcow2',
+                            architecture='x86_64',
+                            container_format='bare'):
+        """Download an image and upload it to glance, validate its status
+        and return an image object pointer. KVM defaults, can override for
+        LXD.
 
-        :param glance: pointer to authenticated glance connection
+        :param glance: pointer to authenticated glance api connection
         :param image_name: display name for new image
+        :param image_url: url to retrieve
+        :param download_dir: directory to store downloaded image file
+        :param hypervisor_type: glance image hypervisor property
+        :param disk_format: glance image disk format
+        :param architecture: glance image architecture property
+        :param container_format: glance image container format
         :returns: glance image pointer
         """
-        self.log.debug('Creating glance cirros image '
+        self.log.debug('Creating glance image ({}) from '
-                       '({})...'.format(image_name))
+                       '{}...'.format(image_name, image_url))
 
-        # Download cirros image
+        # Download image
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -700,31 +712,33 @@ class OpenStackAmuletUtils(AmuletUtils):
         else:
             opener = urllib.FancyURLopener()
 
-        f = opener.open('http://download.cirros-cloud.net/version/released')
+        abs_file_name = os.path.join(download_dir, image_name)
-        version = f.read().strip()
+        if not os.path.exists(abs_file_name):
-        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
+            opener.retrieve(image_url, abs_file_name)
-        local_path = os.path.join('tests', cirros_img)
-
-        if not os.path.exists(local_path):
-            cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
-                                                  version, cirros_img)
-            opener.retrieve(cirros_url, local_path)
-        f.close()
 
+        # Create glance image
+        glance_properties = {
+            'architecture': architecture,
+            'hypervisor_type': hypervisor_type
+        }
         # Create glance image
         if float(glance.version) < 2.0:
-            with open(local_path) as fimage:
+            with open(abs_file_name) as f:
-                image = glance.images.create(name=image_name, is_public=True,
+                image = glance.images.create(
-                                             disk_format='qcow2',
+                    name=image_name,
-                                             container_format='bare',
+                    is_public=True,
-                                             data=fimage)
+                    disk_format=disk_format,
+                    container_format=container_format,
+                    properties=glance_properties,
+                    data=f)
         else:
             image = glance.images.create(
                 name=image_name,
-                disk_format="qcow2",
                 visibility="public",
-                container_format="bare")
+                disk_format=disk_format,
-            glance.images.upload(image.id, open(local_path, 'rb'))
+                container_format=container_format)
+            glance.images.upload(image.id, open(abs_file_name, 'rb'))
+            glance.images.update(image.id, **glance_properties)
 
         # Wait for image to reach active status
         img_id = image.id
@@ -753,15 +767,49 @@ class OpenStackAmuletUtils(AmuletUtils):
                         val_img_stat, val_img_cfmt, val_img_dfmt))
 
         if val_img_name == image_name and val_img_stat == 'active' \
-                and val_img_pub is True and val_img_cfmt == 'bare' \
+                and val_img_pub is True and val_img_cfmt == container_format \
-                and val_img_dfmt == 'qcow2':
+                and val_img_dfmt == disk_format:
             self.log.debug(msg_attr)
         else:
-            msg = ('Volume validation failed, {}'.format(msg_attr))
+            msg = ('Image validation failed, {}'.format(msg_attr))
             amulet.raise_status(amulet.FAIL, msg=msg)
 
         return image
 
+    def create_cirros_image(self, glance, image_name):
+        """Download the latest cirros image and upload it to glance,
+        validate and return a resource pointer.
+
+        :param glance: pointer to authenticated glance connection
+        :param image_name: display name for new image
+        :returns: glance image pointer
+        """
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'glance_create_image instead of '
+                      'create_cirros_image.')
+
+        self.log.debug('Creating glance cirros image '
+                       '({})...'.format(image_name))
+
+        # Get cirros image URL
+        http_proxy = os.getenv('AMULET_HTTP_PROXY')
+        self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
+        if http_proxy:
+            proxies = {'http': http_proxy}
+            opener = urllib.FancyURLopener(proxies)
+        else:
+            opener = urllib.FancyURLopener()
+
+        f = opener.open('http://download.cirros-cloud.net/version/released')
+        version = f.read().strip()
+        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
+        cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
+                                              version, cirros_img)
+        f.close()
+
+        return self.glance_create_image(glance, image_name, cirros_url)
+
     def delete_image(self, glance, image):
         """Delete the specified image."""
 

hooks/charmhelpers/contrib/openstack/context.py

@@ -1523,10 +1523,6 @@ class NeutronAPIContext(OSContextGenerator):
                 'rel_key': 'enable-nsg-logging',
                 'default': False,
             },
-            'nsg_log_output_base': {
-                'rel_key': 'nsg-log-output-base',
-                'default': None,
-            },
         }
         ctxt = self.get_neutron_options({})
         for rid in relation_ids('neutron-plugin-api'):

hooks/charmhelpers/core/host.py

@@ -34,7 +34,7 @@ import six
 
 from contextlib import contextmanager
 from collections import OrderedDict
-from .hookenv import log, DEBUG, local_unit
+from .hookenv import log, INFO, DEBUG, local_unit, charm_name
 from .fstab import Fstab
 from charmhelpers.osplatform import get_platform
 
@@ -1040,3 +1040,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False):
         return modulo * wait
     else:
         return calculated_wait_time
+
+
+def install_ca_cert(ca_cert, name=None):
+    """
+    Install the given cert as a trusted CA.
+
+    The ``name`` is the stem of the filename where the cert is written, and if
+    not provided, it will default to ``juju-{charm_name}``.
+
+    If the cert is empty or None, or is unchanged, nothing is done.
+    """
+    if not ca_cert:
+        return
+    if not isinstance(ca_cert, bytes):
+        ca_cert = ca_cert.encode('utf8')
+    if not name:
+        name = 'juju-{}'.format(charm_name())
+    cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
+    new_hash = hashlib.md5(ca_cert).hexdigest()
+    if file_hash(cert_file) == new_hash:
+        return
+    log("Installing new CA cert at: {}".format(cert_file), level=INFO)
+    write_file(cert_file, ca_cert)
+    subprocess.check_call(['update-ca-certificates', '--fresh'])

hooks/neutron_utils.py

@@ -25,6 +25,9 @@ from charmhelpers.fetch import (
     apt_upgrade,
     apt_update,
     apt_install,
+    apt_autoremove,
+    apt_purge,
+    filter_missing_packages,
 )
 from charmhelpers.contrib.network.ovs import (
     add_bridge,
@@ -176,6 +179,21 @@ GATEWAY_PKGS = {
     ],
 }
 
+PURGE_PACKAGES = [
+    'python-mysqldb',
+    'python-psycopg2',
+    'python-oslo.config',
+    'python-nova',
+    'python-neutron',
+    'python-neutron-fwaas',
+]
+
+PY3_PACKAGES = [
+    'python3-nova',
+    'python3-neutron',
+    'python3-neutron-fwaas',
+]
+
 EARLY_PACKAGES = {
     OVS: ['openvswitch-datapath-dkms'],
     NSX: [],
@@ -253,9 +271,21 @@ def get_packages():
             packages.append('neutron-lbaasv2-agent')
     packages.extend(determine_l3ha_packages())
 
+    if cmp_os_source >= 'rocky':
+        packages = [p for p in packages if not p.startswith('python-')]
+        packages.extend(PY3_PACKAGES)
+
     return packages
 
 
+def get_purge_packages():
+    '''Return a list of packages to purge for the current OS release'''
+    cmp_os_source = CompareOpenStackReleases(os_release('neutron-common'))
+    if cmp_os_source >= 'rocky':
+        return PURGE_PACKAGES
+    return []
+
+
 def determine_l3ha_packages():
     if use_l3ha():
         return L3HA_PACKAGES
@@ -706,12 +736,18 @@ def do_openstack_upgrade(configs):
     apt_update(fatal=True)
     apt_upgrade(options=dpkg_opts,
                 fatal=True, dist=True)
+
     # The cached version of os_release will now be invalid as the pkg version
     # should have changed during the upgrade.
     reset_os_release()
     apt_install(get_early_packages(), fatal=True)
     apt_install(get_packages(), fatal=True)
 
+    installed_packages = filter_missing_packages(get_purge_packages())
+    if installed_packages:
+        apt_purge(installed_packages, fatal=True)
+        apt_autoremove(purge=True, fatal=True)
+
 
 def configure_ovs():
     if config('plugin') in [OVS, OVS_ODL]:

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -680,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils):
             nova.flavors.create(name, ram, vcpus, disk, flavorid,
                                 ephemeral, swap, rxtx_factor, is_public)
 
-    def create_cirros_image(self, glance, image_name):
+    def glance_create_image(self, glance, image_name, image_url,
-        """Download the latest cirros image and upload it to glance,
+                            download_dir='tests',
-        validate and return a resource pointer.
+                            hypervisor_type='qemu',
+                            disk_format='qcow2',
+                            architecture='x86_64',
+                            container_format='bare'):
+        """Download an image and upload it to glance, validate its status
+        and return an image object pointer. KVM defaults, can override for
+        LXD.
 
-        :param glance: pointer to authenticated glance connection
+        :param glance: pointer to authenticated glance api connection
         :param image_name: display name for new image
+        :param image_url: url to retrieve
+        :param download_dir: directory to store downloaded image file
+        :param hypervisor_type: glance image hypervisor property
+        :param disk_format: glance image disk format
+        :param architecture: glance image architecture property
+        :param container_format: glance image container format
         :returns: glance image pointer
         """
-        self.log.debug('Creating glance cirros image '
+        self.log.debug('Creating glance image ({}) from '
-                       '({})...'.format(image_name))
+                       '{}...'.format(image_name, image_url))
 
-        # Download cirros image
+        # Download image
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -700,31 +712,33 @@ class OpenStackAmuletUtils(AmuletUtils):
         else:
             opener = urllib.FancyURLopener()
 
-        f = opener.open('http://download.cirros-cloud.net/version/released')
+        abs_file_name = os.path.join(download_dir, image_name)
-        version = f.read().strip()
+        if not os.path.exists(abs_file_name):
-        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
+            opener.retrieve(image_url, abs_file_name)
-        local_path = os.path.join('tests', cirros_img)
-
-        if not os.path.exists(local_path):
-            cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
-                                                  version, cirros_img)
-            opener.retrieve(cirros_url, local_path)
-        f.close()
 
+        # Create glance image
+        glance_properties = {
+            'architecture': architecture,
+            'hypervisor_type': hypervisor_type
+        }
         # Create glance image
         if float(glance.version) < 2.0:
-            with open(local_path) as fimage:
+            with open(abs_file_name) as f:
-                image = glance.images.create(name=image_name, is_public=True,
+                image = glance.images.create(
-                                             disk_format='qcow2',
+                    name=image_name,
-                                             container_format='bare',
+                    is_public=True,
-                                             data=fimage)
+                    disk_format=disk_format,
+                    container_format=container_format,
+                    properties=glance_properties,
+                    data=f)
         else:
             image = glance.images.create(
                 name=image_name,
-                disk_format="qcow2",
                 visibility="public",
-                container_format="bare")
+                disk_format=disk_format,
-            glance.images.upload(image.id, open(local_path, 'rb'))
+                container_format=container_format)
+            glance.images.upload(image.id, open(abs_file_name, 'rb'))
+            glance.images.update(image.id, **glance_properties)
 
         # Wait for image to reach active status
         img_id = image.id
@@ -753,15 +767,49 @@ class OpenStackAmuletUtils(AmuletUtils):
                         val_img_stat, val_img_cfmt, val_img_dfmt))
 
         if val_img_name == image_name and val_img_stat == 'active' \
-                and val_img_pub is True and val_img_cfmt == 'bare' \
+                and val_img_pub is True and val_img_cfmt == container_format \
-                and val_img_dfmt == 'qcow2':
+                and val_img_dfmt == disk_format:
             self.log.debug(msg_attr)
         else:
-            msg = ('Volume validation failed, {}'.format(msg_attr))
+            msg = ('Image validation failed, {}'.format(msg_attr))
             amulet.raise_status(amulet.FAIL, msg=msg)
 
         return image
 
+    def create_cirros_image(self, glance, image_name):
+        """Download the latest cirros image and upload it to glance,
+        validate and return a resource pointer.
+
+        :param glance: pointer to authenticated glance connection
+        :param image_name: display name for new image
+        :returns: glance image pointer
+        """
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'glance_create_image instead of '
+                      'create_cirros_image.')
+
+        self.log.debug('Creating glance cirros image '
+                       '({})...'.format(image_name))
+
+        # Get cirros image URL
+        http_proxy = os.getenv('AMULET_HTTP_PROXY')
+        self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
+        if http_proxy:
+            proxies = {'http': http_proxy}
+            opener = urllib.FancyURLopener(proxies)
+        else:
+            opener = urllib.FancyURLopener()
+
+        f = opener.open('http://download.cirros-cloud.net/version/released')
+        version = f.read().strip()
+        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
+        cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
+                                              version, cirros_img)
+        f.close()
+
+        return self.glance_create_image(glance, image_name, cirros_url)
+
     def delete_image(self, glance, image):
         """Delete the specified image."""
 

tests/charmhelpers/core/host.py

@@ -34,7 +34,7 @@ import six
 
 from contextlib import contextmanager
 from collections import OrderedDict
-from .hookenv import log, DEBUG, local_unit
+from .hookenv import log, INFO, DEBUG, local_unit, charm_name
 from .fstab import Fstab
 from charmhelpers.osplatform import get_platform
 
@@ -1040,3 +1040,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False):
         return modulo * wait
     else:
         return calculated_wait_time
+
+
+def install_ca_cert(ca_cert, name=None):
+    """
+    Install the given cert as a trusted CA.
+
+    The ``name`` is the stem of the filename where the cert is written, and if
+    not provided, it will default to ``juju-{charm_name}``.
+
+    If the cert is empty or None, or is unchanged, nothing is done.
+    """
+    if not ca_cert:
+        return
+    if not isinstance(ca_cert, bytes):
+        ca_cert = ca_cert.encode('utf8')
+    if not name:
+        name = 'juju-{}'.format(charm_name())
+    cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
+    new_hash = hashlib.md5(ca_cert).hexdigest()
+    if file_hash(cert_file) == new_hash:
+        return
+    log("Installing new CA cert at: {}".format(cert_file), level=INFO)
+    write_file(cert_file, ca_cert)
+    subprocess.check_call(['update-ca-certificates', '--fresh'])

tox.ini

@@ -66,7 +66,7 @@ basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
-    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-rocky --no-destroy
 
 [testenv:func27-dfs]
 # Charm Functional Test

unit_tests/test_neutron_utils.py

@@ -21,6 +21,9 @@ TO_PATCH = [
     'apt_update',
     'apt_upgrade',
     'apt_install',
+    'apt_autoremove',
+    'apt_purge',
+    'filter_missing_packages',
     'configure_installation_source',
     'log',
     'add_bridge',
@@ -148,6 +151,33 @@ class TestNeutronUtils(CharmTestCase):
         self.assertFalse('python-mysqldb' in packages)
         self.assertTrue('python-pymysql' in packages)
 
+    def test_get_packages_ovs_rocky(self):
+        self.config.return_value = 'ovs'
+        self.os_release.return_value = 'rocky'
+        packages = neutron_utils.get_packages()
+        self.assertEqual(
+            len(packages),
+            len([p for p in packages if not p.startswith('python-')])
+        )
+
+    def test_get_purge_packages_ovs(self):
+        self.config.return_value = 'ovs'
+        self.os_release.return_value = 'queens'
+        self.assertEqual([], neutron_utils.get_purge_packages())
+
+    def test_get_purge_packages_ovs_rocky(self):
+        self.config.return_value = 'ovs'
+        self.os_release.return_value = 'rocky'
+        self.assertEqual([
+            'python-mysqldb',
+            'python-psycopg2',
+            'python-oslo.config',
+            'python-nova',
+            'python-neutron',
+            'python-neutron-fwaas'],
+            neutron_utils.get_purge_packages()
+        )
+
     def test_get_packages_ovsodl_icehouse(self):
         self.config.return_value = 'ovs-odl'
         self.os_release.return_value = 'icehouse'
@@ -297,6 +327,7 @@ class TestNeutronUtils(CharmTestCase):
         self.test_config.set('plugin', 'ovs')
         self.get_os_codename_install_source.return_value = 'havana'
         self.os_release.return_value = 'havana'
+        self.filter_missing_packages.side_effect = lambda x: x
         neutron_utils.do_openstack_upgrade(mock_configs)
         mock_register_configs.assert_called_with('havana')
         self.assertTrue(self.log.called)
@@ -311,6 +342,42 @@ class TestNeutronUtils(CharmTestCase):
         self.configure_installation_source.assert_called_with(
             'cloud:precise-havana'
         )
+        self.apt_purge.assert_not_called()
+        self.apt_autoremove.assert_not_called()
+
+    @patch.object(neutron_utils, 'register_configs')
+    @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer')
+    def test_do_openstack_upgrade_rocky(self, mock_renderer,
+                                        mock_register_configs):
+        mock_configs = MagicMock()
+        mock_register_configs.return_value = mock_configs
+        self.config.side_effect = self.test_config.get
+        self.is_relation_made.return_value = False
+        self.test_config.set('openstack-origin', 'cloud:bionic-rocky')
+        self.test_config.set('plugin', 'ovs')
+        self.get_os_codename_install_source.return_value = 'rocky'
+        self.os_release.return_value = 'rocky'
+        self.filter_missing_packages.side_effect = lambda x: x
+        neutron_utils.do_openstack_upgrade(mock_configs)
+        mock_register_configs.assert_called_with('rocky')
+        self.assertTrue(self.log.called)
+        self.apt_update.assert_called_with(fatal=True)
+        dpkg_opts = [
+            '--option', 'Dpkg::Options::=--force-confnew',
+            '--option', 'Dpkg::Options::=--force-confdef',
+        ]
+        self.apt_upgrade.assert_called_with(
+            options=dpkg_opts, fatal=True, dist=True
+        )
+        self.apt_purge.assert_called_with(
+            neutron_utils.PURGE_PACKAGES, fatal=True
+        )
+        self.apt_autoremove.assert_called_with(
+            purge=True, fatal=True
+        )
+        self.configure_installation_source.assert_called_with(
+            'cloud:bionic-rocky'
+        )
 
     @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer')
     def test_register_configs_ovs(self, mock_renderer):