Updates for Ubuntu Yakkety

Yakkety swiches:

a) the name of the libvirt daemon from libvirt-bin -> libvirtd
b) the default libvirt user from libvirtd -> libvirt

Update contexts, restarts, templates and enable yakkety test
to deal with this change.

Resync charm-helpers to support enablement of yakkety amulet test.

Change-Id: I58eb3a5da53d4a12390968c835c0ff408a42d1b5

hooks/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -258,6 +258,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('vivid', 'kilo'),
             ('wily', 'liberty'),
             ('xenial', 'mitaka'),
+            ('yakkety', 'newton'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

hooks/nova_compute_context.py

@@ -137,10 +137,21 @@ class NovaComputeLibvirtContext(context.OSContextGenerator):
             # /etc/libvirt/libvirtd.conf (
             'listen_tls': 0
         }
-        if lsb_release()['DISTRIB_CODENAME'].lower() < "wily":
+        distro_codename = lsb_release()['DISTRIB_CODENAME'].lower()
+
+        # NOTE(jamespage): deal with switch to systemd
+        if distro_codename < "wily":
             ctxt['libvirtd_opts'] = '-d'
         else:
             ctxt['libvirtd_opts'] = ''
+
+        # NOTE(jamespage): deal with alignment with Debian in
+        #                  Ubuntu yakkety and beyond.
+        if distro_codename >= 'yakkety':
+            ctxt['libvirt_user'] = 'libvirt'
+        else:
+            ctxt['libvirt_user'] = 'libvirtd'
+
         # get the processor architecture to use in the nova.conf template
         ctxt['arch'] = platform.machine()
 

hooks/nova_compute_hooks.py

@@ -87,6 +87,7 @@ from nova_compute_utils import (
     set_ppc64_cpu_smt_state,
     destroy_libvirt_network,
     network_manager,
+    libvirt_daemon,
 )
 
 from charmhelpers.contrib.network.ip import (
@@ -327,7 +328,7 @@ def ceph_joined():
     apt_install(filter_installed_packages(['ceph-common']), fatal=True)
     # Bug 1427660
     if not is_unit_paused_set():
-        service_restart('libvirt-bin')
+        service_restart(libvirt_daemon())
 
 
 def get_ceph_request():

hooks/nova_compute_utils.py

@@ -238,9 +238,12 @@ BASE_RESOURCE_MAP = {
     },
 }
 
+LIBVIRTD_DAEMON = 'libvirtd'
+LIBVIRT_BIN_DAEMON = 'libvirt-bin'
+
 LIBVIRT_RESOURCE_MAP = {
     QEMU_CONF: {
-        'services': ['libvirt-bin'],
+        'services': [LIBVIRT_BIN_DAEMON],
         'contexts': [NovaComputeLibvirtContext()],
     },
     QEMU_KVM: {
@@ -248,15 +251,15 @@ LIBVIRT_RESOURCE_MAP = {
         'contexts': [NovaComputeLibvirtContext()],
     },
     LIBVIRTD_CONF: {
-        'services': ['libvirt-bin'],
+        'services': [LIBVIRT_BIN_DAEMON],
         'contexts': [NovaComputeLibvirtContext()],
     },
     LIBVIRT_BIN: {
-        'services': ['libvirt-bin'],
+        'services': [LIBVIRT_BIN_DAEMON],
         'contexts': [NovaComputeLibvirtContext()],
     },
     LIBVIRT_BIN_OVERRIDES: {
-        'services': ['libvirt-bin'],
+        'services': [LIBVIRT_BIN_DAEMON],
         'contexts': [NovaComputeLibvirtOverrideContext()],
     },
 }
@@ -298,6 +301,15 @@ REQUIRED_INTERFACES = {
 }
 
 
+def libvirt_daemon():
+    '''Resolve the correct name of the libvirt daemon service'''
+    distro_codename = lsb_release()['DISTRIB_CODENAME'].lower()
+    if distro_codename >= 'yakkety':
+        return LIBVIRTD_DAEMON
+    else:
+        return LIBVIRT_BIN_DAEMON
+
+
 def resource_map():
     '''
     Dynamically generate a map of resources that will be managed for a single
@@ -321,6 +333,13 @@ def resource_map():
         resource_map.pop(NOVA_API_AA_PROFILE_PATH)
         resource_map.pop(NOVA_NETWORK_AA_PROFILE_PATH)
 
+    distro_codename = lsb_release()['DISTRIB_CODENAME'].lower()
+    if distro_codename >= 'yakkety':
+        for data in resource_map.values():
+            if LIBVIRT_BIN_DAEMON in data['services']:
+                data['services'].remove(LIBVIRT_BIN_DAEMON)
+                data['services'].append(LIBVIRTD_DAEMON)
+
     # Neutron/quantum requires additional contexts, as well as new resources
     # depending on the plugin used.
     # NOTE(james-page): only required for ovs plugin right now

templates/libvirtd.conf

@@ -83,7 +83,7 @@ listen_tls = {{ listen_tls }}
 # without becoming root.
 #
 # This is restricted to 'root' by default.
-unix_sock_group = "libvirtd"
+unix_sock_group = "{{ libvirt_user }}"
 
 # Set the UNIX socket permissions for the R/O socket. This is used
 # for monitoring VM status only

tests/basic_deployment.py

@@ -26,8 +26,25 @@ from charmhelpers.contrib.openstack.amulet.utils import (
     # ERROR
 )
 
+from novaclient import exceptions
+
+
+class NovaOpenStackAmuletUtils(OpenStackAmuletUtils):
+    """Nova based helper extending base helper for creation of flavors"""
+
+    def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto",
+                      ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):
+        """Create the specified flavor."""
+        try:
+            nova.flavors.find(name=name)
+        except exceptions.NotFound, exceptions.NoUniqueMatch:
+            self.log.debug('Creating flavor ({})'.format(name))
+            nova.flavors.create(name, ram, vcpus, disk, flavorid,
+                                ephemeral, swap, rxtx_factor, is_public)
+
+
 # Use DEBUG to turn on debug logging
-u = OpenStackAmuletUtils(DEBUG)
+u = NovaOpenStackAmuletUtils(DEBUG)
 
 
 class NovaBasicDeployment(OpenStackAmuletDeployment):
@@ -158,6 +175,12 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
         # Authenticate admin with glance endpoint
         self.glance = u.authenticate_glance_admin(self.keystone)
 
+        # Authenticate admin with nova endpoint
+        self.nova = u.authenticate_nova_user(self.keystone,
+                                             user='admin',
+                                             password='openstack',
+                                             tenant='admin')
+
         # Create a demo tenant/role/user
         self.demo_tenant = 'demoTenant'
         self.demo_role = 'demoRole'
@@ -504,6 +527,10 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
         if not image:
             amulet.raise_status(amulet.FAIL, msg="Image create failed")
 
+        # NOTE(jamespage): ensure require flavor exists, required for >= newton
+        u.create_flavor(nova=self.nova,
+                        name='m1.tiny', ram=512, vcpus=1, disk=1)
+
         instance = u.create_instance(self.nova_demo, "cirros-image", "cirros",
                                      "m1.tiny")
         if not instance:

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -258,6 +258,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('vivid', 'kilo'),
             ('wily', 'liberty'),
             ('xenial', 'mitaka'),
+            ('yakkety', 'newton'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

tests/gate-basic-yakkety-newton

@@ -0,0 +1,23 @@
+#!/usr/bin/python
+#
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Amulet tests on a basic nova compute deployment on xenial-mitaka."""
+
+from basic_deployment import NovaBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NovaBasicDeployment(series='yakkety')
+    deployment.run_tests()

unit_tests/test_nova_compute_contexts.py

@@ -190,6 +190,21 @@ class NovaComputeContextTests(CharmTestCase):
         qplugin = context.NeutronComputeContext()
         self.assertEquals({}, qplugin())
 
+    def test_libvirt_context_libvirtd(self):
+        self.lsb_release.return_value = {'DISTRIB_CODENAME': 'yakkety'}
+        self.kv.return_value = FakeUnitdata(**{'host_uuid': self.host_uuid})
+        self.test_config.set('enable-live-migration', False)
+        libvirt = context.NovaComputeLibvirtContext()
+
+        self.assertEqual(
+            {'libvirtd_opts': '',
+             'libvirt_user': 'libvirt',
+             'arch': platform.machine(),
+             'kvm_hugepages': 0,
+             'listen_tls': 0,
+             'host_uuid': self.host_uuid,
+             'reserved_host_memory': 512}, libvirt())
+
     def test_libvirt_bin_context_no_migration(self):
         self.kv.return_value = FakeUnitdata(**{'host_uuid': self.host_uuid})
         self.test_config.set('enable-live-migration', False)
@@ -197,6 +212,7 @@ class NovaComputeContextTests(CharmTestCase):
 
         self.assertEqual(
             {'libvirtd_opts': '-d',
+             'libvirt_user': 'libvirtd',
              'arch': platform.machine(),
              'kvm_hugepages': 0,
              'listen_tls': 0,
@@ -210,6 +226,7 @@ class NovaComputeContextTests(CharmTestCase):
 
         self.assertEqual(
             {'libvirtd_opts': '-d -l',
+             'libvirt_user': 'libvirtd',
              'arch': platform.machine(),
              'kvm_hugepages': 0,
              'listen_tls': 0,
@@ -223,6 +240,7 @@ class NovaComputeContextTests(CharmTestCase):
 
         self.assertEqual(
             {'libvirtd_opts': '-d',
+             'libvirt_user': 'libvirtd',
              'disk_cachemodes': 'file=unsafe,block=none',
              'arch': platform.machine(),
              'kvm_hugepages': 0,
@@ -237,6 +255,7 @@ class NovaComputeContextTests(CharmTestCase):
 
         self.assertEqual(
             {'libvirtd_opts': '-d',
+             'libvirt_user': 'libvirtd',
              'arch': platform.machine(),
              'hugepages': True,
              'kvm_hugepages': 1,
@@ -302,6 +321,7 @@ class NovaComputeContextTests(CharmTestCase):
 
         self.assertEqual(
             {'libvirtd_opts': '-d',
+             'libvirt_user': 'libvirtd',
              'arch': platform.machine(),
              'hugepages': True,
              'kvm_hugepages': 1,

unit_tests/test_nova_compute_hooks.py

@@ -77,6 +77,7 @@ TO_PATCH = [
     'git_install',
     'git_install_requested',
     'network_manager',
+    'libvirt_daemon',
     # misc_utils
     'ensure_ceph_keyring',
     'execd_preinstall',
@@ -456,9 +457,11 @@ class NovaComputeRelationsTests(CharmTestCase):
         )
 
     def test_ceph_joined(self):
+        self.libvirt_daemon.return_value = 'libvirt-bin'
         hooks.ceph_joined()
         self.apt_install.assert_called_with(['ceph-common'], fatal=True)
         self.service_restart.assert_called_with('libvirt-bin')
+        self.libvirt_daemon.assert_called()
 
     @patch.object(hooks, 'CONFIGS')
     def test_ceph_changed_missing_relation_data(self, configs):

unit_tests/test_nova_compute_utils.py

@@ -58,6 +58,7 @@ TO_PATCH = [
     'rsync',
     'Fstab',
     'os_application_version_set',
+    'lsb_release',
 ]
 
 openstack_origin_git = \
@@ -76,6 +77,7 @@ class NovaComputeUtilsTests(CharmTestCase):
         super(NovaComputeUtilsTests, self).setUp(utils, TO_PATCH)
         self.config.side_effect = self.test_config.get
         self.charm_dir.return_value = 'mycharm'
+        self.lsb_release.return_value = {'DISTRIB_CODENAME': 'precise'}
 
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'network_manager')
@@ -255,6 +257,70 @@ class NovaComputeUtilsTests(CharmTestCase):
             self.assertEquals(set(ex[k]['services']),
                               set(result[k]['services']))
 
+    def _test_resource_map_neutron(self, net_man, en_meta,
+                                   libvirt_daemon):
+        en_meta.return_value = (False, None)
+        self.test_config.set('multi-host', 'yes')
+        net_man.return_value = 'neutron'
+        result = utils.resource_map()
+
+        ex = {
+            '/etc/default/libvirt-bin': {
+                'contexts': [],
+                'services': [libvirt_daemon]
+            },
+            '/etc/libvirt/qemu.conf': {
+                'contexts': [],
+                'services': [libvirt_daemon]
+            },
+            '/etc/nova/nova.conf': {
+                'contexts': [],
+                'services': ['nova-compute']
+            },
+            '/etc/ceph/secret.xml': {
+                'contexts': [],
+                'services': []
+            },
+            '/var/lib/charm/nova_compute/ceph.conf': {
+                'contexts': [],
+                'services': ['nova-compute']
+            },
+            '/etc/default/qemu-kvm': {
+                'contexts': [],
+                'services': ['qemu-kvm']
+            },
+            '/etc/init/libvirt-bin.override': {
+                'contexts': [],
+                'services': [libvirt_daemon]
+            },
+            '/etc/libvirt/libvirtd.conf': {
+                'contexts': [],
+                'services': [libvirt_daemon]
+            },
+            '/etc/apparmor.d/usr.bin.nova-compute': {
+                'contexts': [],
+                'services': ['nova-compute']
+            },
+        }
+        # Mocking contexts is tricky but we can still test that
+        # the correct files are monitored and the correct services
+        # will be started
+        self.assertEquals(set(ex.keys()), set(result.keys()))
+        for k in ex.keys():
+            self.assertEquals(set(ex[k]['services']),
+                              set(result[k]['services']))
+
+    @patch.object(utils, 'nova_metadata_requirement')
+    @patch.object(utils, 'network_manager')
+    def test_resource_map_neutron(self, net_man, en_meta):
+        self._test_resource_map_neutron(net_man, en_meta, 'libvirt-bin')
+
+    @patch.object(utils, 'nova_metadata_requirement')
+    @patch.object(utils, 'network_manager')
+    def test_resource_map_neutron_yakkety(self, net_man, en_meta,):
+        self.lsb_release.return_value = {'DISTRIB_CODENAME': 'yakkety'}
+        self._test_resource_map_neutron(net_man, en_meta, 'libvirtd')
+
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'neutron_plugin')
     @patch.object(utils, 'network_manager')
@@ -814,3 +880,15 @@ class NovaComputeUtilsTests(CharmTestCase):
         mock_check_output.side_effect = OSError(100, 'Break things')
         with self.assertRaises(OSError):
             utils.destroy_libvirt_network('default')
+
+    def test_libvirt_daemon_yakkety(self):
+        self.lsb_release.return_value = {
+            'DISTRIB_CODENAME': 'yakkety'
+        }
+        self.assertEqual(utils.libvirt_daemon(), utils.LIBVIRTD_DAEMON)
+
+    def test_libvirt_daemon_preyakkety(self):
+        self.lsb_release.return_value = {
+            'DISTRIB_CODENAME': 'xenial'
+        }
+        self.assertEqual(utils.libvirt_daemon(), utils.LIBVIRT_BIN_DAEMON)