Fedora kernel install in ibp mode

Nailgun puts 'kernel_lt' flag in provisioning data and
if this flag is set, it is supposed that we need to install
kernel-lt package which is fedora kernel 3.10.
We assume both kernels are available in centos OS image:
2.6 and 3.10. We can just configure grub to boot with
one of those kernels. If flag 'kernel_lt' is NOT set, we
use 'vmlinuz-2.6' regexp for looking up for kernel file.
Otherwise we use default regexp 'vmlinuz' and sort available
kernels in backward direction, i.e. we use the newest available
kernel.

Co-Authored-By: Vladimir Kozhukalov <vkozhukalov@mirantis.com>
Change-Id: I1f7eee934440ce32d6e733c417e82f578b6d0c18
Partially-closes-bug: #1398643

fuel_agent/drivers/nailgun.py

@@ -78,6 +78,7 @@ class Nailgun(BaseDataDriver):
         self._boot_done = False
 
         self.partition_scheme = self.parse_partition_scheme()
+        self.grub = self.parse_grub()
         self.configdrive_scheme = self.parse_configdrive_scheme()
         # parsing image scheme needs partition scheme has been parsed
         self.image_scheme = self.parse_image_scheme()
@@ -333,10 +334,6 @@ class Nailgun(BaseDataDriver):
                             fs_type=volume.get('file_system', 'xfs'),
                             fs_label=self._getlabel(volume.get('disk_label')))
 
-        LOG.debug('Appending kernel parameters: %s' %
-                  self.data['ks_meta']['pm_data']['kernel_params'])
-        partition_scheme.append_kernel_params(
-            self.data['ks_meta']['pm_data']['kernel_params'])
         return partition_scheme
 
     def parse_configdrive_scheme(self):
@@ -394,6 +391,20 @@ class Nailgun(BaseDataDriver):
         configdrive_scheme.set_profile(profile=data['profile'])
         return configdrive_scheme
 
+    def parse_grub(self):
+        LOG.debug('--- Parse grub settings ---')
+        grub = objects.Grub()
+        LOG.debug('Appending kernel parameters: %s',
+                  self.data['ks_meta']['pm_data']['kernel_params'])
+        grub.append_kernel_params(
+            self.data['ks_meta']['pm_data']['kernel_params'])
+        if 'centos' in self.data['profile'].lower() and \
+                not self.data['ks_meta'].get('kernel_lt'):
+            LOG.debug('Prefered kernel version is 2.6')
+            grub.kernel_regexp = r'^vmlinuz-2\.6.*'
+            grub.initrd_regexp = r'^initramfs-2\.6.*'
+        return grub
+
     def parse_image_scheme(self):
         LOG.debug('--- Preparing image scheme ---')
         data = self.data

fuel_agent/manager.py

@@ -363,7 +363,7 @@ class Manager(object):
             with open(mtab_path, 'wb') as f:
                 f.write(mtab)
 
-    # TODO(kozhukalov): write tests
+    # TODO(kozhukalov): write tests for this method
     def umount_target(self, chroot, pseudo=True):
         LOG.debug('Umounting target file systems')
         if pseudo:
@@ -375,6 +375,8 @@ class Manager(object):
                 continue
             fu.umount_fs(chroot + fs.mount)
 
+    # TODO(kozhukalov): write tests for this method
+    # https://bugs.launchpad.net/fuel/+bug/1449609
     def do_bootloader(self):
         LOG.debug('--- Installing bootloader (do_bootloader) ---')
         chroot = '/tmp/target'
@@ -386,19 +388,27 @@ class Manager(object):
                 'blkid', '-o', 'value', '-s', 'UUID', fs.device,
                 check_exit_code=[0])[0].strip()
 
-        grub_version = gu.guess_grub_version(chroot=chroot)
-        boot_device = self.driver.partition_scheme.boot_device(grub_version)
+        grub = self.driver.grub
+
+        grub.version = gu.guess_grub_version(chroot=chroot)
+        boot_device = self.driver.partition_scheme.boot_device(grub.version)
         install_devices = [d.name for d in self.driver.partition_scheme.parteds
                            if d.install_bootloader]
 
-        kernel_params = self.driver.partition_scheme.kernel_params
-        kernel_params += ' root=UUID=%s ' % mount2uuid['/']
+        grub.append_kernel_params('root=UUID=%s ' % mount2uuid['/'])
 
-        if grub_version == 1:
-            gu.grub1_cfg(kernel_params=kernel_params, chroot=chroot)
+        kernel = grub.kernel_name or \
+            gu.guess_kernel(chroot=chroot, regexp=grub.kernel_regexp)
+        initrd = grub.initrd_name or \
+            gu.guess_initrd(chroot=chroot, regexp=grub.initrd_regexp)
+
+        if grub.version == 1:
+            gu.grub1_cfg(kernel=kernel, initrd=initrd,
+                         kernel_params=grub.kernel_params, chroot=chroot)
             gu.grub1_install(install_devices, boot_device, chroot=chroot)
         else:
-            gu.grub2_cfg(kernel_params=kernel_params, chroot=chroot)
+            gu.grub2_cfg(kernel=kernel, initrd=initrd,
+                         kernel_params=grub.kernel_params, chroot=chroot)
             gu.grub2_install(install_devices, chroot=chroot)
 
         # FIXME(agordeev) There's no convenient way to perfrom NIC remapping in

fuel_agent/objects/__init__.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from fuel_agent.objects.bootloader import Grub
 from fuel_agent.objects.configdrive import ConfigDriveCommon
 from fuel_agent.objects.configdrive import ConfigDriveMcollective
 from fuel_agent.objects.configdrive import ConfigDrivePuppet
@@ -34,7 +35,7 @@ from fuel_agent.objects.repo import Repo
 __all__ = [
     'Partition', 'Pv', 'Vg', 'Lv', 'Md', 'Fs', 'PartitionScheme',
     'ConfigDriveCommon', 'ConfigDrivePuppet', 'ConfigDriveMcollective',
-    'ConfigDriveScheme', 'Image', 'ImageScheme',
+    'ConfigDriveScheme', 'Image', 'ImageScheme', 'Grub',
     'OperatingSystem', 'Ubuntu',
     'Repo', 'DEBRepo',
     'Loop',

fuel_agent/objects/bootloader.py

@@ -0,0 +1,29 @@
+# Copyright 2014 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class Grub(object):
+    def __init__(self, version=None, kernel_params='',
+                 kernel_name=None, kernel_regexp=None,
+                 initrd_name=None, initrd_regexp=None):
+        self.version = version
+        self.kernel_params = kernel_params
+        self.kernel_name = kernel_name
+        self.initrd_name = initrd_name
+        self.kernel_regexp = kernel_regexp
+        self.initrd_regexp = initrd_regexp
+
+    def append_kernel_params(self, *kernel_params):
+        for kp in kernel_params:
+            self.kernel_params = '{0} {1}'.format(self.kernel_params, kp)

fuel_agent/objects/partition.py

@@ -191,7 +191,6 @@ class PartitionScheme(object):
         self.vgs = []
         self.lvs = []
         self.fss = []
-        self.kernel_params = ''
 
     def add_parted(self, **kwargs):
         parted = Parted(**kwargs)
@@ -355,6 +354,3 @@ class PartitionScheme(object):
             for prt in parted.partitions:
                 if prt.configdrive:
                     return prt.name
-
-    def append_kernel_params(self, kernel_params):
-        self.kernel_params += ' ' + kernel_params

fuel_agent/tests/test_grub_utils.py

@@ -205,20 +205,40 @@ class TestGrubUtils(test_base.BaseTestCase):
         self.assertEqual(gu.guess_grub_install('/target'),
                          '/usr/sbin/grub2-install')
 
-    @mock.patch.object(os, 'listdir')
-    def test_guess_kernel(self, mock_listdir):
-        mock_listdir.return_value = ['1', '2', 'vmlinuz-version', '3']
+    @mock.patch('fuel_agent.utils.grub_utils.utils.guess_filename')
+    def test_guess_kernel(self, mock_guess):
+        mock_guess.return_value = 'vmlinuz-version'
         self.assertEqual(gu.guess_kernel('/target'), 'vmlinuz-version')
-        mock_listdir.return_value = ['1', '2', '3']
+        mock_guess.assert_called_once_with(
+            path='/target/boot', regexp=r'^vmlinuz.*')
+        mock_guess.reset_mock()
+
+        mock_guess.return_value = 'vmlinuz-version'
+        self.assertEqual(gu.guess_kernel('/target', r'^vmlinuz-version.*'),
+                         'vmlinuz-version')
+        mock_guess.assert_called_once_with(
+            path='/target/boot', regexp=r'^vmlinuz-version.*')
+        mock_guess.reset_mock()
+
+        mock_guess.return_value = None
         self.assertRaises(errors.GrubUtilsError, gu.guess_kernel, '/target')
 
-    @mock.patch.object(os, 'listdir')
-    def test_guess_initrd(self, mock_listdir):
-        mock_listdir.return_value = ['1', '2', 'initramfs-version', '3']
-        self.assertEqual(gu.guess_initrd('/target'), 'initramfs-version')
-        mock_listdir.return_value = ['1', '2', 'initrd-version', '3']
+    @mock.patch('fuel_agent.utils.grub_utils.utils.guess_filename')
+    def test_guess_initrd(self, mock_guess):
+        mock_guess.return_value = 'initrd-version'
         self.assertEqual(gu.guess_initrd('/target'), 'initrd-version')
-        mock_listdir.return_value = ['1', '2', '3']
+        mock_guess.assert_called_once_with(
+            path='/target/boot', regexp=r'^(initrd|initramfs).*')
+        mock_guess.reset_mock()
+
+        mock_guess.return_value = 'initramfs-version'
+        self.assertEqual(gu.guess_initrd('/target', r'^initramfs-version.*'),
+                         'initramfs-version')
+        mock_guess.assert_called_once_with(
+            path='/target/boot', regexp=r'^initramfs-version.*')
+        mock_guess.reset_mock()
+
+        mock_guess.return_value = None
         self.assertRaises(errors.GrubUtilsError, gu.guess_initrd, '/target')
 
     @mock.patch.object(gu, 'grub1_stage1')

fuel_agent/tests/test_nailgun.py

@@ -699,3 +699,54 @@ class TestNailgun(test_base.BaseTestCase):
         for disk, part in enumerate((-2, -1, -1)):
             self.assertEqual(CEPH_DATA['partition_guid'],
                              p_scheme.parteds[disk].partitions[part].guid)
+
+    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
+    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
+    def test_grub_centos_26(self, mock_lbd, mock_http_req, mock_yaml):
+        data = PROVISION_SAMPLE_DATA.copy()
+        data['profile'] = 'centos'
+        data['ks_meta']['kernel_lt'] = 0
+        mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
+        self.drv = nailgun.Nailgun(data)
+        self.assertEqual(self.drv.grub.kernel_params,
+                         ' ' + data['ks_meta']['pm_data']['kernel_params'])
+        self.assertEqual(self.drv.grub.kernel_regexp, r'^vmlinuz-2\.6.*')
+        self.assertEqual(self.drv.grub.initrd_regexp, r'^initramfs-2\.6.*')
+        self.assertIsNone(self.drv.grub.version)
+        self.assertIsNone(self.drv.grub.kernel_name)
+        self.assertIsNone(self.drv.grub.initrd_name)
+
+    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
+    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
+    def test_grub_centos_lt(self, mock_lbd, mock_http_req, mock_yaml):
+        data = PROVISION_SAMPLE_DATA.copy()
+        data['profile'] = 'centos'
+        data['ks_meta']['kernel_lt'] = 1
+        mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
+        self.drv = nailgun.Nailgun(data)
+        self.assertEqual(self.drv.grub.kernel_params,
+                         ' ' + data['ks_meta']['pm_data']['kernel_params'])
+        self.assertIsNone(self.drv.grub.kernel_regexp)
+        self.assertIsNone(self.drv.grub.initrd_regexp)
+        self.assertIsNone(self.drv.grub.version)
+        self.assertIsNone(self.drv.grub.kernel_name)
+        self.assertIsNone(self.drv.grub.initrd_name)
+
+    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
+    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
+    def test_grub_ubuntu(self, mock_lbd, mock_http_req, mock_yaml):
+        data = PROVISION_SAMPLE_DATA.copy()
+        data['profile'] = 'ubuntu'
+        data['ks_meta']['kernel_lt'] = 0
+        mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
+        self.drv = nailgun.Nailgun(data)
+        self.assertEqual(self.drv.grub.kernel_params,
+                         ' ' + data['ks_meta']['pm_data']['kernel_params'])
+        self.assertIsNone(self.drv.grub.version)
+        self.assertIsNone(self.drv.grub.kernel_regexp)
+        self.assertIsNone(self.drv.grub.initrd_regexp)
+        self.assertIsNone(self.drv.grub.kernel_name)
+        self.assertIsNone(self.drv.grub.initrd_name)

fuel_agent/tests/test_utils.py

@@ -180,3 +180,33 @@ class ExecuteTestCase(testtools.TestCase):
         utils.makedirs_if_not_exists('/fake/path')
         mock_isdir.assert_called_once_with('/fake/path')
         self.assertEqual(mock_makedirs.mock_calls, [])
+
+    @mock.patch('fuel_agent.utils.utils.os.listdir')
+    def test_guess_filename(self, mock_oslistdir):
+        mock_oslistdir.return_value = ['file1', 'file2', 'file3']
+        filename = utils.guess_filename('/some/path', '^file2.*')
+        self.assertEqual(filename, 'file2')
+        mock_oslistdir.assert_called_once_with('/some/path')
+
+    @mock.patch('fuel_agent.utils.utils.os.listdir')
+    def test_guess_filename_not_found(self, mock_oslistdir):
+        mock_oslistdir.return_value = ['file1', 'file2', 'file3']
+        filename = utils.guess_filename('/some/path', '^file4.*')
+        self.assertIsNone(filename)
+        mock_oslistdir.assert_called_once_with('/some/path')
+
+    @mock.patch('fuel_agent.utils.utils.os.listdir')
+    def test_guess_filename_not_exact_match(self, mock_oslistdir):
+        mock_oslistdir.return_value = ['file1', 'file2', 'file3']
+        filename = utils.guess_filename('/some/path', '^file.*')
+        # by default files are sorted in backward direction
+        self.assertEqual(filename, 'file3')
+        mock_oslistdir.assert_called_once_with('/some/path')
+
+    @mock.patch('fuel_agent.utils.utils.os.listdir')
+    def test_guess_filename_not_exact_match_forward_sort(self, mock_oslistdir):
+        mock_oslistdir.return_value = ['file1', 'file2', 'file3']
+        filename = utils.guess_filename('/some/path', '^file.*', reverse=False)
+        # by default files are sorted in backward direction
+        self.assertEqual(filename, 'file1')
+        mock_oslistdir.assert_called_once_with('/some/path')

fuel_agent/utils/grub_utils.py

@@ -83,22 +83,38 @@ def guess_grub1_datadir(chroot='', arch='x86_64'):
             return '/usr/share/grub/' + d
 
 
-def guess_kernel(chroot=''):
-    for filename in sorted(os.listdir(chroot + '/boot'), reverse=True):
-        # We assume kernel name is always starts with vmlinuz.
-        # We use the newest one.
-        if filename.startswith('vmlinuz'):
-            return filename
-    raise errors.GrubUtilsError('Error while trying to find kernel: not found')
+def guess_kernel(chroot='', regexp=None):
+    """Tries to guess kernel by regexp
+    :param chroot: Path to chroot
+    :param regexp: (String) Regular expression (must have python syntax).
+    Default is r'^vmlinuz.*'
+    """
+    kernel = utils.guess_filename(
+        path=os.path.join(chroot, 'boot'),
+        regexp=(regexp or r'^vmlinuz.*'))
+
+    if kernel:
+        return kernel
+
+    raise errors.GrubUtilsError('Error while trying to find kernel: '
+                                'regexp=%s' % regexp)
 
 
-def guess_initrd(chroot=''):
-    for filename in sorted(os.listdir(chroot + '/boot'), reverse=True):
-        # We assume initrd starts either with initramfs or initrd.
-        if filename.startswith('initramfs') or \
-                filename.startswith('initrd'):
-            return filename
-    raise errors.GrubUtilsError('Error while trying to find initrd: not found')
+def guess_initrd(chroot='', regexp=None):
+    """Tries to guess initrd by regexp
+    :param chroot: Path to chroot
+    :param regexp: (String) Regular expression (must have python syntax).
+    Default is r'^(initrd|initramfs).*'
+    """
+    initrd = utils.guess_filename(
+        path=os.path.join(chroot, 'boot'),
+        regexp=(regexp or r'^(initrd|initramfs).*'))
+
+    if initrd:
+        return initrd
+
+    raise errors.GrubUtilsError('Error while trying to find initrd: '
+                                'regexp=%s' % regexp)
 
 
 def grub1_install(install_devices, boot_device, chroot=''):

fuel_agent/utils/utils.py

@@ -236,3 +236,27 @@ def grouper(iterable, n, fillvalue=None):
     """
     args = [iter(iterable)] * n
     return zip_longest(*args, fillvalue=fillvalue)
+
+
+def guess_filename(path, regexp, sort=True, reverse=True):
+    """Tries to find a file by regexp in a given path.
+
+    This method is supposed to be mostly used for looking up
+    for available kernel files which are usually 'vmlinuz-X.Y.Z-foo'.
+    In order to find the newest one we can sort files in backward
+    direction (by default).
+
+    :param path: Directory where to look for a file
+    :param regexp: (String) Regular expression (must have python syntax)
+    :param sort: (Bool) If True (by default), sort files before looking up.
+    It can be necessary when regexp does not unambiguously correspond to file.
+    :param reverse: (Bool) If True (by default), sort files
+    in backward direction.
+    """
+    filenames = os.listdir(path)
+    if sort:
+        filenames = sorted(filenames, reverse=reverse)
+    for filename in filenames:
+        if re.search(regexp, filename):
+            return filename
+    return None