From aadfde20ca3d0defa0c58eed2848354363069f22 Mon Sep 17 00:00:00 2001 From: Denis Egorenko Date: Thu, 23 Apr 2015 18:42:04 +0300 Subject: [PATCH] Fedora kernel install in ibp mode Nailgun puts 'kernel_lt' flag in provisioning data and if this flag is set, it is supposed that we need to install kernel-lt package which is fedora kernel 3.10. We assume both kernels are available in centos OS image: 2.6 and 3.10. We can just configure grub to boot with one of those kernels. If flag 'kernel_lt' is NOT set, we use 'vmlinuz-2.6' regexp for looking up for kernel file. Otherwise we use default regexp 'vmlinuz' and sort available kernels in backward direction, i.e. we use the newest available kernel. Co-Authored-By: Vladimir Kozhukalov Change-Id: I1f7eee934440ce32d6e733c417e82f578b6d0c18 Partially-closes-bug: #1398643 --- fuel_agent/drivers/nailgun.py | 19 ++++++++--- fuel_agent/manager.py | 26 ++++++++++----- fuel_agent/objects/__init__.py | 3 +- fuel_agent/objects/bootloader.py | 29 ++++++++++++++++ fuel_agent/objects/partition.py | 4 --- fuel_agent/tests/test_grub_utils.py | 40 ++++++++++++++++------ fuel_agent/tests/test_nailgun.py | 51 +++++++++++++++++++++++++++++ fuel_agent/tests/test_utils.py | 30 +++++++++++++++++ fuel_agent/utils/grub_utils.py | 44 +++++++++++++++++-------- fuel_agent/utils/utils.py | 24 ++++++++++++++ 10 files changed, 229 insertions(+), 41 deletions(-) create mode 100644 fuel_agent/objects/bootloader.py diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py index f171158..8132df2 100644 --- a/fuel_agent/drivers/nailgun.py +++ b/fuel_agent/drivers/nailgun.py @@ -78,6 +78,7 @@ class Nailgun(BaseDataDriver): self._boot_done = False self.partition_scheme = self.parse_partition_scheme() + self.grub = self.parse_grub() self.configdrive_scheme = self.parse_configdrive_scheme() # parsing image scheme needs partition scheme has been parsed self.image_scheme = self.parse_image_scheme() @@ -333,10 +334,6 @@ class Nailgun(BaseDataDriver): fs_type=volume.get('file_system', 'xfs'), fs_label=self._getlabel(volume.get('disk_label'))) - LOG.debug('Appending kernel parameters: %s' % - self.data['ks_meta']['pm_data']['kernel_params']) - partition_scheme.append_kernel_params( - self.data['ks_meta']['pm_data']['kernel_params']) return partition_scheme def parse_configdrive_scheme(self): @@ -394,6 +391,20 @@ class Nailgun(BaseDataDriver): configdrive_scheme.set_profile(profile=data['profile']) return configdrive_scheme + def parse_grub(self): + LOG.debug('--- Parse grub settings ---') + grub = objects.Grub() + LOG.debug('Appending kernel parameters: %s', + self.data['ks_meta']['pm_data']['kernel_params']) + grub.append_kernel_params( + self.data['ks_meta']['pm_data']['kernel_params']) + if 'centos' in self.data['profile'].lower() and \ + not self.data['ks_meta'].get('kernel_lt'): + LOG.debug('Prefered kernel version is 2.6') + grub.kernel_regexp = r'^vmlinuz-2\.6.*' + grub.initrd_regexp = r'^initramfs-2\.6.*' + return grub + def parse_image_scheme(self): LOG.debug('--- Preparing image scheme ---') data = self.data diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py index 5dcc080..f9674be 100644 --- a/fuel_agent/manager.py +++ b/fuel_agent/manager.py @@ -363,7 +363,7 @@ class Manager(object): with open(mtab_path, 'wb') as f: f.write(mtab) - # TODO(kozhukalov): write tests + # TODO(kozhukalov): write tests for this method def umount_target(self, chroot, pseudo=True): LOG.debug('Umounting target file systems') if pseudo: @@ -375,6 +375,8 @@ class Manager(object): continue fu.umount_fs(chroot + fs.mount) + # TODO(kozhukalov): write tests for this method + # https://bugs.launchpad.net/fuel/+bug/1449609 def do_bootloader(self): LOG.debug('--- Installing bootloader (do_bootloader) ---') chroot = '/tmp/target' @@ -386,19 +388,27 @@ class Manager(object): 'blkid', '-o', 'value', '-s', 'UUID', fs.device, check_exit_code=[0])[0].strip() - grub_version = gu.guess_grub_version(chroot=chroot) - boot_device = self.driver.partition_scheme.boot_device(grub_version) + grub = self.driver.grub + + grub.version = gu.guess_grub_version(chroot=chroot) + boot_device = self.driver.partition_scheme.boot_device(grub.version) install_devices = [d.name for d in self.driver.partition_scheme.parteds if d.install_bootloader] - kernel_params = self.driver.partition_scheme.kernel_params - kernel_params += ' root=UUID=%s ' % mount2uuid['/'] + grub.append_kernel_params('root=UUID=%s ' % mount2uuid['/']) - if grub_version == 1: - gu.grub1_cfg(kernel_params=kernel_params, chroot=chroot) + kernel = grub.kernel_name or \ + gu.guess_kernel(chroot=chroot, regexp=grub.kernel_regexp) + initrd = grub.initrd_name or \ + gu.guess_initrd(chroot=chroot, regexp=grub.initrd_regexp) + + if grub.version == 1: + gu.grub1_cfg(kernel=kernel, initrd=initrd, + kernel_params=grub.kernel_params, chroot=chroot) gu.grub1_install(install_devices, boot_device, chroot=chroot) else: - gu.grub2_cfg(kernel_params=kernel_params, chroot=chroot) + gu.grub2_cfg(kernel=kernel, initrd=initrd, + kernel_params=grub.kernel_params, chroot=chroot) gu.grub2_install(install_devices, chroot=chroot) # FIXME(agordeev) There's no convenient way to perfrom NIC remapping in diff --git a/fuel_agent/objects/__init__.py b/fuel_agent/objects/__init__.py index a701b01..fd7812b 100644 --- a/fuel_agent/objects/__init__.py +++ b/fuel_agent/objects/__init__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from fuel_agent.objects.bootloader import Grub from fuel_agent.objects.configdrive import ConfigDriveCommon from fuel_agent.objects.configdrive import ConfigDriveMcollective from fuel_agent.objects.configdrive import ConfigDrivePuppet @@ -34,7 +35,7 @@ from fuel_agent.objects.repo import Repo __all__ = [ 'Partition', 'Pv', 'Vg', 'Lv', 'Md', 'Fs', 'PartitionScheme', 'ConfigDriveCommon', 'ConfigDrivePuppet', 'ConfigDriveMcollective', - 'ConfigDriveScheme', 'Image', 'ImageScheme', + 'ConfigDriveScheme', 'Image', 'ImageScheme', 'Grub', 'OperatingSystem', 'Ubuntu', 'Repo', 'DEBRepo', 'Loop', diff --git a/fuel_agent/objects/bootloader.py b/fuel_agent/objects/bootloader.py new file mode 100644 index 0000000..62a09c0 --- /dev/null +++ b/fuel_agent/objects/bootloader.py @@ -0,0 +1,29 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Grub(object): + def __init__(self, version=None, kernel_params='', + kernel_name=None, kernel_regexp=None, + initrd_name=None, initrd_regexp=None): + self.version = version + self.kernel_params = kernel_params + self.kernel_name = kernel_name + self.initrd_name = initrd_name + self.kernel_regexp = kernel_regexp + self.initrd_regexp = initrd_regexp + + def append_kernel_params(self, *kernel_params): + for kp in kernel_params: + self.kernel_params = '{0} {1}'.format(self.kernel_params, kp) diff --git a/fuel_agent/objects/partition.py b/fuel_agent/objects/partition.py index f321146..aa32266 100644 --- a/fuel_agent/objects/partition.py +++ b/fuel_agent/objects/partition.py @@ -191,7 +191,6 @@ class PartitionScheme(object): self.vgs = [] self.lvs = [] self.fss = [] - self.kernel_params = '' def add_parted(self, **kwargs): parted = Parted(**kwargs) @@ -355,6 +354,3 @@ class PartitionScheme(object): for prt in parted.partitions: if prt.configdrive: return prt.name - - def append_kernel_params(self, kernel_params): - self.kernel_params += ' ' + kernel_params diff --git a/fuel_agent/tests/test_grub_utils.py b/fuel_agent/tests/test_grub_utils.py index eac0be3..b61c37a 100644 --- a/fuel_agent/tests/test_grub_utils.py +++ b/fuel_agent/tests/test_grub_utils.py @@ -205,20 +205,40 @@ class TestGrubUtils(test_base.BaseTestCase): self.assertEqual(gu.guess_grub_install('/target'), '/usr/sbin/grub2-install') - @mock.patch.object(os, 'listdir') - def test_guess_kernel(self, mock_listdir): - mock_listdir.return_value = ['1', '2', 'vmlinuz-version', '3'] + @mock.patch('fuel_agent.utils.grub_utils.utils.guess_filename') + def test_guess_kernel(self, mock_guess): + mock_guess.return_value = 'vmlinuz-version' self.assertEqual(gu.guess_kernel('/target'), 'vmlinuz-version') - mock_listdir.return_value = ['1', '2', '3'] + mock_guess.assert_called_once_with( + path='/target/boot', regexp=r'^vmlinuz.*') + mock_guess.reset_mock() + + mock_guess.return_value = 'vmlinuz-version' + self.assertEqual(gu.guess_kernel('/target', r'^vmlinuz-version.*'), + 'vmlinuz-version') + mock_guess.assert_called_once_with( + path='/target/boot', regexp=r'^vmlinuz-version.*') + mock_guess.reset_mock() + + mock_guess.return_value = None self.assertRaises(errors.GrubUtilsError, gu.guess_kernel, '/target') - @mock.patch.object(os, 'listdir') - def test_guess_initrd(self, mock_listdir): - mock_listdir.return_value = ['1', '2', 'initramfs-version', '3'] - self.assertEqual(gu.guess_initrd('/target'), 'initramfs-version') - mock_listdir.return_value = ['1', '2', 'initrd-version', '3'] + @mock.patch('fuel_agent.utils.grub_utils.utils.guess_filename') + def test_guess_initrd(self, mock_guess): + mock_guess.return_value = 'initrd-version' self.assertEqual(gu.guess_initrd('/target'), 'initrd-version') - mock_listdir.return_value = ['1', '2', '3'] + mock_guess.assert_called_once_with( + path='/target/boot', regexp=r'^(initrd|initramfs).*') + mock_guess.reset_mock() + + mock_guess.return_value = 'initramfs-version' + self.assertEqual(gu.guess_initrd('/target', r'^initramfs-version.*'), + 'initramfs-version') + mock_guess.assert_called_once_with( + path='/target/boot', regexp=r'^initramfs-version.*') + mock_guess.reset_mock() + + mock_guess.return_value = None self.assertRaises(errors.GrubUtilsError, gu.guess_initrd, '/target') @mock.patch.object(gu, 'grub1_stage1') diff --git a/fuel_agent/tests/test_nailgun.py b/fuel_agent/tests/test_nailgun.py index 4c8b2ea..46c638c 100644 --- a/fuel_agent/tests/test_nailgun.py +++ b/fuel_agent/tests/test_nailgun.py @@ -699,3 +699,54 @@ class TestNailgun(test_base.BaseTestCase): for disk, part in enumerate((-2, -1, -1)): self.assertEqual(CEPH_DATA['partition_guid'], p_scheme.parteds[disk].partitions[part].guid) + + @mock.patch('fuel_agent.drivers.nailgun.yaml.load') + @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') + @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') + def test_grub_centos_26(self, mock_lbd, mock_http_req, mock_yaml): + data = PROVISION_SAMPLE_DATA.copy() + data['profile'] = 'centos' + data['ks_meta']['kernel_lt'] = 0 + mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE + self.drv = nailgun.Nailgun(data) + self.assertEqual(self.drv.grub.kernel_params, + ' ' + data['ks_meta']['pm_data']['kernel_params']) + self.assertEqual(self.drv.grub.kernel_regexp, r'^vmlinuz-2\.6.*') + self.assertEqual(self.drv.grub.initrd_regexp, r'^initramfs-2\.6.*') + self.assertIsNone(self.drv.grub.version) + self.assertIsNone(self.drv.grub.kernel_name) + self.assertIsNone(self.drv.grub.initrd_name) + + @mock.patch('fuel_agent.drivers.nailgun.yaml.load') + @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') + @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') + def test_grub_centos_lt(self, mock_lbd, mock_http_req, mock_yaml): + data = PROVISION_SAMPLE_DATA.copy() + data['profile'] = 'centos' + data['ks_meta']['kernel_lt'] = 1 + mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE + self.drv = nailgun.Nailgun(data) + self.assertEqual(self.drv.grub.kernel_params, + ' ' + data['ks_meta']['pm_data']['kernel_params']) + self.assertIsNone(self.drv.grub.kernel_regexp) + self.assertIsNone(self.drv.grub.initrd_regexp) + self.assertIsNone(self.drv.grub.version) + self.assertIsNone(self.drv.grub.kernel_name) + self.assertIsNone(self.drv.grub.initrd_name) + + @mock.patch('fuel_agent.drivers.nailgun.yaml.load') + @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') + @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') + def test_grub_ubuntu(self, mock_lbd, mock_http_req, mock_yaml): + data = PROVISION_SAMPLE_DATA.copy() + data['profile'] = 'ubuntu' + data['ks_meta']['kernel_lt'] = 0 + mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE + self.drv = nailgun.Nailgun(data) + self.assertEqual(self.drv.grub.kernel_params, + ' ' + data['ks_meta']['pm_data']['kernel_params']) + self.assertIsNone(self.drv.grub.version) + self.assertIsNone(self.drv.grub.kernel_regexp) + self.assertIsNone(self.drv.grub.initrd_regexp) + self.assertIsNone(self.drv.grub.kernel_name) + self.assertIsNone(self.drv.grub.initrd_name) diff --git a/fuel_agent/tests/test_utils.py b/fuel_agent/tests/test_utils.py index 3ebc2d1..5a96a00 100644 --- a/fuel_agent/tests/test_utils.py +++ b/fuel_agent/tests/test_utils.py @@ -180,3 +180,33 @@ class ExecuteTestCase(testtools.TestCase): utils.makedirs_if_not_exists('/fake/path') mock_isdir.assert_called_once_with('/fake/path') self.assertEqual(mock_makedirs.mock_calls, []) + + @mock.patch('fuel_agent.utils.utils.os.listdir') + def test_guess_filename(self, mock_oslistdir): + mock_oslistdir.return_value = ['file1', 'file2', 'file3'] + filename = utils.guess_filename('/some/path', '^file2.*') + self.assertEqual(filename, 'file2') + mock_oslistdir.assert_called_once_with('/some/path') + + @mock.patch('fuel_agent.utils.utils.os.listdir') + def test_guess_filename_not_found(self, mock_oslistdir): + mock_oslistdir.return_value = ['file1', 'file2', 'file3'] + filename = utils.guess_filename('/some/path', '^file4.*') + self.assertIsNone(filename) + mock_oslistdir.assert_called_once_with('/some/path') + + @mock.patch('fuel_agent.utils.utils.os.listdir') + def test_guess_filename_not_exact_match(self, mock_oslistdir): + mock_oslistdir.return_value = ['file1', 'file2', 'file3'] + filename = utils.guess_filename('/some/path', '^file.*') + # by default files are sorted in backward direction + self.assertEqual(filename, 'file3') + mock_oslistdir.assert_called_once_with('/some/path') + + @mock.patch('fuel_agent.utils.utils.os.listdir') + def test_guess_filename_not_exact_match_forward_sort(self, mock_oslistdir): + mock_oslistdir.return_value = ['file1', 'file2', 'file3'] + filename = utils.guess_filename('/some/path', '^file.*', reverse=False) + # by default files are sorted in backward direction + self.assertEqual(filename, 'file1') + mock_oslistdir.assert_called_once_with('/some/path') diff --git a/fuel_agent/utils/grub_utils.py b/fuel_agent/utils/grub_utils.py index 8aefbf3..c2cf120 100644 --- a/fuel_agent/utils/grub_utils.py +++ b/fuel_agent/utils/grub_utils.py @@ -83,22 +83,38 @@ def guess_grub1_datadir(chroot='', arch='x86_64'): return '/usr/share/grub/' + d -def guess_kernel(chroot=''): - for filename in sorted(os.listdir(chroot + '/boot'), reverse=True): - # We assume kernel name is always starts with vmlinuz. - # We use the newest one. - if filename.startswith('vmlinuz'): - return filename - raise errors.GrubUtilsError('Error while trying to find kernel: not found') +def guess_kernel(chroot='', regexp=None): + """Tries to guess kernel by regexp + :param chroot: Path to chroot + :param regexp: (String) Regular expression (must have python syntax). + Default is r'^vmlinuz.*' + """ + kernel = utils.guess_filename( + path=os.path.join(chroot, 'boot'), + regexp=(regexp or r'^vmlinuz.*')) + + if kernel: + return kernel + + raise errors.GrubUtilsError('Error while trying to find kernel: ' + 'regexp=%s' % regexp) -def guess_initrd(chroot=''): - for filename in sorted(os.listdir(chroot + '/boot'), reverse=True): - # We assume initrd starts either with initramfs or initrd. - if filename.startswith('initramfs') or \ - filename.startswith('initrd'): - return filename - raise errors.GrubUtilsError('Error while trying to find initrd: not found') +def guess_initrd(chroot='', regexp=None): + """Tries to guess initrd by regexp + :param chroot: Path to chroot + :param regexp: (String) Regular expression (must have python syntax). + Default is r'^(initrd|initramfs).*' + """ + initrd = utils.guess_filename( + path=os.path.join(chroot, 'boot'), + regexp=(regexp or r'^(initrd|initramfs).*')) + + if initrd: + return initrd + + raise errors.GrubUtilsError('Error while trying to find initrd: ' + 'regexp=%s' % regexp) def grub1_install(install_devices, boot_device, chroot=''): diff --git a/fuel_agent/utils/utils.py b/fuel_agent/utils/utils.py index 7b64f00..8713718 100644 --- a/fuel_agent/utils/utils.py +++ b/fuel_agent/utils/utils.py @@ -236,3 +236,27 @@ def grouper(iterable, n, fillvalue=None): """ args = [iter(iterable)] * n return zip_longest(*args, fillvalue=fillvalue) + + +def guess_filename(path, regexp, sort=True, reverse=True): + """Tries to find a file by regexp in a given path. + + This method is supposed to be mostly used for looking up + for available kernel files which are usually 'vmlinuz-X.Y.Z-foo'. + In order to find the newest one we can sort files in backward + direction (by default). + + :param path: Directory where to look for a file + :param regexp: (String) Regular expression (must have python syntax) + :param sort: (Bool) If True (by default), sort files before looking up. + It can be necessary when regexp does not unambiguously correspond to file. + :param reverse: (Bool) If True (by default), sort files + in backward direction. + """ + filenames = os.listdir(path) + if sort: + filenames = sorted(filenames, reverse=reverse) + for filename in filenames: + if re.search(regexp, filename): + return filename + return None