From cb5d2f5e8e7b2edb2cc22e71755d798fd76d4bb3 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Mon, 13 Oct 2014 20:39:47 +0400 Subject: [PATCH] Some improvements in do_bootloader in fuel_agent 1) Added copying grub stage 1.5 files into /boot/grub. 2) Added fstab related stuff Change-Id: Id540f73d30b51b80d40661e2d59f8382f7a4c901 Implements: blueprint image-based-provisioning --- fuel_agent/manager.py | 38 ++++++++++++++++++++++++++--- fuel_agent/tests/test_fs_utils.py | 13 +++++++++- fuel_agent/tests/test_grub_utils.py | 6 ++++- fuel_agent/utils/fs_utils.py | 12 ++++++++- fuel_agent/utils/grub_utils.py | 30 +++++++++++++++++++++-- 5 files changed, 90 insertions(+), 9 deletions(-) diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py index 19c0031..6b56c09 100644 --- a/fuel_agent/manager.py +++ b/fuel_agent/manager.py @@ -207,6 +207,12 @@ class Manager(object): LOG.debug('Launching image processing chain') processing.process() + LOG.debug('Extending image file systems') + if image.format in ('ext2', 'ext3', 'ext4', 'xfs'): + LOG.debug('Extending %s %s' % + (image.format, image.target_device)) + fu.extend_fs(image.format, image.target_device) + def mount_target(self, chroot): LOG.debug('Mounting target file systems') # Here we are going to mount all file systems in partition scheme. @@ -223,26 +229,40 @@ class Manager(object): fu.mount_bind(chroot, '/sys') fu.mount_bind(chroot, '/dev') fu.mount_bind(chroot, '/proc') + mtab = utils.execute( + 'chroot', chroot, 'grep', '-v', 'rootfs', '/proc/mounts')[0] + with open(chroot + '/etc/mtab', 'wb') as f: + f.write(mtab) def umount_target(self, chroot): LOG.debug('Umounting target file systems') - key = lambda x: len(x.mount.rstrip('/').split('/')) - for fs in sorted(self.partition_scheme.fss, key=key, reverse=True): - fu.umount_fs(fs.device) fu.umount_fs(chroot + '/proc') fu.umount_fs(chroot + '/dev') fu.umount_fs(chroot + '/sys') + key = lambda x: len(x.mount.rstrip('/').split('/')) + for fs in sorted(self.partition_scheme.fss, key=key, reverse=True): + if fs.mount == 'swap': + continue + fu.umount_fs(fs.device) def do_bootloader(self): LOG.debug('--- Installing bootloader (do_bootloader) ---') chroot = '/tmp/target' self.mount_target(chroot) - grub_version = gu.grub_version_guess(chroot=chroot) + mount2uuid = {} + for fs in self.partition_scheme.fss: + mount2uuid[fs.mount], _ = utils.execute( + 'blkid', '-o', 'value', '-s', 'UUID', fs.device, + check_exit_code=[0]) + + grub_version = gu.guess_grub_version(chroot=chroot) boot_device = self.partition_scheme.boot_device(grub_version) install_devices = [d.name for d in self.partition_scheme.parteds if d.install_bootloader] + kernel_params = self.partition_scheme.kernel_params + kernel_params += ' root=UUID=%s ' % mount2uuid['/'] if grub_version == 1: gu.grub1_cfg(kernel_params=kernel_params, chroot=chroot) @@ -251,6 +271,16 @@ class Manager(object): gu.grub2_cfg(kernel_params=kernel_params, chroot=chroot) gu.grub2_install(install_devices, chroot=chroot) + with open(chroot + '/etc/fstab', 'wb') as f: + for fs in self.partition_scheme.fss: + # TODO(kozhukalov): Think of improving the logic so as to + # insert a meaningful fsck order value which is last zero + # at fstab line. Currently we set it into 0 which means + # a corresponding file system will never be checked. We assume + # puppet or other configuration tool will care of it. + f.write('UUID=%s %s %s defaults 0 0\n' % + (mount2uuid[fs.mount], fs.mount, fs.type)) + self.umount_target(chroot) def do_reboot(self): diff --git a/fuel_agent/tests/test_fs_utils.py b/fuel_agent/tests/test_fs_utils.py index 426b1b6..5ab05cd 100644 --- a/fuel_agent/tests/test_fs_utils.py +++ b/fuel_agent/tests/test_fs_utils.py @@ -91,7 +91,18 @@ class TestFSUtils(test_base.BaseTestCase): 'mount', '--bind', '/fake', '/target/fake2', check_exit_code=[0]) @mock.patch.object(utils, 'execute') - def test_umount_fs(self, mock_exec): + def test_umount_fs_ok(self, mock_exec): fu.umount_fs('/fake') mock_exec.assert_called_once_with( 'umount', '/fake', check_exit_code=[0]) + + @mock.patch.object(utils, 'execute') + def test_umount_fs_error(self, mock_exec): + mock_exec.side_effect = [ + errors.ProcessExecutionError('message'), ('', '')] + fu.umount_fs('/fake') + expected_calls = [ + mock.call('umount', '/fake', check_exit_code=[0]), + mock.call('umount', '-l', '/fake', check_exit_code=[0]) + ] + self.assertEqual(expected_calls, mock_exec.call_args_list) diff --git a/fuel_agent/tests/test_grub_utils.py b/fuel_agent/tests/test_grub_utils.py index 30531e0..3798e29 100644 --- a/fuel_agent/tests/test_grub_utils.py +++ b/fuel_agent/tests/test_grub_utils.py @@ -219,8 +219,9 @@ class TestGrubUtils(test_base.BaseTestCase): mock_listdir.return_value = ['1', '2', '3'] self.assertRaises(errors.GrubUtilsError, gu.guess_initrd, '/target') + @mock.patch.object(gu, 'grub1_stage1') @mock.patch.object(gu, 'grub1_mbr') - def test_grub1_install(self, mock_mbr): + def test_grub1_install(self, mock_mbr, mock_stage1): install_devices = ['/dev/foo', '/dev/bar'] expected_calls_mbr = [] for install_device in install_devices: @@ -228,6 +229,7 @@ class TestGrubUtils(test_base.BaseTestCase): mock.call(install_device, '/dev/foo', '0', chroot='/target')) gu.grub1_install(install_devices, '/dev/foo1', '/target') self.assertEqual(expected_calls_mbr, mock_mbr.call_args_list) + mock_stage1.assert_called_once_with(chroot='/target') # should raise exception if boot_device (second argument) # is not a partition but a whole disk self.assertRaises(errors.GrubUtilsError, gu.grub1_install, @@ -239,6 +241,7 @@ class TestGrubUtils(test_base.BaseTestCase): def test_grub1_mbr_install_differs_boot(self, mock_exec, mock_chmod, mock_guess): mock_guess.return_value = '/sbin/grub' + mock_exec.return_value = ('stdout', 'stderr') # install_device != boot_disk batch = 'device (hd0) /dev/foo\n' @@ -274,6 +277,7 @@ class TestGrubUtils(test_base.BaseTestCase): def test_grub1_mbr_install_same_as_boot(self, mock_exec, mock_chmod, mock_guess): mock_guess.return_value = '/sbin/grub' + mock_exec.return_value = ('stdout', 'stderr') # install_device == boot_disk batch = 'device (hd0) /dev/foo\n' diff --git a/fuel_agent/utils/fs_utils.py b/fuel_agent/utils/fs_utils.py index ae7991c..165724f 100644 --- a/fuel_agent/utils/fs_utils.py +++ b/fuel_agent/utils/fs_utils.py @@ -13,8 +13,11 @@ # limitations under the License. from fuel_agent import errors +from fuel_agent.openstack.common import log as logging from fuel_agent.utils import utils +LOG = logging.getLogger(__name__) + def make_fs(fs_type, fs_options, fs_label, dev): # NOTE(agordeev): notice the different flag to force the fs creating @@ -56,4 +59,11 @@ def mount_bind(chroot, path, path2=None): def umount_fs(fs_mount): - utils.execute('umount', fs_mount, check_exit_code=[0]) + try: + LOG.debug('Trying to umount {0}'.format(fs_mount)) + utils.execute('umount', fs_mount, check_exit_code=[0]) + except errors.ProcessExecutionError as e: + LOG.warning('Error while umounting {0} ' + 'exc={1}'.format(fs_mount, e.message)) + LOG.debug('Trying lazy umounting {0}'.format(fs_mount)) + utils.execute('umount', '-l', fs_mount, check_exit_code=[0]) diff --git a/fuel_agent/utils/grub_utils.py b/fuel_agent/utils/grub_utils.py index 081eb44..113516a 100644 --- a/fuel_agent/utils/grub_utils.py +++ b/fuel_agent/utils/grub_utils.py @@ -14,6 +14,7 @@ import os import re +import shutil from fuel_agent import errors from fuel_agent.openstack.common import log as logging @@ -70,6 +71,15 @@ def guess_grub_install(chroot=''): raise errors.GrubUtilsError('grub-install not found') +def guess_grub1_datadir(chroot='', arch='x86_64'): + LOG.debug('Looking for grub data directory') + for d in os.listdir(chroot + '/usr/share/grub'): + if arch in d: + LOG.debug('Looks like grub data directory ' + 'is /usr/share/grub/%s' % d) + return '/usr/share/grub/' + d + + def guess_kernel(chroot=''): for filename in sorted(os.listdir(chroot + '/boot'), reverse=True): # We assume kernel name is always starts with vmlinuz. @@ -98,7 +108,7 @@ def grub1_install(install_devices, boot_device, chroot=''): 'boot device must be a partition') boot_disk = match.group(1) boot_part = str(int(match.group(3)) - 1) - + grub1_stage1(chroot=chroot) for install_device in install_devices: grub1_mbr(install_device, boot_disk, boot_part, chroot=chroot) @@ -139,7 +149,23 @@ def grub1_mbr(install_device, boot_disk, boot_part, chroot=''): cmd = ['/tmp/grub.sh'] if chroot: cmd[:0] = ['chroot', chroot] - utils.execute(*cmd, run_as_root=True, check_exit_code=[0]) + stdout, stderr = utils.execute(*cmd, run_as_root=True, check_exit_code=[0]) + LOG.debug('Grub script stdout: \n%s' % stdout) + LOG.debug('Grub script stderr: \n%s' % stderr) + + +def grub1_stage1(chroot=''): + LOG.debug('Installing grub stage1 files') + for f in os.listdir(chroot + '/boot/grub'): + if f in ('stage1', 'stage2') or 'stage1_5' in f: + LOG.debug('Removing: %s' % chroot + os.path.join('/boot/grub', f)) + os.remove(chroot + os.path.join('/boot/grub', f)) + grub1_datadir = guess_grub1_datadir(chroot=chroot) + for f in os.listdir(chroot + grub1_datadir): + if f in ('stage1', 'stage2') or 'stage1_5' in f: + LOG.debug('Copying %s from %s to /boot/grub' % (f, grub1_datadir)) + shutil.copy(chroot + os.path.join(grub1_datadir, f), + chroot + os.path.join('/boot/grub', f)) def grub1_cfg(kernel=None, initrd=None,