Some improvements in do_bootloader in fuel_agent

1) Added copying grub stage 1.5 files into /boot/grub.
2) Added fstab related stuff

Change-Id: Id540f73d30b51b80d40661e2d59f8382f7a4c901
Implements: blueprint image-based-provisioning

fuel_agent/manager.py

@@ -207,6 +207,12 @@ class Manager(object):
             LOG.debug('Launching image processing chain')
             processing.process()
 
+            LOG.debug('Extending image file systems')
+            if image.format in ('ext2', 'ext3', 'ext4', 'xfs'):
+                LOG.debug('Extending %s %s' %
+                          (image.format, image.target_device))
+                fu.extend_fs(image.format, image.target_device)
+
     def mount_target(self, chroot):
         LOG.debug('Mounting target file systems')
         # Here we are going to mount all file systems in partition scheme.
@@ -223,26 +229,40 @@ class Manager(object):
         fu.mount_bind(chroot, '/sys')
         fu.mount_bind(chroot, '/dev')
         fu.mount_bind(chroot, '/proc')
+        mtab = utils.execute(
+            'chroot', chroot, 'grep', '-v', 'rootfs', '/proc/mounts')[0]
+        with open(chroot + '/etc/mtab', 'wb') as f:
+            f.write(mtab)
 
     def umount_target(self, chroot):
         LOG.debug('Umounting target file systems')
-        key = lambda x: len(x.mount.rstrip('/').split('/'))
-        for fs in sorted(self.partition_scheme.fss, key=key, reverse=True):
-            fu.umount_fs(fs.device)
         fu.umount_fs(chroot + '/proc')
         fu.umount_fs(chroot + '/dev')
         fu.umount_fs(chroot + '/sys')
+        key = lambda x: len(x.mount.rstrip('/').split('/'))
+        for fs in sorted(self.partition_scheme.fss, key=key, reverse=True):
+            if fs.mount == 'swap':
+                continue
+            fu.umount_fs(fs.device)
 
     def do_bootloader(self):
         LOG.debug('--- Installing bootloader (do_bootloader) ---')
         chroot = '/tmp/target'
         self.mount_target(chroot)
 
-        grub_version = gu.grub_version_guess(chroot=chroot)
+        mount2uuid = {}
+        for fs in self.partition_scheme.fss:
+            mount2uuid[fs.mount], _ = utils.execute(
+                'blkid', '-o', 'value', '-s', 'UUID', fs.device,
+                check_exit_code=[0])
+
+        grub_version = gu.guess_grub_version(chroot=chroot)
         boot_device = self.partition_scheme.boot_device(grub_version)
         install_devices = [d.name for d in self.partition_scheme.parteds
                            if d.install_bootloader]
+
         kernel_params = self.partition_scheme.kernel_params
+        kernel_params += ' root=UUID=%s ' % mount2uuid['/']
 
         if grub_version == 1:
             gu.grub1_cfg(kernel_params=kernel_params, chroot=chroot)
@@ -251,6 +271,16 @@ class Manager(object):
             gu.grub2_cfg(kernel_params=kernel_params, chroot=chroot)
             gu.grub2_install(install_devices, chroot=chroot)
 
+        with open(chroot + '/etc/fstab', 'wb') as f:
+            for fs in self.partition_scheme.fss:
+                # TODO(kozhukalov): Think of improving the logic so as to
+                # insert a meaningful fsck order value which is last zero
+                # at fstab line. Currently we set it into 0 which means
+                # a corresponding file system will never be checked. We assume
+                # puppet or other configuration tool will care of it.
+                f.write('UUID=%s %s %s defaults 0 0\n' %
+                        (mount2uuid[fs.mount], fs.mount, fs.type))
+
         self.umount_target(chroot)
 
     def do_reboot(self):

fuel_agent/tests/test_fs_utils.py

@@ -91,7 +91,18 @@ class TestFSUtils(test_base.BaseTestCase):
             'mount', '--bind', '/fake', '/target/fake2', check_exit_code=[0])
 
     @mock.patch.object(utils, 'execute')
-    def test_umount_fs(self, mock_exec):
+    def test_umount_fs_ok(self, mock_exec):
         fu.umount_fs('/fake')
         mock_exec.assert_called_once_with(
             'umount', '/fake', check_exit_code=[0])
+
+    @mock.patch.object(utils, 'execute')
+    def test_umount_fs_error(self, mock_exec):
+        mock_exec.side_effect = [
+            errors.ProcessExecutionError('message'), ('', '')]
+        fu.umount_fs('/fake')
+        expected_calls = [
+            mock.call('umount', '/fake', check_exit_code=[0]),
+            mock.call('umount', '-l', '/fake', check_exit_code=[0])
+        ]
+        self.assertEqual(expected_calls, mock_exec.call_args_list)

fuel_agent/tests/test_grub_utils.py

@@ -219,8 +219,9 @@ class TestGrubUtils(test_base.BaseTestCase):
         mock_listdir.return_value = ['1', '2', '3']
         self.assertRaises(errors.GrubUtilsError, gu.guess_initrd, '/target')
 
+    @mock.patch.object(gu, 'grub1_stage1')
     @mock.patch.object(gu, 'grub1_mbr')
-    def test_grub1_install(self, mock_mbr):
+    def test_grub1_install(self, mock_mbr, mock_stage1):
         install_devices = ['/dev/foo', '/dev/bar']
         expected_calls_mbr = []
         for install_device in install_devices:
@@ -228,6 +229,7 @@ class TestGrubUtils(test_base.BaseTestCase):
                 mock.call(install_device, '/dev/foo', '0', chroot='/target'))
         gu.grub1_install(install_devices, '/dev/foo1', '/target')
         self.assertEqual(expected_calls_mbr, mock_mbr.call_args_list)
+        mock_stage1.assert_called_once_with(chroot='/target')
         # should raise exception if boot_device (second argument)
         # is not a partition but a whole disk
         self.assertRaises(errors.GrubUtilsError, gu.grub1_install,
@@ -239,6 +241,7 @@ class TestGrubUtils(test_base.BaseTestCase):
     def test_grub1_mbr_install_differs_boot(self, mock_exec,
                                             mock_chmod, mock_guess):
         mock_guess.return_value = '/sbin/grub'
+        mock_exec.return_value = ('stdout', 'stderr')
 
         # install_device != boot_disk
         batch = 'device (hd0) /dev/foo\n'
@@ -274,6 +277,7 @@ class TestGrubUtils(test_base.BaseTestCase):
     def test_grub1_mbr_install_same_as_boot(self, mock_exec,
                                             mock_chmod, mock_guess):
         mock_guess.return_value = '/sbin/grub'
+        mock_exec.return_value = ('stdout', 'stderr')
 
         # install_device == boot_disk
         batch = 'device (hd0) /dev/foo\n'

fuel_agent/utils/fs_utils.py

@@ -13,8 +13,11 @@
 # limitations under the License.
 
 from fuel_agent import errors
+from fuel_agent.openstack.common import log as logging
 from fuel_agent.utils import utils
 
+LOG = logging.getLogger(__name__)
+
 
 def make_fs(fs_type, fs_options, fs_label, dev):
     # NOTE(agordeev): notice the different flag to force the fs creating
@@ -56,4 +59,11 @@ def mount_bind(chroot, path, path2=None):
 
 
 def umount_fs(fs_mount):
+    try:
+        LOG.debug('Trying to umount {0}'.format(fs_mount))
         utils.execute('umount', fs_mount, check_exit_code=[0])
+    except errors.ProcessExecutionError as e:
+        LOG.warning('Error while umounting {0} '
+                    'exc={1}'.format(fs_mount, e.message))
+        LOG.debug('Trying lazy umounting {0}'.format(fs_mount))
+        utils.execute('umount', '-l', fs_mount, check_exit_code=[0])

fuel_agent/utils/grub_utils.py

@@ -14,6 +14,7 @@
 
 import os
 import re
+import shutil
 
 from fuel_agent import errors
 from fuel_agent.openstack.common import log as logging
@@ -70,6 +71,15 @@ def guess_grub_install(chroot=''):
     raise errors.GrubUtilsError('grub-install not found')
 
 
+def guess_grub1_datadir(chroot='', arch='x86_64'):
+    LOG.debug('Looking for grub data directory')
+    for d in os.listdir(chroot + '/usr/share/grub'):
+        if arch in d:
+            LOG.debug('Looks like grub data directory '
+                      'is /usr/share/grub/%s' % d)
+            return '/usr/share/grub/' + d
+
+
 def guess_kernel(chroot=''):
     for filename in sorted(os.listdir(chroot + '/boot'), reverse=True):
         # We assume kernel name is always starts with vmlinuz.
@@ -98,7 +108,7 @@ def grub1_install(install_devices, boot_device, chroot=''):
             'boot device must be a partition')
     boot_disk = match.group(1)
     boot_part = str(int(match.group(3)) - 1)
-
+    grub1_stage1(chroot=chroot)
     for install_device in install_devices:
         grub1_mbr(install_device, boot_disk, boot_part, chroot=chroot)
 
@@ -139,7 +149,23 @@ def grub1_mbr(install_device, boot_disk, boot_part, chroot=''):
     cmd = ['/tmp/grub.sh']
     if chroot:
         cmd[:0] = ['chroot', chroot]
-    utils.execute(*cmd, run_as_root=True, check_exit_code=[0])
+    stdout, stderr = utils.execute(*cmd, run_as_root=True, check_exit_code=[0])
+    LOG.debug('Grub script stdout: \n%s' % stdout)
+    LOG.debug('Grub script stderr: \n%s' % stderr)
+
+
+def grub1_stage1(chroot=''):
+    LOG.debug('Installing grub stage1 files')
+    for f in os.listdir(chroot + '/boot/grub'):
+        if f in ('stage1', 'stage2') or 'stage1_5' in f:
+            LOG.debug('Removing: %s' % chroot + os.path.join('/boot/grub', f))
+            os.remove(chroot + os.path.join('/boot/grub', f))
+    grub1_datadir = guess_grub1_datadir(chroot=chroot)
+    for f in os.listdir(chroot + grub1_datadir):
+        if f in ('stage1', 'stage2') or 'stage1_5' in f:
+            LOG.debug('Copying %s from %s to /boot/grub' % (f, grub1_datadir))
+            shutil.copy(chroot + os.path.join(grub1_datadir, f),
+                        chroot + os.path.join('/boot/grub', f))
 
 
 def grub1_cfg(kernel=None, initrd=None,