Add processing of keep_data flag

In case when some partition has keep_data = True
need to elevate flag to file system level and
do not to create new partition table. Do not clean
file system when it has keep_data = True.
There should not be a problem if some raw data
will not be cleaned.

Change-Id: Ifccc9256b12944283219cffeedb8c8da8ab420f8
Implements: blueprint partition-preservation

fuel_agent/drivers/nailgun.py

@@ -262,7 +262,9 @@ class Nailgun(BaseDataDriver):
                     if volume.get('mount') != '/boot':
                         LOG.debug('Adding partition on disk %s: size=%s' %
                                   (disk['name'], volume['size']))
-                        prt = parted.add_partition(size=volume['size'])
+                        prt = parted.add_partition(
+                            size=volume['size'],
+                            keep_data=volume.get('keep_data', False))
                         LOG.debug('Partition name: %s' % prt.name)
 
                     elif volume.get('mount') == '/boot' \
@@ -275,7 +277,9 @@ class Nailgun(BaseDataDriver):
                         # huge disks if it is possible.
                         LOG.debug('Adding /boot partition on disk %s: '
                                   'size=%s', disk['name'], volume['size'])
-                        prt = parted.add_partition(size=volume['size'])
+                        prt = parted.add_partition(
+                            size=volume['size'],
+                            keep_data=volume.get('keep_data', False))
                         LOG.debug('Partition name: %s', prt.name)
                         self._boot_partition_done = True
                     else:
@@ -388,6 +392,7 @@ class Nailgun(BaseDataDriver):
                             fs_type=volume.get('file_system', 'xfs'),
                             fs_label=self._getlabel(volume.get('disk_label')))
 
+        partition_scheme.elevate_keep_data()
         return partition_scheme
 
     def parse_configdrive_scheme(self):

fuel_agent/manager.py

@@ -98,8 +98,27 @@ class Manager(object):
     def __init__(self, data):
         self.driver = utils.get_driver(CONF.data_driver)(data)
 
+    def do_clean_filesystems(self):
+        # All fs without keep_data flag and without image should be cleaned,
+        # mkfs will clean file table and such flags as --force are not needed
+        # TODO(asvechnikov): need to refactor processing keep_flag logic when
+        # data model will become flat
+        for fs in self.driver.partition_scheme.fss:
+            found_images = [img for img in self.driver.image_scheme.images
+                            if img.target_device == fs.device]
+
+            if not fs.keep_data and not found_images:
+                fu.make_fs(fs.type, fs.options, fs.label, fs.device)
+
     def do_partitioning(self):
         LOG.debug('--- Partitioning disks (do_partitioning) ---')
+
+        if self.driver.partition_scheme.skip_partitioning:
+            LOG.debug('Some of fs has keep_data flag, '
+                      'partitioning is skiping')
+            self.do_clean_filesystems()
+            return
+
         # If disks are not wiped out at all, it is likely they contain lvm
         # and md metadata which will prevent re-creating a partition table
         # with 'device is busy' error.

fuel_agent/objects/base.py

@@ -17,8 +17,6 @@
 import abc
 import six
 
-from fuel_agent.utils.decorators import abstractclassmethod
-
 
 @six.add_metaclass(abc.ABCMeta)
 class Serializable(object):
@@ -27,6 +25,6 @@ class Serializable(object):
     def to_dict(self):
         pass
 
-    @abstractclassmethod
+    @classmethod
     def from_dict(cls, data):
-        pass
+        return cls(**data)

fuel_agent/objects/partition.py

@@ -102,6 +102,11 @@ class Parted(base.Serializable):
             separator = 'p'
         return '%s%s%s' % (self.name, separator, self.next_count())
 
+    def partition_by_name(self, name):
+        found = filter(lambda x: (x.name == name), self.partitions)
+        if found:
+            return found[0]
+
     def to_dict(self):
         partitions = [partition.to_dict() for partition in self.partitions]
         return {
@@ -123,7 +128,8 @@ class Parted(base.Serializable):
 class Partition(base.Serializable):
 
     def __init__(self, name, count, device, begin, end, partition_type,
-                 flags=None, guid=None, configdrive=False):
+                 flags=None, guid=None, configdrive=False, keep_data=False):
+        self.keep_data = keep_data
         self.name = name
         self.count = count
         self.device = device
@@ -152,16 +158,15 @@ class Partition(base.Serializable):
             'flags': self.flags,
             'guid': self.guid,
             'configdrive': self.configdrive,
+            'keep_data': self.keep_data,
         }
 
-    @classmethod
-    def from_dict(cls, data):
-        return cls(**data)
-
 
 class PhysicalVolume(base.Serializable):
 
-    def __init__(self, name, metadatasize=16, metadatacopies=2):
+    def __init__(self, name, metadatasize=16,
+                 metadatacopies=2, keep_data=False):
+        self.keep_data = keep_data
         self.name = name
         self.metadatasize = metadatasize
         self.metadatacopies = metadatacopies
@@ -171,19 +176,17 @@ class PhysicalVolume(base.Serializable):
             'name': self.name,
             'metadatasize': self.metadatasize,
             'metadatacopies': self.metadatacopies,
+            'keep_data': self.keep_data,
         }
 
-    @classmethod
-    def from_dict(cls, data):
-        return cls(**data)
-
 
 PV = PhysicalVolume
 
 
 class VolumeGroup(base.Serializable):
 
-    def __init__(self, name, pvnames=None):
+    def __init__(self, name, pvnames=None, keep_data=False):
+        self.keep_data = keep_data
         self.name = name
         self.pvnames = pvnames or []
 
@@ -194,20 +197,18 @@ class VolumeGroup(base.Serializable):
     def to_dict(self):
         return {
             'name': self.name,
-            'pvnames': self.pvnames
+            'pvnames': self.pvnames,
+            'keep_data': self.keep_data,
         }
 
-    @classmethod
-    def from_dict(cls, data):
-        return cls(**data)
-
 
 VG = VolumeGroup
 
 
 class LogicalVolume(base.Serializable):
 
-    def __init__(self, name, vgname, size):
+    def __init__(self, name, vgname, size, keep_data=False):
+        self.keep_data = keep_data
         self.name = name
         self.vgname = vgname
         self.size = size
@@ -222,12 +223,9 @@ class LogicalVolume(base.Serializable):
             'name': self.name,
             'vgname': self.vgname,
             'size': self.size,
+            'keep_data': self.keep_data,
         }
 
-    @classmethod
-    def from_dict(cls, data):
-        return cls(**data)
-
 
 LV = LogicalVolume
 
@@ -235,7 +233,8 @@ LV = LogicalVolume
 class MultipleDevice(base.Serializable):
 
     def __init__(self, name, level,
-                 devices=None, spares=None):
+                 devices=None, spares=None, keep_data=False):
+        self.keep_data = keep_data
         self.name = name
         self.level = level
         self.devices = devices or []
@@ -261,20 +260,18 @@ class MultipleDevice(base.Serializable):
             'level': self.level,
             'devices': self.devices,
             'spares': self.spares,
+            'keep_data': self.keep_data,
         }
 
-    @classmethod
-    def from_dict(cls, data):
-        return cls(**data)
-
 
 MD = MultipleDevice
 
 
 class FileSystem(base.Serializable):
 
-    def __init__(self, device, mount=None,
+    def __init__(self, device, mount=None, fs_type=None,
-                 fs_type=None, fs_options=None, fs_label=None):
+                 fs_options=None, fs_label=None, keep_data=False):
+        self.keep_data = keep_data
         self.device = device
         self.mount = mount
         self.type = fs_type or 'xfs'
@@ -288,12 +285,9 @@ class FileSystem(base.Serializable):
             'fs_type': self.type,
             'fs_options': self.options,
             'fs_label': self.label,
+            'keep_data': self.keep_data,
         }
 
-    @classmethod
-    def from_dict(cls, data):
-        return cls(**data)
-
 
 FS = FileSystem
 
@@ -377,6 +371,11 @@ class PartitionScheme(object):
                     'try to generate md name manually')
             count += 1
 
+    def partition_by_name(self, name):
+        return next((parted.partition_by_name(name)
+                    for parted in self.parteds
+                    if parted.partition_by_name(name)), None)
+
     def vg_by_name(self, vgname):
         found = filter(lambda x: (x.name == vgname), self.vgs)
         if found:
@@ -472,6 +471,56 @@ class PartitionScheme(object):
                 if prt.configdrive:
                     return prt.name
 
+    def elevate_keep_data(self):
+        LOG.debug('Elevate keep_data flag from partitions')
+
+        for vg in self.vgs:
+            for pvname in vg.pvnames:
+                partition = self.partition_by_name(pvname)
+                if partition and partition.keep_data:
+                    partition.keep_data = False
+                    vg.keep_data = True
+                    LOG.debug('Set keep_data to vg=%s' % vg.name)
+
+        for lv in self.lvs:
+            vg = self.vg_by_name(lv.vgname)
+            if vg.keep_data:
+                lv.keep_data = True
+
+        # Need to loop over lv again to remove keep flag from vg
+        for lv in self.lvs:
+            vg = self.vg_by_name(lv.vgname)
+            if vg.keep_data and lv.keep_data:
+                vg.keep_data = False
+
+        for fs in self.fss:
+            lv = self.lv_by_device_name(fs.device)
+            if lv:
+                if lv.keep_data:
+                    lv.keep_data = False
+                    fs.keep_data = True
+                    LOG.debug('Set keep_data to fs=%s from lv=%s' %
+                              (fs.mount, lv.name))
+                continue
+            partition = self.partition_by_name(fs.device)
+            if partition and partition.keep_data:
+                partition.keep_data = False
+                fs.keep_data = True
+                LOG.debug('Set keep flag to fs=%s from partition=%s' %
+                          (fs.mount, partition.name))
+
+    @property
+    def skip_partitioning(self):
+        if any(fs.keep_data for fs in self.fss):
+            return True
+        if any(lv.keep_data for lv in self.lvs):
+            return True
+        if any(vg.keep_data for vg in self.vgs):
+            return True
+        for parted in self.parteds:
+            if any(prt.keep_data for prt in parted.partitions):
+                return True
+
     def to_dict(self):
         return {
             'parteds': [parted.to_dict() for parted in self.parteds],

fuel_agent/tests/test_manager.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import mock
 import os
 import signal
@@ -146,6 +147,29 @@ class TestManager(test_base.BaseTestCase):
         mock_utils.makedirs_if_not_exists.assert_called_once_with(
             '/tmp/target/etc/nailgun-agent')
 
+    @mock.patch('yaml.load')
+    @mock.patch.object(utils, 'init_http_request')
+    @mock.patch.object(hu, 'list_block_devices')
+    @mock.patch.object(fu, 'make_fs')
+    def test_do_partitioning_with_keep_data_flag(self, mock_fu_mf, mock_lbd,
+                                                 mock_http, mock_yaml):
+        mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
+        data = copy.deepcopy(test_nailgun.PROVISION_SAMPLE_DATA)
+
+        for disk in data['ks_meta']['pm_data']['ks_spaces']:
+            for volume in disk['volumes']:
+                if volume['type'] == 'pv' and volume['vg'] == 'image':
+                    volume['keep_data'] = True
+
+        self.mgr = manager.Manager(data)
+
+        self.mgr.do_partitioning()
+        mock_fu_mf_expected_calls = [
+            mock.call('ext2', '', '', '/dev/sda3'),
+            mock.call('ext2', '', '', '/dev/sda4'),
+            mock.call('swap', '', '', '/dev/mapper/os-swap')]
+        self.assertEqual(mock_fu_mf_expected_calls, mock_fu_mf.call_args_list)
+
     @mock.patch('six.moves.builtins.open')
     @mock.patch.object(os, 'symlink')
     @mock.patch.object(os, 'remove')

fuel_agent/tests/test_nailgun.py

@@ -426,7 +426,8 @@ SINGLE_DISK_KS_SPACES = [
                 "size": 200,
                 "type": "partition",
                 "file_system": "ext4",
-                "name": "Root"
+                "name": "Root",
+                "keep_data": True
             },
         ],
         "type": "disk",
@@ -951,6 +952,37 @@ class TestNailgun(test_base.BaseTestCase):
             drv.partition_scheme.fs_by_mount('/boot').device,
             '/dev/sda3')
 
+    @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
+    @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
+    @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
+    def test_elevate_keep_data_single_disk(self, mock_lbd,
+                                           mock_http_req, mock_yaml):
+        data = copy.deepcopy(PROVISION_SAMPLE_DATA)
+        data['ks_meta']['pm_data']['ks_spaces'] = SINGLE_DISK_KS_SPACES
+        mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
+        drv = nailgun.Nailgun(data)
+        self.assertTrue(drv.partition_scheme.fs_by_mount('/').keep_data)
+
+        for parted in drv.partition_scheme.parteds:
+            for partition in parted.partitions:
+                self.assertFalse(partition.keep_data)
+
+        for md in drv.partition_scheme.mds:
+            self.assertFalse(md.keep_data)
+
+        for pv in drv.partition_scheme.pvs:
+            self.assertFalse(pv.keep_data)
+
+        for vg in drv.partition_scheme.vgs:
+            self.assertFalse(vg.keep_data)
+
+        for lv in drv.partition_scheme.lvs:
+            self.assertFalse(lv.keep_data)
+
+        for fs in drv.partition_scheme.fss:
+            if fs.mount != '/':
+                self.assertFalse(fs.keep_data)
+
     @mock.patch('fuel_agent.drivers.nailgun.yaml.load')
     @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
     @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')

fuel_agent/tests/test_partition.py

@@ -69,6 +69,7 @@ class TestMultipleDevice(unittest2.TestCase):
             'level': 'level',
             'devices': ['device_a', ],
             'spares': ['device_b', ],
+            'keep_data': False,
         }
         new_md = partition.MD.from_dict(serialized)
         assert serialized == new_md.to_dict()
@@ -101,6 +102,7 @@ class TestPartition(unittest2.TestCase):
             'guid': 'some_guid',
             'name': 'name',
             'partition_type': 'partition_type',
+            'keep_data': False,
         }
         new_pt = partition.Partition.from_dict(serialized)
         assert serialized == new_pt.to_dict()
@@ -161,6 +163,16 @@ class TestPartitionScheme(unittest2.TestCase):
         self.assertRaises(errors.MDAlreadyExistsError,
                           self.p_scheme.md_next_name)
 
+    def test_partition_by_name(self):
+        parted_1 = self.p_scheme.add_parted(name='name_1', label='label_1')
+        parted_1.add_partition(size=1)
+
+        parted_2 = self.p_scheme.add_parted(name='name_2', label='label_2')
+        expected_prt = parted_2.add_partition(size=1)
+
+        actual_prt = self.p_scheme.partition_by_name(expected_prt.name)
+        self.assertEqual(expected_prt, actual_prt)
+
     def test_md_by_name(self):
         self.assertEqual(0, len(self.p_scheme.mds))
         expected_md = partition.MD('name', 'level')
@@ -204,6 +216,30 @@ class TestPartitionScheme(unittest2.TestCase):
         self.assertEqual('fs_label', self.p_scheme.fss[0].label)
         self.assertEqual('-F', self.p_scheme.fss[0].options)
 
+    def test_elevate_keep_data(self):
+        self.assertEqual(0, len(self.p_scheme.vgs))
+        self.assertEqual(0, len(self.p_scheme.lvs))
+        self.assertEqual(0, len(self.p_scheme.fss))
+        parted = self.p_scheme.add_parted(name='fake_name', label='fake_label')
+        prt = parted.add_partition(size=1, keep_data=True)
+
+        self.p_scheme.vg_attach_by_name(prt.name, 'fake_vg')
+        vg = self.p_scheme.vgs[0]
+
+        self.p_scheme.add_lv(name='fake_lv', vgname=vg.name, size=1)
+        lv = self.p_scheme.lvs[0]
+
+        self.p_scheme.add_fs(device=lv.device_name, mount='fake_mount',
+                             fs_type='xfs', fs_label='fake_label')
+        fs = self.p_scheme.fss[0]
+
+        self.p_scheme.elevate_keep_data()
+
+        self.assertTrue(fs.keep_data)
+        self.assertFalse(lv.keep_data)
+        self.assertFalse(vg.keep_data)
+        self.assertFalse(prt.keep_data)
+
 
 class TestParted(unittest2.TestCase):
     def setUp(self):
@@ -289,6 +325,14 @@ class TestParted(unittest2.TestCase):
         self.prtd.partitions.extend(expected_partitions)
         self.assertEqual(expected_partitions, self.prtd.primary)
 
+    def test_partition_by_name(self):
+        self.prtd.add_partition(size=1)
+        self.prtd.add_partition(size=1)
+        expected_prt = self.prtd.add_partition(size=1)
+
+        actual_prt = self.prtd.partition_by_name(expected_prt.name)
+        self.assertEqual(expected_prt, actual_prt)
+
     def test_conversion(self):
         prt = partition.Partition(
             name='name',
@@ -296,7 +340,8 @@ class TestParted(unittest2.TestCase):
             device='device',
             begin='begin',
             end='end',
-            partition_type='primary'
+            partition_type='primary',
+            keep_data=True
         )
         self.prtd.partitions.append(prt)
         serialized = self.prtd.to_dict()
@@ -325,6 +370,7 @@ class TestLogicalVolume(unittest2.TestCase):
             'name': 'lv-name',
             'vgname': 'vg-name',
             'size': 1234,
+            'keep_data': False,
         }
         new_lv = partition.LV.from_dict(serialized)
         assert serialized == new_lv.to_dict()
@@ -343,6 +389,7 @@ class TestPhisicalVolume(unittest2.TestCase):
             'name': 'pv-name',
             'metadatasize': 987,
             'metadatacopies': 112,
+            'keep_data': False,
         }
         new_pv = partition.PV.from_dict(serialized)
         assert serialized == new_pv.to_dict()
@@ -358,7 +405,8 @@ class TestVolumesGroup(unittest2.TestCase):
         serialized = vg.to_dict()
         assert serialized == {
             'name': 'vg-name',
-            'pvnames': ['pv-name-a', ]
+            'pvnames': ['pv-name-a', ],
+            'keep_data': False,
         }
         new_vg = partition.VG.from_dict(serialized)
         assert serialized == new_vg.to_dict()
@@ -381,6 +429,7 @@ class TestFileSystem(unittest2.TestCase):
             'fs_type': 'type',
             'fs_options': 'some-option',
             'fs_label': 'some-label',
+            'keep_data': False,
         }
         new_fs = partition.FS.from_dict(serialized)
         assert serialized == new_fs.to_dict()