diff --git a/fuel_agent/cmd/agent.py b/fuel_agent/cmd/agent.py index c161aec..90412ae 100644 --- a/fuel_agent/cmd/agent.py +++ b/fuel_agent/cmd/agent.py @@ -43,19 +43,19 @@ def provision(): def partition(): - main(['do_parsing', 'do_partitioning']) + main(['do_partitioning']) def copyimage(): - main(['do_parsing', 'do_copyimage']) + main(['do_copyimage']) def configdrive(): - main(['do_parsing', 'do_configdrive']) + main(['do_configdrive']) def bootloader(): - main(['do_parsing', 'do_bootloader']) + main(['do_bootloader']) def print_err(line): diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py index 10d650a..1c225de 100644 --- a/fuel_agent/drivers/nailgun.py +++ b/fuel_agent/drivers/nailgun.py @@ -14,6 +14,7 @@ import math import os +import six from fuel_agent.drivers import ks_spaces_validator from fuel_agent import errors @@ -70,6 +71,11 @@ class Nailgun(object): # get rid of md over all disks for /boot partition. self._boot_done = False + self.partition_scheme = self.parse_partition_scheme() + self.configdrive_scheme = self.parse_configdrive_scheme() + # parsing image scheme needs partition scheme has been parsed + self.image_scheme = self.parse_image_scheme() + def partition_data(self): return self.data['ks_meta']['pm_data']['ks_spaces'] @@ -129,7 +135,7 @@ class Nailgun(object): def _num_ceph_osds(self): return self._get_partition_count('ceph') - def partition_scheme(self): + def parse_partition_scheme(self): LOG.debug('--- Preparing partition scheme ---') data = self.partition_data() ks_spaces_validator.validate(data) @@ -315,7 +321,7 @@ class Nailgun(object): self.data['ks_meta']['pm_data']['kernel_params']) return partition_scheme - def configdrive_scheme(self): + def parse_configdrive_scheme(self): LOG.debug('--- Preparing configdrive scheme ---') data = self.data configdrive_scheme = objects.ConfigDriveScheme() @@ -370,7 +376,7 @@ class Nailgun(object): configdrive_scheme.set_profile(profile=data['profile']) return configdrive_scheme - def image_scheme(self, partition_scheme): + def parse_image_scheme(self): LOG.debug('--- Preparing image scheme ---') data = self.data image_scheme = objects.ImageScheme() @@ -403,24 +409,19 @@ class Nailgun(object): # /, /boot, /var/lib file systems then we will try to get images # for all those mount points. Images data are to be defined # at provision.json -> ['ks_meta']['image_data'] - LOG.debug('Looping over all file systems in partition scheme') - for fs in partition_scheme.fss: - LOG.debug('Processing fs %s' % fs.mount) - if fs.mount not in data['ks_meta']['image_data']: - LOG.debug('There is no image for fs %s. Skipping.' % fs.mount) - continue - image_data = data['ks_meta']['image_data'][fs.mount] + LOG.debug('Looping over all images in provision data') + for mount_point, image_data in six.iteritems( + data['ks_meta']['image_data']): LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' % - (fs.mount, image_data['uri'], + (mount_point, image_data['uri'], image_data['format'], image_data['container'])) image_scheme.add_image( uri=image_data['uri'], - target_device=fs.device, - # In the future we will get format and container - # from provision.json, but currently it is hard coded. + target_device=self.partition_scheme.fs_by_mount( + mount_point).device, format=image_data['format'], container=image_data['container'], - size=image_meta.get(fs.mount, {}).get('size'), - md5=image_meta.get(fs.mount, {}).get('md5'), + size=image_meta.get(mount_point, {}).get('size'), + md5=image_meta.get(mount_point, {}).get('md5'), ) return image_scheme diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py index c83848e..1abc765 100644 --- a/fuel_agent/manager.py +++ b/fuel_agent/manager.py @@ -78,15 +78,6 @@ LOG = logging.getLogger(__name__) class Manager(object): def __init__(self, data): self.driver = utils.get_driver(CONF.data_driver)(data) - self.partition_scheme = None - self.configdrive_scheme = None - self.image_scheme = None - - def do_parsing(self): - LOG.debug('--- Parsing data (do_parsing) ---') - self.partition_scheme = self.driver.partition_scheme() - self.configdrive_scheme = self.driver.configdrive_scheme() - self.image_scheme = self.driver.image_scheme(self.partition_scheme) def do_partitioning(self): LOG.debug('--- Partitioning disks (do_partitioning) ---') @@ -128,7 +119,7 @@ class Manager(object): utils.execute('udevadm', 'control', '--reload-rules', check_exit_code=[0]) - for parted in self.partition_scheme.parteds: + for parted in self.driver.partition_scheme.parteds: for prt in parted.partitions: # We wipe out the beginning of every new partition # right after creating it. It allows us to avoid possible @@ -146,7 +137,7 @@ class Manager(object): 'seek=%s' % max(prt.end - 3, 0), 'count=5', 'of=%s' % prt.device, check_exit_code=[0, 1]) - for parted in self.partition_scheme.parteds: + for parted in self.driver.partition_scheme.parteds: pu.make_label(parted.name, parted.label) for prt in parted.partitions: pu.make_partition(prt.device, prt.begin, prt.end, prt.type) @@ -197,25 +188,25 @@ class Manager(object): lu.pvremove_all() # creating meta disks - for md in self.partition_scheme.mds: + for md in self.driver.partition_scheme.mds: mu.mdcreate(md.name, md.level, *md.devices) # creating physical volumes - for pv in self.partition_scheme.pvs: + for pv in self.driver.partition_scheme.pvs: lu.pvcreate(pv.name, metadatasize=pv.metadatasize, metadatacopies=pv.metadatacopies) # creating volume groups - for vg in self.partition_scheme.vgs: + for vg in self.driver.partition_scheme.vgs: lu.vgcreate(vg.name, *vg.pvnames) # creating logical volumes - for lv in self.partition_scheme.lvs: + for lv in self.driver.partition_scheme.lvs: lu.lvcreate(lv.vgname, lv.name, lv.size) # making file systems - for fs in self.partition_scheme.fss: - found_images = [img for img in self.image_scheme.images + for fs in self.driver.partition_scheme.fss: + found_images = [img for img in self.driver.image_scheme.images if img.target_device == fs.device] if not found_images: fu.make_fs(fs.type, fs.options, fs.label, fs.device) @@ -231,16 +222,22 @@ class Manager(object): tmpl_dir = CONF.nc_template_path utils.render_and_save( - tmpl_dir, self.configdrive_scheme.template_names('cloud_config'), - self.configdrive_scheme.template_data(), cc_output_path + tmpl_dir, + self.driver.configdrive_scheme.template_names('cloud_config'), + self.driver.configdrive_scheme.template_data(), + cc_output_path ) utils.render_and_save( - tmpl_dir, self.configdrive_scheme.template_names('boothook'), - self.configdrive_scheme.template_data(), bh_output_path + tmpl_dir, + self.driver.configdrive_scheme.template_names('boothook'), + self.driver.configdrive_scheme.template_data(), + bh_output_path ) utils.render_and_save( - tmpl_dir, self.configdrive_scheme.template_names('meta-data'), - self.configdrive_scheme.template_data(), md_output_path + tmpl_dir, + self.driver.configdrive_scheme.template_names('meta-data'), + self.driver.configdrive_scheme.template_data(), + md_output_path ) utils.execute('write-mime-multipart', '--output=%s' % ud_output_path, @@ -250,14 +247,14 @@ class Manager(object): '-volid', 'cidata', '-joliet', '-rock', ud_output_path, md_output_path) - configdrive_device = self.partition_scheme.configdrive_device() + configdrive_device = self.driver.partition_scheme.configdrive_device() if configdrive_device is None: raise errors.WrongPartitionSchemeError( 'Error while trying to get configdrive device: ' 'configdrive device not found') size = os.path.getsize(CONF.config_drive_path) md5 = utils.calculate_md5(CONF.config_drive_path, size) - self.image_scheme.add_image( + self.driver.image_scheme.add_image( uri='file://%s' % CONF.config_drive_path, target_device=configdrive_device, format='iso9660', @@ -268,7 +265,7 @@ class Manager(object): def do_copyimage(self): LOG.debug('--- Copying images (do_copyimage) ---') - for image in self.image_scheme.images: + for image in self.driver.image_scheme.images: LOG.debug('Processing image: %s' % image.uri) processing = au.Chain() @@ -320,7 +317,7 @@ class Manager(object): # Shorter paths earlier. We sort all mount points by their depth. # ['/', '/boot', '/var', '/var/lib/mysql'] key = lambda x: len(x.mount.rstrip('/').split('/')) - for fs in sorted(self.partition_scheme.fss, key=key): + for fs in sorted(self.driver.partition_scheme.fss, key=key): if fs.mount == 'swap': continue mount = chroot + fs.mount @@ -344,7 +341,8 @@ class Manager(object): fu.umount_fs(chroot + '/dev') fu.umount_fs(chroot + '/sys') key = lambda x: len(x.mount.rstrip('/').split('/')) - for fs in sorted(self.partition_scheme.fss, key=key, reverse=True): + for fs in sorted(self.driver.partition_scheme.fss, + key=key, reverse=True): if fs.mount == 'swap': continue fu.umount_fs(fs.device) @@ -355,17 +353,17 @@ class Manager(object): self.mount_target(chroot) mount2uuid = {} - for fs in self.partition_scheme.fss: + for fs in self.driver.partition_scheme.fss: mount2uuid[fs.mount] = utils.execute( 'blkid', '-o', 'value', '-s', 'UUID', fs.device, check_exit_code=[0])[0].strip() grub_version = gu.guess_grub_version(chroot=chroot) - boot_device = self.partition_scheme.boot_device(grub_version) - install_devices = [d.name for d in self.partition_scheme.parteds + boot_device = self.driver.partition_scheme.boot_device(grub_version) + install_devices = [d.name for d in self.driver.partition_scheme.parteds if d.install_bootloader] - kernel_params = self.partition_scheme.kernel_params + kernel_params = self.driver.partition_scheme.kernel_params kernel_params += ' root=UUID=%s ' % mount2uuid['/'] if grub_version == 1: @@ -381,7 +379,8 @@ class Manager(object): 'w') as f: f.write('# Generated by fuel-agent during provisioning: BEGIN\n') # pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1 - for mapping in self.configdrive_scheme.common.udevrules.split(','): + for mapping in self.driver.configdrive_scheme.\ + common.udevrules.split(','): mac_addr, nic_name = mapping.split('_') f.write('SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ' 'ATTR{address}=="%s", ATTR{type}=="1", KERNEL=="eth*",' @@ -396,7 +395,7 @@ class Manager(object): '# DO NOT DELETE. It is needed to disable net-generator\n') with open(chroot + '/etc/fstab', 'wb') as f: - for fs in self.partition_scheme.fss: + for fs in self.driver.partition_scheme.fss: # TODO(kozhukalov): Think of improving the logic so as to # insert a meaningful fsck order value which is last zero # at fstab line. Currently we set it into 0 which means @@ -413,7 +412,6 @@ class Manager(object): def do_provisioning(self): LOG.debug('--- Provisioning (do_provisioning) ---') - self.do_parsing() self.do_partitioning() self.do_configdrive() self.do_copyimage() diff --git a/fuel_agent/tests/test_manager.py b/fuel_agent/tests/test_manager.py index faeea13..72d3b50 100644 --- a/fuel_agent/tests/test_manager.py +++ b/fuel_agent/tests/test_manager.py @@ -34,21 +34,14 @@ CONF = cfg.CONF class TestManager(test_base.BaseTestCase): - def setUp(self): - super(TestManager, self).setUp() - self.mgr = manager.Manager(test_nailgun.PROVISION_SAMPLE_DATA) @mock.patch('yaml.load') @mock.patch.object(utils, 'init_http_request') @mock.patch.object(hu, 'list_block_devices') - def test_do_parsing(self, mock_lbd, mock_http_req, mock_yaml): + def setUp(self, mock_lbd, mock_http, mock_yaml): + super(TestManager, self).setUp() mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE - self.mgr.do_parsing() - #NOTE(agordeev): there's no need for deeper assertions as all schemes - # thoroughly tested in test_nailgun - self.assertFalse(self.mgr.partition_scheme is None) - self.assertFalse(self.mgr.configdrive_scheme is None) - self.assertFalse(self.mgr.image_scheme is None) + self.mgr = manager.Manager(test_nailgun.PROVISION_SAMPLE_DATA) @mock.patch('six.moves.builtins.open') @mock.patch.object(os, 'symlink') @@ -79,7 +72,6 @@ class TestManager(test_base.BaseTestCase): mock_os_ld.return_value = ['not_a_rule', 'fake.rules'] mock_os_p.exists.return_value = True mock_hu_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE - self.mgr.do_parsing() self.mgr.do_partitioning() mock_pu_ml_expected_calls = [mock.call('/dev/sda', 'gpt'), mock.call('/dev/sdb', 'gpt'), @@ -150,8 +142,7 @@ class TestManager(test_base.BaseTestCase): mock_get_size.return_value = 123 mock_md5.return_value = 'fakemd5' mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE - self.mgr.do_parsing() - self.assertEqual(1, len(self.mgr.image_scheme.images)) + self.assertEqual(1, len(self.mgr.driver.image_scheme.images)) self.mgr.do_configdrive() mock_u_ras_expected_calls = [ mock.call(CONF.nc_template_path, @@ -187,11 +178,11 @@ class TestManager(test_base.BaseTestCase): '%s/%s' % (CONF.tmp_path, 'user-data'), '%s/%s' % (CONF.tmp_path, 'meta-data'))] self.assertEqual(mock_u_e_expected_calls, mock_u_e.call_args_list) - self.assertEqual(2, len(self.mgr.image_scheme.images)) - cf_drv_img = self.mgr.image_scheme.images[-1] + self.assertEqual(2, len(self.mgr.driver.image_scheme.images)) + cf_drv_img = self.mgr.driver.image_scheme.images[-1] self.assertEqual('file://%s' % CONF.config_drive_path, cf_drv_img.uri) self.assertEqual('/dev/sda7', - self.mgr.partition_scheme.configdrive_device()) + self.mgr.driver.partition_scheme.configdrive_device()) self.assertEqual('iso9660', cf_drv_img.format) self.assertEqual('raw', cf_drv_img.container) self.assertEqual('fakemd5', cf_drv_img.md5) @@ -207,7 +198,6 @@ class TestManager(test_base.BaseTestCase): mock_u_e, mock_p_ps_cd, mock_http_req, mock_yaml): mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE - self.mgr.do_parsing() mock_p_ps_cd.return_value = None self.assertRaises(errors.WrongPartitionSchemeError, self.mgr.do_configdrive) @@ -239,10 +229,9 @@ class TestManager(test_base.BaseTestCase): mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE mock_au_c.return_value = FakeChain() - self.mgr.do_parsing() self.mgr.do_configdrive() self.mgr.do_copyimage() - imgs = self.mgr.image_scheme.images + imgs = self.mgr.driver.image_scheme.images self.assertEqual(2, len(imgs)) expected_processors_list = [] for img in imgs[:-1]: @@ -293,11 +282,10 @@ class TestManager(test_base.BaseTestCase): mock_md5.side_effect = ['fakemd5', 'really_fakemd5', 'fakemd5'] mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE mock_au_c.return_value = FakeChain() - self.mgr.do_parsing() - self.mgr.image_scheme.images[0].size = 1234 - self.mgr.image_scheme.images[0].md5 = 'really_fakemd5' + self.mgr.driver.image_scheme.images[0].size = 1234 + self.mgr.driver.image_scheme.images[0].md5 = 'really_fakemd5' self.mgr.do_configdrive() - self.assertEqual(2, len(self.mgr.image_scheme.images)) + self.assertEqual(2, len(self.mgr.driver.image_scheme.images)) self.mgr.do_copyimage() expected_md5_calls = [mock.call('/tmp/config-drive.img', 123), mock.call('/dev/mapper/os-root', 1234), @@ -334,10 +322,9 @@ class TestManager(test_base.BaseTestCase): mock_md5.side_effect = ['fakemd5', 'really_fakemd5', 'fakemd5'] mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE mock_au_c.return_value = FakeChain() - self.mgr.do_parsing() - self.mgr.image_scheme.images[0].size = 1234 - self.mgr.image_scheme.images[0].md5 = 'fakemd5' + self.mgr.driver.image_scheme.images[0].size = 1234 + self.mgr.driver.image_scheme.images[0].md5 = 'fakemd5' self.mgr.do_configdrive() - self.assertEqual(2, len(self.mgr.image_scheme.images)) + self.assertEqual(2, len(self.mgr.driver.image_scheme.images)) self.assertRaises(errors.ImageChecksumMismatchError, self.mgr.do_copyimage) diff --git a/fuel_agent/tests/test_nailgun.py b/fuel_agent/tests/test_nailgun.py index 799950f..e254e61 100644 --- a/fuel_agent/tests/test_nailgun.py +++ b/fuel_agent/tests/test_nailgun.py @@ -405,8 +405,13 @@ LIST_BLOCK_DEVICES_SAMPLE = [ class TestNailgun(test_base.BaseTestCase): - def setUp(self): + + @mock.patch('yaml.load') + @mock.patch.object(utils, 'init_http_request') + @mock.patch.object(hu, 'list_block_devices') + def setUp(self, mock_lbd, mock_http, mock_yaml): super(TestNailgun, self).setUp() + mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE self.drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA) def test_match_device_by_id_matches(self): @@ -521,7 +526,7 @@ class TestNailgun(test_base.BaseTestCase): self.assertFalse(nailgun.match_device(fake_hu_disk, fake_ks_disk)) def test_configdrive_scheme(self): - cd_scheme = self.drv.configdrive_scheme() + cd_scheme = self.drv.configdrive_scheme self.assertEqual(['fake_authorized_key1', 'fake_authorized_key2', 'fake_auth_key'], cd_scheme.common.ssh_auth_keys) self.assertEqual('node-1.domain.tld', cd_scheme.common.hostname) @@ -571,7 +576,7 @@ class TestNailgun(test_base.BaseTestCase): @mock.patch.object(hu, 'list_block_devices') def test_partition_scheme(self, mock_lbd): mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE - p_scheme = self.drv.partition_scheme() + p_scheme = self.drv.partition_scheme self.assertEqual(5, len(p_scheme.fss)) self.assertEqual(4, len(p_scheme.pvs)) self.assertEqual(3, len(p_scheme.lvs)) @@ -583,8 +588,8 @@ class TestNailgun(test_base.BaseTestCase): @mock.patch.object(hu, 'list_block_devices') def test_image_scheme(self, mock_lbd, mock_http_req, mock_yaml): mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE - p_scheme = self.drv.partition_scheme() - i_scheme = self.drv.image_scheme(p_scheme) + p_scheme = self.drv.partition_scheme + i_scheme = self.drv.image_scheme expected_images = [] for fs in p_scheme.fss: if fs.mount not in PROVISION_SAMPLE_DATA['ks_meta']['image_data']: @@ -615,8 +620,10 @@ class TestNailgun(test_base.BaseTestCase): prop_mock = mock.PropertyMock(return_value=yaml.dump(fake_image_meta)) type(mock_http_req.return_value).text = prop_mock mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE - p_scheme = self.drv.partition_scheme() - i_scheme = self.drv.image_scheme(p_scheme) + p_data = PROVISION_SAMPLE_DATA.copy() + self.drv = nailgun.Nailgun(p_data) + p_scheme = self.drv.partition_scheme + i_scheme = self.drv.image_scheme mock_http_req.assert_called_once_with( 'http://fake.host.org:123/imgs/fake_image.yaml') expected_images = [] @@ -665,8 +672,10 @@ class TestNailgun(test_base.BaseTestCase): self.assertEqual(3, self.drv._get_partition_count('Boot')) self.assertEqual(1, self.drv._get_partition_count('TMP')) + @mock.patch('yaml.load') + @mock.patch.object(utils, 'init_http_request') @mock.patch.object(hu, 'list_block_devices') - def test_partition_scheme_ceph(self, mock_lbd): + def test_partition_scheme_ceph(self, mock_lbd, mock_http_req, mock_yaml): #TODO(agordeev): perform better testing of ceph logic p_data = PROVISION_SAMPLE_DATA.copy() for i in range(0, 3): @@ -674,9 +683,9 @@ class TestNailgun(test_base.BaseTestCase): CEPH_JOURNAL) p_data['ks_meta']['pm_data']['ks_spaces'][i]['volumes'].append( CEPH_DATA) - self.drv = nailgun.Nailgun(p_data) mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE - p_scheme = self.drv.partition_scheme() + self.drv = nailgun.Nailgun(p_data) + p_scheme = self.drv.partition_scheme self.assertEqual(5, len(p_scheme.fss)) self.assertEqual(4, len(p_scheme.pvs)) self.assertEqual(3, len(p_scheme.lvs))