x/bareon
Code Issues Proposed changes

IBP: Improved data parsing flow

Browse Source 
It is better when manager does not need to
care about calling data driver parse method but
instead manager just instantiates data driver
with raw data and then asks data driver for
ready to use objects.

Even more, it is necessary to encapsulate this logic
into a data driver itself because which schemes we
actually need (PartitionScheme, ConfigdriveScheme)
depend on the manager's do_* method we use. Some
do_* methods don't even need PartitionScheme or
ConfigdriveScheme and a user who calls for example
do_reboot method does not assume she needs to
provide fuel-agent with partitioning data.

Related-bug: #1433193
Implements: blueprint ibp-build-ubuntu-images
Change-Id: I79b73d63c1ecb1dd0af24bb446003cb6585d206f
This commit is contained in:
Vladimir Kozhukalov 2015-03-25 17:13:52 +03:00
parent c93e8b05a3
commit 90e4e7ebe2
5 changed files with 87 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
fuel_agent/cmd/agent.py
View File

@ -43,19 +43,19 @@ def provision():
def partition():
main(['do_parsing', 'do_partitioning'])
main(['do_partitioning'])
def copyimage():
main(['do_parsing', 'do_copyimage'])
main(['do_copyimage'])
def configdrive():
main(['do_parsing', 'do_configdrive'])
main(['do_configdrive'])
def bootloader():
main(['do_parsing', 'do_bootloader'])
main(['do_bootloader'])
def print_err(line):

33
fuel_agent/drivers/nailgun.py
View File

@ -14,6 +14,7 @@
import math
import os
import six
from fuel_agent.drivers import ks_spaces_validator
from fuel_agent import errors
@ -70,6 +71,11 @@ class Nailgun(object):
# get rid of md over all disks for /boot partition.
self._boot_done = False
self.partition_scheme = self.parse_partition_scheme()
self.configdrive_scheme = self.parse_configdrive_scheme()
# parsing image scheme needs partition scheme has been parsed
self.image_scheme = self.parse_image_scheme()
def partition_data(self):
return self.data['ks_meta']['pm_data']['ks_spaces']
@ -129,7 +135,7 @@ class Nailgun(object):
def _num_ceph_osds(self):
return self._get_partition_count('ceph')
def partition_scheme(self):
def parse_partition_scheme(self):
LOG.debug('--- Preparing partition scheme ---')
data = self.partition_data()
ks_spaces_validator.validate(data)
@ -315,7 +321,7 @@ class Nailgun(object):
self.data['ks_meta']['pm_data']['kernel_params'])
return partition_scheme
def configdrive_scheme(self):
def parse_configdrive_scheme(self):
LOG.debug('--- Preparing configdrive scheme ---')
data = self.data
configdrive_scheme = objects.ConfigDriveScheme()
@ -370,7 +376,7 @@ class Nailgun(object):
configdrive_scheme.set_profile(profile=data['profile'])
return configdrive_scheme
def image_scheme(self, partition_scheme):
def parse_image_scheme(self):
LOG.debug('--- Preparing image scheme ---')
data = self.data
image_scheme = objects.ImageScheme()
@ -403,24 +409,19 @@ class Nailgun(object):
# /, /boot, /var/lib file systems then we will try to get images
# for all those mount points. Images data are to be defined
# at provision.json -> ['ks_meta']['image_data']
LOG.debug('Looping over all file systems in partition scheme')
for fs in partition_scheme.fss:
LOG.debug('Processing fs %s' % fs.mount)
if fs.mount not in data['ks_meta']['image_data']:
LOG.debug('There is no image for fs %s. Skipping.' % fs.mount)
continue
image_data = data['ks_meta']['image_data'][fs.mount]
LOG.debug('Looping over all images in provision data')
for mount_point, image_data in six.iteritems(
data['ks_meta']['image_data']):
LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' %
(fs.mount, image_data['uri'],
(mount_point, image_data['uri'],
image_data['format'], image_data['container']))
image_scheme.add_image(
uri=image_data['uri'],
target_device=fs.device,
# In the future we will get format and container
# from provision.json, but currently it is hard coded.
target_device=self.partition_scheme.fs_by_mount(
mount_point).device,
format=image_data['format'],
container=image_data['container'],
size=image_meta.get(fs.mount, {}).get('size'),
md5=image_meta.get(fs.mount, {}).get('md5'),
size=image_meta.get(mount_point, {}).get('size'),
md5=image_meta.get(mount_point, {}).get('md5'),
)
return image_scheme

68
fuel_agent/manager.py
View File

@ -78,15 +78,6 @@ LOG = logging.getLogger(__name__)
class Manager(object):
def __init__(self, data):
self.driver = utils.get_driver(CONF.data_driver)(data)
self.partition_scheme = None
self.configdrive_scheme = None
self.image_scheme = None
def do_parsing(self):
LOG.debug('--- Parsing data (do_parsing) ---')
self.partition_scheme = self.driver.partition_scheme()
self.configdrive_scheme = self.driver.configdrive_scheme()
self.image_scheme = self.driver.image_scheme(self.partition_scheme)
def do_partitioning(self):
LOG.debug('--- Partitioning disks (do_partitioning) ---')
@ -128,7 +119,7 @@ class Manager(object):
utils.execute('udevadm', 'control', '--reload-rules',
check_exit_code=[0])
for parted in self.partition_scheme.parteds:
for parted in self.driver.partition_scheme.parteds:
for prt in parted.partitions:
# We wipe out the beginning of every new partition
# right after creating it. It allows us to avoid possible
@ -146,7 +137,7 @@ class Manager(object):
'seek=%s' % max(prt.end - 3, 0), 'count=5',
'of=%s' % prt.device, check_exit_code=[0, 1])
for parted in self.partition_scheme.parteds:
for parted in self.driver.partition_scheme.parteds:
pu.make_label(parted.name, parted.label)
for prt in parted.partitions:
pu.make_partition(prt.device, prt.begin, prt.end, prt.type)
@ -197,25 +188,25 @@ class Manager(object):
lu.pvremove_all()
# creating meta disks
for md in self.partition_scheme.mds:
for md in self.driver.partition_scheme.mds:
mu.mdcreate(md.name, md.level, *md.devices)
# creating physical volumes
for pv in self.partition_scheme.pvs:
for pv in self.driver.partition_scheme.pvs:
lu.pvcreate(pv.name, metadatasize=pv.metadatasize,
metadatacopies=pv.metadatacopies)
# creating volume groups
for vg in self.partition_scheme.vgs:
for vg in self.driver.partition_scheme.vgs:
lu.vgcreate(vg.name, *vg.pvnames)
# creating logical volumes
for lv in self.partition_scheme.lvs:
for lv in self.driver.partition_scheme.lvs:
lu.lvcreate(lv.vgname, lv.name, lv.size)
# making file systems
for fs in self.partition_scheme.fss:
found_images = [img for img in self.image_scheme.images
for fs in self.driver.partition_scheme.fss:
found_images = [img for img in self.driver.image_scheme.images
if img.target_device == fs.device]
if not found_images:
fu.make_fs(fs.type, fs.options, fs.label, fs.device)
@ -231,16 +222,22 @@ class Manager(object):
tmpl_dir = CONF.nc_template_path
utils.render_and_save(
tmpl_dir, self.configdrive_scheme.template_names('cloud_config'),
self.configdrive_scheme.template_data(), cc_output_path
tmpl_dir,
self.driver.configdrive_scheme.template_names('cloud_config'),
self.driver.configdrive_scheme.template_data(),
cc_output_path
)
utils.render_and_save(
tmpl_dir, self.configdrive_scheme.template_names('boothook'),
self.configdrive_scheme.template_data(), bh_output_path
tmpl_dir,
self.driver.configdrive_scheme.template_names('boothook'),
self.driver.configdrive_scheme.template_data(),
bh_output_path
)
utils.render_and_save(
tmpl_dir, self.configdrive_scheme.template_names('meta-data'),
self.configdrive_scheme.template_data(), md_output_path
tmpl_dir,
self.driver.configdrive_scheme.template_names('meta-data'),
self.driver.configdrive_scheme.template_data(),
md_output_path
)
utils.execute('write-mime-multipart', '--output=%s' % ud_output_path,
@ -250,14 +247,14 @@ class Manager(object):
'-volid', 'cidata', '-joliet', '-rock', ud_output_path,
md_output_path)
configdrive_device = self.partition_scheme.configdrive_device()
configdrive_device = self.driver.partition_scheme.configdrive_device()
if configdrive_device is None:
raise errors.WrongPartitionSchemeError(
'Error while trying to get configdrive device: '
'configdrive device not found')
size = os.path.getsize(CONF.config_drive_path)
md5 = utils.calculate_md5(CONF.config_drive_path, size)
self.image_scheme.add_image(
self.driver.image_scheme.add_image(
uri='file://%s' % CONF.config_drive_path,
target_device=configdrive_device,
format='iso9660',
@ -268,7 +265,7 @@ class Manager(object):
def do_copyimage(self):
LOG.debug('--- Copying images (do_copyimage) ---')
for image in self.image_scheme.images:
for image in self.driver.image_scheme.images:
LOG.debug('Processing image: %s' % image.uri)
processing = au.Chain()
@ -320,7 +317,7 @@ class Manager(object):
# Shorter paths earlier. We sort all mount points by their depth.
# ['/', '/boot', '/var', '/var/lib/mysql']
key = lambda x: len(x.mount.rstrip('/').split('/'))
for fs in sorted(self.partition_scheme.fss, key=key):
for fs in sorted(self.driver.partition_scheme.fss, key=key):
if fs.mount == 'swap':
continue
mount = chroot + fs.mount
@ -344,7 +341,8 @@ class Manager(object):
fu.umount_fs(chroot + '/dev')
fu.umount_fs(chroot + '/sys')
key = lambda x: len(x.mount.rstrip('/').split('/'))
for fs in sorted(self.partition_scheme.fss, key=key, reverse=True):
for fs in sorted(self.driver.partition_scheme.fss,
key=key, reverse=True):
if fs.mount == 'swap':
continue
fu.umount_fs(fs.device)
@ -355,17 +353,17 @@ class Manager(object):
self.mount_target(chroot)
mount2uuid = {}
for fs in self.partition_scheme.fss:
for fs in self.driver.partition_scheme.fss:
mount2uuid[fs.mount] = utils.execute(
'blkid', '-o', 'value', '-s', 'UUID', fs.device,
check_exit_code=[0])[0].strip()
grub_version = gu.guess_grub_version(chroot=chroot)
boot_device = self.partition_scheme.boot_device(grub_version)
install_devices = [d.name for d in self.partition_scheme.parteds
boot_device = self.driver.partition_scheme.boot_device(grub_version)
install_devices = [d.name for d in self.driver.partition_scheme.parteds
if d.install_bootloader]
kernel_params = self.partition_scheme.kernel_params
kernel_params = self.driver.partition_scheme.kernel_params
kernel_params += ' root=UUID=%s ' % mount2uuid['/']
if grub_version == 1:
@ -381,7 +379,8 @@ class Manager(object):
'w') as f:
f.write('# Generated by fuel-agent during provisioning: BEGIN\n')
# pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1
for mapping in self.configdrive_scheme.common.udevrules.split(','):
for mapping in self.driver.configdrive_scheme.\
common.udevrules.split(','):
mac_addr, nic_name = mapping.split('_')
f.write('SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '
'ATTR{address}=="%s", ATTR{type}=="1", KERNEL=="eth*",'
@ -396,7 +395,7 @@ class Manager(object):
'# DO NOT DELETE. It is needed to disable net-generator\n')
with open(chroot + '/etc/fstab', 'wb') as f:
for fs in self.partition_scheme.fss:
for fs in self.driver.partition_scheme.fss:
# TODO(kozhukalov): Think of improving the logic so as to
# insert a meaningful fsck order value which is last zero
# at fstab line. Currently we set it into 0 which means
@ -413,7 +412,6 @@ class Manager(object):
def do_provisioning(self):
LOG.debug('--- Provisioning (do_provisioning) ---')
self.do_parsing()
self.do_partitioning()
self.do_configdrive()
self.do_copyimage()

41
fuel_agent/tests/test_manager.py
View File

@ -34,21 +34,14 @@ CONF = cfg.CONF
class TestManager(test_base.BaseTestCase):
def setUp(self):
super(TestManager, self).setUp()
self.mgr = manager.Manager(test_nailgun.PROVISION_SAMPLE_DATA)
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(hu, 'list_block_devices')
def test_do_parsing(self, mock_lbd, mock_http_req, mock_yaml):
def setUp(self, mock_lbd, mock_http, mock_yaml):
super(TestManager, self).setUp()
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
self.mgr.do_parsing()
#NOTE(agordeev): there's no need for deeper assertions as all schemes
# thoroughly tested in test_nailgun
self.assertFalse(self.mgr.partition_scheme is None)
self.assertFalse(self.mgr.configdrive_scheme is None)
self.assertFalse(self.mgr.image_scheme is None)
self.mgr = manager.Manager(test_nailgun.PROVISION_SAMPLE_DATA)
@mock.patch('six.moves.builtins.open')
@mock.patch.object(os, 'symlink')
@ -79,7 +72,6 @@ class TestManager(test_base.BaseTestCase):
mock_os_ld.return_value = ['not_a_rule', 'fake.rules']
mock_os_p.exists.return_value = True
mock_hu_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
self.mgr.do_parsing()
self.mgr.do_partitioning()
mock_pu_ml_expected_calls = [mock.call('/dev/sda', 'gpt'),
mock.call('/dev/sdb', 'gpt'),
@ -150,8 +142,7 @@ class TestManager(test_base.BaseTestCase):
mock_get_size.return_value = 123
mock_md5.return_value = 'fakemd5'
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
self.mgr.do_parsing()
self.assertEqual(1, len(self.mgr.image_scheme.images))
self.assertEqual(1, len(self.mgr.driver.image_scheme.images))
self.mgr.do_configdrive()
mock_u_ras_expected_calls = [
mock.call(CONF.nc_template_path,
@ -187,11 +178,11 @@ class TestManager(test_base.BaseTestCase):
'%s/%s' % (CONF.tmp_path, 'user-data'),
'%s/%s' % (CONF.tmp_path, 'meta-data'))]
self.assertEqual(mock_u_e_expected_calls, mock_u_e.call_args_list)
self.assertEqual(2, len(self.mgr.image_scheme.images))
cf_drv_img = self.mgr.image_scheme.images[-1]
self.assertEqual(2, len(self.mgr.driver.image_scheme.images))
cf_drv_img = self.mgr.driver.image_scheme.images[-1]
self.assertEqual('file://%s' % CONF.config_drive_path, cf_drv_img.uri)
self.assertEqual('/dev/sda7',
self.mgr.partition_scheme.configdrive_device())
self.mgr.driver.partition_scheme.configdrive_device())
self.assertEqual('iso9660', cf_drv_img.format)
self.assertEqual('raw', cf_drv_img.container)
self.assertEqual('fakemd5', cf_drv_img.md5)
@ -207,7 +198,6 @@ class TestManager(test_base.BaseTestCase):
mock_u_e, mock_p_ps_cd,
mock_http_req, mock_yaml):
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
self.mgr.do_parsing()
mock_p_ps_cd.return_value = None
self.assertRaises(errors.WrongPartitionSchemeError,
self.mgr.do_configdrive)
@ -239,10 +229,9 @@ class TestManager(test_base.BaseTestCase):
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
mock_au_c.return_value = FakeChain()
self.mgr.do_parsing()
self.mgr.do_configdrive()
self.mgr.do_copyimage()
imgs = self.mgr.image_scheme.images
imgs = self.mgr.driver.image_scheme.images
self.assertEqual(2, len(imgs))
expected_processors_list = []
for img in imgs[:-1]:
@ -293,11 +282,10 @@ class TestManager(test_base.BaseTestCase):
mock_md5.side_effect = ['fakemd5', 'really_fakemd5', 'fakemd5']
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
mock_au_c.return_value = FakeChain()
self.mgr.do_parsing()
self.mgr.image_scheme.images[0].size = 1234
self.mgr.image_scheme.images[0].md5 = 'really_fakemd5'
self.mgr.driver.image_scheme.images[0].size = 1234
self.mgr.driver.image_scheme.images[0].md5 = 'really_fakemd5'
self.mgr.do_configdrive()
self.assertEqual(2, len(self.mgr.image_scheme.images))
self.assertEqual(2, len(self.mgr.driver.image_scheme.images))
self.mgr.do_copyimage()
expected_md5_calls = [mock.call('/tmp/config-drive.img', 123),
mock.call('/dev/mapper/os-root', 1234),
@ -334,10 +322,9 @@ class TestManager(test_base.BaseTestCase):
mock_md5.side_effect = ['fakemd5', 'really_fakemd5', 'fakemd5']
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
mock_au_c.return_value = FakeChain()
self.mgr.do_parsing()
self.mgr.image_scheme.images[0].size = 1234
self.mgr.image_scheme.images[0].md5 = 'fakemd5'
self.mgr.driver.image_scheme.images[0].size = 1234
self.mgr.driver.image_scheme.images[0].md5 = 'fakemd5'
self.mgr.do_configdrive()
self.assertEqual(2, len(self.mgr.image_scheme.images))
self.assertEqual(2, len(self.mgr.driver.image_scheme.images))
self.assertRaises(errors.ImageChecksumMismatchError,
self.mgr.do_copyimage)

29
fuel_agent/tests/test_nailgun.py
View File

@ -405,8 +405,13 @@ LIST_BLOCK_DEVICES_SAMPLE = [
class TestNailgun(test_base.BaseTestCase):
def setUp(self):
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(hu, 'list_block_devices')
def setUp(self, mock_lbd, mock_http, mock_yaml):
super(TestNailgun, self).setUp()
mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
self.drv = nailgun.Nailgun(PROVISION_SAMPLE_DATA)
def test_match_device_by_id_matches(self):
@ -521,7 +526,7 @@ class TestNailgun(test_base.BaseTestCase):
self.assertFalse(nailgun.match_device(fake_hu_disk, fake_ks_disk))
def test_configdrive_scheme(self):
cd_scheme = self.drv.configdrive_scheme()
cd_scheme = self.drv.configdrive_scheme
self.assertEqual(['fake_authorized_key1', 'fake_authorized_key2',
'fake_auth_key'], cd_scheme.common.ssh_auth_keys)
self.assertEqual('node-1.domain.tld', cd_scheme.common.hostname)
@ -571,7 +576,7 @@ class TestNailgun(test_base.BaseTestCase):
@mock.patch.object(hu, 'list_block_devices')
def test_partition_scheme(self, mock_lbd):
mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
p_scheme = self.drv.partition_scheme()
p_scheme = self.drv.partition_scheme
self.assertEqual(5, len(p_scheme.fss))
self.assertEqual(4, len(p_scheme.pvs))
self.assertEqual(3, len(p_scheme.lvs))
@ -583,8 +588,8 @@ class TestNailgun(test_base.BaseTestCase):
@mock.patch.object(hu, 'list_block_devices')
def test_image_scheme(self, mock_lbd, mock_http_req, mock_yaml):
mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
p_scheme = self.drv.partition_scheme()
i_scheme = self.drv.image_scheme(p_scheme)
p_scheme = self.drv.partition_scheme
i_scheme = self.drv.image_scheme
expected_images = []
for fs in p_scheme.fss:
if fs.mount not in PROVISION_SAMPLE_DATA['ks_meta']['image_data']:
@ -615,8 +620,10 @@ class TestNailgun(test_base.BaseTestCase):
prop_mock = mock.PropertyMock(return_value=yaml.dump(fake_image_meta))
type(mock_http_req.return_value).text = prop_mock
mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
p_scheme = self.drv.partition_scheme()
i_scheme = self.drv.image_scheme(p_scheme)
p_data = PROVISION_SAMPLE_DATA.copy()
self.drv = nailgun.Nailgun(p_data)
p_scheme = self.drv.partition_scheme
i_scheme = self.drv.image_scheme
mock_http_req.assert_called_once_with(
'http://fake.host.org:123/imgs/fake_image.yaml')
expected_images = []
@ -665,8 +672,10 @@ class TestNailgun(test_base.BaseTestCase):
self.assertEqual(3, self.drv._get_partition_count('Boot'))
self.assertEqual(1, self.drv._get_partition_count('TMP'))
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(hu, 'list_block_devices')
def test_partition_scheme_ceph(self, mock_lbd):
def test_partition_scheme_ceph(self, mock_lbd, mock_http_req, mock_yaml):
#TODO(agordeev): perform better testing of ceph logic
p_data = PROVISION_SAMPLE_DATA.copy()
for i in range(0, 3):
@ -674,9 +683,9 @@ class TestNailgun(test_base.BaseTestCase):
CEPH_JOURNAL)
p_data['ks_meta']['pm_data']['ks_spaces'][i]['volumes'].append(
CEPH_DATA)
self.drv = nailgun.Nailgun(p_data)
mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
p_scheme = self.drv.partition_scheme()
self.drv = nailgun.Nailgun(p_data)
p_scheme = self.drv.partition_scheme
self.assertEqual(5, len(p_scheme.fss))
self.assertEqual(4, len(p_scheme.pvs))
self.assertEqual(3, len(p_scheme.lvs))