diff --git a/bareon/actions/configdrive.py b/bareon/actions/configdrive.py index a5d2ec3..71a5b3c 100644 --- a/bareon/actions/configdrive.py +++ b/bareon/actions/configdrive.py @@ -14,12 +14,16 @@ import os +import shutil +import tempfile from oslo_config import cfg +import six from bareon.actions import base from bareon import errors from bareon.openstack.common import log as logging +from bareon.utils import fs as fu from bareon.utils import utils opts = [ @@ -54,7 +58,7 @@ LOG = logging.getLogger(__name__) class ConfigDriveAction(base.BaseAction): """ConfigDriveAction - creates No cloud datasource image for cloud-init + creates ConfigDrive datasource image for cloud-init """ def validate(self): @@ -64,43 +68,78 @@ class ConfigDriveAction(base.BaseAction): def execute(self): self.do_configdrive() + def _make_configdrive_image(self, src_files): + bs = 4096 + configdrive_device = self.driver.partition_scheme.configdrive_device() + size = utils.execute('blockdev', '--getsize64', configdrive_device)[0] + size = int(size.strip()) + + utils.execute('truncate', '--size=%d' % size, CONF.config_drive_path) + fu.make_fs( + fs_type='ext2', + fs_options=' -b %d -F ' % bs, + fs_label='config-2', + dev=six.text_type(CONF.config_drive_path)) + + mount_point = tempfile.mkdtemp(dir=CONF.tmp_path) + try: + fu.mount_fs('ext2', CONF.config_drive_path, mount_point) + for file_path in src_files: + name = os.path.basename(file_path) + if os.path.isdir(file_path): + shutil.copytree(file_path, os.path.join(mount_point, name)) + else: + shutil.copy2(file_path, mount_point) + except Exception as exc: + LOG.error('Error copying files to configdrive: %s', exc) + raise + finally: + fu.umount_fs(mount_point) + os.rmdir(mount_point) + + def _prepare_configdrive_files(self): + # see data sources part of cloud-init documentation + # for directory structure + cd_root = tempfile.mkdtemp(dir=CONF.tmp_path) + cd_latest = os.path.join(cd_root, 'openstack', 'latest') + md_output_path = os.path.join(cd_latest, 'meta_data.json') + ud_output_path = os.path.join(cd_latest, 'user_data') + os.makedirs(cd_latest) + + cc_output_path = os.path.join(CONF.tmp_path, 'cloud_config.txt') + bh_output_path = os.path.join(CONF.tmp_path, 'boothook.txt') + + tmpl_dir = CONF.nc_template_path + utils.render_and_save( + tmpl_dir, + self.driver.configdrive_scheme.template_names('cloud_config'), + self.driver.configdrive_scheme.template_data(), + cc_output_path + ) + utils.render_and_save( + tmpl_dir, + self.driver.configdrive_scheme.template_names('boothook'), + self.driver.configdrive_scheme.template_data(), + bh_output_path + ) + utils.render_and_save( + tmpl_dir, + self.driver.configdrive_scheme.template_names('meta_data_json'), + self.driver.configdrive_scheme.template_data(), + md_output_path + ) + + utils.execute( + 'write-mime-multipart', '--output=%s' % ud_output_path, + '%s:text/cloud-boothook' % bh_output_path, + '%s:text/cloud-config' % cc_output_path) + return [os.path.join(cd_root, 'openstack')] + def do_configdrive(self): LOG.debug('--- Creating configdrive (do_configdrive) ---') if CONF.prepare_configdrive: - cc_output_path = os.path.join(CONF.tmp_path, 'cloud_config.txt') - bh_output_path = os.path.join(CONF.tmp_path, 'boothook.txt') - # NOTE:file should be strictly named as 'user-data' - # the same is for meta-data as well - ud_output_path = os.path.join(CONF.tmp_path, 'user-data') - md_output_path = os.path.join(CONF.tmp_path, 'meta-data') - - tmpl_dir = CONF.nc_template_path - utils.render_and_save( - tmpl_dir, - self.driver.configdrive_scheme.template_names('cloud_config'), - self.driver.configdrive_scheme.template_data(), - cc_output_path - ) - utils.render_and_save( - tmpl_dir, - self.driver.configdrive_scheme.template_names('boothook'), - self.driver.configdrive_scheme.template_data(), - bh_output_path - ) - utils.render_and_save( - tmpl_dir, - self.driver.configdrive_scheme.template_names('meta_data'), - self.driver.configdrive_scheme.template_data(), - md_output_path - ) - - utils.execute( - 'write-mime-multipart', '--output=%s' % ud_output_path, - '%s:text/cloud-boothook' % bh_output_path, - '%s:text/cloud-config' % cc_output_path) - utils.execute('genisoimage', '-output', CONF.config_drive_path, - '-volid', 'cidata', '-joliet', '-rock', - ud_output_path, md_output_path) + files = self._prepare_configdrive_files() + self._make_configdrive_image(files) if CONF.prepare_configdrive or os.path.isfile(CONF.config_drive_path): self._add_configdrive_image() @@ -114,10 +153,11 @@ class ConfigDriveAction(base.BaseAction): 'configdrive device not found') size = os.path.getsize(CONF.config_drive_path) md5 = utils.calculate_md5(CONF.config_drive_path, size) + fs_type = fu.get_fs_type(CONF.config_drive_path) self.driver.image_scheme.add_image( uri='file://%s' % CONF.config_drive_path, target_device=configdrive_device, - format='iso9660', + format=fs_type, container='raw', size=size, md5=md5, diff --git a/bareon/tests/test_do_configdrive.py b/bareon/tests/test_do_configdrive.py index 10ea7f0..cef016a 100644 --- a/bareon/tests/test_do_configdrive.py +++ b/bareon/tests/test_do_configdrive.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import six import unittest2 @@ -40,16 +41,30 @@ class TestConfigDriveAction(unittest2.TestCase): profile='pro_fi-le') self.drv.configdrive_scheme.template_data = mock.Mock() self.drv.image_scheme = objects.ImageScheme() + self.drv.partition_scheme.configdrive_device.return_value = '/dev/sda7' + def test_do_configdrive(self): + with mock.patch.multiple(self.action, + _prepare_configdrive_files=mock.DEFAULT, + _make_configdrive_image=mock.DEFAULT, + _add_configdrive_image=mock.DEFAULT) as mocks: + mocks['_prepare_configdrive_files'].return_value = 'x' + self.action.execute() + mocks['_prepare_configdrive_files'].assert_called_once_with() + mocks['_make_configdrive_image'].assert_called_once_with('x') + mocks['_add_configdrive_image'].assert_called_once_with() + + @mock.patch.object(configdrive, 'tempfile', autospec=True) @mock.patch.object(configdrive, 'os', autospec=True) @mock.patch.object(configdrive, 'utils', autospec=True) - def test_do_configdrive(self, mock_utils, mock_os): - self.drv.partition_scheme.configdrive_device.return_value = '/dev/sda7' - mock_os.path.getsize.return_value = 123 - mock_os.path.join = lambda x, y: '%s/%s' % (x, y) - mock_utils.calculate_md5.return_value = 'fakemd5' - self.assertEqual(0, len(self.drv.image_scheme.images)) - self.action.execute() + def test_prepare_configdrive_files(self, mock_utils, mock_os, mock_temp): + mock_os.path.join = os.path.join + mock_temp.mkdtemp.return_value = '/tmp/qwe' + ret = self.action._prepare_configdrive_files() + self.assertEqual(ret, ['/tmp/qwe/openstack']) + mock_temp.mkdtemp.assert_called_once_with(dir=CONF.tmp_path) + mock_os.makedirs.assert_called_once_with('/tmp/qwe/openstack/latest') + mock_u_ras_expected_calls = [ mock.call(CONF.nc_template_path, ['cloud_config_pro_fi-le.jinja2', @@ -64,41 +79,83 @@ class TestConfigDriveAction(unittest2.TestCase): 'boothook.jinja2'], mock.ANY, '%s/%s' % (CONF.tmp_path, 'boothook.txt')), mock.call(CONF.nc_template_path, - ['meta_data_pro_fi-le.jinja2', - 'meta_data_pro.jinja2', - 'meta_data_pro_fi.jinja2', - 'meta_data.jinja2'], - mock.ANY, '%s/%s' % (CONF.tmp_path, 'meta-data'))] + ['meta_data_json_pro_fi-le.jinja2', + 'meta_data_json_pro.jinja2', + 'meta_data_json_pro_fi.jinja2', + 'meta_data_json.jinja2'], + mock.ANY, '/tmp/qwe/openstack/latest/meta_data.json')] self.assertEqual(mock_u_ras_expected_calls, mock_utils.render_and_save.call_args_list) - mock_u_e_expected_calls = [ - mock.call('write-mime-multipart', - '--output=%s' % ('%s/%s' % (CONF.tmp_path, 'user-data')), - '%s:text/cloud-boothook' % ('%s/%s' % (CONF.tmp_path, - 'boothook.txt')), - '%s:text/cloud-config' % ('%s/%s' % (CONF.tmp_path, - 'cloud_config.txt')) - ), - mock.call('genisoimage', '-output', CONF.config_drive_path, - '-volid', 'cidata', '-joliet', '-rock', - '%s/%s' % (CONF.tmp_path, 'user-data'), - '%s/%s' % (CONF.tmp_path, 'meta-data'))] - self.assertEqual(mock_u_e_expected_calls, - mock_utils.execute.call_args_list) + mock_utils.execute.assert_called_once_with( + 'write-mime-multipart', + '--output=/tmp/qwe/openstack/latest/user_data', + '%s/%s:text/cloud-boothook' % (CONF.tmp_path, 'boothook.txt'), + '%s/%s:text/cloud-config' % (CONF.tmp_path, 'cloud_config.txt')) + + @mock.patch.object(configdrive, 'tempfile', autospec=True) + @mock.patch.object(configdrive, 'shutil', autospec=True) + @mock.patch.object(configdrive, 'fu', autospec=True) + @mock.patch.object(configdrive, 'os', autospec=True) + @mock.patch.object(configdrive, 'utils', autospec=True) + def test_make_configdrive_image(self, mock_utils, mock_os, mock_fu, + mock_shutil, mock_temp): + mock_utils.execute.side_effect = [(' 795648', ''), None] + mock_os.path.isdir.side_effect = [True, False] + mock_os.path.join = os.path.join + mock_os.path.basename = os.path.basename + + mock_temp.mkdtemp.return_value = '/tmp/mount_point' + + self.action._make_configdrive_image(['/tmp/openstack', + '/tmp/somefile']) + + mock_u_e_calls = [ + mock.call('blockdev', '--getsize64', '/dev/sda7'), + mock.call('truncate', '--size=795648', CONF.config_drive_path)] + + self.assertEqual(mock_u_e_calls, mock_utils.execute.call_args_list, + str(mock_utils.execute.call_args_list)) + + mock_fu.make_fs.assert_called_with(fs_type='ext2', + fs_options=' -b 4096 -F ', + fs_label='config-2', + dev=CONF.config_drive_path) + mock_fu.mount_fs.assert_called_with('ext2', + CONF.config_drive_path, + '/tmp/mount_point') + mock_fu.umount_fs.assert_called_with('/tmp/mount_point') + mock_os.rmdir.assert_called_with('/tmp/mount_point') + mock_shutil.copy2.assert_called_with('/tmp/somefile', + '/tmp/mount_point') + mock_shutil.copytree.assert_called_with('/tmp/openstack', + '/tmp/mount_point/openstack') + + @mock.patch.object(configdrive, 'fu', autospec=True) + @mock.patch.object(configdrive, 'os', autospec=True) + @mock.patch.object(configdrive, 'utils', autospec=True) + def test_add_configdrive_image(self, mock_utils, mock_os, mock_fu): + mock_fu.get_fs_type.return_value = 'ext999' + mock_utils.calculate_md5.return_value = 'fakemd5' + mock_os.path.getsize.return_value = 123 + + self.action._add_configdrive_image() + self.assertEqual(1, len(self.drv.image_scheme.images)) - cf_drv_img = self.drv.image_scheme.images[-1] + cf_drv_img = self.drv.image_scheme.images[0] self.assertEqual('file://%s' % CONF.config_drive_path, cf_drv_img.uri) - self.assertEqual('/dev/sda7', - self.drv.partition_scheme.configdrive_device()) - self.assertEqual('iso9660', cf_drv_img.format) + self.assertEqual('/dev/sda7', cf_drv_img.target_device) + self.assertEqual('ext999', cf_drv_img.format) self.assertEqual('raw', cf_drv_img.container) self.assertEqual('fakemd5', cf_drv_img.md5) self.assertEqual(123, cf_drv_img.size) @mock.patch.object(configdrive, 'os', autospec=True) @mock.patch.object(configdrive, 'utils', autospec=True) - def test_do_configdrive_no_configdrive_device(self, mock_utils, mock_os): + def test_add_configdrive_image_no_configdrive_device(self, mock_utils, + mock_os): self.drv.partition_scheme.configdrive_device.return_value = None + mock_utils.calculate_md5.return_value = 'fakemd5' + mock_os.path.getsize.return_value = 123 self.assertRaises(errors.WrongPartitionSchemeError, - self.action.execute) + self.action._add_configdrive_image) diff --git a/bareon/tests/test_fs_utils.py b/bareon/tests/test_fs_utils.py index b6d1761..3bb2e4d 100644 --- a/bareon/tests/test_fs_utils.py +++ b/bareon/tests/test_fs_utils.py @@ -161,6 +161,15 @@ class TestFSUtils(unittest2.TestCase): self.assertEqual(fu.format_fs_label(long_label), template.format(long_label_trimmed)) + def test_get_fs_type(self, mock_exec): + output = "megafs\n" + mock_exec.return_value = (output, '') + ret = fu.get_fs_type('/dev/sda4') + mock_exec.assert_called_once_with('blkid', '-o', 'value', + '-s', 'TYPE', '-c', '/dev/null', + '/dev/sda4') + self.assertEqual(ret, 'megafs') + class TestFSRetry(unittest2.TestCase): diff --git a/bareon/utils/fs.py b/bareon/utils/fs.py index 6cb8e9d..27573f8 100644 --- a/bareon/utils/fs.py +++ b/bareon/utils/fs.py @@ -128,3 +128,9 @@ def umount_fs(fs_mount, try_lazy_umount=False): utils.execute('umount', '-l', fs_mount, check_exit_code=[0]) else: raise + + +def get_fs_type(device): + output = utils.execute('blkid', '-o', 'value', '-s', 'TYPE', + '-c', '/dev/null', device)[0] + return output.strip() diff --git a/cloud-init-templates/meta_data_json.jinja2 b/cloud-init-templates/meta_data_json.jinja2 new file mode 100644 index 0000000..14e8a50 --- /dev/null +++ b/cloud-init-templates/meta_data_json.jinja2 @@ -0,0 +1,4 @@ +{ + "hostname": "{{ common.hostname }}", + "uuid": "some-unused-id" +}