[IBP] Add build_utils
Build utils is providing all necessary low-level functions to build target image. Related-Bug: #1433193 Partially implements: blueprint ibp-build-ubuntu-images Change-Id: I8d89377fb38073968a580f7c8e3d04e61ce8ed60
This commit is contained in:
Alexander Gordeev
Vladimir Kozhukalov
parent
524d6ff54b
commit
c93e8b05a3
@ -142,3 +142,7 @@ class HttpUrlInvalidContentLength(BaseError):
|
|||||||
|
|
||||||
class ImageChecksumMismatchError(BaseError):
|
class ImageChecksumMismatchError(BaseError):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class NoFreeLoopDevices(BaseError):
|
||||||
|
pass
|
||||||
|
|||||||
356
fuel_agent/tests/test_build_utils.py
Normal file
356
fuel_agent/tests/test_build_utils.py
Normal file
@ -0,0 +1,356 @@
|
|||||||
|
# Copyright 2015 Mirantis, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import testtools
|
||||||
|
|
||||||
|
import mock
|
||||||
|
from oslo.config import cfg
|
||||||
|
|
||||||
|
from fuel_agent import errors
|
||||||
|
from fuel_agent.utils import build_utils as bu
|
||||||
|
from fuel_agent.utils import hardware_utils as hu
|
||||||
|
from fuel_agent.utils import utils
|
||||||
|
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
|
||||||
|
class BuildUtilsTestCase(testtools.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
super(BuildUtilsTestCase, self).setUp()
|
||||||
|
|
||||||
|
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||||
|
def test_run_debootstrap(self, mock_exec):
|
||||||
|
bu.run_debootstrap('uri', 'suite', 'chroot', 'arch')
|
||||||
|
mock_exec.assert_called_once_with('debootstrap', '--verbose',
|
||||||
|
'--no-check-gpg', '--arch=arch',
|
||||||
|
'suite', 'chroot', 'uri')
|
||||||
|
|
||||||
|
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||||
|
def test_run_debootstrap_eatmydata(self, mock_exec):
|
||||||
|
bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', eatmydata=True)
|
||||||
|
mock_exec.assert_called_once_with('debootstrap', '--verbose',
|
||||||
|
'--no-check-gpg', '--arch=arch',
|
||||||
|
'--include=eatmydata', 'suite',
|
||||||
|
'chroot', 'uri')
|
||||||
|
|
||||||
|
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||||
|
def test_run_apt_get(self, mock_exec):
|
||||||
|
bu.run_apt_get('chroot', ['package1', 'package2'])
|
||||||
|
mock_exec_expected_calls = [
|
||||||
|
mock.call('chroot', 'chroot', 'apt-get', '-y', 'update'),
|
||||||
|
mock.call('chroot', 'chroot', 'apt-get', '-y', 'install',
|
||||||
|
'package1 package2')]
|
||||||
|
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||||
|
def test_run_apt_get_eatmydata(self, mock_exec):
|
||||||
|
bu.run_apt_get('chroot', ['package1', 'package2'], eatmydata=True)
|
||||||
|
mock_exec_expected_calls = [
|
||||||
|
mock.call('chroot', 'chroot', 'apt-get', '-y', 'update'),
|
||||||
|
mock.call('chroot', 'chroot', 'eatmydata', 'apt-get', '-y',
|
||||||
|
'install', 'package1 package2')]
|
||||||
|
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'fchmod')
|
||||||
|
@mock.patch.object(os, 'makedirs')
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
def test_suppress_services_start(self, mock_path, mock_mkdir, mock_fchmod):
|
||||||
|
mock_path.join.return_value = 'fake_path'
|
||||||
|
mock_path.exists.return_value = False
|
||||||
|
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||||
|
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||||
|
file_handle_mock.fileno.return_value = 'fake_fileno'
|
||||||
|
bu.suppress_services_start('chroot')
|
||||||
|
mock_open.assert_called_once_with('fake_path', 'w')
|
||||||
|
expected = '#!/bin/sh\n# prevent any service from being started\n'\
|
||||||
|
'exit 101\n'
|
||||||
|
file_handle_mock.write.assert_called_once_with(expected)
|
||||||
|
mock_fchmod.assert_called_once_with('fake_fileno', 0o755)
|
||||||
|
mock_mkdir.assert_called_once_with('fake_path')
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'fchmod')
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
def test_suppress_services_start_nomkdir(self, mock_path, mock_fchmod):
|
||||||
|
mock_path.join.return_value = 'fake_path'
|
||||||
|
mock_path.exists.return_value = True
|
||||||
|
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||||
|
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||||
|
file_handle_mock.fileno.return_value = 'fake_fileno'
|
||||||
|
bu.suppress_services_start('chroot')
|
||||||
|
mock_open.assert_called_once_with('fake_path', 'w')
|
||||||
|
expected = '#!/bin/sh\n# prevent any service from being started\n'\
|
||||||
|
'exit 101\n'
|
||||||
|
file_handle_mock.write.assert_called_once_with(expected)
|
||||||
|
mock_fchmod.assert_called_once_with('fake_fileno', 0o755)
|
||||||
|
|
||||||
|
@mock.patch.object(shutil, 'rmtree')
|
||||||
|
@mock.patch.object(os, 'makedirs')
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
def test_clean_dirs(self, mock_path, mock_mkdir, mock_rmtree):
|
||||||
|
mock_path.isdir.return_value = True
|
||||||
|
dirs = ['dir1', 'dir2', 'dir3']
|
||||||
|
mock_path.join.side_effect = dirs
|
||||||
|
bu.clean_dirs('chroot', dirs)
|
||||||
|
for m in (mock_rmtree, mock_mkdir):
|
||||||
|
self.assertEqual([mock.call(d) for d in dirs], m.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
def test_clean_dirs_not_isdir(self, mock_path):
|
||||||
|
mock_path.isdir.return_value = False
|
||||||
|
dirs = ['dir1', 'dir2', 'dir3']
|
||||||
|
mock_path.join.side_effect = dirs
|
||||||
|
bu.clean_dirs('chroot', dirs)
|
||||||
|
self.assertEqual([mock.call('chroot', d) for d in dirs],
|
||||||
|
mock_path.join.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'remove')
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
def test_remove_files(self, mock_path, mock_remove):
|
||||||
|
mock_path.exists.return_value = True
|
||||||
|
files = ['file1', 'file2', 'dir3']
|
||||||
|
mock_path.join.side_effect = files
|
||||||
|
bu.remove_files('chroot', files)
|
||||||
|
self.assertEqual([mock.call(f) for f in files],
|
||||||
|
mock_remove.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
def test_remove_files_not_exists(self, mock_path):
|
||||||
|
mock_path.exists.return_value = False
|
||||||
|
files = ['file1', 'file2', 'dir3']
|
||||||
|
mock_path.join.side_effect = files
|
||||||
|
bu.remove_files('chroot', files)
|
||||||
|
self.assertEqual([mock.call('chroot', f) for f in files],
|
||||||
|
mock_path.join.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(bu, 'remove_files')
|
||||||
|
@mock.patch.object(bu, 'clean_dirs')
|
||||||
|
def test_clean_apt_settings(self, mock_dirs, mock_files):
|
||||||
|
bu.clean_apt_settings('chroot', 'unsigned')
|
||||||
|
mock_dirs.assert_called_once_with(
|
||||||
|
'chroot', ['etc/apt/preferences.d', 'etc/apt/sources.list.d'])
|
||||||
|
mock_files.assert_called_once_with(
|
||||||
|
'chroot', ['etc/apt/sources.list', 'etc/apt/preferences',
|
||||||
|
'etc/apt/apt.conf.d/%s' % 'unsigned'])
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
@mock.patch.object(bu, 'clean_apt_settings')
|
||||||
|
@mock.patch.object(bu, 'remove_files')
|
||||||
|
@mock.patch.object(utils, 'execute')
|
||||||
|
def test_do_post_inst(self, mock_exec, mock_files, mock_clean, mock_path):
|
||||||
|
mock_path.join.return_value = 'fake_path'
|
||||||
|
bu.do_post_inst('chroot')
|
||||||
|
mock_exec.assert_called_once_with(
|
||||||
|
'sed', '-i', 's%root:[\*,\!]%root:$6$IInX3Cqo$5xytL1VZbZTusO'
|
||||||
|
'ewFnG6couuF0Ia61yS3rbC6P5YbZP2TYclwHqMq9e3Tg8rvQxhxSlBXP1DZ'
|
||||||
|
'hdUamxdOBXK0.%', 'fake_path')
|
||||||
|
mock_files.assert_called_once_with('chroot', ['usr/sbin/policy-rc.d'])
|
||||||
|
mock_clean.assert_called_once_with('chroot')
|
||||||
|
mock_path.join.assert_called_once_with('chroot', 'etc/shadow')
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'kill')
|
||||||
|
@mock.patch.object(os, 'readlink', return_value='chroot')
|
||||||
|
@mock.patch.object(utils, 'execute')
|
||||||
|
def test_send_signal_to_chrooted_processes(self, mock_exec, mock_link,
|
||||||
|
mock_kill):
|
||||||
|
mock_exec.return_value = ('kernel 951 1641 1700 1920 3210 4104',
|
||||||
|
'')
|
||||||
|
bu.send_signal_to_chrooted_processes('chroot', 'signal')
|
||||||
|
mock_exec.assert_called_once_with('fuser', '-v', 'chroot',
|
||||||
|
check_exit_code=False)
|
||||||
|
expected_mock_link_calls = [
|
||||||
|
mock.call('/proc/951/root'),
|
||||||
|
mock.call('/proc/1641/root'),
|
||||||
|
mock.call('/proc/1700/root'),
|
||||||
|
mock.call('/proc/1920/root'),
|
||||||
|
mock.call('/proc/3210/root'),
|
||||||
|
mock.call('/proc/4104/root')]
|
||||||
|
expected_mock_kill_calls = [
|
||||||
|
mock.call(951, 'signal'),
|
||||||
|
mock.call(1641, 'signal'),
|
||||||
|
mock.call(1700, 'signal'),
|
||||||
|
mock.call(1920, 'signal'),
|
||||||
|
mock.call(3210, 'signal'),
|
||||||
|
mock.call(4104, 'signal')]
|
||||||
|
self.assertEqual(expected_mock_link_calls, mock_link.call_args_list)
|
||||||
|
self.assertEqual(expected_mock_kill_calls, mock_kill.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'makedev', return_value='fake_dev')
|
||||||
|
@mock.patch.object(os, 'mknod')
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
@mock.patch.object(utils, 'execute', return_value=('/dev/loop123\n', ''))
|
||||||
|
def test_get_free_loop_device_ok(self, mock_exec, mock_path, mock_mknod,
|
||||||
|
mock_mkdev):
|
||||||
|
mock_path.exists.return_value = False
|
||||||
|
self.assertEqual('/dev/loop123', bu.get_free_loop_device(1))
|
||||||
|
mock_exec.assert_called_once_with('losetup', '--find')
|
||||||
|
mock_path.exists.assert_called_once_with('/dev/loop0')
|
||||||
|
mock_mknod.assert_called_once_with('/dev/loop0', 25008, 'fake_dev')
|
||||||
|
mock_mkdev.assert_called_once_with(1, 0)
|
||||||
|
|
||||||
|
def test_set_apt_get_env(self):
|
||||||
|
with mock.patch.dict('os.environ', {}):
|
||||||
|
bu.set_apt_get_env()
|
||||||
|
self.assertEqual('noninteractive', os.environ['DEBIAN_FRONTEND'])
|
||||||
|
self.assertEqual('true', os.environ['DEBCONF_NONINTERACTIVE_SEEN'])
|
||||||
|
for var in ('LC_ALL', 'LANG', 'LANGUAGE'):
|
||||||
|
self.assertEqual('C', os.environ[var])
|
||||||
|
|
||||||
|
def test_strip_filename(self):
|
||||||
|
self.assertEqual('safe_Tex.-98',
|
||||||
|
bu.strip_filename('!@$^^^safe _Tex.?-98;'))
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'makedev', return_value='fake_dev')
|
||||||
|
@mock.patch.object(os, 'mknod')
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
@mock.patch.object(utils, 'execute', return_value=('', 'Error!!!'))
|
||||||
|
def test_get_free_loop_device_not_found(self, mock_exec, mock_path,
|
||||||
|
mock_mknod, mock_mkdev):
|
||||||
|
mock_path.exists.return_value = False
|
||||||
|
self.assertRaises(errors.NoFreeLoopDevices, bu.get_free_loop_device)
|
||||||
|
|
||||||
|
@mock.patch('tempfile.NamedTemporaryFile')
|
||||||
|
@mock.patch.object(utils, 'execute')
|
||||||
|
def test_create_sparse_tmp_file(self, mock_exec, mock_temp):
|
||||||
|
tmp_file = mock.Mock()
|
||||||
|
tmp_file.name = 'fake_name'
|
||||||
|
mock_temp.return_value = tmp_file
|
||||||
|
bu.create_sparse_tmp_file('dir', 'suffix', 1)
|
||||||
|
mock_temp.assert_called_once_with(dir='dir', suffix='suffix',
|
||||||
|
delete=False)
|
||||||
|
mock_exec.assert_called_once_with('truncate', '-s', '1M',
|
||||||
|
tmp_file.name)
|
||||||
|
|
||||||
|
@mock.patch.object(utils, 'execute')
|
||||||
|
def test_attach_file_to_loop(self, mock_exec):
|
||||||
|
bu.attach_file_to_loop('file', 'loop')
|
||||||
|
mock_exec.assert_called_once_with('losetup', 'loop', 'file')
|
||||||
|
|
||||||
|
@mock.patch.object(utils, 'execute')
|
||||||
|
def test_deattach_loop(self, mock_exec):
|
||||||
|
bu.deattach_loop('loop')
|
||||||
|
mock_exec.assert_called_once_with('losetup', '-d', 'loop')
|
||||||
|
|
||||||
|
@mock.patch.object(hu, 'parse_simple_kv')
|
||||||
|
@mock.patch.object(utils, 'execute')
|
||||||
|
def test_shrink_sparse_file(self, mock_exec, mock_parse):
|
||||||
|
mock_parse.return_value = {'block count': 1, 'block size': 2}
|
||||||
|
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||||
|
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||||
|
bu.shrink_sparse_file('file')
|
||||||
|
mock_open.assert_called_once_with('file', 'rwb+')
|
||||||
|
file_handle_mock.truncate.assert_called_once_with(1 * 2)
|
||||||
|
expected_mock_exec_calls = [mock.call('e2fsck', '-y', '-f', 'file'),
|
||||||
|
mock.call('resize2fs', '-F', '-M', 'file')]
|
||||||
|
mock_parse.assert_called_once_with('dumpe2fs', 'file')
|
||||||
|
self.assertEqual(expected_mock_exec_calls, mock_exec.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
def test_add_apt_source(self, mock_path):
|
||||||
|
mock_path.return_value = 'fake_path'
|
||||||
|
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||||
|
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||||
|
bu.add_apt_source('name1', 'uri1', 'suite1', 'section1', 'chroot')
|
||||||
|
expected_calls = [mock.call('deb uri1 suite1 section1\n')]
|
||||||
|
self.assertEqual(expected_calls,
|
||||||
|
file_handle_mock.write.call_args_list)
|
||||||
|
expected_mock_path_calls = [
|
||||||
|
mock.call('chroot', 'etc/apt/sources.list.d',
|
||||||
|
'fuel-image-name1.list')]
|
||||||
|
self.assertEqual(expected_mock_path_calls,
|
||||||
|
mock_path.join.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
def test_add_apt_source_no_section(self, mock_path):
|
||||||
|
mock_path.return_value = 'fake_path'
|
||||||
|
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||||
|
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||||
|
bu.add_apt_source('name2', 'uri2', 'suite2', None, 'chroot')
|
||||||
|
expected_calls = [mock.call('deb uri2 suite2\n')]
|
||||||
|
self.assertEqual(expected_calls,
|
||||||
|
file_handle_mock.write.call_args_list)
|
||||||
|
expected_mock_path_calls = [
|
||||||
|
mock.call('chroot', 'etc/apt/sources.list.d',
|
||||||
|
'fuel-image-name2.list')]
|
||||||
|
self.assertEqual(expected_mock_path_calls,
|
||||||
|
mock_path.join.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
def test_add_apt_preference(self, mock_path):
|
||||||
|
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||||
|
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||||
|
bu.add_apt_preference('name1', 123, 'suite1', 'section1', 'chroot')
|
||||||
|
expected_calls = [
|
||||||
|
mock.call('Package: *\n'),
|
||||||
|
mock.call('Pin: release a=suite1,c=section1\n'),
|
||||||
|
mock.call('Pin-Priority: 123\n')]
|
||||||
|
self.assertEqual(expected_calls,
|
||||||
|
file_handle_mock.write.call_args_list)
|
||||||
|
expected_mock_path_calls = [
|
||||||
|
mock.call('chroot', 'etc/apt/preferences.d',
|
||||||
|
'fuel-image-name1.pref')]
|
||||||
|
self.assertEqual(expected_mock_path_calls,
|
||||||
|
mock_path.join.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
def test_add_apt_preference_multuple_sections(self, mock_path):
|
||||||
|
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||||
|
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||||
|
bu.add_apt_preference('name3', 234, 'suite1', 'section2 section3',
|
||||||
|
'chroot')
|
||||||
|
expected_calls = [
|
||||||
|
mock.call('Package: *\n'),
|
||||||
|
mock.call('Pin: release a=suite1,c=section2\n'),
|
||||||
|
mock.call('Pin: release a=suite1,c=section3\n'),
|
||||||
|
mock.call('Pin-Priority: 234\n')]
|
||||||
|
self.assertEqual(expected_calls,
|
||||||
|
file_handle_mock.write.call_args_list)
|
||||||
|
expected_mock_path_calls = [
|
||||||
|
mock.call('chroot', 'etc/apt/preferences.d',
|
||||||
|
'fuel-image-name3.pref')]
|
||||||
|
self.assertEqual(expected_mock_path_calls,
|
||||||
|
mock_path.join.call_args_list)
|
||||||
|
|
||||||
|
@mock.patch.object(bu, 'clean_apt_settings')
|
||||||
|
@mock.patch.object(os, 'path')
|
||||||
|
def test_pre_apt_get(self, mock_path, mock_clean):
|
||||||
|
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||||
|
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||||
|
bu.pre_apt_get('chroot')
|
||||||
|
file_handle_mock.write.assert_called_once_with(
|
||||||
|
'APT::Get::AllowUnauthenticated 1;\n')
|
||||||
|
mock_clean.assert_called_once_with('chroot')
|
||||||
|
mock_path.join.assert_called_once_with('chroot', 'etc/apt/apt.conf.d',
|
||||||
|
CONF.allow_unsigned_file)
|
||||||
|
|
||||||
|
@mock.patch('gzip.open')
|
||||||
|
@mock.patch.object(os, 'remove')
|
||||||
|
def test_containerize_gzip(self, mock_remove, mock_gzip):
|
||||||
|
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||||
|
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||||
|
file_handle_mock.read.side_effect = ['test data', '']
|
||||||
|
g = mock.Mock()
|
||||||
|
mock_gzip.return_value = g
|
||||||
|
self.assertEqual('file.gz', bu.containerize('file', 'gzip', 1))
|
||||||
|
g.write.assert_called_once_with('test data')
|
||||||
|
expected_calls = [mock.call(1), mock.call(1)]
|
||||||
|
self.assertEqual(expected_calls,
|
||||||
|
file_handle_mock.read.call_args_list)
|
||||||
|
mock_remove.assert_called_once_with('file')
|
||||||
|
|
||||||
|
def test_containerize_bad_container(self):
|
||||||
|
self.assertRaises(errors.WrongImageDataError, bu.containerize, 'file',
|
||||||
|
'fake')
|
||||||
303
fuel_agent/utils/build_utils.py
Normal file
303
fuel_agent/utils/build_utils.py
Normal file
@ -0,0 +1,303 @@
|
|||||||
|
# Copyright 2015 Mirantis, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import gzip
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from oslo.config import cfg
|
||||||
|
|
||||||
|
from fuel_agent import errors
|
||||||
|
from fuel_agent.openstack.common import log as logging
|
||||||
|
from fuel_agent.utils import hardware_utils as hu
|
||||||
|
from fuel_agent.utils import utils
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
bu_opts = [
|
||||||
|
cfg.IntOpt(
|
||||||
|
'max_loop_devices_count',
|
||||||
|
default=255,
|
||||||
|
# NOTE(agordeev): up to 256 loop devices could be allocated up to
|
||||||
|
# kernel version 2.6.23, and the limit (from version 2.6.24 onwards)
|
||||||
|
# isn't theoretically present anymore.
|
||||||
|
help='Maximum allowed loop devices count to use'
|
||||||
|
),
|
||||||
|
cfg.IntOpt(
|
||||||
|
'sparse_file_size',
|
||||||
|
default=2048,
|
||||||
|
help='Size of sparse file in MiBs'
|
||||||
|
),
|
||||||
|
cfg.IntOpt(
|
||||||
|
'loop_device_major_number',
|
||||||
|
default=7,
|
||||||
|
help='System-wide major number for loop device'
|
||||||
|
),
|
||||||
|
cfg.StrOpt(
|
||||||
|
'allow_unsigned_file',
|
||||||
|
default='allow_unsigned_packages',
|
||||||
|
help='File where to store apt setting for unsigned packages'
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
|
CONF.register_opts(bu_opts)
|
||||||
|
DEFAULT_APT_PATH = {
|
||||||
|
'sources_file': 'etc/apt/sources.list',
|
||||||
|
'sources_dir': 'etc/apt/sources.list.d',
|
||||||
|
'preferences_file': 'etc/apt/preferences',
|
||||||
|
'preferences_dir': 'etc/apt/preferences.d',
|
||||||
|
'conf_dir': 'etc/apt/apt.conf.d',
|
||||||
|
}
|
||||||
|
# NOTE(agordeev): hardcoded to r00tme
|
||||||
|
ROOT_PASSWORD = '$6$IInX3Cqo$5xytL1VZbZTusOewFnG6couuF0Ia61yS3rbC6P5YbZP2TYcl'\
|
||||||
|
'wHqMq9e3Tg8rvQxhxSlBXP1DZhdUamxdOBXK0.'
|
||||||
|
|
||||||
|
|
||||||
|
def run_debootstrap(uri, suite, chroot, arch='amd64', eatmydata=False):
|
||||||
|
"""Builds initial base system.
|
||||||
|
|
||||||
|
debootstrap builds initial base system which is capable to run apt-get.
|
||||||
|
debootstrap is well known for its glithcy resolving of package dependecies,
|
||||||
|
so the rest of packages will be installed later by run_apt_get.
|
||||||
|
"""
|
||||||
|
# TODO(agordeev): do retry!
|
||||||
|
cmds = ['debootstrap', '--verbose', '--no-check-gpg', '--arch=%s' % arch,
|
||||||
|
suite, chroot, uri]
|
||||||
|
if eatmydata:
|
||||||
|
cmds.insert(4, '--include=eatmydata')
|
||||||
|
stdout, stderr = utils.execute(*cmds)
|
||||||
|
LOG.debug('Running deboostrap completed.\nstdout: %s\nstderr: %s', stdout,
|
||||||
|
stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def set_apt_get_env():
|
||||||
|
# NOTE(agordeev): disable any confirmations/questions from apt-get side
|
||||||
|
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
|
||||||
|
os.environ['DEBCONF_NONINTERACTIVE_SEEN'] = 'true'
|
||||||
|
os.environ['LC_ALL'] = os.environ['LANG'] = os.environ['LANGUAGE'] = 'C'
|
||||||
|
|
||||||
|
|
||||||
|
def run_apt_get(chroot, packages, eatmydata=False):
|
||||||
|
"""Runs apt-get install <packages>.
|
||||||
|
|
||||||
|
Unlike debootstrap, apt-get has a perfect package dependecies resolver
|
||||||
|
under the hood.
|
||||||
|
eatmydata could be used to totally ignore the storm of sync() calls from
|
||||||
|
dpkg/apt-get tools. It's dangerous, but could decrease package install
|
||||||
|
time in X times.
|
||||||
|
"""
|
||||||
|
# TODO(agordeev): do retry!
|
||||||
|
cmds = ['chroot', chroot, 'apt-get', '-y', 'update']
|
||||||
|
stdout, stderr = utils.execute(*cmds)
|
||||||
|
LOG.debug('Running apt-get update completed.\nstdout: %s\nstderr: %s',
|
||||||
|
stdout, stderr)
|
||||||
|
cmds = ['chroot', chroot, 'apt-get', '-y', 'install', ' '.join(packages)]
|
||||||
|
if eatmydata:
|
||||||
|
cmds.insert(2, 'eatmydata')
|
||||||
|
stdout, stderr = utils.execute(*cmds)
|
||||||
|
LOG.debug('Running apt-get install completed.\nstdout: %s\nstderr: %s',
|
||||||
|
stdout, stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def suppress_services_start(chroot):
|
||||||
|
"""Suppresses services start.
|
||||||
|
|
||||||
|
Prevents start of any service such as udev/ssh/etc in chrooted environment
|
||||||
|
while image is being built.
|
||||||
|
"""
|
||||||
|
path = os.path.join(chroot, 'usr/sbin')
|
||||||
|
if not os.path.exists(path):
|
||||||
|
os.makedirs(path)
|
||||||
|
with open(os.path.join(path, 'policy-rc.d'), 'w') as f:
|
||||||
|
f.write('#!/bin/sh\n'
|
||||||
|
'# prevent any service from being started\n'
|
||||||
|
'exit 101\n')
|
||||||
|
os.fchmod(f.fileno(), 0o755)
|
||||||
|
|
||||||
|
|
||||||
|
def clean_dirs(chroot, dirs):
|
||||||
|
for d in dirs:
|
||||||
|
path = os.path.join(chroot, d)
|
||||||
|
if os.path.isdir(path):
|
||||||
|
shutil.rmtree(path)
|
||||||
|
os.makedirs(path)
|
||||||
|
LOG.debug('Removed dir: %s', path)
|
||||||
|
|
||||||
|
|
||||||
|
def remove_files(chroot, files):
|
||||||
|
for f in files:
|
||||||
|
path = os.path.join(chroot, f)
|
||||||
|
if os.path.exists(path):
|
||||||
|
os.remove(path)
|
||||||
|
LOG.debug('Removed file: %s', path)
|
||||||
|
|
||||||
|
|
||||||
|
def clean_apt_settings(chroot, allow_unsigned_file=CONF.allow_unsigned_file):
|
||||||
|
"""Cleans apt settings such as package sources and repo pinning."""
|
||||||
|
files = [DEFAULT_APT_PATH['sources_file'],
|
||||||
|
DEFAULT_APT_PATH['preferences_file'],
|
||||||
|
os.path.join(DEFAULT_APT_PATH['conf_dir'], allow_unsigned_file)]
|
||||||
|
remove_files(chroot, files)
|
||||||
|
dirs = [DEFAULT_APT_PATH['preferences_dir'],
|
||||||
|
DEFAULT_APT_PATH['sources_dir']]
|
||||||
|
clean_dirs(chroot, dirs)
|
||||||
|
|
||||||
|
|
||||||
|
def do_post_inst(chroot):
|
||||||
|
# NOTE(agordeev): set up password for root
|
||||||
|
utils.execute('sed', '-i',
|
||||||
|
's%root:[\*,\!]%root:' + ROOT_PASSWORD + '%',
|
||||||
|
os.path.join(chroot, 'etc/shadow'))
|
||||||
|
# NOTE(agordeev): remove custom policy-rc.d which is needed to disable
|
||||||
|
# execution of post/pre-install package hooks and start of services
|
||||||
|
remove_files(chroot, ['usr/sbin/policy-rc.d'])
|
||||||
|
clean_apt_settings(chroot)
|
||||||
|
|
||||||
|
|
||||||
|
def send_signal_to_chrooted_processes(chroot, signal):
|
||||||
|
"""Sends signal to all processes, which are running inside of chroot."""
|
||||||
|
for p in utils.execute('fuser', '-v', chroot,
|
||||||
|
check_exit_code=False)[0].split():
|
||||||
|
try:
|
||||||
|
pid = int(p)
|
||||||
|
if os.readlink('/proc/%s/root' % pid) == chroot:
|
||||||
|
LOG.debug('Sending %s to chrooted process %s', signal, pid)
|
||||||
|
os.kill(pid, signal)
|
||||||
|
except (OSError, ValueError):
|
||||||
|
LOG.warning('Skipping non pid %s from fuser output' % p)
|
||||||
|
|
||||||
|
|
||||||
|
def get_free_loop_device(
|
||||||
|
loop_device_major_number=CONF.loop_device_major_number,
|
||||||
|
max_loop_devices_count=CONF.max_loop_devices_count):
|
||||||
|
"""Returns the name of free loop device.
|
||||||
|
|
||||||
|
It should return the name of free loop device or raise an exception.
|
||||||
|
Unfortunately, free loop device couldn't be reversed for the later usage,
|
||||||
|
so we must start to use it as fast as we can.
|
||||||
|
If there's no free loop it will try to create new one and ask a system for
|
||||||
|
free loop again.
|
||||||
|
"""
|
||||||
|
for minor in range(0, max_loop_devices_count):
|
||||||
|
cur_loop = "/dev/loop%s" % minor
|
||||||
|
if not os.path.exists(cur_loop):
|
||||||
|
os.mknod(cur_loop, 0o660 | stat.S_IFBLK,
|
||||||
|
os.makedev(loop_device_major_number, minor))
|
||||||
|
try:
|
||||||
|
return utils.execute('losetup', '--find')[0].split()[0]
|
||||||
|
except (IndexError, errors.ProcessExecutionError):
|
||||||
|
LOG.debug("Couldn't find free loop device, trying again")
|
||||||
|
raise errors.NoFreeLoopDevices('Free loop device not found')
|
||||||
|
|
||||||
|
|
||||||
|
def create_sparse_tmp_file(dir, suffix, size=CONF.sparse_file_size):
|
||||||
|
"""Creates sparse file.
|
||||||
|
|
||||||
|
Creates file which consumes disk space more efficiently when the file
|
||||||
|
itself is mostly empty.
|
||||||
|
"""
|
||||||
|
tf = tempfile.NamedTemporaryFile(dir=dir, suffix=suffix, delete=False)
|
||||||
|
utils.execute('truncate', '-s', '%sM' % size, tf.name)
|
||||||
|
return tf.name
|
||||||
|
|
||||||
|
|
||||||
|
def attach_file_to_loop(filename, loop):
|
||||||
|
utils.execute('losetup', loop, filename)
|
||||||
|
|
||||||
|
|
||||||
|
def deattach_loop(loop):
|
||||||
|
utils.execute('losetup', '-d', loop)
|
||||||
|
|
||||||
|
|
||||||
|
def shrink_sparse_file(filename):
|
||||||
|
"""Shrinks file to its size of actual data. Only ext fs are supported."""
|
||||||
|
utils.execute('e2fsck', '-y', '-f', filename)
|
||||||
|
utils.execute('resize2fs', '-F', '-M', filename)
|
||||||
|
data = hu.parse_simple_kv('dumpe2fs', filename)
|
||||||
|
block_count = int(data['block count'])
|
||||||
|
block_size = int(data['block size'])
|
||||||
|
with open(filename, 'rwb+') as f:
|
||||||
|
f.truncate(block_count * block_size)
|
||||||
|
|
||||||
|
|
||||||
|
def strip_filename(name):
|
||||||
|
"""Strips filename for apt settings.
|
||||||
|
|
||||||
|
The name could only contain alphanumeric, hyphen (-), underscore (_) and
|
||||||
|
period (.) characters.
|
||||||
|
"""
|
||||||
|
return re.sub(r"[^a-zA-Z0-9-_.]*", "", name)
|
||||||
|
|
||||||
|
|
||||||
|
def add_apt_source(name, uri, suite, section, chroot):
|
||||||
|
# NOTE(agordeev): The files have either no or "list" as filename extension
|
||||||
|
filename = 'fuel-image-{name}.list'.format(name=strip_filename(name))
|
||||||
|
if section:
|
||||||
|
entry = 'deb {uri} {suite} {section}\n'.format(uri=uri, suite=suite,
|
||||||
|
section=section)
|
||||||
|
else:
|
||||||
|
entry = 'deb {uri} {suite}\n'.format(uri=uri, suite=suite)
|
||||||
|
with open(os.path.join(chroot, DEFAULT_APT_PATH['sources_dir'], filename),
|
||||||
|
'w') as f:
|
||||||
|
f.write(entry)
|
||||||
|
|
||||||
|
|
||||||
|
def add_apt_preference(name, priority, suite, section, chroot):
|
||||||
|
# NOTE(agordeev): The files have either no or "pref" as filename extension
|
||||||
|
filename = 'fuel-image-{name}.pref'.format(name=strip_filename(name))
|
||||||
|
# NOTE(agordeev): priotity=None means that there's no specific pinning for
|
||||||
|
# particular repo and nothing to process.
|
||||||
|
# Default system-wide preferences (priority=500) will be used instead.
|
||||||
|
if priority:
|
||||||
|
sections = section.split()
|
||||||
|
with open(os.path.join(chroot, DEFAULT_APT_PATH['preferences_dir'],
|
||||||
|
filename), 'w') as f:
|
||||||
|
f.write('Package: *\n')
|
||||||
|
if sections:
|
||||||
|
for section in sections:
|
||||||
|
f.write('Pin: release a={suite},c={section}\n'.format(
|
||||||
|
suite=suite, section=section))
|
||||||
|
else:
|
||||||
|
f.write('Pin: release a={suite}\n'.format(suite=suite))
|
||||||
|
f.write('Pin-Priority: {priority}\n'.format(priority=priority))
|
||||||
|
|
||||||
|
|
||||||
|
def pre_apt_get(chroot, allow_unsigned_file=CONF.allow_unsigned_file):
|
||||||
|
"""It must be called prior run_apt_get."""
|
||||||
|
clean_apt_settings(chroot)
|
||||||
|
# NOTE(agordeev): allow to install packages without gpg digest
|
||||||
|
with open(os.path.join(chroot, DEFAULT_APT_PATH['conf_dir'],
|
||||||
|
allow_unsigned_file), 'w') as f:
|
||||||
|
f.write('APT::Get::AllowUnauthenticated 1;\n')
|
||||||
|
|
||||||
|
|
||||||
|
def containerize(filename, container, chunk_size=CONF.data_chunk_size):
|
||||||
|
if container == 'gzip':
|
||||||
|
output_file = filename + '.gz'
|
||||||
|
with open(filename, 'rb') as f:
|
||||||
|
# NOTE(agordeev): gzip in python2.6 doesn't have context manager
|
||||||
|
# support
|
||||||
|
g = gzip.open(output_file, 'wb')
|
||||||
|
for chunk in iter(lambda: f.read(chunk_size), ''):
|
||||||
|
g.write(chunk)
|
||||||
|
g.close()
|
||||||
|
os.remove(filename)
|
||||||
|
return output_file
|
||||||
|
raise errors.WrongImageDataError(
|
||||||
|
'Error while image initialization: '
|
||||||
|
'unsupported image container: {container}'.format(container=container))
|
||||||
Loading…
x
Reference in New Issue
Block a user