[IBP] Add build_utils
Build utils is providing all necessary low-level functions to build target image. Related-Bug: #1433193 Partially implements: blueprint ibp-build-ubuntu-images Change-Id: I8d89377fb38073968a580f7c8e3d04e61ce8ed60
This commit is contained in:
Alexander Gordeev
Vladimir Kozhukalov
parent
524d6ff54b
commit
c93e8b05a3
@ -142,3 +142,7 @@ class HttpUrlInvalidContentLength(BaseError):
|
||||
|
||||
class ImageChecksumMismatchError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class NoFreeLoopDevices(BaseError):
|
||||
pass
|
||||
|
||||
356
fuel_agent/tests/test_build_utils.py
Normal file
356
fuel_agent/tests/test_build_utils.py
Normal file
@ -0,0 +1,356 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import testtools
|
||||
|
||||
import mock
|
||||
from oslo.config import cfg
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.utils import build_utils as bu
|
||||
from fuel_agent.utils import hardware_utils as hu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class BuildUtilsTestCase(testtools.TestCase):
|
||||
def setUp(self):
|
||||
super(BuildUtilsTestCase, self).setUp()
|
||||
|
||||
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||
def test_run_debootstrap(self, mock_exec):
|
||||
bu.run_debootstrap('uri', 'suite', 'chroot', 'arch')
|
||||
mock_exec.assert_called_once_with('debootstrap', '--verbose',
|
||||
'--no-check-gpg', '--arch=arch',
|
||||
'suite', 'chroot', 'uri')
|
||||
|
||||
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||
def test_run_debootstrap_eatmydata(self, mock_exec):
|
||||
bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', eatmydata=True)
|
||||
mock_exec.assert_called_once_with('debootstrap', '--verbose',
|
||||
'--no-check-gpg', '--arch=arch',
|
||||
'--include=eatmydata', 'suite',
|
||||
'chroot', 'uri')
|
||||
|
||||
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||
def test_run_apt_get(self, mock_exec):
|
||||
bu.run_apt_get('chroot', ['package1', 'package2'])
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('chroot', 'chroot', 'apt-get', '-y', 'update'),
|
||||
mock.call('chroot', 'chroot', 'apt-get', '-y', 'install',
|
||||
'package1 package2')]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
|
||||
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||
def test_run_apt_get_eatmydata(self, mock_exec):
|
||||
bu.run_apt_get('chroot', ['package1', 'package2'], eatmydata=True)
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('chroot', 'chroot', 'apt-get', '-y', 'update'),
|
||||
mock.call('chroot', 'chroot', 'eatmydata', 'apt-get', '-y',
|
||||
'install', 'package1 package2')]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'fchmod')
|
||||
@mock.patch.object(os, 'makedirs')
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_suppress_services_start(self, mock_path, mock_mkdir, mock_fchmod):
|
||||
mock_path.join.return_value = 'fake_path'
|
||||
mock_path.exists.return_value = False
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
file_handle_mock.fileno.return_value = 'fake_fileno'
|
||||
bu.suppress_services_start('chroot')
|
||||
mock_open.assert_called_once_with('fake_path', 'w')
|
||||
expected = '#!/bin/sh\n# prevent any service from being started\n'\
|
||||
'exit 101\n'
|
||||
file_handle_mock.write.assert_called_once_with(expected)
|
||||
mock_fchmod.assert_called_once_with('fake_fileno', 0o755)
|
||||
mock_mkdir.assert_called_once_with('fake_path')
|
||||
|
||||
@mock.patch.object(os, 'fchmod')
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_suppress_services_start_nomkdir(self, mock_path, mock_fchmod):
|
||||
mock_path.join.return_value = 'fake_path'
|
||||
mock_path.exists.return_value = True
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
file_handle_mock.fileno.return_value = 'fake_fileno'
|
||||
bu.suppress_services_start('chroot')
|
||||
mock_open.assert_called_once_with('fake_path', 'w')
|
||||
expected = '#!/bin/sh\n# prevent any service from being started\n'\
|
||||
'exit 101\n'
|
||||
file_handle_mock.write.assert_called_once_with(expected)
|
||||
mock_fchmod.assert_called_once_with('fake_fileno', 0o755)
|
||||
|
||||
@mock.patch.object(shutil, 'rmtree')
|
||||
@mock.patch.object(os, 'makedirs')
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_clean_dirs(self, mock_path, mock_mkdir, mock_rmtree):
|
||||
mock_path.isdir.return_value = True
|
||||
dirs = ['dir1', 'dir2', 'dir3']
|
||||
mock_path.join.side_effect = dirs
|
||||
bu.clean_dirs('chroot', dirs)
|
||||
for m in (mock_rmtree, mock_mkdir):
|
||||
self.assertEqual([mock.call(d) for d in dirs], m.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_clean_dirs_not_isdir(self, mock_path):
|
||||
mock_path.isdir.return_value = False
|
||||
dirs = ['dir1', 'dir2', 'dir3']
|
||||
mock_path.join.side_effect = dirs
|
||||
bu.clean_dirs('chroot', dirs)
|
||||
self.assertEqual([mock.call('chroot', d) for d in dirs],
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'remove')
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_remove_files(self, mock_path, mock_remove):
|
||||
mock_path.exists.return_value = True
|
||||
files = ['file1', 'file2', 'dir3']
|
||||
mock_path.join.side_effect = files
|
||||
bu.remove_files('chroot', files)
|
||||
self.assertEqual([mock.call(f) for f in files],
|
||||
mock_remove.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_remove_files_not_exists(self, mock_path):
|
||||
mock_path.exists.return_value = False
|
||||
files = ['file1', 'file2', 'dir3']
|
||||
mock_path.join.side_effect = files
|
||||
bu.remove_files('chroot', files)
|
||||
self.assertEqual([mock.call('chroot', f) for f in files],
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(bu, 'remove_files')
|
||||
@mock.patch.object(bu, 'clean_dirs')
|
||||
def test_clean_apt_settings(self, mock_dirs, mock_files):
|
||||
bu.clean_apt_settings('chroot', 'unsigned')
|
||||
mock_dirs.assert_called_once_with(
|
||||
'chroot', ['etc/apt/preferences.d', 'etc/apt/sources.list.d'])
|
||||
mock_files.assert_called_once_with(
|
||||
'chroot', ['etc/apt/sources.list', 'etc/apt/preferences',
|
||||
'etc/apt/apt.conf.d/%s' % 'unsigned'])
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
@mock.patch.object(bu, 'clean_apt_settings')
|
||||
@mock.patch.object(bu, 'remove_files')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_do_post_inst(self, mock_exec, mock_files, mock_clean, mock_path):
|
||||
mock_path.join.return_value = 'fake_path'
|
||||
bu.do_post_inst('chroot')
|
||||
mock_exec.assert_called_once_with(
|
||||
'sed', '-i', 's%root:[\*,\!]%root:$6$IInX3Cqo$5xytL1VZbZTusO'
|
||||
'ewFnG6couuF0Ia61yS3rbC6P5YbZP2TYclwHqMq9e3Tg8rvQxhxSlBXP1DZ'
|
||||
'hdUamxdOBXK0.%', 'fake_path')
|
||||
mock_files.assert_called_once_with('chroot', ['usr/sbin/policy-rc.d'])
|
||||
mock_clean.assert_called_once_with('chroot')
|
||||
mock_path.join.assert_called_once_with('chroot', 'etc/shadow')
|
||||
|
||||
@mock.patch.object(os, 'kill')
|
||||
@mock.patch.object(os, 'readlink', return_value='chroot')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_send_signal_to_chrooted_processes(self, mock_exec, mock_link,
|
||||
mock_kill):
|
||||
mock_exec.return_value = ('kernel 951 1641 1700 1920 3210 4104',
|
||||
'')
|
||||
bu.send_signal_to_chrooted_processes('chroot', 'signal')
|
||||
mock_exec.assert_called_once_with('fuser', '-v', 'chroot',
|
||||
check_exit_code=False)
|
||||
expected_mock_link_calls = [
|
||||
mock.call('/proc/951/root'),
|
||||
mock.call('/proc/1641/root'),
|
||||
mock.call('/proc/1700/root'),
|
||||
mock.call('/proc/1920/root'),
|
||||
mock.call('/proc/3210/root'),
|
||||
mock.call('/proc/4104/root')]
|
||||
expected_mock_kill_calls = [
|
||||
mock.call(951, 'signal'),
|
||||
mock.call(1641, 'signal'),
|
||||
mock.call(1700, 'signal'),
|
||||
mock.call(1920, 'signal'),
|
||||
mock.call(3210, 'signal'),
|
||||
mock.call(4104, 'signal')]
|
||||
self.assertEqual(expected_mock_link_calls, mock_link.call_args_list)
|
||||
self.assertEqual(expected_mock_kill_calls, mock_kill.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'makedev', return_value='fake_dev')
|
||||
@mock.patch.object(os, 'mknod')
|
||||
@mock.patch.object(os, 'path')
|
||||
@mock.patch.object(utils, 'execute', return_value=('/dev/loop123\n', ''))
|
||||
def test_get_free_loop_device_ok(self, mock_exec, mock_path, mock_mknod,
|
||||
mock_mkdev):
|
||||
mock_path.exists.return_value = False
|
||||
self.assertEqual('/dev/loop123', bu.get_free_loop_device(1))
|
||||
mock_exec.assert_called_once_with('losetup', '--find')
|
||||
mock_path.exists.assert_called_once_with('/dev/loop0')
|
||||
mock_mknod.assert_called_once_with('/dev/loop0', 25008, 'fake_dev')
|
||||
mock_mkdev.assert_called_once_with(1, 0)
|
||||
|
||||
def test_set_apt_get_env(self):
|
||||
with mock.patch.dict('os.environ', {}):
|
||||
bu.set_apt_get_env()
|
||||
self.assertEqual('noninteractive', os.environ['DEBIAN_FRONTEND'])
|
||||
self.assertEqual('true', os.environ['DEBCONF_NONINTERACTIVE_SEEN'])
|
||||
for var in ('LC_ALL', 'LANG', 'LANGUAGE'):
|
||||
self.assertEqual('C', os.environ[var])
|
||||
|
||||
def test_strip_filename(self):
|
||||
self.assertEqual('safe_Tex.-98',
|
||||
bu.strip_filename('!@$^^^safe _Tex.?-98;'))
|
||||
|
||||
@mock.patch.object(os, 'makedev', return_value='fake_dev')
|
||||
@mock.patch.object(os, 'mknod')
|
||||
@mock.patch.object(os, 'path')
|
||||
@mock.patch.object(utils, 'execute', return_value=('', 'Error!!!'))
|
||||
def test_get_free_loop_device_not_found(self, mock_exec, mock_path,
|
||||
mock_mknod, mock_mkdev):
|
||||
mock_path.exists.return_value = False
|
||||
self.assertRaises(errors.NoFreeLoopDevices, bu.get_free_loop_device)
|
||||
|
||||
@mock.patch('tempfile.NamedTemporaryFile')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_create_sparse_tmp_file(self, mock_exec, mock_temp):
|
||||
tmp_file = mock.Mock()
|
||||
tmp_file.name = 'fake_name'
|
||||
mock_temp.return_value = tmp_file
|
||||
bu.create_sparse_tmp_file('dir', 'suffix', 1)
|
||||
mock_temp.assert_called_once_with(dir='dir', suffix='suffix',
|
||||
delete=False)
|
||||
mock_exec.assert_called_once_with('truncate', '-s', '1M',
|
||||
tmp_file.name)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_attach_file_to_loop(self, mock_exec):
|
||||
bu.attach_file_to_loop('file', 'loop')
|
||||
mock_exec.assert_called_once_with('losetup', 'loop', 'file')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_deattach_loop(self, mock_exec):
|
||||
bu.deattach_loop('loop')
|
||||
mock_exec.assert_called_once_with('losetup', '-d', 'loop')
|
||||
|
||||
@mock.patch.object(hu, 'parse_simple_kv')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_shrink_sparse_file(self, mock_exec, mock_parse):
|
||||
mock_parse.return_value = {'block count': 1, 'block size': 2}
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
bu.shrink_sparse_file('file')
|
||||
mock_open.assert_called_once_with('file', 'rwb+')
|
||||
file_handle_mock.truncate.assert_called_once_with(1 * 2)
|
||||
expected_mock_exec_calls = [mock.call('e2fsck', '-y', '-f', 'file'),
|
||||
mock.call('resize2fs', '-F', '-M', 'file')]
|
||||
mock_parse.assert_called_once_with('dumpe2fs', 'file')
|
||||
self.assertEqual(expected_mock_exec_calls, mock_exec.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_add_apt_source(self, mock_path):
|
||||
mock_path.return_value = 'fake_path'
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
bu.add_apt_source('name1', 'uri1', 'suite1', 'section1', 'chroot')
|
||||
expected_calls = [mock.call('deb uri1 suite1 section1\n')]
|
||||
self.assertEqual(expected_calls,
|
||||
file_handle_mock.write.call_args_list)
|
||||
expected_mock_path_calls = [
|
||||
mock.call('chroot', 'etc/apt/sources.list.d',
|
||||
'fuel-image-name1.list')]
|
||||
self.assertEqual(expected_mock_path_calls,
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_add_apt_source_no_section(self, mock_path):
|
||||
mock_path.return_value = 'fake_path'
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
bu.add_apt_source('name2', 'uri2', 'suite2', None, 'chroot')
|
||||
expected_calls = [mock.call('deb uri2 suite2\n')]
|
||||
self.assertEqual(expected_calls,
|
||||
file_handle_mock.write.call_args_list)
|
||||
expected_mock_path_calls = [
|
||||
mock.call('chroot', 'etc/apt/sources.list.d',
|
||||
'fuel-image-name2.list')]
|
||||
self.assertEqual(expected_mock_path_calls,
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_add_apt_preference(self, mock_path):
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
bu.add_apt_preference('name1', 123, 'suite1', 'section1', 'chroot')
|
||||
expected_calls = [
|
||||
mock.call('Package: *\n'),
|
||||
mock.call('Pin: release a=suite1,c=section1\n'),
|
||||
mock.call('Pin-Priority: 123\n')]
|
||||
self.assertEqual(expected_calls,
|
||||
file_handle_mock.write.call_args_list)
|
||||
expected_mock_path_calls = [
|
||||
mock.call('chroot', 'etc/apt/preferences.d',
|
||||
'fuel-image-name1.pref')]
|
||||
self.assertEqual(expected_mock_path_calls,
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_add_apt_preference_multuple_sections(self, mock_path):
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
bu.add_apt_preference('name3', 234, 'suite1', 'section2 section3',
|
||||
'chroot')
|
||||
expected_calls = [
|
||||
mock.call('Package: *\n'),
|
||||
mock.call('Pin: release a=suite1,c=section2\n'),
|
||||
mock.call('Pin: release a=suite1,c=section3\n'),
|
||||
mock.call('Pin-Priority: 234\n')]
|
||||
self.assertEqual(expected_calls,
|
||||
file_handle_mock.write.call_args_list)
|
||||
expected_mock_path_calls = [
|
||||
mock.call('chroot', 'etc/apt/preferences.d',
|
||||
'fuel-image-name3.pref')]
|
||||
self.assertEqual(expected_mock_path_calls,
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(bu, 'clean_apt_settings')
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_pre_apt_get(self, mock_path, mock_clean):
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
bu.pre_apt_get('chroot')
|
||||
file_handle_mock.write.assert_called_once_with(
|
||||
'APT::Get::AllowUnauthenticated 1;\n')
|
||||
mock_clean.assert_called_once_with('chroot')
|
||||
mock_path.join.assert_called_once_with('chroot', 'etc/apt/apt.conf.d',
|
||||
CONF.allow_unsigned_file)
|
||||
|
||||
@mock.patch('gzip.open')
|
||||
@mock.patch.object(os, 'remove')
|
||||
def test_containerize_gzip(self, mock_remove, mock_gzip):
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
file_handle_mock.read.side_effect = ['test data', '']
|
||||
g = mock.Mock()
|
||||
mock_gzip.return_value = g
|
||||
self.assertEqual('file.gz', bu.containerize('file', 'gzip', 1))
|
||||
g.write.assert_called_once_with('test data')
|
||||
expected_calls = [mock.call(1), mock.call(1)]
|
||||
self.assertEqual(expected_calls,
|
||||
file_handle_mock.read.call_args_list)
|
||||
mock_remove.assert_called_once_with('file')
|
||||
|
||||
def test_containerize_bad_container(self):
|
||||
self.assertRaises(errors.WrongImageDataError, bu.containerize, 'file',
|
||||
'fake')
|
||||
303
fuel_agent/utils/build_utils.py
Normal file
303
fuel_agent/utils/build_utils.py
Normal file
@ -0,0 +1,303 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import gzip
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent.utils import hardware_utils as hu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
bu_opts = [
|
||||
cfg.IntOpt(
|
||||
'max_loop_devices_count',
|
||||
default=255,
|
||||
# NOTE(agordeev): up to 256 loop devices could be allocated up to
|
||||
# kernel version 2.6.23, and the limit (from version 2.6.24 onwards)
|
||||
# isn't theoretically present anymore.
|
||||
help='Maximum allowed loop devices count to use'
|
||||
),
|
||||
cfg.IntOpt(
|
||||
'sparse_file_size',
|
||||
default=2048,
|
||||
help='Size of sparse file in MiBs'
|
||||
),
|
||||
cfg.IntOpt(
|
||||
'loop_device_major_number',
|
||||
default=7,
|
||||
help='System-wide major number for loop device'
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'allow_unsigned_file',
|
||||
default='allow_unsigned_packages',
|
||||
help='File where to store apt setting for unsigned packages'
|
||||
),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(bu_opts)
|
||||
DEFAULT_APT_PATH = {
|
||||
'sources_file': 'etc/apt/sources.list',
|
||||
'sources_dir': 'etc/apt/sources.list.d',
|
||||
'preferences_file': 'etc/apt/preferences',
|
||||
'preferences_dir': 'etc/apt/preferences.d',
|
||||
'conf_dir': 'etc/apt/apt.conf.d',
|
||||
}
|
||||
# NOTE(agordeev): hardcoded to r00tme
|
||||
ROOT_PASSWORD = '$6$IInX3Cqo$5xytL1VZbZTusOewFnG6couuF0Ia61yS3rbC6P5YbZP2TYcl'\
|
||||
'wHqMq9e3Tg8rvQxhxSlBXP1DZhdUamxdOBXK0.'
|
||||
|
||||
|
||||
def run_debootstrap(uri, suite, chroot, arch='amd64', eatmydata=False):
|
||||
"""Builds initial base system.
|
||||
|
||||
debootstrap builds initial base system which is capable to run apt-get.
|
||||
debootstrap is well known for its glithcy resolving of package dependecies,
|
||||
so the rest of packages will be installed later by run_apt_get.
|
||||
"""
|
||||
# TODO(agordeev): do retry!
|
||||
cmds = ['debootstrap', '--verbose', '--no-check-gpg', '--arch=%s' % arch,
|
||||
suite, chroot, uri]
|
||||
if eatmydata:
|
||||
cmds.insert(4, '--include=eatmydata')
|
||||
stdout, stderr = utils.execute(*cmds)
|
||||
LOG.debug('Running deboostrap completed.\nstdout: %s\nstderr: %s', stdout,
|
||||
stderr)
|
||||
|
||||
|
||||
def set_apt_get_env():
|
||||
# NOTE(agordeev): disable any confirmations/questions from apt-get side
|
||||
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
|
||||
os.environ['DEBCONF_NONINTERACTIVE_SEEN'] = 'true'
|
||||
os.environ['LC_ALL'] = os.environ['LANG'] = os.environ['LANGUAGE'] = 'C'
|
||||
|
||||
|
||||
def run_apt_get(chroot, packages, eatmydata=False):
|
||||
"""Runs apt-get install <packages>.
|
||||
|
||||
Unlike debootstrap, apt-get has a perfect package dependecies resolver
|
||||
under the hood.
|
||||
eatmydata could be used to totally ignore the storm of sync() calls from
|
||||
dpkg/apt-get tools. It's dangerous, but could decrease package install
|
||||
time in X times.
|
||||
"""
|
||||
# TODO(agordeev): do retry!
|
||||
cmds = ['chroot', chroot, 'apt-get', '-y', 'update']
|
||||
stdout, stderr = utils.execute(*cmds)
|
||||
LOG.debug('Running apt-get update completed.\nstdout: %s\nstderr: %s',
|
||||
stdout, stderr)
|
||||
cmds = ['chroot', chroot, 'apt-get', '-y', 'install', ' '.join(packages)]
|
||||
if eatmydata:
|
||||
cmds.insert(2, 'eatmydata')
|
||||
stdout, stderr = utils.execute(*cmds)
|
||||
LOG.debug('Running apt-get install completed.\nstdout: %s\nstderr: %s',
|
||||
stdout, stderr)
|
||||
|
||||
|
||||
def suppress_services_start(chroot):
|
||||
"""Suppresses services start.
|
||||
|
||||
Prevents start of any service such as udev/ssh/etc in chrooted environment
|
||||
while image is being built.
|
||||
"""
|
||||
path = os.path.join(chroot, 'usr/sbin')
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
with open(os.path.join(path, 'policy-rc.d'), 'w') as f:
|
||||
f.write('#!/bin/sh\n'
|
||||
'# prevent any service from being started\n'
|
||||
'exit 101\n')
|
||||
os.fchmod(f.fileno(), 0o755)
|
||||
|
||||
|
||||
def clean_dirs(chroot, dirs):
|
||||
for d in dirs:
|
||||
path = os.path.join(chroot, d)
|
||||
if os.path.isdir(path):
|
||||
shutil.rmtree(path)
|
||||
os.makedirs(path)
|
||||
LOG.debug('Removed dir: %s', path)
|
||||
|
||||
|
||||
def remove_files(chroot, files):
|
||||
for f in files:
|
||||
path = os.path.join(chroot, f)
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
LOG.debug('Removed file: %s', path)
|
||||
|
||||
|
||||
def clean_apt_settings(chroot, allow_unsigned_file=CONF.allow_unsigned_file):
|
||||
"""Cleans apt settings such as package sources and repo pinning."""
|
||||
files = [DEFAULT_APT_PATH['sources_file'],
|
||||
DEFAULT_APT_PATH['preferences_file'],
|
||||
os.path.join(DEFAULT_APT_PATH['conf_dir'], allow_unsigned_file)]
|
||||
remove_files(chroot, files)
|
||||
dirs = [DEFAULT_APT_PATH['preferences_dir'],
|
||||
DEFAULT_APT_PATH['sources_dir']]
|
||||
clean_dirs(chroot, dirs)
|
||||
|
||||
|
||||
def do_post_inst(chroot):
|
||||
# NOTE(agordeev): set up password for root
|
||||
utils.execute('sed', '-i',
|
||||
's%root:[\*,\!]%root:' + ROOT_PASSWORD + '%',
|
||||
os.path.join(chroot, 'etc/shadow'))
|
||||
# NOTE(agordeev): remove custom policy-rc.d which is needed to disable
|
||||
# execution of post/pre-install package hooks and start of services
|
||||
remove_files(chroot, ['usr/sbin/policy-rc.d'])
|
||||
clean_apt_settings(chroot)
|
||||
|
||||
|
||||
def send_signal_to_chrooted_processes(chroot, signal):
|
||||
"""Sends signal to all processes, which are running inside of chroot."""
|
||||
for p in utils.execute('fuser', '-v', chroot,
|
||||
check_exit_code=False)[0].split():
|
||||
try:
|
||||
pid = int(p)
|
||||
if os.readlink('/proc/%s/root' % pid) == chroot:
|
||||
LOG.debug('Sending %s to chrooted process %s', signal, pid)
|
||||
os.kill(pid, signal)
|
||||
except (OSError, ValueError):
|
||||
LOG.warning('Skipping non pid %s from fuser output' % p)
|
||||
|
||||
|
||||
def get_free_loop_device(
|
||||
loop_device_major_number=CONF.loop_device_major_number,
|
||||
max_loop_devices_count=CONF.max_loop_devices_count):
|
||||
"""Returns the name of free loop device.
|
||||
|
||||
It should return the name of free loop device or raise an exception.
|
||||
Unfortunately, free loop device couldn't be reversed for the later usage,
|
||||
so we must start to use it as fast as we can.
|
||||
If there's no free loop it will try to create new one and ask a system for
|
||||
free loop again.
|
||||
"""
|
||||
for minor in range(0, max_loop_devices_count):
|
||||
cur_loop = "/dev/loop%s" % minor
|
||||
if not os.path.exists(cur_loop):
|
||||
os.mknod(cur_loop, 0o660 | stat.S_IFBLK,
|
||||
os.makedev(loop_device_major_number, minor))
|
||||
try:
|
||||
return utils.execute('losetup', '--find')[0].split()[0]
|
||||
except (IndexError, errors.ProcessExecutionError):
|
||||
LOG.debug("Couldn't find free loop device, trying again")
|
||||
raise errors.NoFreeLoopDevices('Free loop device not found')
|
||||
|
||||
|
||||
def create_sparse_tmp_file(dir, suffix, size=CONF.sparse_file_size):
|
||||
"""Creates sparse file.
|
||||
|
||||
Creates file which consumes disk space more efficiently when the file
|
||||
itself is mostly empty.
|
||||
"""
|
||||
tf = tempfile.NamedTemporaryFile(dir=dir, suffix=suffix, delete=False)
|
||||
utils.execute('truncate', '-s', '%sM' % size, tf.name)
|
||||
return tf.name
|
||||
|
||||
|
||||
def attach_file_to_loop(filename, loop):
|
||||
utils.execute('losetup', loop, filename)
|
||||
|
||||
|
||||
def deattach_loop(loop):
|
||||
utils.execute('losetup', '-d', loop)
|
||||
|
||||
|
||||
def shrink_sparse_file(filename):
|
||||
"""Shrinks file to its size of actual data. Only ext fs are supported."""
|
||||
utils.execute('e2fsck', '-y', '-f', filename)
|
||||
utils.execute('resize2fs', '-F', '-M', filename)
|
||||
data = hu.parse_simple_kv('dumpe2fs', filename)
|
||||
block_count = int(data['block count'])
|
||||
block_size = int(data['block size'])
|
||||
with open(filename, 'rwb+') as f:
|
||||
f.truncate(block_count * block_size)
|
||||
|
||||
|
||||
def strip_filename(name):
|
||||
"""Strips filename for apt settings.
|
||||
|
||||
The name could only contain alphanumeric, hyphen (-), underscore (_) and
|
||||
period (.) characters.
|
||||
"""
|
||||
return re.sub(r"[^a-zA-Z0-9-_.]*", "", name)
|
||||
|
||||
|
||||
def add_apt_source(name, uri, suite, section, chroot):
|
||||
# NOTE(agordeev): The files have either no or "list" as filename extension
|
||||
filename = 'fuel-image-{name}.list'.format(name=strip_filename(name))
|
||||
if section:
|
||||
entry = 'deb {uri} {suite} {section}\n'.format(uri=uri, suite=suite,
|
||||
section=section)
|
||||
else:
|
||||
entry = 'deb {uri} {suite}\n'.format(uri=uri, suite=suite)
|
||||
with open(os.path.join(chroot, DEFAULT_APT_PATH['sources_dir'], filename),
|
||||
'w') as f:
|
||||
f.write(entry)
|
||||
|
||||
|
||||
def add_apt_preference(name, priority, suite, section, chroot):
|
||||
# NOTE(agordeev): The files have either no or "pref" as filename extension
|
||||
filename = 'fuel-image-{name}.pref'.format(name=strip_filename(name))
|
||||
# NOTE(agordeev): priotity=None means that there's no specific pinning for
|
||||
# particular repo and nothing to process.
|
||||
# Default system-wide preferences (priority=500) will be used instead.
|
||||
if priority:
|
||||
sections = section.split()
|
||||
with open(os.path.join(chroot, DEFAULT_APT_PATH['preferences_dir'],
|
||||
filename), 'w') as f:
|
||||
f.write('Package: *\n')
|
||||
if sections:
|
||||
for section in sections:
|
||||
f.write('Pin: release a={suite},c={section}\n'.format(
|
||||
suite=suite, section=section))
|
||||
else:
|
||||
f.write('Pin: release a={suite}\n'.format(suite=suite))
|
||||
f.write('Pin-Priority: {priority}\n'.format(priority=priority))
|
||||
|
||||
|
||||
def pre_apt_get(chroot, allow_unsigned_file=CONF.allow_unsigned_file):
|
||||
"""It must be called prior run_apt_get."""
|
||||
clean_apt_settings(chroot)
|
||||
# NOTE(agordeev): allow to install packages without gpg digest
|
||||
with open(os.path.join(chroot, DEFAULT_APT_PATH['conf_dir'],
|
||||
allow_unsigned_file), 'w') as f:
|
||||
f.write('APT::Get::AllowUnauthenticated 1;\n')
|
||||
|
||||
|
||||
def containerize(filename, container, chunk_size=CONF.data_chunk_size):
|
||||
if container == 'gzip':
|
||||
output_file = filename + '.gz'
|
||||
with open(filename, 'rb') as f:
|
||||
# NOTE(agordeev): gzip in python2.6 doesn't have context manager
|
||||
# support
|
||||
g = gzip.open(output_file, 'wb')
|
||||
for chunk in iter(lambda: f.read(chunk_size), ''):
|
||||
g.write(chunk)
|
||||
g.close()
|
||||
os.remove(filename)
|
||||
return output_file
|
||||
raise errors.WrongImageDataError(
|
||||
'Error while image initialization: '
|
||||
'unsupported image container: {container}'.format(container=container))
|
||||
Loading…
Reference in New Issue
Block a user