x/bareon
Code Issues Proposed changes

Allow to set proxy parameters:

Browse Source 
* https/http/ftp for:
 - debootstrap
 - apt configs

Change-Id: Ia6a01fab402a112eddc3ec741f1707749d9c64c3
Implements: blueprint dynamically-build-bootstrap
This commit is contained in:
alexz 2015-11-05 17:50:13 +02:00
parent 7027b9c31f
commit 07560a9fc3
10 changed files with 138 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
fuel_agent/drivers/nailgun.py
View File

@ -720,7 +720,17 @@ class NailgunBuildImage(BaseDataDriver):
suite=repo['suite'],
section=repo['section'],
priority=repo['priority']))
os = objects.Ubuntu(repos=repos, packages=packages, major=14, minor=4)
proxies = objects.RepoProxies()
proxy_dict = self.data.get('proxies', {})
for protocol, uri in six.iteritems(proxy_dict.get('protocols', {})):
proxies.add_proxy(protocol, uri)
proxies.add_direct_repo_addrs(proxy_dict.get(
'direct_repo_addr_list', []))
os = objects.Ubuntu(repos=repos, packages=packages, major=14, minor=4,
proxies=proxies)
return os
def parse_schemes(self):

16
fuel_agent/manager.py
View File

@ -652,17 +652,23 @@ class Manager(object):
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot,
attempts=CONF.fetch_packages_attempts)
proxies = self.driver.operating_system.proxies
bu.run_debootstrap(
uri=uri, suite=suite, chroot=chroot,
attempts=CONF.fetch_packages_attempts,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
bu.pre_apt_get(
chroot, allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
for repo in self.driver.operating_system.repos:
LOG.debug('Adding repository source: name={name}, uri={uri},'

2
fuel_agent/objects/__init__.py
View File

@ -32,6 +32,7 @@ from fuel_agent.objects.partition.scheme import PartitionScheme
from fuel_agent.objects.partition.vg import VolumeGroup
from fuel_agent.objects.repo import DEBRepo
from fuel_agent.objects.repo import Repo
from fuel_agent.objects.repo import RepoProxies
PV = PhysicalVolume
@ -68,4 +69,5 @@ __all__ = [
'Repo',
'DEBRepo',
'Loop',
'RepoProxies'
]

4
fuel_agent/objects/operating_system.py
View File

@ -14,11 +14,13 @@
class OperatingSystem(object):
def __init__(self, repos, packages, major='unknown', minor='unknown'):
def __init__(self, repos, packages, major='unknown', minor='unknown',
proxies=None):
self.repos = repos
self.packages = packages
self.major = major
self.minor = minor
self.proxies = proxies
def to_dict(self):
return {'major': self.major,

18
fuel_agent/objects/repo.py
View File

@ -26,3 +26,21 @@ class DEBRepo(Repo):
self.suite = suite
self.section = section
self.meta = meta
class RepoProxies(object):
def __init__(self, proxies=None, direct_repo_addr_list=None):
"""RepoProxies object
:param proxies: dict with proto:uri format
:param direct_repo_addr: list of addr
:return:
"""
self.proxies = proxies or {}
self.direct_repo_addr_list = direct_repo_addr_list or []
def add_proxy(self, protocol, uri):
self.proxies[protocol] = uri
def add_direct_repo_addrs(self, repo_addr_list):
self.direct_repo_addr_list.extend(repo_addr_list)

24
fuel_agent/tests/test_build_utils.py
View File

@ -37,21 +37,25 @@ class BuildUtilsTestCase(unittest2.TestCase):
def setUp(self):
super(BuildUtilsTestCase, self).setUp()
@mock.patch('fuel_agent.utils.build.os', environ={})
@mock.patch.object(utils, 'execute', return_value=(None, None))
def test_run_debootstrap(self, mock_exec):
def test_run_debootstrap(self, mock_exec, mock_environ):
bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', attempts=2)
mock_exec.assert_called_once_with('debootstrap', '--verbose',
'--no-check-gpg', '--arch=arch',
'suite', 'chroot', 'uri', attempts=2)
'suite', 'chroot', 'uri', attempts=2,
env_variables={})
@mock.patch('fuel_agent.utils.build.os', environ={})
@mock.patch.object(utils, 'execute', return_value=(None, None))
def test_run_debootstrap_eatmydata(self, mock_exec):
def test_run_debootstrap_eatmydata(self, mock_exec, mock_environ):
bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', eatmydata=True,
attempts=2)
mock_exec.assert_called_once_with('debootstrap', '--verbose',
'--no-check-gpg', '--arch=arch',
'--include=eatmydata', 'suite',
'chroot', 'uri', attempts=2)
'chroot', 'uri', attempts=2,
env_variables={})
@mock.patch.object(utils, 'execute', return_value=(None, None))
def test_run_apt_get(self, mock_exec):
@ -155,10 +159,14 @@ class BuildUtilsTestCase(unittest2.TestCase):
bu.clean_apt_settings('chroot', 'unsigned', 'force_ipv4')
mock_dirs.assert_called_once_with(
'chroot', ['etc/apt/preferences.d', 'etc/apt/sources.list.d'])
mock_files.assert_called_once_with(
'chroot', ['etc/apt/sources.list', 'etc/apt/preferences',
'etc/apt/apt.conf.d/%s' % 'force_ipv4',
'etc/apt/apt.conf.d/%s' % 'unsigned'])
files = set(['etc/apt/sources.list', 'etc/apt/preferences',
'etc/apt/apt.conf.d/%s' % 'force_ipv4',
'etc/apt/apt.conf.d/%s' % 'unsigned',
'etc/apt/apt.conf.d/01fuel_agent-use-proxy-ftp',
'etc/apt/apt.conf.d/01fuel_agent-use-proxy-http',
'etc/apt/apt.conf.d/01fuel_agent-use-proxy-https'])
self.assertEqual('chroot', mock_files.call_args[0][0])
self.assertEqual(files, set(mock_files.call_args[0][1]))
@mock.patch('fuel_agent.utils.build.open',
create=True, new_callable=mock.mock_open)

9
fuel_agent/tests/test_manager.py
View File

@ -839,6 +839,9 @@ class TestImageBuild(unittest2.TestCase):
objects.DEBRepo('mos', 'http://fakemos',
'mosX.Y', 'fakesection', priority=1000)],
packages=['fakepackage1', 'fakepackage2'])
self.mgr.driver.operating_system.proxies = objects.RepoProxies(
proxies={'fake': 'fake'},
direct_repo_addr_list='fake_addr')
self.mgr.driver.operating_system.minor = 4
self.mgr.driver.operating_system.major = 14
mock_os.path.exists.return_value = False
@ -893,11 +896,13 @@ class TestImageBuild(unittest2.TestCase):
mock_bu.suppress_services_start.call_args_list)
mock_bu.run_debootstrap.assert_called_once_with(
uri='http://fakeubuntu', suite='trusty', chroot='/tmp/imgdir',
attempts=CONF.fetch_packages_attempts)
attempts=CONF.fetch_packages_attempts,
proxies={'fake': 'fake'}, direct_repo_addr='fake_addr')
mock_bu.set_apt_get_env.assert_called_once_with()
mock_bu.pre_apt_get.assert_called_once_with(
'/tmp/imgdir', allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
force_ipv4_file=CONF.force_ipv4_file, proxies={'fake': 'fake'},
direct_repo_addr='fake_addr')
self.assertEqual([
mock.call(name='ubuntu',
uri='http://fakeubuntu',

16
fuel_agent/tests/test_nailgun_build_image.py
View File

@ -114,10 +114,11 @@ class TestNailgunBuildImage(unittest2.TestCase):
data = {'codename': 'not-trusty'}
NailgunBuildImage(data)
@mock.patch('fuel_agent.objects.RepoProxies')
@mock.patch('fuel_agent.objects.Ubuntu')
@mock.patch.object(NailgunBuildImage, 'parse_schemes')
def test_parse_operating_system_packages_given(self, mock_parse_schemes,
mock_ub):
mock_ub, mock_proxies):
data = {
'repos': [],
'codename': 'trusty',
@ -127,13 +128,15 @@ class TestNailgunBuildImage(unittest2.TestCase):
mock_ub_instance.packages = data['packages']
driver = NailgunBuildImage(data)
mock_ub.assert_called_once_with(repos=[], packages=data['packages'],
major=14, minor=4)
major=14, minor=4,
proxies=mock_proxies.return_value)
self.assertEqual(driver.operating_system.packages, data['packages'])
@mock.patch('fuel_agent.objects.RepoProxies')
@mock.patch('fuel_agent.objects.Ubuntu')
@mock.patch.object(NailgunBuildImage, 'parse_schemes')
def test_parse_operating_system_packages_not_given(
self, mock_parse_schemes, mock_ub):
self, mock_parse_schemes, mock_ub, mock_proxies):
data = {
'repos': [],
'codename': 'trusty'
@ -143,15 +146,16 @@ class TestNailgunBuildImage(unittest2.TestCase):
driver = NailgunBuildImage(data)
mock_ub.assert_called_once_with(
repos=[], packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES,
major=14, minor=4)
major=14, minor=4, proxies=mock_proxies.return_value)
self.assertEqual(driver.operating_system.packages,
NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES)
@mock.patch('fuel_agent.objects.RepoProxies')
@mock.patch('fuel_agent.objects.DEBRepo')
@mock.patch('fuel_agent.objects.Ubuntu')
@mock.patch.object(NailgunBuildImage, 'parse_schemes')
def test_parse_operating_system_repos(self, mock_parse_schemes, mock_ub,
mock_deb):
mock_deb, mock_proxies):
data = {
'repos': REPOS_SAMPLE,
'codename': 'trusty'
@ -174,7 +178,7 @@ class TestNailgunBuildImage(unittest2.TestCase):
mock_ub_instance.repos = repos
mock_ub.assert_called_once_with(
repos=repos, packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES,
major=14, minor=4)
major=14, minor=4, proxies=mock_proxies.return_value)
self.assertEqual(mock_deb_expected_calls,
mock_deb.call_args_list[:len(REPOS_SAMPLE)])
self.assertEqual(driver.operating_system.repos, repos)

63
fuel_agent/utils/build.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import gzip
import os
import re
@ -38,24 +39,45 @@ DEFAULT_APT_PATH = {
'preferences_dir': 'etc/apt/preferences.d',
'conf_dir': 'etc/apt/apt.conf.d',
}
# protocol : conf_file_name
# FIXME(azvyagintsev): Move to oslo_config
# Bug: https://bugs.launchpad.net/fuel/+bug/1514772
PROXY_PROTOCOLS = {
'ftp': '01fuel_agent-use-proxy-ftp',
'http': '01fuel_agent-use-proxy-http',
'https': '01fuel_agent-use-proxy-https'
}
# NOTE(agordeev): hardcoded to r00tme
ROOT_PASSWORD = '$6$IInX3Cqo$5xytL1VZbZTusOewFnG6couuF0Ia61yS3rbC6P5YbZP2TYcl'\
'wHqMq9e3Tg8rvQxhxSlBXP1DZhdUamxdOBXK0.'
def run_debootstrap(uri, suite, chroot, arch='amd64', eatmydata=False,
attempts=10):
attempts=10, proxies=None, direct_repo_addr=None):
"""Builds initial base system.
debootstrap builds initial base system which is capable to run apt-get.
debootstrap is well known for its glithcy resolving of package dependecies,
so the rest of packages will be installed later by run_apt_get.
"""
cmds = ['debootstrap', '--verbose', '--no-check-gpg', '--arch=%s' % arch,
suite, chroot, uri]
env_vars = copy.deepcopy(os.environ)
for proto in six.iterkeys(PROXY_PROTOCOLS):
if proto in (proxies or {}):
LOG.debug('Using {0} proxy {1} for debootstrap'.format(
proto, proxies[proto]))
env_vars['{0}_proxy'.format(proto)] = proxies[proto]
if direct_repo_addr:
env_vars['no_proxy'] = ','.join(direct_repo_addr)
LOG.debug('Setting no_proxy for: {0}'.format(env_vars['no_proxy']))
cmds = ['debootstrap', '--verbose', '--no-check-gpg',
'--arch={0}'.format(arch)]
if eatmydata:
cmds.insert(4, '--include=eatmydata')
stdout, stderr = utils.execute(*cmds, attempts=attempts)
cmds.extend(['--include=eatmydata'])
cmds.extend([suite, chroot, uri])
stdout, stderr = utils.execute(*cmds, attempts=attempts,
env_variables=env_vars)
LOG.debug('Running deboostrap completed.\nstdout: %s\nstderr: %s', stdout,
stderr)
@ -138,6 +160,9 @@ def clean_apt_settings(chroot, allow_unsigned_file='allow_unsigned_packages',
DEFAULT_APT_PATH['preferences_file'],
os.path.join(DEFAULT_APT_PATH['conf_dir'], force_ipv4_file),
os.path.join(DEFAULT_APT_PATH['conf_dir'], allow_unsigned_file)]
# also remove proxies
for p_file in six.itervalues(PROXY_PROTOCOLS):
files.append(os.path.join(DEFAULT_APT_PATH['conf_dir'], p_file))
remove_files(chroot, files)
dirs = [DEFAULT_APT_PATH['preferences_dir'],
DEFAULT_APT_PATH['sources_dir']]
@ -426,8 +451,31 @@ def add_apt_preference(name, priority, suite, section, chroot, uri):
f.write('Pin-Priority: {priority}\n'.format(priority=priority))
def set_apt_proxy(chroot, proxies, direct_repo_addr=None):
"""Configure proxy for apt-config
direct_repo_addr:: direct apt address:
access to it bypass proxies.
"""
for protocol in six.iterkeys(proxies):
with open(os.path.join(chroot, DEFAULT_APT_PATH['conf_dir'],
PROXY_PROTOCOLS[protocol]), 'w') as f:
f.write('Acquire::{0}::proxy "{1}";\n'
''.format(protocol, proxies[protocol]))
LOG.debug('Apply apt-proxy: \nprotocol: {0}\nurl: {1}'
''.format(protocol, proxies[protocol]))
if direct_repo_addr:
for addr in direct_repo_addr:
f.write('Acquire::{0}::proxy::{1} "DIRECT";\n'
''.format(protocol, addr))
LOG.debug('Set DIRECT repo: \nprotocol:'
' {0}\nurl: {1}'.format(protocol, addr))
def pre_apt_get(chroot, allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file='force_ipv4'):
force_ipv4_file='force_ipv4',
proxies=None, direct_repo_addr=None):
"""It must be called prior run_apt_get."""
clean_apt_settings(chroot, allow_unsigned_file=allow_unsigned_file,
force_ipv4_file=force_ipv4_file)
@ -439,6 +487,9 @@ def pre_apt_get(chroot, allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file), 'w') as f:
f.write('Acquire::ForceIPv4 "true";\n')
if proxies:
set_apt_proxy(chroot, proxies, direct_repo_addr)
def containerize(filename, container, chunk_size=1048576):
if container == 'gzip':

3
fuel_agent/utils/utils.py
View File

@ -77,6 +77,9 @@ def execute(*cmd, **kwargs):
command = ' '.join(cmd)
LOG.debug('Trying to execute command: %s', command)
commands = [c.strip() for c in re.split(r'\|', command)]
if kwargs.get('env_variables'):
LOG.debug('Env variables: {0}'.
format(kwargs.get('env_variables')))
env = kwargs.pop('env_variables', copy.deepcopy(os.environ))
env['PATH'] = '/bin:/usr/bin:/sbin:/usr/sbin'
env['LC_ALL'] = env['LANG'] = env['LANGUAGE'] = kwargs.pop('language', 'C')