From 283624a1a66544bc1b880aad98e7ca99d007c0ad Mon Sep 17 00:00:00 2001 From: Artur Svechnikov Date: Mon, 23 Nov 2015 18:17:51 +0300 Subject: [PATCH] Add build command to fuel-bootstrap Change-Id: I8da4a27cae791aa46ee455c74dcb2a89cfd2814c Implements: blueprint bootstrap-images-support-in-cli --- .../fuel_bootstrap/commands/build.py | 172 +++++++++++++ .../fuel_bootstrap/consts.py | 50 +++- .../fuel_bootstrap/utils/bootstrap_image.py | 18 ++ .../fuel_bootstrap/utils/data.py | 238 ++++++++++++++++++ .../fuel_bootstrap_cli/requirements.txt | 1 + 5 files changed, 477 insertions(+), 2 deletions(-) create mode 100644 contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py create mode 100644 contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/data.py diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py new file mode 100644 index 0000000..b9be4d9 --- /dev/null +++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from cliff import command + +from fuel_bootstrap.utils import bootstrap_image as bs_image + + +class BuildCommand(command.Command): + """Build new bootstrap image with specified parameters.""" + + def get_parser(self, prog_name): + parser = super(BuildCommand, self).get_parser(prog_name) + parser.add_argument( + '--ubuntu-release', + type=str, + help="Choose the Ubuntu release (currently supports" + " only trusty).", + ) + parser.add_argument( + '--ubuntu-repo', + type=str, + metavar='REPOSITORY', + help="Use the specified Ubuntu repository. Format" + " 'uri codename'.", + ) + parser.add_argument( + '--mos-repo', + type=str, + metavar='REPOSITORY', + help="Add link to repository with fuel* packages. That" + " should either http://mirror.fuel-infra.org/mos-repos" + " or its mirror. Format 'uri codename'.", + ) + parser.add_argument( + '--repo', + dest='repos', + type=str, + metavar='REPOSITORY', + help="Add one more repository. format 'type uri" + " codename [sections][,priority]'.", + action='append' + ) + parser.add_argument( + '--http-proxy', + type=str, + metavar='URL', + help="Pass http-proxy URL." + ) + parser.add_argument( + '--https-proxy', + type=str, + metavar='URL', + help="Pass https-proxy URL." + ) + parser.add_argument( + '--direct-repo-addr', + metavar='ADDR', + help="Use a direct connection to repository(address)" + " bypass proxy.", + action='append' + ) + parser.add_argument( + '--script', + dest='post_script_file', + type=str, + metavar='FILE', + help="The script is executed after installing packages (both" + " mandatory and user specified ones) and before creating" + " initramfs." + ) + parser.add_argument( + '--include-kernel-module', + help="Make sure the given modules are included into initramfs" + " image. (by adding module into /etc/initramfs-tools/" + "modules) **NOTE** If the module in question is not" + " shipped with the kernel itself please add the package" + " providing it (see the `--packege` option). Keep in mind" + " that initramfs image should be kept as small as" + " possible. This option is intended to include uncommon" + " network interface cards' drivers so the initramfs can" + " fetch the root filesystem image via the network." + ) + parser.add_argument( + '--blacklist-kernel-module', + help="Make sure the given modules never get loaded" + " automatically. **NOTE** Direct injection of files into" + " the image is not recommended, and a proper way to" + " customize an image is adding (custom) packages." + ) + parser.add_argument( + '--package', + dest='packages', + type=str, + metavar='PKGNAME', + help="The option can be given multiple times, all specified" + " packages and their dependencies will be installed.", + action='append' + ) + parser.add_argument( + '--label', + type=str, + metavar='LABEL', + help="Custom string, which will be presented in bootstrap" + " listing." + ) + parser.add_argument( + '--extra-file', + dest='extra_files', + type=str, + metavar='PATH', + help="Directory that will be injected to the image" + " root filesystem. **NOTE** Files/packages will be" + " injected after installing all packages, but before" + " generating system initramfs - thus it's possible to" + " adjust initramfs.", + action='append' + ) + parser.add_argument( + '--extend-kopts', + type=str, + metavar='OPTS', + help="Extend default kernel options" + ) + parser.add_argument( + '--kernel-flavor', + type=str, + help="Defines kernel version. 'linux-image-generic-lts-trusty'" + " will be used by default.", + default='linux-image-generic-lts-trusty' + ) + parser.add_argument( + '--root-ssh-authorized-file', + type=str, + metavar='FILE', + help="Copy public ssh key into image - makes it possible" + " to login as root into any bootstrap node using the" + " key in question." + ) + parser.add_argument( + '--output-dir', + type=str, + metavar='DIR', + help="Which directory should contain built image. /tmp/" + " is used by default.", + default="/tmp/" + ) + parser.add_argument( + '--image-build-dir', + type=str, + metavar='DIR', + help="Which directory should be used for building image." + " /tmp/ will be used by default.", + default="/tmp/" + ) + return parser + + def take_action(self, parsed_args): + bs_image.make_bootstrap(parsed_args) diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/consts.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/consts.py index 81faf4e..93dd0b2 100644 --- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/consts.py +++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/consts.py @@ -16,7 +16,53 @@ import os -# FIXME: make the image directory configurable -BOOTSTRAP_IMAGES_DIR = "/var/www/nailgun/bootstrap" +# FIXME: Move configurable consts to settings.yaml + +BOOTSTRAP_IMAGES_DIR = "/var/www/nailgun/bootstraps/" METADATA_FILE = "metadata.yaml" SYMLINK = os.path.join(BOOTSTRAP_IMAGES_DIR, "active_bootstrap") +ASTUTE_FILE = "/etc/fuel/astute.yaml" +CONTAINER_FORMAT = "tar.gz" +ROOTFS = {'name': 'rootfs', + 'mask': 'rootfs', + 'compress_format': 'xz', + 'uri': 'http://127.0.0.1:8080/bootstraps/{uuid}/root.squashfs', + 'format': 'ext4', + 'container': 'raw'} +BOOTSTRAP_MODULES = [ + {'name': 'kernel', + 'mask': 'kernel', + 'uri': 'http://127.0.0.1:8080/bootstraps/{uuid}/vmlinuz'}, + {'name': 'initrd', + 'mask': 'initrd', + 'compress_format': 'xz', + 'uri': 'http://127.0.0.1:8080/bootstraps/{uuid}/initrd.img'}, + ROOTFS +] + +IMAGE_DATA = {'/': ROOTFS} + +UBUNTU_RELEASE = 'trusty' + +# Packages required for the master node to discover a bootstrap node +# Hardcoded list used for disable user-factor : when user can accidentally +# remove fuel-required packages, and create totally non-working bootstrap +DEFAULT_PACKAGES = [ + "openssh-client", + "openssh-server", + "ntp", + "mcollective", + "nailgun-agent", + "nailgun-mcagents", + "network-checker", + "fuel-agent" + "ubuntu-minimal", + "live-boot", + "live-boot-initramfs-tools", + "wget", + "linux-firmware", + "linux-firmware-nonfree", + "xz-utils", + "squashfs-tools", + "msmtp-mta" +] diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py index 1476b23..023e155 100644 --- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py +++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py @@ -21,8 +21,11 @@ import tarfile import tempfile import yaml +from fuel_agent.utils import utils + from fuel_bootstrap import consts from fuel_bootstrap import errors +from fuel_bootstrap.utils import data as data_util LOG = logging.getLogger(__name__) @@ -117,3 +120,18 @@ def import_image(arch_path): def extract_to_dir(arch_path, extract_path): LOG.info("Try extract %s to %s", arch_path, extract_path) tarfile.open(arch_path, 'r').extractall(extract_path) + + +def make_bootstrap(params): + bootdata_builder = data_util.BootstrapDataBuilder(params) + bootdata = bootdata_builder.build() + + LOG.info("Try to build image with data:\n%s", yaml.safe_dump(bootdata)) + + with tempfile.NamedTemporaryFile() as f: + f.write(yaml.safe_dump(bootdata)) + f.flush() + utils.execute('fa_mkbootstrap', '--nouse-syslog', '--data_driver', + 'bootstrap_build_image', '--nodebug', '-v', + '--image_build_dir', params.image_build_dir, + '--input_data_file', f.name) diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/data.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/data.py new file mode 100644 index 0000000..7155f84 --- /dev/null +++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/data.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import os +import re +import six +import uuid +import yaml + +from fuel_bootstrap import consts +from fuel_bootstrap import errors + + +class BootstrapDataBuilder(object): + + def __init__(self, data): + self.astute = self._parse_astute() + + self.uuid = six.text_type(uuid.uuid4()) + + self.container_format = consts.CONTAINER_FORMAT + + self.ubuntu_release = data.ubuntu_release or consts.UBUNTU_RELEASE + self.ubuntu_repo = data.ubuntu_repo + self.mos_repo = data.mos_repo + self.repos = data.repos or [] + + self.http_proxy = data.http_proxy or \ + self.astute['BOOTSTRAP']['HTTP_PROXY'] + self.https_proxy = data.https_proxy or \ + self.astute['BOOTSTRAP']['HTTPS_PROXY'] + self.direct_repo_addr = data.direct_repo_addr + + self.post_script_file = data.post_script_file + self.root_ssh_authorized_file = data.root_ssh_authorized_file + self.extra_files = data.extra_files + + self.include_kernel_module = data.include_kernel_module + self.blacklist_kernel_module = data.blacklist_kernel_module + + self.packages = data.packages + + self.label = data.label + self.extend_kopts = data.extend_kopts + self.kernel_flavor = data.kernel_flavor + self.output = os.path.join( + data.output_dir, + "{uuid}.{format}".format( + uuid=self.uuid, + format=self.container_format)) + + def _parse_astute(self): + with open(consts.ASTUTE_FILE) as f: + data = yaml.safe_load(f) + return data + + def build(self): + return { + 'bootstrap': { + 'modules': self._prepare_modules(), + 'extend_kopts': self.extend_kopts, + 'post_script_file': self.post_script_file, + 'uuid': self.uuid, + 'extra_files': self.extra_files, + 'root_ssh_authorized_file': self.root_ssh_authorized_file, + 'container': { + 'meta_file': consts.METADATA_FILE, + 'format': self.container_format + } + }, + 'repos': self._get_repos(), + 'proxies': self._get_proxy_settings(), + 'codename': self.ubuntu_release, + 'output': self.output, + 'packages': self._get_packages(), + 'image_data': self._prepare_image_data() + } + + def _prepare_modules(self): + modules = copy.copy(consts.BOOTSTRAP_MODULES) + for module in modules: + module['uri'] = module['uri'].format(uuid=self.uuid) + return modules + + def _prepare_image_data(self): + image_data = copy.copy(consts.IMAGE_DATA) + image_data['/']['uri'] = image_data['/']['uri'].format(uuid=self.uuid) + return image_data + + def _get_proxy_settings(self): + if self.http_proxy or self.https_proxy: + return {'protocols': {'http': self.http_proxy, + 'https': self.https_proxy}, + 'direct_repo_addr_list': self._get_direct_repo_addr()} + return {} + + def _get_direct_repo_addr(self): + addrs = set() + if self.direct_repo_addr: + addrs |= set(self.direct_repo_addr) + + addrs.add(self.astute['ADMIN_NETWORK']['ipaddress']) + + return list(addrs) + + def _get_repos(self): + repos = [] + if self.ubuntu_repo: + repos.extend(self._parse_ubuntu_repos(self.ubuntu_repo)) + else: + repos.extend(self.astute['BOOTSTRAP']['MIRROR_DISTRO']) + + if self.mos_repo: + repos.extend(self._parse_mos_repos(self.mos_repo)) + else: + repos.extend(self.astute['BOOTSTRAP']['MIRROR_MOS']) + + repo_count = 0 + for repo in self.repos: + repo_count += 1 + repos.append(self._parse_repo( + repo, + name="extra_repo{0}".format(repo_count))) + + if not self.repos: + repos.extend(self.astute['BOOTSTRAP']['EXTRA_DEB_REPOS']) + + return sorted(repos, key=lambda repo: repo['priority'] or 500) + + def _get_packages(self): + result = set(consts.DEFAULT_PACKAGES) + result.add(self.kernel_flavor) + if self.packages: + result |= set(self.packages) + return list(result) + + @classmethod + def _parse_not_extra_repo(cls, repo): + regexp = r"(?P[^\s]+) (?P[^\s]+)" + + match = re.match(regexp, repo) + + if not match: + raise errors.IncorrectRepository( + "Coulnd't parse ubuntu repository {0}". + format(repo) + ) + + return match.group('uri', 'suite') + + @classmethod + def _parse_mos_repos(cls, repo): + uri, suite = cls._parse_not_extra_repo(repo) + + result = cls._generate_repos_from_uri( + uri=uri, + codename=suite, + name='mos', + components=['', '-updates', '-security'], + section='main restricted', + priority='1050' + ) + result += cls._generate_repos_from_uri( + uri=uri, + codename=suite, + name='mos', + components=['-holdback'], + section='main restricted', + priority='1100' + ) + return result + + @classmethod + def _parse_ubuntu_repos(cls, repo): + uri, suite = cls._parse_not_extra_repo(repo) + + return cls._generate_repos_from_uri( + uri=uri, + codename=cls.ubuntu_release, + name='ubuntu', + components=['', '-updates', '-security'], + section='main universe multiverse' + ) + + @classmethod + def _generate_repos_from_uri(cls, uri, codename, name, components=None, + section=None, type_=None, priority=None): + if not components: + components = [''] + result = [] + for component in components: + result.append({ + "name": "{0}{1}".format(name, component), + "type": type_ or "deb", + "uri": uri, + "priority": priority, + "section": section, + "suite": "{0}{1}".format(codename, component) + }) + return result + + @classmethod + def _parse_repo(cls, repo, name=None): + regexp = r"(?Pdeb(-src)?) (?P[^\s]+) (?P[^\s]+)( "\ + r"(?P
[\w\s]*))?(,(?P[\d]+))?" + + match = re.match(regexp, repo) + + if not match: + raise errors.IncorrectRepository("Couldn't parse repository '{0}'" + .format(repo)) + + repo_type = match.group('type') + repo_suite = match.group('suite') + repo_section = match.group('section') + repo_uri = match.group('uri') + repo_priority = match.group('priority') + + return {'name': name, + 'type': repo_type, + 'uri': repo_uri, + 'priority': repo_priority, + 'suite': repo_suite, + 'section': repo_section or ''} diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/requirements.txt b/contrib/fuel_bootstrap/fuel_bootstrap_cli/requirements.txt index 9e33c61..ce8417a 100644 --- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/requirements.txt +++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/requirements.txt @@ -4,3 +4,4 @@ stevedore pbr>=0.6 cliff>=1.7.0 six>=1.7.0 +python-fuelclient