From 620d610eaaf03217f3a038c031fe28ce43132166 Mon Sep 17 00:00:00 2001 From: Jeffrey Zhang Date: Wed, 30 Dec 2015 18:49:56 +0800 Subject: [PATCH] Generate the kolla-build.conf file using oslo_config Closes-Bug: #1530122 Change-Id: I96039eade2aacf1bcd9d14d958cdddd915fc9a5c --- .gitignore | 4 + etc/oslo-config-generator/kolla-build.conf | 3 + kolla/cmd/build.py | 158 +------------ kolla/common/__init__.py | 0 kolla/common/config.py | 263 +++++++++++++++++++++ kolla/opts.py | 17 ++ kolla/version.py | 15 ++ setup.cfg | 2 + tox.ini | 6 + 9 files changed, 317 insertions(+), 151 deletions(-) create mode 100644 etc/oslo-config-generator/kolla-build.conf create mode 100644 kolla/common/__init__.py create mode 100644 kolla/common/config.py create mode 100644 kolla/opts.py create mode 100644 kolla/version.py diff --git a/.gitignore b/.gitignore index c16fe78c11..ee78f4f390 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,10 @@ .tox/ .buildconf +# Files generated by setup.py +kolla.egg-info +build + # Files generated by tools/genenv openrc openstack.env diff --git a/etc/oslo-config-generator/kolla-build.conf b/etc/oslo-config-generator/kolla-build.conf new file mode 100644 index 0000000000..a59c08d7f8 --- /dev/null +++ b/etc/oslo-config-generator/kolla-build.conf @@ -0,0 +1,3 @@ +[DEFAULT] +output_file = etc/kolla/kolla-build.conf +namespace = kolla diff --git a/kolla/cmd/build.py b/kolla/cmd/build.py index cf20c72d3d..fa2bbd7ab8 100755 --- a/kolla/cmd/build.py +++ b/kolla/cmd/build.py @@ -34,22 +34,19 @@ import docker import git import jinja2 from oslo_config import cfg -from oslo_config import types from requests.exceptions import ConnectionError import six from six.moves import range +from kolla.common import config as common_config +from kolla import version + logging.basicConfig() LOG = logging.getLogger(__name__) LOG.setLevel(logging.INFO) signal.signal(signal.SIGINT, signal.SIG_DFL) -RDO_MIRROR = "http://trunk.rdoproject.org/centos7" -DELOREAN = "{}/current/delorean.repo".format(RDO_MIRROR) -DELOREAN_DEPS = "{}/delorean-deps.repo".format(RDO_MIRROR) -INSTALL_TYPE_CHOICES = ['binary', 'source', 'rdo', 'rhos'] - class KollaDirNotFoundException(Exception): pass @@ -279,14 +276,6 @@ class WorkerThread(Thread): self.push_queue.put(image) -def get_kolla_version(): - local_conf_path = os.path.join(find_base_dir(), "setup.cfg") - config = six.moves.configparser.RawConfigParser() - config.read(local_conf_path) - version = config.get("metadata", "version") - return version - - def find_os_type(): return platform.linux_distribution() @@ -307,21 +296,6 @@ def find_base_dir(): ) -def find_config_file(filename): - global_conf_path = os.path.join('/etc/kolla', filename) - local_conf_path = os.path.join(find_base_dir(), 'etc', 'kolla', filename) - - if os.access(global_conf_path, os.R_OK): - return global_conf_path - elif os.access(local_conf_path, os.R_OK): - return local_conf_path - else: - raise KollaDirNotFoundException( - 'Cant find kolla config. Searched at: %s and %s' % - (global_conf_path, local_conf_path) - ) - - class KollaWorker(object): def __init__(self, conf): @@ -417,6 +391,7 @@ class KollaWorker(object): LOG.debug('Set atime and mtime to 0 for all content in working dir') def create_dockerfiles(self): + kolla_version = version.version_info.cached_version_string() for path in self.docker_build_paths: template_name = "Dockerfile.j2" env = jinja2.Environment(loader=jinja2.FileSystemLoader(path)) @@ -429,7 +404,7 @@ class KollaWorker(object): 'namespace': self.namespace, 'tag': self.tag, 'maintainer': self.maintainer, - 'kolla_version': get_kolla_version(), + 'kolla_version': kolla_version, 'rpm_setup': self.rpm_setup} if self.include_header: with open(self.include_header, 'r') as f: @@ -616,128 +591,9 @@ class KollaWorker(object): return queue -def get_conf(): - conf = cfg.ConfigOpts() - - _kolla_profile_opts = [ - cfg.ListOpt('infra', - default=['ceph', 'data', 'mariadb', 'haproxy', - 'keepalived', 'kolla-ansible', 'memcached', - 'mongodb', 'openvswitch', 'rabbitmq', 'rsyslog']), - cfg.ListOpt('main', - default=['cinder', 'ceilometer', 'glance', 'heat', - 'horizon', 'keystone', 'neutron', 'nova', - 'swift']), - cfg.ListOpt('aux', - default=['aodh', 'designate', 'gnocchi', 'ironic', - 'magnum', 'mistral', 'trove,' 'zaqar']), - cfg.ListOpt('default', - default=['data', 'kolla-ansible', 'glance', 'haproxy', - 'heat', 'horizon', 'keepalived', 'keystone', - 'memcached', 'mariadb', 'neutron', 'nova', - 'openvswitch', 'rabbitmq', 'rsyslog']), - cfg.ListOpt('gate', - default=['ceph', 'cinder', 'data', 'dind', 'glance', - 'haproxy', 'heat', 'horizon', 'keepalived', - 'keystone', 'kolla-ansible', 'mariadb', - 'memcached', 'neutron', 'nova', 'openvswitch', - 'rabbitmq', 'rsyslog']) - ] - - _kolla_cli_opts = [ - cfg.StrOpt('base', short='b', default='centos', - deprecated_group='kolla-build', - help='The base distro to use when building'), - cfg.StrOpt('base_tag', default='latest', - deprecated_group='kolla-build', - help='The base distro image tag'), - cfg.BoolOpt('debug', short='d', default=False, - deprecated_group='kolla-build', - help='Turn on debugging log level'), - cfg.StrOpt('include-header', short='i', - deprecated_group='kolla-build', - help=('Path to custom file to be added at ' - 'beginning of base Dockerfile')), - cfg.StrOpt('include-footer', short='I', - deprecated_group='kolla-build', - help=('Path to custom file to be added at ' - 'end of Dockerfiles for final images')), - cfg.BoolOpt('keep', default=False, - deprecated_group='kolla-build', - help='Keep failed intermediate containers'), - cfg.StrOpt('namespace', short='n', default='kollaglue', - deprecated_group='kolla-build', - help='The Docker namespace name'), - cfg.BoolOpt('cache', default=True, - help='Use the Docker cache when building', - ), - cfg.BoolOpt('no-cache', default=False, - help='Do not use the Docker cache when building', - deprecated_for_removal=True), - cfg.MultiOpt('profile', types.String(), short='p', - deprecated_group='kolla-build', - help=('Build a pre-defined set of images, see [profiles]' - ' section in {}. The default profiles are:' - ' {}'.format( - find_config_file('kolla-build.conf'), - ', '.join( - [opt.name for opt in _kolla_profile_opts]) - ))), - cfg.BoolOpt('push', default=False, - deprecated_group='kolla-build', - help='Push images after building'), - cfg.IntOpt('push-threads', default=1, min=1, - deprecated_group='kolla-build', - help=('The number of threads to user while pushing' - ' Images. Note: Docker can not handle threading' - ' push properly.')), - cfg.IntOpt('retries', short='r', default=3, min=0, - deprecated_group='kolla-build', - help='The number of times to retry while building'), - cfg.MultiOpt('regex', types.String(), positional=True, - help=('Build only images matching regex and its' - ' dependencies')), - cfg.StrOpt('registry', deprecated_group='kolla-build', - help=('The docker registry host. The default registry host' - ' is Docker Hub')), - cfg.StrOpt('type', short='t', default='binary', - choices=INSTALL_TYPE_CHOICES, - dest='install_type', deprecated_group='kolla-build', - help=('The method of the Openstack install. The valid' - ' types are: {}'.format( - ', '.join(INSTALL_TYPE_CHOICES)))), - cfg.IntOpt('threads', short='T', default=8, min=1, - deprecated_group='kolla-build', - help=('The number of threads to use while building.' - ' (Note: setting to one will allow real time' - ' logging.)')), - cfg.StrOpt('tag', default=get_kolla_version(), - deprecated_group='kolla-build', - help='The Docker tag'), - cfg.BoolOpt('template-only', default=False, - deprecated_group='kolla-build', - help=("Don't build images. Generate Dockerfile only")), - ] - - _kolla_base_opts = [ - cfg.StrOpt('maintainer', deprecated_group='kolla-build', - default='Kolla Project (https://launchpad.net/kolla)', - help='The MAINTAINER field'), - cfg.ListOpt('rpm_setup_config', default=[DELOREAN, DELOREAN_DEPS], - deprecated_group='kolla-build', - help=('Comma separated list of .rpm or .repo file(s)' - 'or URL(s) to install before building containers')) - ] - conf.register_cli_opts(_kolla_cli_opts) - conf.register_opts(_kolla_profile_opts, group='profiles') - conf.register_opts(_kolla_base_opts) - conf(sys.argv[1:], - default_config_files=[find_config_file('kolla-build.conf')]) - return conf - - def main(): - conf = get_conf() + conf = cfg.ConfigOpts() + common_config.parse(conf, sys.argv[1:], prog='kolla-build') if conf.debug: LOG.setLevel(logging.DEBUG) diff --git a/kolla/common/__init__.py b/kolla/common/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kolla/common/config.py b/kolla/common/config.py new file mode 100644 index 0000000000..68e1f7e5ed --- /dev/null +++ b/kolla/common/config.py @@ -0,0 +1,263 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools + +from oslo_config import cfg +from oslo_config import types + +from kolla import version + + +RDO_MIRROR = "http://trunk.rdoproject.org/centos7" +DELOREAN = "{}/current/delorean.repo".format(RDO_MIRROR) +DELOREAN_DEPS = "{}/delorean-deps.repo".format(RDO_MIRROR) +INSTALL_TYPE_CHOICES = ['binary', 'source', 'rdo', 'rhos'] + +_PROFILE_OPTS = [ + cfg.ListOpt('infra', + default=['ceph', 'data', 'mariadb', 'haproxy', + 'keepalived', 'kolla-ansible', 'memcached', + 'mongodb', 'openvswitch', 'rabbitmq', 'rsyslog'], + help='Infra images'), + cfg.ListOpt('main', + default=['cinder', 'ceilometer', 'glance', 'heat', + 'horizon', 'keystone', 'neutron', 'nova', + 'swift'], + help='Main images'), + cfg.ListOpt('aux', + default=['aodh', 'designate', 'gnocchi', 'ironic', + 'magnum', 'mistral', 'trove,' 'zaqar'], + help='Aux Images'), + cfg.ListOpt('default', + default=['data', 'kolla-ansible', 'glance', 'haproxy', + 'heat', 'horizon', 'keepalived', 'keystone', + 'memcached', 'mariadb', 'neutron', 'nova', + 'openvswitch', 'rabbitmq', 'rsyslog'], + help='Default images'), + cfg.ListOpt('gate', + default=['ceph', 'cinder', 'data', 'dind', 'glance', + 'haproxy', 'heat', 'horizon', 'keepalived', + 'keystone', 'kolla-ansible', 'mariadb', + 'memcached', 'neutron', 'nova', 'openvswitch', + 'rabbitmq', 'rsyslog'], + help='Gate images') +] + +_CLI_OPTS = [ + cfg.StrOpt('base', short='b', default='centos', + deprecated_group='kolla-build', + help='The base distro to use when building'), + cfg.StrOpt('base_tag', default='latest', + deprecated_group='kolla-build', + help='The base distro image tag'), + cfg.BoolOpt('debug', short='d', default=False, + deprecated_group='kolla-build', + help='Turn on debugging log level'), + cfg.StrOpt('include-header', short='i', + deprecated_group='kolla-build', + help=('Path to custom file to be added at ' + 'beginning of base Dockerfile')), + cfg.StrOpt('include-footer', short='I', + deprecated_group='kolla-build', + help=('Path to custom file to be added at ' + 'end of Dockerfiles for final images')), + cfg.BoolOpt('keep', default=False, + deprecated_group='kolla-build', + help='Keep failed intermediate containers'), + cfg.StrOpt('namespace', short='n', default='kollaglue', + deprecated_group='kolla-build', + help='The Docker namespace name'), + cfg.BoolOpt('cache', default=True, + help='Use the Docker cache when building', + ), + cfg.BoolOpt('no-cache', default=False, + help='Do not use the Docker cache when building', + deprecated_for_removal=True), + cfg.MultiOpt('profile', types.String(), short='p', + deprecated_group='kolla-build', + help=('Build a pre-defined set of images, see [profiles]' + ' section in config. The default profiles are:' + ' {}'.format(', '.join( + [opt.name for opt in _PROFILE_OPTS]) + ))), + cfg.BoolOpt('push', default=False, + deprecated_group='kolla-build', + help='Push images after building'), + cfg.IntOpt('push-threads', default=1, min=1, + deprecated_group='kolla-build', + help=('The number of threads to user while pushing' + ' Images. Note: Docker can not handle threading' + ' push properly.')), + cfg.IntOpt('retries', short='r', default=3, min=0, + deprecated_group='kolla-build', + help='The number of times to retry while building'), + cfg.MultiOpt('regex', types.String(), positional=True, + help=('Build only images matching regex and its' + ' dependencies')), + cfg.StrOpt('registry', deprecated_group='kolla-build', + help=('The docker registry host. The default registry host' + ' is Docker Hub')), + cfg.StrOpt('type', short='t', default='binary', + choices=INSTALL_TYPE_CHOICES, + dest='install_type', deprecated_group='kolla-build', + help=('The method of the Openstack install. The valid' + ' types are: {}'.format( + ', '.join(INSTALL_TYPE_CHOICES)))), + cfg.IntOpt('threads', short='T', default=8, min=1, + deprecated_group='kolla-build', + help=('The number of threads to use while building.' + ' (Note: setting to one will allow real time' + ' logging.)')), + cfg.StrOpt('tag', default=version.version_info.cached_version_string(), + deprecated_group='kolla-build', + help='The Docker tag'), + cfg.BoolOpt('template-only', default=False, + deprecated_group='kolla-build', + help=("Don't build images. Generate Dockerfile only")), +] + +_BASE_OPTS = [ + cfg.StrOpt('maintainer', deprecated_group='kolla-build', + default='Kolla Project (https://launchpad.net/kolla)', + help='The MAINTAINER field'), + cfg.ListOpt('rpm_setup_config', default=[DELOREAN, DELOREAN_DEPS], + deprecated_group='kolla-build', + help=('Comma separated list of .rpm or .repo file(s)' + 'or URL(s) to install before building containers')) +] + + +SOURCES = { + 'openstack-base': { + 'type': 'url', + 'location': ('https://github.com/openstack/requirements/tarball/' + 'master')}, + 'aodh-base': { + 'type': 'url', + 'location': 'http://tarballs.openstack.org/aodh/aodh-master.tar.gz'}, + 'ceilometer-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/ceilometer/' + 'ceilometer-master.tar.gz')}, + 'cinder-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/cinder/' + 'cinder-master.tar.gz')}, + 'designate-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/designate/' + 'designate-master.tar.gz')}, + 'glance-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/glance/' + 'glance-master.tar.gz')}, + 'gnocchi-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/gnocchi/' + 'gnocchi-master.tar.gz')}, + 'heat-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/heat/heat-master.tar.gz')}, + 'horizon': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/horizon/' + 'horizon-master.tar.gz')}, + 'ironic-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/ironic/' + 'ironic-master.tar.gz')}, + 'keystone': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/keystone/' + 'keystone-master.tar.gz')}, + 'magnum-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/magnum/' + 'magnum-master.tar.gz')}, + 'mistral-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/mistral/' + 'mistral-master.tar.gz')}, + 'murano-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/murano/' + 'murano-master.tar.gz')}, + 'neutron-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/neutron/' + 'neutron-master.tar.gz')}, + 'nova-base': { + 'type': 'url', + 'location': 'http://tarballs.openstack.org/nova/nova-master.tar.gz'}, + 'nova-spicehtml5proxy': { + 'type': 'url', + 'location': ('http://github.com/SPICE/spice-html5/tarball/' + 'spice-html5-0.1.6')}, + 'nova-novncproxy': { + 'type': 'url', + 'location': 'http://github.com/kanaka/noVNC/tarball/v0.5.1'}, + 'swift-base': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/swift/' + 'swift-master.tar.gz')}, + 'tempest': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/tempest/' + 'tempest-master.tar.gz')}, + 'zaqar': { + 'type': 'url', + 'location': ('http://tarballs.openstack.org/zaqar/' + 'zaqar-master.tar.gz')} + +} + + +def _get_source_opt(type_, location, reference=None): + return [cfg.StrOpt('type', choices=['git', 'url'], + default=type_, + help='Source location type'), + cfg.StrOpt('location', default=location, + help='The location for source install'), + cfg.StrOpt('reference', default=reference, + help=('Git reference to pull, commit sha, tag' + 'or branch name'))] + + +def gen_source_opts(): + for name, params in SOURCES.iteritems(): + type_ = params['type'] + location = params['location'] + reference = params.get('reference') + yield name, _get_source_opt(type_, location, reference) + + +def list_opts(): + return itertools.chain([(None, _CLI_OPTS), + (None, _BASE_OPTS), + ('profiles', _PROFILE_OPTS)], + gen_source_opts(), + ) + + +def parse(conf, args, usage=None, prog=None, + default_config_files=None): + conf.register_cli_opts(_CLI_OPTS) + conf.register_opts(_BASE_OPTS) + conf.register_opts(_PROFILE_OPTS, group='profiles') + + conf(args=args, + project='kolla', + usage=usage, + prog=prog, + version=version.version_info.cached_version_string(), + default_config_files=default_config_files) diff --git a/kolla/opts.py b/kolla/opts.py new file mode 100644 index 0000000000..0571d7fbe2 --- /dev/null +++ b/kolla/opts.py @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from kolla.common import config + + +def list_opts(): + return config.list_opts() diff --git a/kolla/version.py b/kolla/version.py new file mode 100644 index 0000000000..4430b70c55 --- /dev/null +++ b/kolla/version.py @@ -0,0 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pbr.version + +version_info = pbr.version.VersionInfo('kolla') diff --git a/setup.cfg b/setup.cfg index e5196cf201..74bb08c160 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,8 @@ scripts = [entry_points] console_scripts = kolla-build = kolla.cmd.build:main +oslo.config.opts = + kolla = kolla.opts:list_opts [global] setup-hooks = diff --git a/tox.ini b/tox.ini index 5d778cade5..ea86928d50 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,7 @@ skipsdist = True envlist = functional,pep8 [testenv] +usedevelop=True install_command = pip install -U {opts} {packages} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt @@ -102,6 +103,11 @@ commands = bash -c "if [ ! -d .testrepository ]; then testr init; fi" sudo -g docker testr run test_build.BuildTestUbuntuSource +[testenv:genconfig] +whitelist_externals = which +commands= + oslo-config-generator --config-file etc/oslo-config-generator/kolla-build.conf + [flake8] show-source = True exclude=.eggs,.git,.tox,doc