Generate the kolla-build.conf file using oslo_config

Closes-Bug: #1530122
Change-Id: I96039eade2aacf1bcd9d14d958cdddd915fc9a5c
This commit is contained in:
Jeffrey Zhang 2015-12-30 18:49:56 +08:00
parent d1b0a8b3c2
commit 620d610eaa
9 changed files with 317 additions and 151 deletions

4
.gitignore vendored
View File

@ -9,6 +9,10 @@
.tox/
.buildconf
# Files generated by setup.py
kolla.egg-info
build
# Files generated by tools/genenv
openrc
openstack.env

View File

@ -0,0 +1,3 @@
[DEFAULT]
output_file = etc/kolla/kolla-build.conf
namespace = kolla

View File

@ -34,22 +34,19 @@ import docker
import git
import jinja2
from oslo_config import cfg
from oslo_config import types
from requests.exceptions import ConnectionError
import six
from six.moves import range
from kolla.common import config as common_config
from kolla import version
logging.basicConfig()
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.INFO)
signal.signal(signal.SIGINT, signal.SIG_DFL)
RDO_MIRROR = "http://trunk.rdoproject.org/centos7"
DELOREAN = "{}/current/delorean.repo".format(RDO_MIRROR)
DELOREAN_DEPS = "{}/delorean-deps.repo".format(RDO_MIRROR)
INSTALL_TYPE_CHOICES = ['binary', 'source', 'rdo', 'rhos']
class KollaDirNotFoundException(Exception):
pass
@ -279,14 +276,6 @@ class WorkerThread(Thread):
self.push_queue.put(image)
def get_kolla_version():
local_conf_path = os.path.join(find_base_dir(), "setup.cfg")
config = six.moves.configparser.RawConfigParser()
config.read(local_conf_path)
version = config.get("metadata", "version")
return version
def find_os_type():
return platform.linux_distribution()
@ -307,21 +296,6 @@ def find_base_dir():
)
def find_config_file(filename):
global_conf_path = os.path.join('/etc/kolla', filename)
local_conf_path = os.path.join(find_base_dir(), 'etc', 'kolla', filename)
if os.access(global_conf_path, os.R_OK):
return global_conf_path
elif os.access(local_conf_path, os.R_OK):
return local_conf_path
else:
raise KollaDirNotFoundException(
'Cant find kolla config. Searched at: %s and %s' %
(global_conf_path, local_conf_path)
)
class KollaWorker(object):
def __init__(self, conf):
@ -417,6 +391,7 @@ class KollaWorker(object):
LOG.debug('Set atime and mtime to 0 for all content in working dir')
def create_dockerfiles(self):
kolla_version = version.version_info.cached_version_string()
for path in self.docker_build_paths:
template_name = "Dockerfile.j2"
env = jinja2.Environment(loader=jinja2.FileSystemLoader(path))
@ -429,7 +404,7 @@ class KollaWorker(object):
'namespace': self.namespace,
'tag': self.tag,
'maintainer': self.maintainer,
'kolla_version': get_kolla_version(),
'kolla_version': kolla_version,
'rpm_setup': self.rpm_setup}
if self.include_header:
with open(self.include_header, 'r') as f:
@ -616,128 +591,9 @@ class KollaWorker(object):
return queue
def get_conf():
conf = cfg.ConfigOpts()
_kolla_profile_opts = [
cfg.ListOpt('infra',
default=['ceph', 'data', 'mariadb', 'haproxy',
'keepalived', 'kolla-ansible', 'memcached',
'mongodb', 'openvswitch', 'rabbitmq', 'rsyslog']),
cfg.ListOpt('main',
default=['cinder', 'ceilometer', 'glance', 'heat',
'horizon', 'keystone', 'neutron', 'nova',
'swift']),
cfg.ListOpt('aux',
default=['aodh', 'designate', 'gnocchi', 'ironic',
'magnum', 'mistral', 'trove,' 'zaqar']),
cfg.ListOpt('default',
default=['data', 'kolla-ansible', 'glance', 'haproxy',
'heat', 'horizon', 'keepalived', 'keystone',
'memcached', 'mariadb', 'neutron', 'nova',
'openvswitch', 'rabbitmq', 'rsyslog']),
cfg.ListOpt('gate',
default=['ceph', 'cinder', 'data', 'dind', 'glance',
'haproxy', 'heat', 'horizon', 'keepalived',
'keystone', 'kolla-ansible', 'mariadb',
'memcached', 'neutron', 'nova', 'openvswitch',
'rabbitmq', 'rsyslog'])
]
_kolla_cli_opts = [
cfg.StrOpt('base', short='b', default='centos',
deprecated_group='kolla-build',
help='The base distro to use when building'),
cfg.StrOpt('base_tag', default='latest',
deprecated_group='kolla-build',
help='The base distro image tag'),
cfg.BoolOpt('debug', short='d', default=False,
deprecated_group='kolla-build',
help='Turn on debugging log level'),
cfg.StrOpt('include-header', short='i',
deprecated_group='kolla-build',
help=('Path to custom file to be added at '
'beginning of base Dockerfile')),
cfg.StrOpt('include-footer', short='I',
deprecated_group='kolla-build',
help=('Path to custom file to be added at '
'end of Dockerfiles for final images')),
cfg.BoolOpt('keep', default=False,
deprecated_group='kolla-build',
help='Keep failed intermediate containers'),
cfg.StrOpt('namespace', short='n', default='kollaglue',
deprecated_group='kolla-build',
help='The Docker namespace name'),
cfg.BoolOpt('cache', default=True,
help='Use the Docker cache when building',
),
cfg.BoolOpt('no-cache', default=False,
help='Do not use the Docker cache when building',
deprecated_for_removal=True),
cfg.MultiOpt('profile', types.String(), short='p',
deprecated_group='kolla-build',
help=('Build a pre-defined set of images, see [profiles]'
' section in {}. The default profiles are:'
' {}'.format(
find_config_file('kolla-build.conf'),
', '.join(
[opt.name for opt in _kolla_profile_opts])
))),
cfg.BoolOpt('push', default=False,
deprecated_group='kolla-build',
help='Push images after building'),
cfg.IntOpt('push-threads', default=1, min=1,
deprecated_group='kolla-build',
help=('The number of threads to user while pushing'
' Images. Note: Docker can not handle threading'
' push properly.')),
cfg.IntOpt('retries', short='r', default=3, min=0,
deprecated_group='kolla-build',
help='The number of times to retry while building'),
cfg.MultiOpt('regex', types.String(), positional=True,
help=('Build only images matching regex and its'
' dependencies')),
cfg.StrOpt('registry', deprecated_group='kolla-build',
help=('The docker registry host. The default registry host'
' is Docker Hub')),
cfg.StrOpt('type', short='t', default='binary',
choices=INSTALL_TYPE_CHOICES,
dest='install_type', deprecated_group='kolla-build',
help=('The method of the Openstack install. The valid'
' types are: {}'.format(
', '.join(INSTALL_TYPE_CHOICES)))),
cfg.IntOpt('threads', short='T', default=8, min=1,
deprecated_group='kolla-build',
help=('The number of threads to use while building.'
' (Note: setting to one will allow real time'
' logging.)')),
cfg.StrOpt('tag', default=get_kolla_version(),
deprecated_group='kolla-build',
help='The Docker tag'),
cfg.BoolOpt('template-only', default=False,
deprecated_group='kolla-build',
help=("Don't build images. Generate Dockerfile only")),
]
_kolla_base_opts = [
cfg.StrOpt('maintainer', deprecated_group='kolla-build',
default='Kolla Project (https://launchpad.net/kolla)',
help='The MAINTAINER field'),
cfg.ListOpt('rpm_setup_config', default=[DELOREAN, DELOREAN_DEPS],
deprecated_group='kolla-build',
help=('Comma separated list of .rpm or .repo file(s)'
'or URL(s) to install before building containers'))
]
conf.register_cli_opts(_kolla_cli_opts)
conf.register_opts(_kolla_profile_opts, group='profiles')
conf.register_opts(_kolla_base_opts)
conf(sys.argv[1:],
default_config_files=[find_config_file('kolla-build.conf')])
return conf
def main():
conf = get_conf()
conf = cfg.ConfigOpts()
common_config.parse(conf, sys.argv[1:], prog='kolla-build')
if conf.debug:
LOG.setLevel(logging.DEBUG)

0
kolla/common/__init__.py Normal file
View File

263
kolla/common/config.py Normal file
View File

@ -0,0 +1,263 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import itertools
from oslo_config import cfg
from oslo_config import types
from kolla import version
RDO_MIRROR = "http://trunk.rdoproject.org/centos7"
DELOREAN = "{}/current/delorean.repo".format(RDO_MIRROR)
DELOREAN_DEPS = "{}/delorean-deps.repo".format(RDO_MIRROR)
INSTALL_TYPE_CHOICES = ['binary', 'source', 'rdo', 'rhos']
_PROFILE_OPTS = [
cfg.ListOpt('infra',
default=['ceph', 'data', 'mariadb', 'haproxy',
'keepalived', 'kolla-ansible', 'memcached',
'mongodb', 'openvswitch', 'rabbitmq', 'rsyslog'],
help='Infra images'),
cfg.ListOpt('main',
default=['cinder', 'ceilometer', 'glance', 'heat',
'horizon', 'keystone', 'neutron', 'nova',
'swift'],
help='Main images'),
cfg.ListOpt('aux',
default=['aodh', 'designate', 'gnocchi', 'ironic',
'magnum', 'mistral', 'trove,' 'zaqar'],
help='Aux Images'),
cfg.ListOpt('default',
default=['data', 'kolla-ansible', 'glance', 'haproxy',
'heat', 'horizon', 'keepalived', 'keystone',
'memcached', 'mariadb', 'neutron', 'nova',
'openvswitch', 'rabbitmq', 'rsyslog'],
help='Default images'),
cfg.ListOpt('gate',
default=['ceph', 'cinder', 'data', 'dind', 'glance',
'haproxy', 'heat', 'horizon', 'keepalived',
'keystone', 'kolla-ansible', 'mariadb',
'memcached', 'neutron', 'nova', 'openvswitch',
'rabbitmq', 'rsyslog'],
help='Gate images')
]
_CLI_OPTS = [
cfg.StrOpt('base', short='b', default='centos',
deprecated_group='kolla-build',
help='The base distro to use when building'),
cfg.StrOpt('base_tag', default='latest',
deprecated_group='kolla-build',
help='The base distro image tag'),
cfg.BoolOpt('debug', short='d', default=False,
deprecated_group='kolla-build',
help='Turn on debugging log level'),
cfg.StrOpt('include-header', short='i',
deprecated_group='kolla-build',
help=('Path to custom file to be added at '
'beginning of base Dockerfile')),
cfg.StrOpt('include-footer', short='I',
deprecated_group='kolla-build',
help=('Path to custom file to be added at '
'end of Dockerfiles for final images')),
cfg.BoolOpt('keep', default=False,
deprecated_group='kolla-build',
help='Keep failed intermediate containers'),
cfg.StrOpt('namespace', short='n', default='kollaglue',
deprecated_group='kolla-build',
help='The Docker namespace name'),
cfg.BoolOpt('cache', default=True,
help='Use the Docker cache when building',
),
cfg.BoolOpt('no-cache', default=False,
help='Do not use the Docker cache when building',
deprecated_for_removal=True),
cfg.MultiOpt('profile', types.String(), short='p',
deprecated_group='kolla-build',
help=('Build a pre-defined set of images, see [profiles]'
' section in config. The default profiles are:'
' {}'.format(', '.join(
[opt.name for opt in _PROFILE_OPTS])
))),
cfg.BoolOpt('push', default=False,
deprecated_group='kolla-build',
help='Push images after building'),
cfg.IntOpt('push-threads', default=1, min=1,
deprecated_group='kolla-build',
help=('The number of threads to user while pushing'
' Images. Note: Docker can not handle threading'
' push properly.')),
cfg.IntOpt('retries', short='r', default=3, min=0,
deprecated_group='kolla-build',
help='The number of times to retry while building'),
cfg.MultiOpt('regex', types.String(), positional=True,
help=('Build only images matching regex and its'
' dependencies')),
cfg.StrOpt('registry', deprecated_group='kolla-build',
help=('The docker registry host. The default registry host'
' is Docker Hub')),
cfg.StrOpt('type', short='t', default='binary',
choices=INSTALL_TYPE_CHOICES,
dest='install_type', deprecated_group='kolla-build',
help=('The method of the Openstack install. The valid'
' types are: {}'.format(
', '.join(INSTALL_TYPE_CHOICES)))),
cfg.IntOpt('threads', short='T', default=8, min=1,
deprecated_group='kolla-build',
help=('The number of threads to use while building.'
' (Note: setting to one will allow real time'
' logging.)')),
cfg.StrOpt('tag', default=version.version_info.cached_version_string(),
deprecated_group='kolla-build',
help='The Docker tag'),
cfg.BoolOpt('template-only', default=False,
deprecated_group='kolla-build',
help=("Don't build images. Generate Dockerfile only")),
]
_BASE_OPTS = [
cfg.StrOpt('maintainer', deprecated_group='kolla-build',
default='Kolla Project (https://launchpad.net/kolla)',
help='The MAINTAINER field'),
cfg.ListOpt('rpm_setup_config', default=[DELOREAN, DELOREAN_DEPS],
deprecated_group='kolla-build',
help=('Comma separated list of .rpm or .repo file(s)'
'or URL(s) to install before building containers'))
]
SOURCES = {
'openstack-base': {
'type': 'url',
'location': ('https://github.com/openstack/requirements/tarball/'
'master')},
'aodh-base': {
'type': 'url',
'location': 'http://tarballs.openstack.org/aodh/aodh-master.tar.gz'},
'ceilometer-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/ceilometer/'
'ceilometer-master.tar.gz')},
'cinder-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/cinder/'
'cinder-master.tar.gz')},
'designate-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/designate/'
'designate-master.tar.gz')},
'glance-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/glance/'
'glance-master.tar.gz')},
'gnocchi-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/gnocchi/'
'gnocchi-master.tar.gz')},
'heat-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/heat/heat-master.tar.gz')},
'horizon': {
'type': 'url',
'location': ('http://tarballs.openstack.org/horizon/'
'horizon-master.tar.gz')},
'ironic-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/ironic/'
'ironic-master.tar.gz')},
'keystone': {
'type': 'url',
'location': ('http://tarballs.openstack.org/keystone/'
'keystone-master.tar.gz')},
'magnum-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/magnum/'
'magnum-master.tar.gz')},
'mistral-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/mistral/'
'mistral-master.tar.gz')},
'murano-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/murano/'
'murano-master.tar.gz')},
'neutron-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/neutron/'
'neutron-master.tar.gz')},
'nova-base': {
'type': 'url',
'location': 'http://tarballs.openstack.org/nova/nova-master.tar.gz'},
'nova-spicehtml5proxy': {
'type': 'url',
'location': ('http://github.com/SPICE/spice-html5/tarball/'
'spice-html5-0.1.6')},
'nova-novncproxy': {
'type': 'url',
'location': 'http://github.com/kanaka/noVNC/tarball/v0.5.1'},
'swift-base': {
'type': 'url',
'location': ('http://tarballs.openstack.org/swift/'
'swift-master.tar.gz')},
'tempest': {
'type': 'url',
'location': ('http://tarballs.openstack.org/tempest/'
'tempest-master.tar.gz')},
'zaqar': {
'type': 'url',
'location': ('http://tarballs.openstack.org/zaqar/'
'zaqar-master.tar.gz')}
}
def _get_source_opt(type_, location, reference=None):
return [cfg.StrOpt('type', choices=['git', 'url'],
default=type_,
help='Source location type'),
cfg.StrOpt('location', default=location,
help='The location for source install'),
cfg.StrOpt('reference', default=reference,
help=('Git reference to pull, commit sha, tag'
'or branch name'))]
def gen_source_opts():
for name, params in SOURCES.iteritems():
type_ = params['type']
location = params['location']
reference = params.get('reference')
yield name, _get_source_opt(type_, location, reference)
def list_opts():
return itertools.chain([(None, _CLI_OPTS),
(None, _BASE_OPTS),
('profiles', _PROFILE_OPTS)],
gen_source_opts(),
)
def parse(conf, args, usage=None, prog=None,
default_config_files=None):
conf.register_cli_opts(_CLI_OPTS)
conf.register_opts(_BASE_OPTS)
conf.register_opts(_PROFILE_OPTS, group='profiles')
conf(args=args,
project='kolla',
usage=usage,
prog=prog,
version=version.version_info.cached_version_string(),
default_config_files=default_config_files)

17
kolla/opts.py Normal file
View File

@ -0,0 +1,17 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kolla.common import config
def list_opts():
return config.list_opts()

15
kolla/version.py Normal file
View File

@ -0,0 +1,15 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pbr.version
version_info = pbr.version.VersionInfo('kolla')

View File

@ -37,6 +37,8 @@ scripts =
[entry_points]
console_scripts =
kolla-build = kolla.cmd.build:main
oslo.config.opts =
kolla = kolla.opts:list_opts
[global]
setup-hooks =

View File

@ -4,6 +4,7 @@ skipsdist = True
envlist = functional,pep8
[testenv]
usedevelop=True
install_command = pip install -U {opts} {packages}
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
@ -102,6 +103,11 @@ commands =
bash -c "if [ ! -d .testrepository ]; then testr init; fi"
sudo -g docker testr run test_build.BuildTestUbuntuSource
[testenv:genconfig]
whitelist_externals = which
commands=
oslo-config-generator --config-file etc/oslo-config-generator/kolla-build.conf
[flake8]
show-source = True
exclude=.eggs,.git,.tox,doc