x/packetary
Code Issues Proposed changes

Merge "Introduced fuel-mirror"

Browse Source
This commit is contained in:
Jenkins 2015-11-26 14:46:55 +00:00 committed by Gerrit Code Review
commit a31465e82e
43 changed files with 2043 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
contrib/fuel_mirror/MANIFEST.in Normal file
View File

@ -0,0 +1,8 @@
include AUTHORS
include ChangeLog
recursive-include etc *
exclude .gitignore
exclude .gitreview
global-exclude *.pyc

16
contrib/fuel_mirror/README.rst Normal file
View File

@ -0,0 +1,16 @@
===========
fuel_mirror
===========
The fuel-mirror is utility, that allows to create local repositories
with packages are required for the OpenStack deployment.
* Free software: Apache license
* Documentation: http://docs.openstack.org/developer/fuel-mirror
* Source: http://git.openstack.org/cgit/openstack/fuel-mirror/
* Bugs: http://bugs.launchpad.net/fuel
Features
--------
* TODO

2
contrib/fuel_mirror/babel.cfg Normal file
View File

@ -0,0 +1,2 @@
[python: **.py]

58
contrib/fuel_mirror/data/centos.yaml Normal file
View File

@ -0,0 +1,58 @@
centos_baseurl: &centos_baseurl
mos_baseurl: &mos_baseurl
fuel_release_match:
version: $openstack_version
operating_system: CentOS
repos:
- &centos
name: "centos"
uri: "http://mirror.centos.org/centos/6/os/x86_64"
type: "rpm"
priority: null
- &centos_updates
name: "centos-updates"
uri: "http://mirror.centos.org/centos/6/updates/x86_64"
type: "rpm"
priority: null
- &mos
name: "mos"
uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/os/x86_64"
type: "rpm"
priority: null
- &mos_updates
name: "mos-updates"
uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/updates/x86_64"
type: "rpm"
priority: null
- &mos_security
name: "mos-security"
uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/security/x86_64"
type: "rpm"
priority: null
- &mos_holdback
name: "mos-holdback"
uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/holdback/x86_64"
type: "rpm"
priority: null
groups:
mos:
- *mos
- *mos_updates
- *mos_security
- *mos_holdback
centos:
- *centos
- *centos_updates
inheritance:
centos: mos

140
contrib/fuel_mirror/data/ubuntu.yaml Normal file
View File

@ -0,0 +1,140 @@
# GLOBAL variables
ubuntu_baseurl: &ubuntu_baseurl http://archive.ubuntu.com/ubuntu
mos_baseurl: &mos_baseurl http://mirror.fuel-infra.org/mos-repos/ubuntu/$mos_version
fuel_release_match:
version: $openstack_version
operating_system: Ubuntu
repos:
- &ubuntu
name: "ubuntu"
uri: *ubuntu_baseurl
suite: "trusty"
section: "main multiverse restricted universe"
type: "deb"
priority: null
- &ubuntu_updates
name: "ubuntu-updates"
uri: *ubuntu_baseurl
suite: "trusty-updates"
section: "main multiverse restricted universe"
type: "deb"
priority: null
- &ubuntu_security
name: "ubuntu-security"
uri: *ubuntu_baseurl
suite: "trusty-security"
section: "main multiverse restricted universe"
type: "deb"
priority: null
- &mos
name: "mos"
uri: *mos_baseurl
suite: "mos$mos_version"
section: "main restricted"
type: "deb"
priority: 1000
- &mos_updates
name: "mos-updates"
uri: *mos_baseurl
suite: "mos$mos_version-updates"
section: "main restricted"
type: "deb"
priority: 1000
- &mos_security
name: "mos-security"
uri: *mos_baseurl
suite: "mos$mos_version-security"
section: "main restricted"
type: "deb"
priority: 1000
- &mos_holdback
name: "mos-holdback"
uri: *mos_baseurl
suite: "mos$mos_version"
section: "main restricted"
type: "deb"
priority: 1000
packages: &packages
- "acpi-support"
- "anacron"
- "aptitude"
- "atop"
- "bash-completion"
- "bc"
- "build-essential"
- "cloud-init"
- "conntrackd"
- "cpu-checker"
- "cpufrequtils"
- "debconf-utils"
- "devscripts"
- "fping"
- "git"
- "grub-pc"
- "htop"
- "ifenslave"
- "iperf"
- "iptables-persistent"
- "irqbalance"
- "language-pack-en"
- "linux-firmware-nonfree"
- "linux-headers-generic-lts-trusty"
- "linux-image-generic-lts-trusty"
- "livecd-rootfs"
- "memcached"
- "monit"
- "nginx"
- "ntp"
- "openssh-server"
- "percona-toolkit"
- "percona-xtrabackup"
- "pm-utils"
- "python-lesscpy"
- "python-pip"
- "puppet"
- "rsyslog-gnutls"
- "rsyslog-relp"
- "screen"
- "swift-plugin-s3"
- "sysfsutils"
- "sysstat"
- "telnet"
- "tmux"
- "traceroute"
- "ubuntu-standard"
- "vim"
- "virt-what"
- "xinetd"
groups:
mos:
- *mos
- *mos_updates
- *mos_security
- *mos_holdback
ubuntu:
- *ubuntu
- *ubuntu_updates
- *ubuntu_security
inheritance:
ubuntu: mos
osnames:
mos: ubuntu
requirements:
ubuntu: *packages

75
contrib/fuel_mirror/doc/source/conf.py Normal file
View File

@ -0,0 +1,75 @@
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
sys.path.insert(0, os.path.abspath('../..'))
# -- General configuration ----------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.autodoc',
#'sphinx.ext.intersphinx',
'oslosphinx'
]
# autodoc generation is a bit aggressive and a nuisance when doing heavy
# text edit cycles.
# execute "export SPHINX_DEBUG=1" in your terminal to disable
# The suffix of source filenames.
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'fuel_mirror'
copyright = u'2015, Mirantis, Inc'
# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = True
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# -- Options for HTML output --------------------------------------------------
# The theme to use for HTML and HTML Help pages. Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
# html_theme_path = ["."]
# html_theme = '_theme'
# html_static_path = ['static']
# Output file base name for HTML help builder.
htmlhelp_basename = '%sdoc' % project
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass
# [howto/manual]).
latex_documents = [
('index',
'%s.tex' % project,
u'%s Documentation' % project,
u'OpenStack Foundation', 'manual'),
]
# Example configuration for intersphinx: refer to the Python standard library.
#intersphinx_mapping = {'http://docs.python.org/': None}

4
contrib/fuel_mirror/doc/source/contributing.rst Normal file
View File

@ -0,0 +1,4 @@
============
Contributing
============
.. include:: ../../../../CONTRIBUTING.rst

25
contrib/fuel_mirror/doc/source/index.rst Normal file
View File

@ -0,0 +1,25 @@
.. fuel_mirror documentation master file, created by
sphinx-quickstart on Tue Jul 9 22:26:36 2013.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to fuel_mirror's documentation!
========================================================
Contents:
.. toctree::
:maxdepth: 2
readme
installation
usage
contributing
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

12
contrib/fuel_mirror/doc/source/installation.rst Normal file
View File

@ -0,0 +1,12 @@
============
Installation
============
At the command line::
$ pip install fuel_mirror
Or, if you have virtualenvwrapper installed::
$ mkvirtualenv fuel_mirror
$ pip install fuel_mirror

1
contrib/fuel_mirror/doc/source/readme.rst Normal file
View File

@ -0,0 +1 @@
.. include:: ../../README.rst

7
contrib/fuel_mirror/doc/source/usage.rst Normal file
View File

@ -0,0 +1,7 @@
========
Usage
========
To use fuel_mirror in a project::
import fuel_createmirror

11
contrib/fuel_mirror/etc/config.yaml Normal file
View File

@ -0,0 +1,11 @@
threads_num: 10
ignore_errors_num: 2
retries_num: 3
target_dir: "/var/www/nailgun/mirrors"
pattern_dir: "/usr/share/fuel-mirror"
# uncomment if need
# http_proxy: null
# https_proxy: null
# fuel_server: 10.20.0.2
# base_url: http://10.20.0.2:8080

22
contrib/fuel_mirror/fuel_mirror/__init__.py Normal file
View File

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import pbr.version
__version__ = pbr.version.VersionInfo(
'fuel_mirror').version_string()

143
contrib/fuel_mirror/fuel_mirror/app.py Normal file
View File

@ -0,0 +1,143 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from cliff import app
from cliff.commandmanager import CommandManager
import yaml
import fuel_mirror
from fuel_mirror.common import accessors
from fuel_mirror.common import utils
_FUEL_DEFAULT_HTTP_PORT = 8080
class Application(app.App):
"""Main cliff application class.
Performs initialization of the command manager and
configuration of basic engines.
"""
config = None
fuel = None
repo_manager_accessor = None
sources = None
versions = None
def build_option_parser(self, description, version, argparse_kwargs=None):
"""Specifies common cmdline arguments."""
p_inst = super(Application, self)
parser = p_inst.build_option_parser(description=description,
version=version,
argparse_kwargs=argparse_kwargs)
parser.add_argument(
"--config",
default="/etc/fuel-mirror/config.yaml",
metavar="PATH",
help="Path to config file."
)
parser.add_argument(
"-S", "--fuel-server",
metavar="FUEL-SERVER",
help="The public address of Fuel Master."
)
parser.add_argument(
"--fuel-user",
help="Fuel Master admin login."
" Alternatively, use env var KEYSTONE_USER)."
)
parser.add_argument(
"--fuel-password",
help="Fuel Master admin password."
" Alternatively, use env var KEYSTONE_PASSWORD)."
)
return parser
def initialize_app(self, argv):
"""Initialises common options."""
with open(self.options.config, "r") as stream:
self.config = yaml.load(stream)
fuel_default = utils.get_fuel_settings()
fuel_server = utils.first(
self.options.fuel_server,
self.config.get("fuel_server"),
fuel_default["server"]
)
fuel_user = utils.first(
self.options.fuel_user,
fuel_default["user"]
)
fuel_password = utils.first(
self.options.fuel_password,
fuel_default["password"]
)
self.config.setdefault(
"base_url", "http://{0}:{1}".format(
fuel_server.split(":", 1)[0],
_FUEL_DEFAULT_HTTP_PORT
)
)
self.fuel = accessors.get_fuel_api_accessor(
fuel_server,
fuel_user,
fuel_password
)
fuel_ver = self.fuel.FuelVersion.get_all_data()
self.config.setdefault(
'mos_version', fuel_ver['release']
)
self.config.setdefault(
'openstack_version', fuel_ver['openstack_version']
)
self.repo_manager_accessor = accessors.get_packetary_accessor(
threads_num=int(self.config.get('threads_num', 0)),
retries_num=int(self.config.get('retries_num', 0)),
ignore_errors_num=int(self.config.get('ignore_errors_num', 0)),
http_proxy=self.config.get('http_proxy'),
https_proxy=self.config.get('https_proxy'),
)
def main(argv=None):
"""Entry point."""
return Application(
description="The utility to create local mirrors.",
version=fuel_mirror.__version__,
command_manager=CommandManager("fuel_mirror", convert_underscores=True)
).run(argv)
def debug(name, cmd_class, argv=None):
"""Helps to debug command."""
import sys
if argv is None:
argv = sys.argv[1:]
argv = [name] + argv + ["-v", "-v", "--debug"]
cmd_mgr = CommandManager("test_fuel_mirror", convert_underscores=True)
cmd_mgr.add_command(name, cmd_class)
return Application(
description="The fuel mirror utility test.",
version="0.0.1",
command_manager=cmd_mgr
).run(argv)

0
contrib/fuel_mirror/fuel_mirror/commands/__init__.py Normal file
View File

158
contrib/fuel_mirror/fuel_mirror/commands/apply.py Normal file
View File

@ -0,0 +1,158 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
from packetary.library.utils import localize_repo_url
from fuel_mirror.commands.base import BaseCommand
from fuel_mirror.common.utils import is_subdict
from fuel_mirror.common.utils import lists_merge
class ApplyCommand(BaseCommand):
"""Applies local mirrors for Fuel-environments."""
def get_parser(self, prog_name):
parser = super(ApplyCommand, self).get_parser(prog_name)
parser.add_argument(
"--default",
dest="set_default",
action="store_true",
default=False,
help="Set as default repository."
)
parser.add_argument(
"-e", "--env",
dest="env", nargs="+",
help="Fuel environment ID to update, "
"by default applies for all environments."
)
return parser
def take_action(self, parsed_args):
data = self.load_data(parsed_args)
base_url = self.app.config["base_url"]
localized_repos = []
for _, repos in self.get_groups(parsed_args, data):
for repo_data in repos:
new_data = repo_data.copy()
new_data['uri'] = localize_repo_url(
base_url, repo_data['uri']
)
localized_repos.append(new_data)
release_match = data["fuel_release_match"]
self.update_clusters(parsed_args.env, localized_repos, release_match)
if parsed_args.set_default:
self.update_default_repos(localized_repos, release_match)
self.app.stdout.write(
"Operations have been completed successfully.\n"
)
def update_clusters(self, ids, repositories, release_match):
"""Applies repositories for existing clusters.
:param ids: the cluster ids.
:param repositories: the meta information of repositories
:param release_match: The pattern to check Fuel Release
"""
self.app.stdout.write("Updating the Cluster repositories...\n")
if ids:
clusters = self.app.fuel.Environment.get_by_ids(ids)
else:
clusters = self.app.fuel.Environment.get_all()
for cluster in clusters:
releases = six.moves.filter(
lambda x: is_subdict(release_match, x.data),
self.app.fuel.Release.get_by_ids([cluster.data["release_id"]])
)
if next(releases, None) is None:
continue
modified = self._update_repository_settings(
cluster.get_settings_data(),
repositories
)
if modified:
self.app.LOG.info(
"Try to update the Cluster '%s'",
cluster.data['name']
)
self.app.LOG.debug(
"The modified cluster attributes: %s",
modified
)
cluster.set_settings_data(modified)
def update_default_repos(self, repositories, release_match):
"""Applies repositories for existing default settings.
:param repositories: the meta information of repositories
:param release_match: The pattern to check Fuel Release
"""
self.app.stdout.write("Updating the default repositories...\n")
releases = six.moves.filter(
lambda x: is_subdict(release_match, x.data),
self.app.fuel.Release.get_all()
)
for release in releases:
if self._update_repository_settings(
release.data["attributes_metadata"], repositories
):
self.app.LOG.info(
"Try to update the Release '%s'",
release.data['name']
)
self.app.LOG.debug(
"The modified release attributes: %s",
release.data
)
# TODO(need to add method for release object)
release.connection.put_request(
release.instance_api_path.format(release.id),
release.data
)
def _update_repository_settings(self, settings, repositories):
"""Updates repository settings.
:param settings: the target settings
:param repositories: the meta of repositories
"""
editable = settings["editable"]
if 'repo_setup' not in editable:
self.app.LOG.info('Attributes is read-only.')
return
repos_attr = editable["repo_setup"]["repos"]
lists_merge(repos_attr['value'], repositories, "name")
return {"editable": {"repo_setup": {"repos": repos_attr}}}
def debug(argv=None):
"""Helper for debugging Apply command."""
from fuel_mirror.app import debug
debug("apply", ApplyCommand, argv)
if __name__ == "__main__":
debug()

97
contrib/fuel_mirror/fuel_mirror/commands/base.py Normal file
View File

@ -0,0 +1,97 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os.path
from string import Template
from cliff import command
import yaml
class BaseCommand(command.Command):
"""The Base command for fuel-mirror."""
REPO_ARCH = "x86_64"
@property
def stdout(self):
"""Shortcut for self.app.stdout."""
return self.app.stdout
def get_parser(self, prog_name):
"""Specifies common options."""
parser = super(BaseCommand, self).get_parser(prog_name)
input_group = parser.add_mutually_exclusive_group(required=True)
input_group.add_argument(
'-I', '--input-file',
metavar='PATH',
help='The path to file with input data.')
input_group.add_argument(
'-P', '--pattern',
metavar='NAME',
help='The builtin input file name.'
)
parser.add_argument(
"-G", "--group",
dest="groups",
required=True,
nargs='+',
help="The name of repository groups."
)
return parser
def resolve_input_pattern(self, pattern):
"""Gets the full path to input file by pattern.
:param pattern: the config file name without ext
:return: the full path
"""
return os.path.join(
self.app.config['pattern_dir'], pattern + ".yaml"
)
def load_data(self, parsed_args):
"""Load the input data.
:param parsed_args: the command-line arguments
:return: the input data
"""
if parsed_args.pattern:
input_file = self.resolve_input_pattern(parsed_args.pattern)
else:
input_file = parsed_args.input_file
# TODO(add input data validation scheme)
with open(input_file, "r") as fd:
return yaml.load(Template(fd.read()).safe_substitute(
mos_version=self.app.config["mos_version"],
openstack_version=self.app.config["openstack_version"],
))
@classmethod
def get_groups(cls, parsed_args, data):
"""Gets repository groups from input data.
:param parsed_args: the command-line arguments
:param data: the input data
:return: the sequence of pairs (group_name, repositories)
"""
all_groups = data['groups']
return (
(x, all_groups[x]) for x in parsed_args.groups if x in all_groups
)

75
contrib/fuel_mirror/fuel_mirror/commands/create.py Normal file
View File

@ -0,0 +1,75 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from fuel_mirror.commands.base import BaseCommand
from fuel_mirror.common.url_builder import get_url_builder
class CreateCommand(BaseCommand):
"""Creates a new local mirrors."""
def take_action(self, parsed_args):
"""See the Command.take_action."""
data = self.load_data(parsed_args)
repos_reqs = data.get('requirements', {})
inheritance = data.get('inheritance', {})
target_dir = self.app.config["target_dir"]
total_stats = None
for group_name, repos in self.get_groups(parsed_args, data):
url_builder = get_url_builder(repos[0]["type"])
repo_manager = self.app.repo_manager_accessor(
repos[0]["type"], self.REPO_ARCH
)
if group_name in inheritance:
child_group = inheritance[group_name]
dependencies = [
url_builder.get_repo_url(x)
for x in data['groups'][child_group]
]
else:
dependencies = None
stat = repo_manager.clone_repositories(
[url_builder.get_repo_url(x) for x in repos],
target_dir,
dependencies,
repos_reqs.get(group_name)
)
if total_stats is None:
total_stats = stat
else:
total_stats += stat
if total_stats is not None:
self.stdout.write(
"Packages processed: {0.copied}/{0.total}\n"
.format(total_stats)
)
else:
self.stdout.write(
"No packages.\n"
)
def debug(argv=None):
"""Helper for debugging Create command."""
from fuel_mirror.app import debug
debug("create", CreateCommand, argv)
if __name__ == "__main__":
debug()

0
contrib/fuel_mirror/fuel_mirror/common/__init__.py Normal file
View File

54
contrib/fuel_mirror/fuel_mirror/common/accessors.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import functools
import os
def get_packetary_accessor(**kwargs):
"""Gets the configured repository manager.
:param kwargs: The packetary configuration parameters.
"""
import packetary
return functools.partial(
packetary.RepositoryApi.create,
packetary.Context(packetary.Configuration(**kwargs))
)
def get_fuel_api_accessor(address=None, user=None, password=None):
"""Gets the fuel client api accessor.
:param address: The address of Fuel Master node.
:param user: The username to access to the Fuel Master node.
:param user: The password to access to the Fuel Master node.
"""
if address:
host_and_port = address.split(":")
os.environ["SERVER_ADDRESS"] = host_and_port[0]
if len(host_and_port) > 1:
os.environ["LISTEN_PORT"] = host_and_port[1]
if user is not None:
os.environ["KEYSTONE_USER"] = user
if password is not None:
os.environ["KEYSTONE_PASS"] = password
# import fuelclient.ClientAPI after configuring
# environment variables
from fuelclient import objects
return objects

68
contrib/fuel_mirror/fuel_mirror/common/url_builder.py Normal file
View File

@ -0,0 +1,68 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def get_url_builder(repotype):
"""Gets the instance of RepoUrlBuilder.
:param repotype: the type of repository: rpm|deb
:return: the RepoBuilder implementation
"""
return {
"deb": AptRepoUrlBuilder,
"rpm": YumRepoUrlBuilder
}[repotype]
class RepoUrlBuilder(object):
REPO_FOLDER = "mirror"
@classmethod
def get_repo_url(cls, repo_data):
"""Gets the url with replaced variable holders.
:param repo_data: the repositories`s meta data
:return: the full repository`s url
"""
class AptRepoUrlBuilder(RepoUrlBuilder):
"""URL builder for apt-repository(es)."""
@classmethod
def get_repo_url(cls, repo_data):
return " ".join(
repo_data[x] for x in ("uri", "suite", "section")
)
class YumRepoUrlBuilder(RepoUrlBuilder):
"""URL builder for Yum repository(es)."""
@classmethod
def split_url(cls, url, maxsplit=2):
"""Splits url to baseurl, reponame adn architecture.
:param url: the repository`s URL
:param maxsplit: the number of expected components
:return the components of url
"""
# TODO(need generic url building algorithm)
# there is used assumption that url has following format
# $baseurl/$reponame/$repoarch
return url.rstrip("/").rsplit("/", maxsplit)
@classmethod
def get_repo_url(cls, repo_data):
return cls.split_url(repo_data["uri"], 1)[0]

96
contrib/fuel_mirror/fuel_mirror/common/utils.py Normal file
View File

@ -0,0 +1,96 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
import yaml
def lists_merge(main, patch, key):
"""Merges the list of dicts with same keys.
>>> lists_merge([{"a": 1, "c": 2}], [{"a": 1, "c": 3}], key="a")
[{'a': 1, 'c': 3}]
:param main: the main list
:type main: list
:param patch: the list of additional elements
:type patch: list
:param key: the key for compare
"""
main_idx = dict(
(x[key], i) for i, x in enumerate(main)
)
patch_idx = dict(
(x[key], i) for i, x in enumerate(patch)
)
for k in sorted(patch_idx):
if k in main_idx:
main[main_idx[k]].update(patch[patch_idx[k]])
else:
main.append(patch[patch_idx[k]])
return main
def is_subdict(dict1, dict2):
"""Checks that dict1 is subdict of dict2.
>>> is_subdict({"a": 1}, {'a': 1, 'b': 1})
True
:param dict1: the candidate
:param dict2: the super dict
:return: True if all keys from dict1 are present
and has same value in dict2 otherwise False
"""
for k, v in six.iteritems(dict1):
if k not in dict2 or dict2[k] != v:
return False
return True
def first(*args):
"""Get first not empty value.
>>> first(0, 1) == next(iter(filter(None, [0, 1])))
True
:param args: the list of arguments
:return first value that bool(v) is True, None if not found.
"""
for arg in args:
if arg:
return arg
def get_fuel_settings():
"""Gets the fuel settings from astute container, if it is available."""
_DEFAULT_SETTINGS = {
"server": "10.20.0.2",
"user": None,
"password": None,
}
try:
with open("/etc/fuel/astute.yaml", "r") as fd:
settings = yaml.load(fd)
return {
"server": settings.get("ADMIN_NETWORK", {}).get("ipaddress"),
"user": settings.get("FUEL_ACCESS", {}).get("user"),
"password": settings.get("FUEL_ACCESS", {}).get("password")
}
except (OSError, IOError):
pass
return _DEFAULT_SETTINGS

0
contrib/fuel_mirror/fuel_mirror/tests/__init__.py Normal file
View File

24
contrib/fuel_mirror/fuel_mirror/tests/base.py Normal file
View File

@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
try:
import unittest2 as unittest
except ImportError:
import unittest
class TestCase(unittest.TestCase):
"""Test case base class for all unit tests."""

22
contrib/fuel_mirror/fuel_mirror/tests/data/test_centos.yaml Normal file
View File

@ -0,0 +1,22 @@
fuel_release_match:
operating_system: CentOS
inheritance:
centos: mos
groups:
mos:
- name: "mos"
type: "rpm"
uri: "http://localhost/mos$mos_version/x86_64"
priority: 10
centos:
- name: "centos"
type: "rpm"
uri: "http://localhost/centos/os/x86_64"
priority: 5
requirements:
centos:
- "package_rpm"

6
contrib/fuel_mirror/fuel_mirror/tests/data/test_config.yaml Normal file
View File

@ -0,0 +1,6 @@
threads_num: 1
ignore_errors_num: 2
retries_num: 3
http_proxy: "http://localhost"
https_proxy: "https://localhost"
target_dir: "/var/www/"

26
contrib/fuel_mirror/fuel_mirror/tests/data/test_ubuntu.yaml Normal file
View File

@ -0,0 +1,26 @@
fuel_release_match:
operating_system: Ubuntu
inheritance:
ubuntu: mos
groups:
mos:
- name: "mos"
type: "deb"
uri: "http://localhost/mos"
suite: "mos$mos_version"
section: "main restricted"
priority: 1000
ubuntu:
- name: "ubuntu"
type: "deb"
uri: "http://localhost/ubuntu"
suite: "trusty"
section: "main multiverse restricted universe"
priority: 500
requirements:
ubuntu:
- "package_deb"

90
contrib/fuel_mirror/fuel_mirror/tests/test_accessors.py Normal file
View File

@ -0,0 +1,90 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from fuel_mirror.common import accessors
from fuel_mirror.tests import base
class TestAccessors(base.TestCase):
def test_get_packetary_accessor(self):
packetary = mock.MagicMock()
with mock.patch.dict("sys.modules", packetary=packetary):
accessor = accessors.get_packetary_accessor(
http_proxy="http://localhost",
https_proxy="https://localhost",
retries_num=1,
threads_num=2,
ignore_errors_num=3
)
accessor("deb")
accessor("yum")
packetary.Configuration.assert_called_once_with(
http_proxy="http://localhost",
https_proxy="https://localhost",
retries_num=1,
threads_num=2,
ignore_errors_num=3
)
packetary.Context.assert_called_once_with(
packetary.Configuration()
)
self.assertEqual(2, packetary.RepositoryApi.create.call_count)
packetary.RepositoryApi.create.assert_any_call(
packetary.Context(), "deb"
)
packetary.RepositoryApi.create.assert_any_call(
packetary.Context(), "yum"
)
@mock.patch("fuel_mirror.common.accessors.os")
def test_get_fuel_api_accessor(self, os):
fuelclient = mock.MagicMock()
patch = {
"fuelclient": fuelclient,
"fuelclient.objects": fuelclient.objects
}
with mock.patch.dict("sys.modules", patch):
accessor = accessors.get_fuel_api_accessor(
"localhost:8080", "guest", "123"
)
accessor.Environment.get_all()
os.environ.__setitem__.asseert_any_call(
"SERVER_ADDRESS", "localhost"
)
os.environ.__setitem__.asseert_any_call(
"LISTEN_PORT", "8080"
)
os.environ.__setitem__.asseert_any_call(
"KEYSTONE_USER", "guest"
)
os.environ.__setitem__.asseert_any_call(
"KEYSTONE_PASS", "123"
)
fuelclient.objects.Environment.get_all.assert_called_once_with()
@mock.patch("fuel_mirror.common.accessors.os")
def test_get_fuel_api_accessor_with_default_parameters(self, os):
fuelclient = mock.MagicMock()
patch = {
"fuelclient": fuelclient,
"fuelclient.objects": fuelclient.objects
}
with mock.patch.dict("sys.modules", patch):
accessors.get_fuel_api_accessor()
os.environ.__setitem__.assert_not_called()

317
contrib/fuel_mirror/fuel_mirror/tests/test_cli_commands.py Normal file
View File

@ -0,0 +1,317 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
import os.path
import subprocess
# The cmd2 does not work with python3.5
# because it tries to get access to the property mswindows,
# that was removed in 3.5
subprocess.mswindows = False
from fuel_mirror.commands import apply
from fuel_mirror.commands import create
from fuel_mirror.tests import base
CONFIG_PATH = os.path.join(
os.path.dirname(__file__), "data", "test_config.yaml"
)
UBUNTU_PATH = os.path.join(
os.path.dirname(__file__), "data", "test_ubuntu.yaml"
)
CENTOS_PATH = os.path.join(
os.path.dirname(__file__), "data", "test_centos.yaml"
)
@mock.patch.multiple(
"fuel_mirror.app",
accessors=mock.DEFAULT,
)
class TestCliCommands(base.TestCase):
common_argv = [
"--config", CONFIG_PATH,
"--fuel-server=10.25.0.10",
"--fuel-user=test",
"--fuel-password=test1"
]
def start_cmd(self, cmd, argv, data_file):
cmd.debug(
argv + self.common_argv + ["--input-file", data_file]
)
def _setup_fuel_versions(self, fuel_mock):
fuel_mock.FuelVersion.get_all_data.return_value = {
"release": "1",
"openstack_version": "2"
}
def _create_fuel_release(self, fuel_mock, osname):
release = mock.MagicMock(data={
"name": "test release",
"operating_system": osname,
"attributes_metadata": {
"editable": {"repo_setup": {"repos": {"value": []}}}
}
})
fuel_mock.Release.get_by_ids.return_value = [release]
fuel_mock.Release.get_all.return_value = [release]
return release
def _create_fuel_env(self, fuel_mock):
env = mock.MagicMock(data={
"name": "test",
"release_id": 1
})
env.get_settings_data.return_value = {
"editable": {"repo_setup": {"repos": {"value": []}}}
}
fuel_mock.Environment.get_by_ids.return_value = [env]
fuel_mock.Environment.get_all.return_value = [env]
return env
def test_create_mos_ubuntu(self, accessors):
self._setup_fuel_versions(accessors.get_fuel_api_accessor())
packetary = accessors.get_packetary_accessor()
self.start_cmd(create, ["--group", "mos"], UBUNTU_PATH)
accessors.get_packetary_accessor.assert_called_with(
threads_num=1,
ignore_errors_num=2,
retries_num=3,
http_proxy="http://localhost",
https_proxy="https://localhost",
)
packetary.assert_called_with("deb", "x86_64")
api = packetary()
api.clone_repositories.assert_called_once_with(
['http://localhost/mos mos1 main restricted'],
'/var/www/',
None, None
)
def test_create_partial_ubuntu(self, accessors):
self._setup_fuel_versions(accessors.get_fuel_api_accessor())
packetary = accessors.get_packetary_accessor()
self.start_cmd(create, ["--group", "ubuntu"], UBUNTU_PATH)
accessors.get_packetary_accessor.assert_called_with(
threads_num=1,
ignore_errors_num=2,
retries_num=3,
http_proxy="http://localhost",
https_proxy="https://localhost",
)
packetary.assert_called_with("deb", "x86_64")
api = packetary()
api.clone_repositories.assert_called_once_with(
['http://localhost/ubuntu trusty '
'main multiverse restricted universe'],
'/var/www/',
['http://localhost/mos mos1 main restricted'],
['package_deb']
)
def test_create_mos_centos(self, accessors):
self._setup_fuel_versions(accessors.get_fuel_api_accessor())
packetary = accessors.get_packetary_accessor()
self.start_cmd(create, ["--group", "mos"], CENTOS_PATH)
accessors.get_packetary_accessor.assert_called_with(
threads_num=1,
ignore_errors_num=2,
retries_num=3,
http_proxy="http://localhost",
https_proxy="https://localhost",
)
packetary.assert_called_with("rpm", "x86_64")
api = packetary()
api.clone_repositories.assert_called_once_with(
['http://localhost/mos1'],
'/var/www/',
None, None
)
def test_create_partial_centos(self, accessors):
self._setup_fuel_versions(accessors.get_fuel_api_accessor())
packetary = accessors.get_packetary_accessor()
self.start_cmd(create, ["--group", "centos"], CENTOS_PATH)
accessors.get_packetary_accessor.assert_called_with(
threads_num=1,
ignore_errors_num=2,
retries_num=3,
http_proxy="http://localhost",
https_proxy="https://localhost",
)
packetary.assert_called_with("rpm", "x86_64")
api = packetary()
api.clone_repositories.assert_called_once_with(
['http://localhost/centos/os'],
'/var/www/',
['http://localhost/mos1'],
["package_rpm"]
)
def test_apply_for_ubuntu_based_env(self, accessors):
fuel = accessors.get_fuel_api_accessor()
self._setup_fuel_versions(fuel)
env = self._create_fuel_env(fuel)
self._create_fuel_release(fuel, "Ubuntu")
self.start_cmd(
apply, ['--group', 'mos', 'ubuntu', '--env', '1'],
UBUNTU_PATH
)
accessors.get_fuel_api_accessor.assert_called_with(
"10.25.0.10", "test", "test1"
)
fuel.FuelVersion.get_all_data.assert_called_once_with()
env.set_settings_data.assert_called_with(
{'editable': {'repo_setup': {'repos': {'value': [
{
'priority': 1000,
'name': 'mos',
'suite': 'mos1',
'section': 'main restricted',
'type': 'deb',
'uri': 'http://10.25.0.10:8080/mos'
},
{
'priority': 500,
'name': 'ubuntu',
'suite': 'trusty',
'section': 'main multiverse restricted universe',
'type': 'deb',
'uri': 'http://10.25.0.10:8080/ubuntu'
}
]}}}}
)
def test_apply_for_centos_based_env(self, accessors):
fuel = accessors.get_fuel_api_accessor()
self._setup_fuel_versions(fuel)
env = self._create_fuel_env(fuel)
self._create_fuel_release(fuel, "CentOS")
self.start_cmd(
apply, ['--group', 'mos', 'centos', '--env', '1'],
CENTOS_PATH
)
accessors.get_fuel_api_accessor.assert_called_with(
"10.25.0.10", "test", "test1"
)
fuel.FuelVersion.get_all_data.assert_called_once_with()
env.set_settings_data.assert_called_with(
{'editable': {'repo_setup': {'repos': {'value': [
{
'priority': 5,
'name': 'centos',
'type': 'rpm',
'uri': 'http://10.25.0.10:8080/centos/os/x86_64'
},
{
'priority': 10,
'name': 'mos',
'type': 'rpm',
'uri': 'http://10.25.0.10:8080/mos1/x86_64'
}]
}}}}
)
def test_apply_for_ubuntu_release(self, accessors):
fuel = accessors.get_fuel_api_accessor()
self._setup_fuel_versions(fuel)
env = self._create_fuel_env(fuel)
release = self._create_fuel_release(fuel, "Ubuntu")
self.start_cmd(
apply, ['--group', 'mos', 'ubuntu', '--default'],
UBUNTU_PATH
)
accessors.get_fuel_api_accessor.assert_called_with(
"10.25.0.10", "test", "test1"
)
fuel.FuelVersion.get_all_data.assert_called_once_with()
self.assertEqual(1, env.set_settings_data.call_count)
release.connection.put_request.assert_called_once_with(
release.instance_api_path.format(),
{
'name': "test release",
'operating_system': 'Ubuntu',
'attributes_metadata': {
'editable': {'repo_setup': {'repos': {'value': [
{
'name': 'mos',
'priority': 1000,
'suite': 'mos1',
'section': 'main restricted',
'type': 'deb',
'uri': 'http://10.25.0.10:8080/mos'
},
{
'name': 'ubuntu',
'priority': 500,
'suite': 'trusty',
'section': 'main multiverse restricted universe',
'type': 'deb',
'uri': 'http://10.25.0.10:8080/ubuntu'
}
]}}}
}
}
)
def test_apply_for_centos_release(self, accessors):
fuel = accessors.get_fuel_api_accessor()
self._setup_fuel_versions(fuel)
env = self._create_fuel_env(fuel)
release = self._create_fuel_release(fuel, "CentOS")
self.start_cmd(
apply, ['--group', 'mos', 'centos', '--default'],
CENTOS_PATH
)
accessors.get_fuel_api_accessor.assert_called_with(
"10.25.0.10", "test", "test1"
)
fuel.FuelVersion.get_all_data.assert_called_once_with()
self.assertEqual(1, env.set_settings_data.call_count)
release.connection.put_request.assert_called_once_with(
release.instance_api_path.format(),
{
'name': "test release",
'operating_system': 'CentOS',
'attributes_metadata': {
'editable': {'repo_setup': {'repos': {'value': [
{
'name': 'centos',
'priority': 5,
'type': 'rpm',
'uri': 'http://10.25.0.10:8080/centos/os/x86_64'
},
{
'name': 'mos',
'priority': 10,
'type': 'rpm',
'uri': 'http://10.25.0.10:8080/mos1/x86_64'
},
]}}}
}
}
)

68
contrib/fuel_mirror/fuel_mirror/tests/test_url_builder.py Normal file
View File

@ -0,0 +1,68 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from fuel_mirror.common import url_builder
from fuel_mirror.tests import base
class TestUrlBuilder(base.TestCase):
def test_get_url_builder(self):
self.assertTrue(issubclass(
url_builder.get_url_builder("deb"),
url_builder.AptRepoUrlBuilder
))
self.assertTrue(issubclass(
url_builder.get_url_builder("rpm"),
url_builder.YumRepoUrlBuilder
))
with self.assertRaises(KeyError):
url_builder.get_url_builder("unknown")
class TestAptUrlBuilder(base.TestCase):
@classmethod
def setUpClass(cls):
cls.builder = url_builder.get_url_builder("deb")
cls.repo_data = {
"name": "ubuntu",
"suite": "trusty",
"section": "main restricted",
"type": "deb",
"uri": "http://localhost/ubuntu"
}
def test_get_repo_url(self):
self.assertEqual(
"http://localhost/ubuntu trusty main restricted",
self.builder.get_repo_url(self.repo_data)
)
class TestYumUrlBuilder(base.TestCase):
@classmethod
def setUpClass(cls):
cls.builder = url_builder.get_url_builder("rpm")
cls.repo_data = {
"name": "centos",
"type": "rpm",
"uri": "http://localhost/os/x86_64"
}
def test_get_repo_url(self):
self.assertEqual(
"http://localhost/os",
self.builder.get_repo_url(self.repo_data)
)

106
contrib/fuel_mirror/fuel_mirror/tests/test_utils.py Normal file
View File

@ -0,0 +1,106 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
import six
from fuel_mirror.common import utils
from fuel_mirror.tests import base
class DictAsObj(object):
def __init__(self, d):
self.__dict__.update(d)
def __eq__(self, other):
return self.__dict__ == other.__dict__
class TestUtils(base.TestCase):
def test_lists_merge(self):
main = [{"a": 1, "b": 2, "c": 0}, {"a": 2, "b": 3, "c": 1}]
patch = [{"a": 2, "b": 4}, {"a": 3, "b": 5}]
utils.lists_merge(
main,
patch,
key="a"
)
self.assertItemsEqual(
[{"a": 1, "b": 2, "c": 0},
{"a": 2, "b": 4, "c": 1},
{"a": 3, "b": 5}],
main
)
def test_first(self):
self.assertEqual(
1,
utils.first(0, 1, 0),
)
self.assertEqual(
1,
utils.first(None, [], '', 1),
)
self.assertIsNone(
utils.first(None, [], 0, ''),
)
self.assertIsNone(
utils.first(),
)
def test_is_subdict(self):
self.assertFalse(utils.is_subdict({"c": 1}, {"a": 1, "b": 1}))
self.assertFalse(utils.is_subdict({"a": 1, "b": 2}, {"a": 1, "b": 1}))
self.assertFalse(
utils.is_subdict({"a": 1, "b": 1, "c": 2}, {"a": 1, "b": 1})
)
self.assertFalse(
utils.is_subdict({"a": 1, "b": None}, {"a": 1})
)
self.assertTrue(utils.is_subdict({}, {"a": 1}))
self.assertTrue(utils.is_subdict({"a": 1}, {"a": 1, "b": 1}))
self.assertTrue(utils.is_subdict({"a": 1, "b": 1}, {"a": 1, "b": 1}))
@mock.patch("fuel_mirror.common.utils.open")
def test_get_fuel_settings(self, m_open):
m_open().__enter__.side_effect = [
six.StringIO(
'ADMIN_NETWORK:\n'
' ipaddress: "10.20.0.4"\n'
'FUEL_ACCESS:\n'
' user: "test"\n'
' password: "test_pwd"\n',
),
OSError
]
self.assertEqual(
{
"server": "10.20.0.4",
"user": "test",
"password": "test_pwd",
},
utils.get_fuel_settings()
)
self.assertEqual(
{
"server": "10.20.0.2",
"user": None,
"password": None,
},
utils.get_fuel_settings()
)

33
contrib/fuel_mirror/fuel_mirror/tests/test_validate_config.py Normal file
View File

@ -0,0 +1,33 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os.path
import yaml
from fuel_mirror.tests import base
DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "data")
class TestValidateConfigs(base.TestCase):
def test_validate_data_files(self):
for f in os.listdir(DATA_DIR):
with open(os.path.join(DATA_DIR, f), "r") as fd:
data = yaml.load(fd)
# TODO(add input data validation scheme)
self.assertIn("groups", data)
self.assertIn("fuel_release_match", data)

9
contrib/fuel_mirror/requirements.txt Normal file
View File

@ -0,0 +1,9 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
pbr>=0.8
Babel>=1.3
cliff>=1.7.0
six>=1.5.2
packetary

79
contrib/fuel_mirror/scripts/fuel-createmirror Executable file
View File

@ -0,0 +1,79 @@
#!/bin/bash
echo "This script is DEPRECATED. Please use fuel-mirror utility!"
# This shell script was wraps the fuel-mirror utility to provide backward compatibility
# with previous version of tool.
usage() {
cat <<EOF
Usage: `basename $0` [options]
Create and update local mirrors of MOS and/or Ubuntu.
IMPORTANT!
If NO parameters specified, this script will:
- Create/Update both MOS and Ubuntu local mirrors
- Set them as repositories for existing NEW environments in Fuel UI
- Set them as DEFAULT repositories for new environments
Options:
-h| --help This help screen.
-d| --no-default Don't change default repositories for new environments
-a| --no-apply Don't apply changes to Fuel environments
-M| --mos Create/Update MOS local mirror only
-U| --ubuntu Create/Update Ubuntu local mirror only
-p| --password Fuel Master admin password (defaults to admin)
EOF
}
# Parse options
OPTS=`getopt -o hdaMUNp: -l help,no-default,no-apply,mos,ubuntu,password:,dry-run -- "$@"`
if [ $? != 0 ]; then
usage
exit 1
fi
eval set -- "$OPTS"
CMD_OPTS="--pattern=ubuntu"
while true ; do
case "$1" in
-h| --help ) usage ; exit 0;;
-d | --no-default ) OPT_NO_DEFAULT=1; shift;;
-a | --no-apply ) OPT_NO_APPLY=1; shift;;
-N | --dry-run ) EXEC_PREFIX="echo EXEC "; shift;;
-M | --mos ) GROUPS="$GROUPS mos"; shift;;
-U | --ubuntu ) GROUPS="$GROUPS ubuntu"; shift;;
-p | --password ) CMD_OPTS="$CMD_OPTS --fuel-password=$2"; shift; shift;;
-- ) shift; break;;
* ) break;;
esac
done
if [[ "$@" != "" ]]; then
echo "Invalid option -- $@"
usage
exit 1
fi
if [[ "$GROUPS" == "" ]]; then
GROUPS="mos ubuntu"
fi
CMD_OPTS="CMD_OPTS --group $GROUPS"
$EXEC_PREFIX fuel-mirror create ${CMD_OPTS}
if [[ "$OPT_NO_DEFAULT" == "" ]]; then
CMD_OPTS="$CMD_OPTS --default"
fi
if [[ "OPT_NO_APPLY" == "" ]]; then
CMD_OPTS="$CMD_OPTS --apply"
fi
$EXEC_PREFIX fuel-mirror apply ${CMD_OPTS}

61
contrib/fuel_mirror/setup.cfg Normal file
View File

@ -0,0 +1,61 @@
[metadata]
name = fuel_mirror
summary = The Utility to create local repositories with packages is
required for openstack deployment.
description-file =
README.rst
author = Mirantis Inc.
author_email = product@mirantis.com
url = http://mirantis.com
home-page = http://mirantis.com
classifier =
Development Status :: 4 - Beta
Environment :: OpenStack
Intended Audience :: Information Technology
Intended Audience :: System Administrators
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.3
Programming Language :: Python :: 3.4
Topic :: Utilities
[files]
packages =
fuel_mirror
data_files =
etc/fuel-mirror = etc/*
share/fuel-mirror = data/*
[build_sphinx]
source-dir = doc/source
build-dir = doc/build
all_files = 1
[entry_points]
console_scripts =
fuel-mirror=fuel_mirror.app:main
fuel_mirror =
apply=fuel_mirror.commands.apply:ApplyCommand
create=fuel_mirror.commands.create:CreateCommand
[upload_sphinx]
upload-dir = doc/build/html
[compile_catalog]
directory = locale
domain = fuel_mirror
[update_catalog]
domain = fuel_mirror
output_dir = locale
input_file = locale/fuel_mirror.pot
[extract_messages]
keywords = _ gettext ngettext l_ lazy_gettext
mapping_file = babel.cfg
output_file = locale/fuel_mirror.pot

28
contrib/fuel_mirror/setup.py Normal file
View File

@ -0,0 +1,28 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools
# In python < 2.7.4, a lazy loading of package `pbr` will break
# setuptools if some other modules registered functions in `atexit`.
# solution from: http://bugs.python.org/issue15881#msg170215
try:
import multiprocessing # noqa
except ImportError:
pass
setuptools.setup(
setup_requires=['pbr'],
pbr=True)

17
contrib/fuel_mirror/test-requirements.txt Normal file
View File

@ -0,0 +1,17 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
hacking<0.11,>=0.10.0
coverage>=3.6
discover
python-subunit>=0.0.18
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
oslosphinx>=2.5.0 # Apache-2.0
oslotest>=1.10.0 # Apache-2.0
testrepository>=0.0.18
testscenarios>=0.4
testtools>=1.4.0
cliff>=1.7.0
six>=1.5.2

35
contrib/fuel_mirror/tox.ini Normal file
View File

@ -0,0 +1,35 @@
[tox]
minversion = 1.6
envlist = py34,py27,py26,pep8
skipsdist = True
[testenv]
usedevelop = True
install_command = pip install -U {opts} {packages}
setenv =
VIRTUAL_ENV={envdir}
deps = -r{toxinidir}/test-requirements.txt
commands = python setup.py test --slowest --testr-args='{posargs:fuel_mirror}'
[testenv:pep8]
commands = flake8
[testenv:venv]
commands = {posargs}
[testenv:cover]
commands = python setup.py test --coverage --testr-args='{posargs:fuel_mirror}'
[testenv:docs]
commands = python setup.py build_sphinx
[testenv:debug]
commands = oslo_debug_helper {posargs}
[flake8]
# E123, E125 skipped as they are invalid PEP-8.
show-source = True
ignore = E123,E125
builtins = _
exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build

18
packetary/drivers/deb_driver.py
View File

@ -180,21 +180,19 @@ class DebRepositoryDriver(RepositoryDriverBase):
def fork_repository(self, connection, repository, destination, def fork_repository(self, connection, repository, destination,
source=False, locale=False): source=False, locale=False):
"""Overrides method of superclass."""
# TODO(download gpk) # TODO(download gpk)
# TODO(sources and locales) # TODO(sources and locales)
if not destination.endswith(os.path.sep): new_repo = copy.copy(repository)
destination += os.path.sep new_repo.url = utils.localize_repo_url(destination, repository.url)
clone = copy.copy(repository)
clone.url = destination
packages_file = utils.get_path_from_url( packages_file = utils.get_path_from_url(
self._get_url_of_metafile(clone, "Packages") self._get_url_of_metafile(new_repo, "Packages")
) )
release_file = utils.get_path_from_url( release_file = utils.get_path_from_url(
self._get_url_of_metafile(clone, "Release") self._get_url_of_metafile(new_repo, "Release")
)
self.logger.info(
"clone repository %s to %s", repository, new_repo.url
) )
self.logger.info("clone repository %s to %s", repository, destination)
utils.ensure_dir_exist(os.path.dirname(release_file)) utils.ensure_dir_exist(os.path.dirname(release_file))
release = deb822.Release() release = deb822.Release()
@ -208,7 +206,7 @@ class DebRepositoryDriver(RepositoryDriverBase):
open(packages_file, "ab").close() open(packages_file, "ab").close()
gzip.open(packages_file + ".gz", "ab").close() gzip.open(packages_file + ".gz", "ab").close()
return clone return new_repo
def _update_suite_index(self, repository): def _update_suite_index(self, repository):
"""Updates the Release file in the suite.""" """Updates the Release file in the suite."""

22
packetary/drivers/rpm_driver.py
View File

@ -72,15 +72,10 @@ class RpmRepositoryDriver(RepositoryDriverBase):
return (url.rstrip("/") for url in urls) return (url.rstrip("/") for url in urls)
def get_repository(self, connection, url, arch, consumer): def get_repository(self, connection, url, arch, consumer):
"""Overrides method of superclass.""" name = utils.get_path_from_url(url, False)
# Currently supported repositories, that has URL in following format:
# baseurl/{name}/{architecture}
# because the architecture is sentetic part of rpm repository URL
name = url.rsplit("/", 1)[-1]
baseurl = "/".join((url, arch, ""))
consumer(Repository( consumer(Repository(
name=name, name=name,
url=baseurl, url=url + "/",
architecture=arch, architecture=arch,
origin="" origin=""
)) ))
@ -165,17 +160,14 @@ class RpmRepositoryDriver(RepositoryDriverBase):
def fork_repository(self, connection, repository, destination, def fork_repository(self, connection, repository, destination,
source=False, locale=False): source=False, locale=False):
"""Overrides method of superclass."""
# TODO(download gpk) # TODO(download gpk)
# TODO(sources and locales) # TODO(sources and locales)
destination = os.path.join(
destination, repository.name,
repository.architecture, ""
)
new_repo = copy.copy(repository) new_repo = copy.copy(repository)
new_repo.url = destination new_repo.url = utils.localize_repo_url(destination, repository.url)
self.logger.info("clone repository %s to %s", repository, destination) self.logger.info(
utils.ensure_dir_exist(destination) "clone repository %s to %s", repository, new_repo.url
)
utils.ensure_dir_exist(new_repo.url)
self.rebuild_repository(new_repo, set()) self.rebuild_repository(new_repo, set())
return new_repo return new_repo

10
packetary/library/utils.py
View File

@ -90,6 +90,16 @@ def get_path_from_url(url, ensure_file=True):
return comps.path return comps.path
def localize_repo_url(localurl, repo_url):
"""Gets local repository url.
:param localurl: the base local URL
:param repo_url: the origin URL of repository
:return: localurl + get_path_from_url(repo_url)
"""
return localurl.rstrip("/") + urlparse(repo_url).path
def ensure_dir_exist(path): def ensure_dir_exist(path):
"""Creates directory if it does not exist. """Creates directory if it does not exist.

22
packetary/tests/test_deb_driver.py
View File

@ -20,6 +20,7 @@ import six
from packetary.drivers import deb_driver from packetary.drivers import deb_driver
from packetary.library.utils import localize_repo_url
from packetary.tests import base from packetary.tests import base
from packetary.tests.stubs.generator import gen_package from packetary.tests.stubs.generator import gen_package
from packetary.tests.stubs.generator import gen_repository from packetary.tests.stubs.generator import gen_repository
@ -188,26 +189,29 @@ class TestDebDriver(base.TestCase):
os.path.sep = "/" os.path.sep = "/"
os.path.join = lambda *x: "/".join(x) os.path.join = lambda *x: "/".join(x)
utils.get_path_from_url = lambda x: x utils.get_path_from_url = lambda x: x
repo = gen_repository(name=("trusty", "main"), url="http://localhost") utils.localize_repo_url = localize_repo_url
repo = gen_repository(
name=("trusty", "main"), url="http://localhost/test/"
)
files = [ files = [
mock.MagicMock(), mock.MagicMock(),
mock.MagicMock() mock.MagicMock()
] ]
open.side_effect = files open.side_effect = files
clone = self.driver.fork_repository(self.connection, repo, "/root") new_repo = self.driver.fork_repository(self.connection, repo, "/root")
self.assertEqual(repo.name, clone.name) self.assertEqual(repo.name, new_repo.name)
self.assertEqual(repo.architecture, clone.architecture) self.assertEqual(repo.architecture, new_repo.architecture)
self.assertEqual(repo.origin, clone.origin) self.assertEqual(repo.origin, new_repo.origin)
self.assertEqual("/root/", clone.url) self.assertEqual("/root/test/", new_repo.url)
utils.ensure_dir_exist.assert_called_once_with(os.path.dirname()) utils.ensure_dir_exist.assert_called_once_with(os.path.dirname())
open.assert_any_call( open.assert_any_call(
"/root/dists/trusty/main/binary-amd64/Release", "wb" "/root/test/dists/trusty/main/binary-amd64/Release", "wb"
) )
open.assert_any_call( open.assert_any_call(
"/root/dists/trusty/main/binary-amd64/Packages", "ab" "/root/test/dists/trusty/main/binary-amd64/Packages", "ab"
) )
gzip.open.assert_called_once_with( gzip.open.assert_called_once_with(
"/root/dists/trusty/main/binary-amd64/Packages.gz", "ab" "/root/test/dists/trusty/main/binary-amd64/Packages.gz", "ab"
) )
@mock.patch.multiple( @mock.patch.multiple(

19
packetary/tests/test_rpm_driver.py
View File

@ -20,6 +20,7 @@ import sys
import six import six
from packetary.library.utils import localize_repo_url
from packetary.objects import FileChecksum from packetary.objects import FileChecksum
from packetary.tests import base from packetary.tests import base
from packetary.tests.stubs.generator import gen_repository from packetary.tests.stubs.generator import gen_repository
@ -65,12 +66,15 @@ class TestRpmDriver(base.TestCase):
repos = [] repos = []
self.driver.get_repository( self.driver.get_repository(
self.connection, "http://host/centos/os", "x86_64", repos.append self.connection,
"http://host/centos/os/x86_64",
"x86_64",
repos.append
) )
self.assertEqual(1, len(repos)) self.assertEqual(1, len(repos))
repo = repos[0] repo = repos[0]
self.assertEqual("os", repo.name) self.assertEqual("/centos/os/x86_64", repo.name)
self.assertEqual("", repo.origin) self.assertEqual("", repo.origin)
self.assertEqual("x86_64", repo.architecture) self.assertEqual("x86_64", repo.architecture)
self.assertEqual("http://host/centos/os/x86_64/", repo.url) self.assertEqual("http://host/centos/os/x86_64/", repo.url)
@ -189,16 +193,17 @@ class TestRpmDriver(base.TestCase):
@mock.patch("packetary.drivers.rpm_driver.utils") @mock.patch("packetary.drivers.rpm_driver.utils")
def test_fork_repository(self, utils): def test_fork_repository(self, utils):
repo = gen_repository("os", url="http://localhost/os/x86_64") repo = gen_repository("os", url="http://localhost/os/x86_64/")
clone = self.driver.fork_repository( utils.localize_repo_url = localize_repo_url
new_repo = self.driver.fork_repository(
self.connection, self.connection,
repo, repo,
"/repo" "/repo"
) )
utils.ensure_dir_exist.assert_called_once_with("/repo/os/x86_64/") utils.ensure_dir_exist.assert_called_once_with("/repo/os/x86_64/")
self.assertEqual(repo.name, clone.name) self.assertEqual(repo.name, new_repo.name)
self.assertEqual(repo.architecture, clone.architecture) self.assertEqual(repo.architecture, new_repo.architecture)
self.assertEqual("/repo/os/x86_64/", clone.url) self.assertEqual("/repo/os/x86_64/", new_repo.url)
self.createrepo.MetaDataGenerator()\ self.createrepo.MetaDataGenerator()\
.doFinalMove.assert_called_once_with() .doFinalMove.assert_called_once_with()