Merge "Introduced fuel-mirror"

contrib/fuel_mirror/MANIFEST.in

@@ -0,0 +1,8 @@
+include AUTHORS
+include ChangeLog
+recursive-include etc *
+
+exclude .gitignore
+exclude .gitreview
+
+global-exclude *.pyc

contrib/fuel_mirror/README.rst

@@ -0,0 +1,16 @@
+===========
+fuel_mirror
+===========
+
+The fuel-mirror is utility, that allows to create local repositories
+with packages are required for the OpenStack deployment.
+
+* Free software: Apache license
+* Documentation: http://docs.openstack.org/developer/fuel-mirror
+* Source: http://git.openstack.org/cgit/openstack/fuel-mirror/
+* Bugs: http://bugs.launchpad.net/fuel
+
+Features
+--------
+
+* TODO

contrib/fuel_mirror/babel.cfg

@@ -0,0 +1,2 @@
+[python: **.py]
+

contrib/fuel_mirror/data/centos.yaml

@@ -0,0 +1,58 @@
+centos_baseurl: &centos_baseurl
+mos_baseurl: &mos_baseurl
+
+fuel_release_match:
+   version: $openstack_version
+   operating_system: CentOS
+
+repos:
+    -   &centos
+        name: "centos"
+        uri: "http://mirror.centos.org/centos/6/os/x86_64"
+        type: "rpm"
+        priority: null
+
+    -   &centos_updates
+        name: "centos-updates"
+        uri: "http://mirror.centos.org/centos/6/updates/x86_64"
+        type: "rpm"
+        priority: null
+
+    -   &mos
+        name: "mos"
+        uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/os/x86_64"
+        type: "rpm"
+        priority: null
+
+    -   &mos_updates
+        name: "mos-updates"
+        uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/updates/x86_64"
+        type: "rpm"
+        priority: null
+
+    -   &mos_security
+        name: "mos-security"
+        uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/security/x86_64"
+        type: "rpm"
+        priority: null
+
+    -   &mos_holdback
+        name: "mos-holdback"
+        uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/holdback/x86_64"
+        type: "rpm"
+        priority: null
+
+groups:
+    mos:
+        - *mos
+        - *mos_updates
+        - *mos_security
+        - *mos_holdback
+
+    centos:
+        - *centos
+        - *centos_updates
+
+
+inheritance:
+    centos: mos

contrib/fuel_mirror/data/ubuntu.yaml

@@ -0,0 +1,140 @@
+# GLOBAL variables
+ubuntu_baseurl: &ubuntu_baseurl http://archive.ubuntu.com/ubuntu
+mos_baseurl: &mos_baseurl http://mirror.fuel-infra.org/mos-repos/ubuntu/$mos_version
+
+fuel_release_match:
+   version: $openstack_version
+   operating_system: Ubuntu
+
+repos:
+    -   &ubuntu
+        name: "ubuntu"
+        uri: *ubuntu_baseurl
+        suite: "trusty"
+        section: "main multiverse restricted universe"
+        type: "deb"
+        priority: null
+
+    -   &ubuntu_updates
+        name: "ubuntu-updates"
+        uri: *ubuntu_baseurl
+        suite: "trusty-updates"
+        section: "main multiverse restricted universe"
+        type: "deb"
+        priority: null
+
+    -   &ubuntu_security
+        name: "ubuntu-security"
+        uri: *ubuntu_baseurl
+        suite: "trusty-security"
+        section: "main multiverse restricted universe"
+        type: "deb"
+        priority: null
+
+    -   &mos
+        name: "mos"
+        uri: *mos_baseurl
+        suite: "mos$mos_version"
+        section: "main restricted"
+        type: "deb"
+        priority: 1000
+
+    -   &mos_updates
+        name: "mos-updates"
+        uri: *mos_baseurl
+        suite: "mos$mos_version-updates"
+        section: "main restricted"
+        type: "deb"
+        priority: 1000
+
+    -   &mos_security
+        name: "mos-security"
+        uri: *mos_baseurl
+        suite: "mos$mos_version-security"
+        section: "main restricted"
+        type: "deb"
+        priority: 1000
+
+    -   &mos_holdback
+        name: "mos-holdback"
+        uri: *mos_baseurl
+        suite: "mos$mos_version"
+        section: "main restricted"
+        type: "deb"
+        priority: 1000
+
+
+packages: &packages
+     - "acpi-support"
+     - "anacron"
+     - "aptitude"
+     - "atop"
+     - "bash-completion"
+     - "bc"
+     - "build-essential"
+     - "cloud-init"
+     - "conntrackd"
+     - "cpu-checker"
+     - "cpufrequtils"
+     - "debconf-utils"
+     - "devscripts"
+     - "fping"
+     - "git"
+     - "grub-pc"
+     - "htop"
+     - "ifenslave"
+     - "iperf"
+     - "iptables-persistent"
+     - "irqbalance"
+     - "language-pack-en"
+     - "linux-firmware-nonfree"
+     - "linux-headers-generic-lts-trusty"
+     - "linux-image-generic-lts-trusty"
+     - "livecd-rootfs"
+     - "memcached"
+     - "monit"
+     - "nginx"
+     - "ntp"
+     - "openssh-server"
+     - "percona-toolkit"
+     - "percona-xtrabackup"
+     - "pm-utils"
+     - "python-lesscpy"
+     - "python-pip"
+     - "puppet"
+     - "rsyslog-gnutls"
+     - "rsyslog-relp"
+     - "screen"
+     - "swift-plugin-s3"
+     - "sysfsutils"
+     - "sysstat"
+     - "telnet"
+     - "tmux"
+     - "traceroute"
+     - "ubuntu-standard"
+     - "vim"
+     - "virt-what"
+     - "xinetd"
+
+
+groups:
+    mos:
+        - *mos
+        - *mos_updates
+        - *mos_security
+        - *mos_holdback
+
+    ubuntu:
+        - *ubuntu
+        - *ubuntu_updates
+        - *ubuntu_security
+
+
+inheritance:
+    ubuntu: mos
+
+osnames:
+    mos: ubuntu
+
+requirements:
+    ubuntu: *packages

contrib/fuel_mirror/doc/source/conf.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('../..'))
+# -- General configuration ----------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    #'sphinx.ext.intersphinx',
+    'oslosphinx'
+]
+
+# autodoc generation is a bit aggressive and a nuisance when doing heavy
+# text edit cycles.
+# execute "export SPHINX_DEBUG=1" in your terminal to disable
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'fuel_mirror'
+copyright = u'2015, Mirantis, Inc'
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+add_module_names = True
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# -- Options for HTML output --------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  Major themes that come with
+# Sphinx are currently 'default' and 'sphinxdoc'.
+# html_theme_path = ["."]
+# html_theme = '_theme'
+# html_static_path = ['static']
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = '%sdoc' % project
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass
+# [howto/manual]).
+latex_documents = [
+    ('index',
+     '%s.tex' % project,
+     u'%s Documentation' % project,
+     u'OpenStack Foundation', 'manual'),
+]
+
+# Example configuration for intersphinx: refer to the Python standard library.
+#intersphinx_mapping = {'http://docs.python.org/': None}

contrib/fuel_mirror/doc/source/contributing.rst

@@ -0,0 +1,4 @@
+============
+Contributing
+============
+.. include:: ../../../../CONTRIBUTING.rst

contrib/fuel_mirror/doc/source/index.rst

@@ -0,0 +1,25 @@
+.. fuel_mirror documentation master file, created by
+   sphinx-quickstart on Tue Jul  9 22:26:36 2013.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to fuel_mirror's documentation!
+========================================================
+
+Contents:
+
+.. toctree::
+   :maxdepth: 2
+
+   readme
+   installation
+   usage
+   contributing
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

contrib/fuel_mirror/doc/source/installation.rst

@@ -0,0 +1,12 @@
+============
+Installation
+============
+
+At the command line::
+
+    $ pip install fuel_mirror
+
+Or, if you have virtualenvwrapper installed::
+
+    $ mkvirtualenv fuel_mirror
+    $ pip install fuel_mirror

contrib/fuel_mirror/doc/source/readme.rst

@@ -0,0 +1 @@
+.. include:: ../../README.rst

contrib/fuel_mirror/doc/source/usage.rst

@@ -0,0 +1,7 @@
+========
+Usage
+========
+
+To use fuel_mirror in a project::
+
+    import fuel_createmirror

contrib/fuel_mirror/etc/config.yaml

@@ -0,0 +1,11 @@
+threads_num: 10
+ignore_errors_num: 2
+retries_num: 3
+target_dir: "/var/www/nailgun/mirrors"
+pattern_dir: "/usr/share/fuel-mirror"
+
+# uncomment if need
+#   http_proxy: null
+#   https_proxy: null
+#   fuel_server: 10.20.0.2
+#   base_url: http://10.20.0.2:8080

contrib/fuel_mirror/fuel_mirror/__init__.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import pbr.version
+
+
+__version__ = pbr.version.VersionInfo(
+    'fuel_mirror').version_string()

contrib/fuel_mirror/fuel_mirror/app.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from cliff import app
+from cliff.commandmanager import CommandManager
+import yaml
+
+
+import fuel_mirror
+from fuel_mirror.common import accessors
+from fuel_mirror.common import utils
+
+
+_FUEL_DEFAULT_HTTP_PORT = 8080
+
+
+class Application(app.App):
+    """Main cliff application class.
+
+    Performs initialization of the command manager and
+    configuration of basic engines.
+    """
+
+    config = None
+    fuel = None
+    repo_manager_accessor = None
+    sources = None
+    versions = None
+
+    def build_option_parser(self, description, version, argparse_kwargs=None):
+        """Specifies common cmdline arguments."""
+        p_inst = super(Application, self)
+        parser = p_inst.build_option_parser(description=description,
+                                            version=version,
+                                            argparse_kwargs=argparse_kwargs)
+
+        parser.add_argument(
+            "--config",
+            default="/etc/fuel-mirror/config.yaml",
+            metavar="PATH",
+            help="Path to config file."
+        )
+        parser.add_argument(
+            "-S", "--fuel-server",
+            metavar="FUEL-SERVER",
+            help="The public address of Fuel Master."
+        )
+        parser.add_argument(
+            "--fuel-user",
+            help="Fuel Master admin login."
+                 " Alternatively, use env var KEYSTONE_USER)."
+        )
+        parser.add_argument(
+            "--fuel-password",
+            help="Fuel Master admin password."
+                 " Alternatively, use env var KEYSTONE_PASSWORD)."
+        )
+        return parser
+
+    def initialize_app(self, argv):
+        """Initialises common options."""
+        with open(self.options.config, "r") as stream:
+            self.config = yaml.load(stream)
+        fuel_default = utils.get_fuel_settings()
+
+        fuel_server = utils.first(
+            self.options.fuel_server,
+            self.config.get("fuel_server"),
+            fuel_default["server"]
+        )
+        fuel_user = utils.first(
+            self.options.fuel_user,
+            fuel_default["user"]
+        )
+        fuel_password = utils.first(
+            self.options.fuel_password,
+            fuel_default["password"]
+        )
+        self.config.setdefault(
+            "base_url", "http://{0}:{1}".format(
+                fuel_server.split(":", 1)[0],
+                _FUEL_DEFAULT_HTTP_PORT
+            )
+        )
+        self.fuel = accessors.get_fuel_api_accessor(
+            fuel_server,
+            fuel_user,
+            fuel_password
+        )
+        fuel_ver = self.fuel.FuelVersion.get_all_data()
+        self.config.setdefault(
+            'mos_version', fuel_ver['release']
+        )
+        self.config.setdefault(
+            'openstack_version', fuel_ver['openstack_version']
+        )
+
+        self.repo_manager_accessor = accessors.get_packetary_accessor(
+            threads_num=int(self.config.get('threads_num', 0)),
+            retries_num=int(self.config.get('retries_num', 0)),
+            ignore_errors_num=int(self.config.get('ignore_errors_num', 0)),
+            http_proxy=self.config.get('http_proxy'),
+            https_proxy=self.config.get('https_proxy'),
+        )
+
+
+def main(argv=None):
+    """Entry point."""
+    return Application(
+        description="The utility to create local mirrors.",
+        version=fuel_mirror.__version__,
+        command_manager=CommandManager("fuel_mirror", convert_underscores=True)
+    ).run(argv)
+
+
+def debug(name, cmd_class, argv=None):
+    """Helps to debug command."""
+    import sys
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    argv = [name] + argv + ["-v", "-v", "--debug"]
+    cmd_mgr = CommandManager("test_fuel_mirror", convert_underscores=True)
+    cmd_mgr.add_command(name, cmd_class)
+    return Application(
+        description="The fuel mirror utility test.",
+        version="0.0.1",
+        command_manager=cmd_mgr
+    ).run(argv)

contrib/fuel_mirror/fuel_mirror/commands/apply.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import six
+
+from packetary.library.utils import localize_repo_url
+
+from fuel_mirror.commands.base import BaseCommand
+from fuel_mirror.common.utils import is_subdict
+from fuel_mirror.common.utils import lists_merge
+
+
+class ApplyCommand(BaseCommand):
+    """Applies local mirrors for Fuel-environments."""
+
+    def get_parser(self, prog_name):
+        parser = super(ApplyCommand, self).get_parser(prog_name)
+        parser.add_argument(
+            "--default",
+            dest="set_default",
+            action="store_true",
+            default=False,
+            help="Set as default repository."
+        )
+        parser.add_argument(
+            "-e", "--env",
+            dest="env", nargs="+",
+            help="Fuel environment ID to update, "
+                 "by default applies for all environments."
+        )
+
+        return parser
+
+    def take_action(self, parsed_args):
+        data = self.load_data(parsed_args)
+        base_url = self.app.config["base_url"]
+        localized_repos = []
+        for _, repos in self.get_groups(parsed_args, data):
+            for repo_data in repos:
+                new_data = repo_data.copy()
+                new_data['uri'] = localize_repo_url(
+                    base_url, repo_data['uri']
+                )
+                localized_repos.append(new_data)
+
+        release_match = data["fuel_release_match"]
+        self.update_clusters(parsed_args.env, localized_repos, release_match)
+        if parsed_args.set_default:
+            self.update_default_repos(localized_repos, release_match)
+
+        self.app.stdout.write(
+            "Operations have been completed successfully.\n"
+        )
+
+    def update_clusters(self, ids, repositories, release_match):
+        """Applies repositories for existing clusters.
+
+        :param ids: the cluster ids.
+        :param repositories: the meta information of repositories
+        :param release_match: The pattern to check Fuel Release
+        """
+        self.app.stdout.write("Updating the Cluster repositories...\n")
+
+        if ids:
+            clusters = self.app.fuel.Environment.get_by_ids(ids)
+        else:
+            clusters = self.app.fuel.Environment.get_all()
+
+        for cluster in clusters:
+            releases = six.moves.filter(
+                lambda x: is_subdict(release_match, x.data),
+                self.app.fuel.Release.get_by_ids([cluster.data["release_id"]])
+            )
+            if next(releases, None) is None:
+                continue
+
+            modified = self._update_repository_settings(
+                cluster.get_settings_data(),
+                repositories
+            )
+            if modified:
+                self.app.LOG.info(
+                    "Try to update the Cluster '%s'",
+                    cluster.data['name']
+                )
+                self.app.LOG.debug(
+                    "The modified cluster attributes: %s",
+                    modified
+                )
+                cluster.set_settings_data(modified)
+
+    def update_default_repos(self, repositories, release_match):
+        """Applies repositories for existing default settings.
+
+        :param repositories: the meta information of repositories
+        :param release_match: The pattern to check Fuel Release
+        """
+        self.app.stdout.write("Updating the default repositories...\n")
+        releases = six.moves.filter(
+            lambda x: is_subdict(release_match, x.data),
+            self.app.fuel.Release.get_all()
+        )
+        for release in releases:
+            if self._update_repository_settings(
+                release.data["attributes_metadata"], repositories
+            ):
+                self.app.LOG.info(
+                    "Try to update the Release '%s'",
+                    release.data['name']
+                )
+                self.app.LOG.debug(
+                    "The modified release attributes: %s",
+                    release.data
+                )
+                # TODO(need to add method for release object)
+                release.connection.put_request(
+                    release.instance_api_path.format(release.id),
+                    release.data
+                )
+
+    def _update_repository_settings(self, settings, repositories):
+        """Updates repository settings.
+
+        :param settings: the target settings
+        :param repositories: the meta of repositories
+        """
+        editable = settings["editable"]
+        if 'repo_setup' not in editable:
+            self.app.LOG.info('Attributes is read-only.')
+            return
+
+        repos_attr = editable["repo_setup"]["repos"]
+        lists_merge(repos_attr['value'], repositories, "name")
+        return {"editable": {"repo_setup": {"repos": repos_attr}}}
+
+
+def debug(argv=None):
+    """Helper for debugging Apply command."""
+    from fuel_mirror.app import debug
+
+    debug("apply", ApplyCommand, argv)
+
+
+if __name__ == "__main__":
+    debug()

contrib/fuel_mirror/fuel_mirror/commands/base.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os.path
+from string import Template
+
+from cliff import command
+import yaml
+
+
+class BaseCommand(command.Command):
+    """The Base command for fuel-mirror."""
+    REPO_ARCH = "x86_64"
+
+    @property
+    def stdout(self):
+        """Shortcut for self.app.stdout."""
+        return self.app.stdout
+
+    def get_parser(self, prog_name):
+        """Specifies common options."""
+        parser = super(BaseCommand, self).get_parser(prog_name)
+
+        input_group = parser.add_mutually_exclusive_group(required=True)
+        input_group.add_argument(
+            '-I', '--input-file',
+            metavar='PATH',
+            help='The path to file with input data.')
+
+        input_group.add_argument(
+            '-P', '--pattern',
+            metavar='NAME',
+            help='The builtin input file name.'
+        )
+
+        parser.add_argument(
+            "-G", "--group",
+            dest="groups",
+            required=True,
+            nargs='+',
+            help="The name of repository groups."
+        )
+        return parser
+
+    def resolve_input_pattern(self, pattern):
+        """Gets the full path to input file by pattern.
+
+        :param pattern: the config file name without ext
+        :return: the full path
+        """
+        return os.path.join(
+            self.app.config['pattern_dir'], pattern + ".yaml"
+        )
+
+    def load_data(self, parsed_args):
+        """Load the input data.
+
+        :param parsed_args: the command-line arguments
+        :return: the input data
+        """
+        if parsed_args.pattern:
+            input_file = self.resolve_input_pattern(parsed_args.pattern)
+        else:
+            input_file = parsed_args.input_file
+
+        # TODO(add input data validation scheme)
+        with open(input_file, "r") as fd:
+            return yaml.load(Template(fd.read()).safe_substitute(
+                mos_version=self.app.config["mos_version"],
+                openstack_version=self.app.config["openstack_version"],
+            ))
+
+    @classmethod
+    def get_groups(cls, parsed_args, data):
+        """Gets repository groups from input data.
+
+        :param parsed_args: the command-line arguments
+        :param data: the input data
+        :return: the sequence of pairs (group_name, repositories)
+        """
+        all_groups = data['groups']
+        return (
+            (x, all_groups[x]) for x in parsed_args.groups if x in all_groups
+        )

contrib/fuel_mirror/fuel_mirror/commands/create.py

@@ -0,0 +1,75 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from fuel_mirror.commands.base import BaseCommand
+from fuel_mirror.common.url_builder import get_url_builder
+
+
+class CreateCommand(BaseCommand):
+    """Creates a new local mirrors."""
+
+    def take_action(self, parsed_args):
+        """See the Command.take_action."""
+        data = self.load_data(parsed_args)
+        repos_reqs = data.get('requirements', {})
+        inheritance = data.get('inheritance', {})
+        target_dir = self.app.config["target_dir"]
+
+        total_stats = None
+        for group_name, repos in self.get_groups(parsed_args, data):
+            url_builder = get_url_builder(repos[0]["type"])
+            repo_manager = self.app.repo_manager_accessor(
+                repos[0]["type"], self.REPO_ARCH
+            )
+            if group_name in inheritance:
+                child_group = inheritance[group_name]
+                dependencies = [
+                    url_builder.get_repo_url(x)
+                    for x in data['groups'][child_group]
+                ]
+            else:
+                dependencies = None
+
+            stat = repo_manager.clone_repositories(
+                [url_builder.get_repo_url(x) for x in repos],
+                target_dir,
+                dependencies,
+                repos_reqs.get(group_name)
+            )
+
+            if total_stats is None:
+                total_stats = stat
+            else:
+                total_stats += stat
+
+        if total_stats is not None:
+            self.stdout.write(
+                "Packages processed: {0.copied}/{0.total}\n"
+                .format(total_stats)
+            )
+        else:
+            self.stdout.write(
+                "No packages.\n"
+            )
+
+
+def debug(argv=None):
+    """Helper for debugging Create command."""
+    from fuel_mirror.app import debug
+
+    debug("create", CreateCommand, argv)
+
+
+if __name__ == "__main__":
+    debug()

contrib/fuel_mirror/fuel_mirror/common/accessors.py

@@ -0,0 +1,54 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import functools
+import os
+
+
+def get_packetary_accessor(**kwargs):
+    """Gets the configured repository manager.
+
+    :param kwargs: The packetary configuration parameters.
+    """
+
+    import packetary
+
+    return functools.partial(
+        packetary.RepositoryApi.create,
+        packetary.Context(packetary.Configuration(**kwargs))
+    )
+
+
+def get_fuel_api_accessor(address=None, user=None, password=None):
+    """Gets the fuel client api accessor.
+
+    :param address: The address of Fuel Master node.
+    :param user: The username to access to the Fuel Master node.
+    :param user: The password to access to the Fuel Master node.
+    """
+    if address:
+        host_and_port = address.split(":")
+        os.environ["SERVER_ADDRESS"] = host_and_port[0]
+        if len(host_and_port) > 1:
+            os.environ["LISTEN_PORT"] = host_and_port[1]
+
+    if user is not None:
+        os.environ["KEYSTONE_USER"] = user
+    if password is not None:
+        os.environ["KEYSTONE_PASS"] = password
+
+    # import fuelclient.ClientAPI after configuring
+    # environment variables
+    from fuelclient import objects
+    return objects

contrib/fuel_mirror/fuel_mirror/common/url_builder.py

@@ -0,0 +1,68 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+def get_url_builder(repotype):
+    """Gets the instance of RepoUrlBuilder.
+
+    :param repotype: the type of repository: rpm|deb
+    :return: the RepoBuilder implementation
+    """
+    return {
+        "deb": AptRepoUrlBuilder,
+        "rpm": YumRepoUrlBuilder
+    }[repotype]
+
+
+class RepoUrlBuilder(object):
+    REPO_FOLDER = "mirror"
+
+    @classmethod
+    def get_repo_url(cls, repo_data):
+        """Gets the url with replaced variable holders.
+
+        :param repo_data: the repositories`s meta data
+        :return: the full repository`s url
+        """
+
+
+class AptRepoUrlBuilder(RepoUrlBuilder):
+    """URL builder for apt-repository(es)."""
+
+    @classmethod
+    def get_repo_url(cls, repo_data):
+        return " ".join(
+            repo_data[x] for x in ("uri", "suite", "section")
+        )
+
+
+class YumRepoUrlBuilder(RepoUrlBuilder):
+    """URL builder for Yum repository(es)."""
+
+    @classmethod
+    def split_url(cls, url, maxsplit=2):
+        """Splits url to baseurl, reponame adn architecture.
+
+        :param url: the repository`s URL
+        :param maxsplit: the number of expected components
+        :return the components of url
+        """
+        # TODO(need generic url building algorithm)
+        # there is used assumption that url has following format
+        # $baseurl/$reponame/$repoarch
+        return url.rstrip("/").rsplit("/", maxsplit)
+
+    @classmethod
+    def get_repo_url(cls, repo_data):
+        return cls.split_url(repo_data["uri"], 1)[0]

contrib/fuel_mirror/fuel_mirror/common/utils.py

@@ -0,0 +1,96 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import six
+import yaml
+
+
+def lists_merge(main, patch, key):
+    """Merges the list of dicts with same keys.
+
+    >>> lists_merge([{"a": 1, "c": 2}], [{"a": 1, "c": 3}], key="a")
+    [{'a': 1, 'c': 3}]
+
+    :param main: the main list
+    :type main: list
+    :param patch: the list of additional elements
+    :type patch: list
+    :param key: the key for compare
+    """
+    main_idx = dict(
+        (x[key], i) for i, x in enumerate(main)
+    )
+
+    patch_idx = dict(
+        (x[key], i) for i, x in enumerate(patch)
+    )
+
+    for k in sorted(patch_idx):
+        if k in main_idx:
+            main[main_idx[k]].update(patch[patch_idx[k]])
+        else:
+            main.append(patch[patch_idx[k]])
+    return main
+
+
+def is_subdict(dict1, dict2):
+    """Checks that dict1 is subdict of dict2.
+
+    >>> is_subdict({"a": 1}, {'a': 1, 'b': 1})
+    True
+
+    :param dict1: the candidate
+    :param dict2: the super dict
+    :return: True if all keys from dict1 are present
+             and has same value in dict2 otherwise False
+    """
+    for k, v in six.iteritems(dict1):
+        if k not in dict2 or dict2[k] != v:
+            return False
+    return True
+
+
+def first(*args):
+    """Get first not empty value.
+
+    >>> first(0, 1) == next(iter(filter(None, [0, 1])))
+    True
+
+    :param args: the list of arguments
+    :return first value that bool(v) is True, None if not found.
+    """
+    for arg in args:
+        if arg:
+            return arg
+
+
+def get_fuel_settings():
+    """Gets the fuel settings from astute container, if it is available."""
+
+    _DEFAULT_SETTINGS = {
+        "server": "10.20.0.2",
+        "user": None,
+        "password": None,
+    }
+    try:
+        with open("/etc/fuel/astute.yaml", "r") as fd:
+            settings = yaml.load(fd)
+        return {
+            "server": settings.get("ADMIN_NETWORK", {}).get("ipaddress"),
+            "user": settings.get("FUEL_ACCESS", {}).get("user"),
+            "password": settings.get("FUEL_ACCESS", {}).get("password")
+        }
+    except (OSError, IOError):
+        pass
+    return _DEFAULT_SETTINGS

contrib/fuel_mirror/fuel_mirror/tests/base.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+
+class TestCase(unittest.TestCase):
+    """Test case base class for all unit tests."""

contrib/fuel_mirror/fuel_mirror/tests/data/test_centos.yaml

@@ -0,0 +1,22 @@
+fuel_release_match:
+   operating_system: CentOS
+
+inheritance:
+    centos: mos
+
+groups:
+    mos:
+        -   name: "mos"
+            type: "rpm"
+            uri: "http://localhost/mos$mos_version/x86_64"
+            priority: 10
+
+    centos:
+        -   name: "centos"
+            type: "rpm"
+            uri: "http://localhost/centos/os/x86_64"
+            priority: 5
+
+requirements:
+    centos:
+        -   "package_rpm"

contrib/fuel_mirror/fuel_mirror/tests/data/test_config.yaml

@@ -0,0 +1,6 @@
+threads_num: 1
+ignore_errors_num: 2
+retries_num: 3
+http_proxy: "http://localhost"
+https_proxy: "https://localhost"
+target_dir: "/var/www/"

contrib/fuel_mirror/fuel_mirror/tests/data/test_ubuntu.yaml

@@ -0,0 +1,26 @@
+fuel_release_match:
+   operating_system: Ubuntu
+
+inheritance:
+    ubuntu: mos
+
+groups:
+    mos:
+        -   name: "mos"
+            type: "deb"
+            uri: "http://localhost/mos"
+            suite: "mos$mos_version"
+            section: "main restricted"
+            priority: 1000
+
+    ubuntu:
+        -   name: "ubuntu"
+            type: "deb"
+            uri: "http://localhost/ubuntu"
+            suite: "trusty"
+            section: "main multiverse restricted universe"
+            priority: 500
+
+requirements:
+    ubuntu:
+        - "package_deb"

contrib/fuel_mirror/fuel_mirror/tests/test_accessors.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from fuel_mirror.common import accessors
+from fuel_mirror.tests import base
+
+
+class TestAccessors(base.TestCase):
+    def test_get_packetary_accessor(self):
+        packetary = mock.MagicMock()
+        with mock.patch.dict("sys.modules", packetary=packetary):
+            accessor = accessors.get_packetary_accessor(
+                http_proxy="http://localhost",
+                https_proxy="https://localhost",
+                retries_num=1,
+                threads_num=2,
+                ignore_errors_num=3
+            )
+            accessor("deb")
+            accessor("yum")
+            packetary.Configuration.assert_called_once_with(
+                http_proxy="http://localhost",
+                https_proxy="https://localhost",
+                retries_num=1,
+                threads_num=2,
+                ignore_errors_num=3
+            )
+            packetary.Context.assert_called_once_with(
+                packetary.Configuration()
+            )
+            self.assertEqual(2, packetary.RepositoryApi.create.call_count)
+            packetary.RepositoryApi.create.assert_any_call(
+                packetary.Context(), "deb"
+            )
+            packetary.RepositoryApi.create.assert_any_call(
+                packetary.Context(), "yum"
+            )
+
+    @mock.patch("fuel_mirror.common.accessors.os")
+    def test_get_fuel_api_accessor(self, os):
+        fuelclient = mock.MagicMock()
+        patch = {
+            "fuelclient": fuelclient,
+            "fuelclient.objects": fuelclient.objects
+        }
+        with mock.patch.dict("sys.modules", patch):
+            accessor = accessors.get_fuel_api_accessor(
+                "localhost:8080", "guest", "123"
+            )
+            accessor.Environment.get_all()
+
+            os.environ.__setitem__.asseert_any_call(
+                "SERVER_ADDRESS", "localhost"
+            )
+            os.environ.__setitem__.asseert_any_call(
+                "LISTEN_PORT", "8080"
+            )
+            os.environ.__setitem__.asseert_any_call(
+                "KEYSTONE_USER", "guest"
+            )
+            os.environ.__setitem__.asseert_any_call(
+                "KEYSTONE_PASS", "123"
+            )
+            fuelclient.objects.Environment.get_all.assert_called_once_with()
+
+    @mock.patch("fuel_mirror.common.accessors.os")
+    def test_get_fuel_api_accessor_with_default_parameters(self, os):
+        fuelclient = mock.MagicMock()
+        patch = {
+            "fuelclient": fuelclient,
+            "fuelclient.objects": fuelclient.objects
+        }
+        with mock.patch.dict("sys.modules", patch):
+            accessors.get_fuel_api_accessor()
+            os.environ.__setitem__.assert_not_called()

contrib/fuel_mirror/fuel_mirror/tests/test_cli_commands.py

@@ -0,0 +1,317 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+import os.path
+import subprocess
+
+# The cmd2 does not work with python3.5
+# because it tries to get access to the property mswindows,
+# that was removed in 3.5
+subprocess.mswindows = False
+
+from fuel_mirror.commands import apply
+from fuel_mirror.commands import create
+from fuel_mirror.tests import base
+
+
+CONFIG_PATH = os.path.join(
+    os.path.dirname(__file__), "data", "test_config.yaml"
+)
+
+UBUNTU_PATH = os.path.join(
+    os.path.dirname(__file__), "data", "test_ubuntu.yaml"
+)
+
+CENTOS_PATH = os.path.join(
+    os.path.dirname(__file__), "data", "test_centos.yaml"
+)
+
+
+@mock.patch.multiple(
+    "fuel_mirror.app",
+    accessors=mock.DEFAULT,
+)
+class TestCliCommands(base.TestCase):
+    common_argv = [
+        "--config", CONFIG_PATH,
+        "--fuel-server=10.25.0.10",
+        "--fuel-user=test",
+        "--fuel-password=test1"
+    ]
+
+    def start_cmd(self, cmd, argv, data_file):
+        cmd.debug(
+            argv + self.common_argv + ["--input-file", data_file]
+        )
+
+    def _setup_fuel_versions(self, fuel_mock):
+        fuel_mock.FuelVersion.get_all_data.return_value = {
+            "release": "1",
+            "openstack_version": "2"
+        }
+
+    def _create_fuel_release(self, fuel_mock, osname):
+        release = mock.MagicMock(data={
+            "name": "test release",
+            "operating_system": osname,
+            "attributes_metadata": {
+                "editable": {"repo_setup": {"repos": {"value": []}}}
+            }
+        })
+
+        fuel_mock.Release.get_by_ids.return_value = [release]
+        fuel_mock.Release.get_all.return_value = [release]
+        return release
+
+    def _create_fuel_env(self, fuel_mock):
+        env = mock.MagicMock(data={
+            "name": "test",
+            "release_id": 1
+        })
+        env.get_settings_data.return_value = {
+            "editable": {"repo_setup": {"repos": {"value": []}}}
+        }
+        fuel_mock.Environment.get_by_ids.return_value = [env]
+        fuel_mock.Environment.get_all.return_value = [env]
+        return env
+
+    def test_create_mos_ubuntu(self, accessors):
+        self._setup_fuel_versions(accessors.get_fuel_api_accessor())
+        packetary = accessors.get_packetary_accessor()
+
+        self.start_cmd(create, ["--group", "mos"], UBUNTU_PATH)
+        accessors.get_packetary_accessor.assert_called_with(
+            threads_num=1,
+            ignore_errors_num=2,
+            retries_num=3,
+            http_proxy="http://localhost",
+            https_proxy="https://localhost",
+        )
+        packetary.assert_called_with("deb", "x86_64")
+        api = packetary()
+        api.clone_repositories.assert_called_once_with(
+            ['http://localhost/mos mos1 main restricted'],
+            '/var/www/',
+            None, None
+        )
+
+    def test_create_partial_ubuntu(self, accessors):
+        self._setup_fuel_versions(accessors.get_fuel_api_accessor())
+        packetary = accessors.get_packetary_accessor()
+
+        self.start_cmd(create, ["--group", "ubuntu"], UBUNTU_PATH)
+        accessors.get_packetary_accessor.assert_called_with(
+            threads_num=1,
+            ignore_errors_num=2,
+            retries_num=3,
+            http_proxy="http://localhost",
+            https_proxy="https://localhost",
+        )
+        packetary.assert_called_with("deb", "x86_64")
+        api = packetary()
+        api.clone_repositories.assert_called_once_with(
+            ['http://localhost/ubuntu trusty '
+             'main multiverse restricted universe'],
+            '/var/www/',
+            ['http://localhost/mos mos1 main restricted'],
+            ['package_deb']
+        )
+
+    def test_create_mos_centos(self, accessors):
+        self._setup_fuel_versions(accessors.get_fuel_api_accessor())
+        packetary = accessors.get_packetary_accessor()
+
+        self.start_cmd(create, ["--group", "mos"], CENTOS_PATH)
+        accessors.get_packetary_accessor.assert_called_with(
+            threads_num=1,
+            ignore_errors_num=2,
+            retries_num=3,
+            http_proxy="http://localhost",
+            https_proxy="https://localhost",
+        )
+        packetary.assert_called_with("rpm", "x86_64")
+        api = packetary()
+        api.clone_repositories.assert_called_once_with(
+            ['http://localhost/mos1'],
+            '/var/www/',
+            None, None
+        )
+
+    def test_create_partial_centos(self, accessors):
+        self._setup_fuel_versions(accessors.get_fuel_api_accessor())
+        packetary = accessors.get_packetary_accessor()
+
+        self.start_cmd(create, ["--group", "centos"], CENTOS_PATH)
+        accessors.get_packetary_accessor.assert_called_with(
+            threads_num=1,
+            ignore_errors_num=2,
+            retries_num=3,
+            http_proxy="http://localhost",
+            https_proxy="https://localhost",
+        )
+        packetary.assert_called_with("rpm", "x86_64")
+        api = packetary()
+        api.clone_repositories.assert_called_once_with(
+            ['http://localhost/centos/os'],
+            '/var/www/',
+            ['http://localhost/mos1'],
+            ["package_rpm"]
+        )
+
+    def test_apply_for_ubuntu_based_env(self, accessors):
+        fuel = accessors.get_fuel_api_accessor()
+        self._setup_fuel_versions(fuel)
+        env = self._create_fuel_env(fuel)
+        self._create_fuel_release(fuel, "Ubuntu")
+        self.start_cmd(
+            apply, ['--group', 'mos', 'ubuntu', '--env', '1'],
+            UBUNTU_PATH
+        )
+        accessors.get_fuel_api_accessor.assert_called_with(
+            "10.25.0.10", "test", "test1"
+        )
+        fuel.FuelVersion.get_all_data.assert_called_once_with()
+        env.set_settings_data.assert_called_with(
+            {'editable': {'repo_setup': {'repos': {'value': [
+                {
+                    'priority': 1000,
+                    'name': 'mos',
+                    'suite': 'mos1',
+                    'section': 'main restricted',
+                    'type': 'deb',
+                    'uri': 'http://10.25.0.10:8080/mos'
+                },
+                {
+                    'priority': 500,
+                    'name': 'ubuntu',
+                    'suite': 'trusty',
+                    'section': 'main multiverse restricted universe',
+                    'type': 'deb',
+                    'uri': 'http://10.25.0.10:8080/ubuntu'
+                }
+            ]}}}}
+        )
+
+    def test_apply_for_centos_based_env(self, accessors):
+        fuel = accessors.get_fuel_api_accessor()
+        self._setup_fuel_versions(fuel)
+        env = self._create_fuel_env(fuel)
+        self._create_fuel_release(fuel, "CentOS")
+        self.start_cmd(
+            apply, ['--group', 'mos', 'centos', '--env', '1'],
+            CENTOS_PATH
+        )
+        accessors.get_fuel_api_accessor.assert_called_with(
+            "10.25.0.10", "test", "test1"
+        )
+        fuel.FuelVersion.get_all_data.assert_called_once_with()
+        env.set_settings_data.assert_called_with(
+            {'editable': {'repo_setup': {'repos': {'value': [
+                {
+                    'priority': 5,
+                    'name': 'centos',
+                    'type': 'rpm',
+                    'uri': 'http://10.25.0.10:8080/centos/os/x86_64'
+                },
+                {
+                    'priority': 10,
+                    'name': 'mos',
+                    'type': 'rpm',
+                    'uri': 'http://10.25.0.10:8080/mos1/x86_64'
+                }]
+            }}}}
+        )
+
+    def test_apply_for_ubuntu_release(self, accessors):
+        fuel = accessors.get_fuel_api_accessor()
+        self._setup_fuel_versions(fuel)
+        env = self._create_fuel_env(fuel)
+        release = self._create_fuel_release(fuel, "Ubuntu")
+        self.start_cmd(
+            apply, ['--group', 'mos', 'ubuntu', '--default'],
+            UBUNTU_PATH
+        )
+        accessors.get_fuel_api_accessor.assert_called_with(
+            "10.25.0.10", "test", "test1"
+        )
+        fuel.FuelVersion.get_all_data.assert_called_once_with()
+        self.assertEqual(1, env.set_settings_data.call_count)
+        release.connection.put_request.assert_called_once_with(
+            release.instance_api_path.format(),
+            {
+                'name': "test release",
+                'operating_system': 'Ubuntu',
+                'attributes_metadata': {
+                    'editable': {'repo_setup': {'repos': {'value': [
+                        {
+                            'name': 'mos',
+                            'priority': 1000,
+                            'suite': 'mos1',
+                            'section': 'main restricted',
+                            'type': 'deb',
+                            'uri': 'http://10.25.0.10:8080/mos'
+                        },
+                        {
+                            'name': 'ubuntu',
+                            'priority': 500,
+                            'suite': 'trusty',
+                            'section': 'main multiverse restricted universe',
+                            'type': 'deb',
+                            'uri': 'http://10.25.0.10:8080/ubuntu'
+                        }
+                    ]}}}
+                }
+            }
+        )
+
+    def test_apply_for_centos_release(self, accessors):
+        fuel = accessors.get_fuel_api_accessor()
+        self._setup_fuel_versions(fuel)
+        env = self._create_fuel_env(fuel)
+        release = self._create_fuel_release(fuel, "CentOS")
+        self.start_cmd(
+            apply, ['--group', 'mos', 'centos', '--default'],
+            CENTOS_PATH
+        )
+        accessors.get_fuel_api_accessor.assert_called_with(
+            "10.25.0.10", "test", "test1"
+        )
+        fuel.FuelVersion.get_all_data.assert_called_once_with()
+        self.assertEqual(1, env.set_settings_data.call_count)
+        release.connection.put_request.assert_called_once_with(
+            release.instance_api_path.format(),
+            {
+                'name': "test release",
+                'operating_system': 'CentOS',
+                'attributes_metadata': {
+                    'editable': {'repo_setup': {'repos': {'value': [
+                        {
+                            'name': 'centos',
+                            'priority': 5,
+                            'type': 'rpm',
+                            'uri': 'http://10.25.0.10:8080/centos/os/x86_64'
+                        },
+                        {
+                            'name': 'mos',
+                            'priority': 10,
+                            'type': 'rpm',
+                            'uri': 'http://10.25.0.10:8080/mos1/x86_64'
+                        },
+                    ]}}}
+                }
+            }
+        )

contrib/fuel_mirror/fuel_mirror/tests/test_url_builder.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from fuel_mirror.common import url_builder
+from fuel_mirror.tests import base
+
+
+class TestUrlBuilder(base.TestCase):
+    def test_get_url_builder(self):
+        self.assertTrue(issubclass(
+            url_builder.get_url_builder("deb"),
+            url_builder.AptRepoUrlBuilder
+        ))
+        self.assertTrue(issubclass(
+            url_builder.get_url_builder("rpm"),
+            url_builder.YumRepoUrlBuilder
+        ))
+        with self.assertRaises(KeyError):
+            url_builder.get_url_builder("unknown")
+
+
+class TestAptUrlBuilder(base.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.builder = url_builder.get_url_builder("deb")
+        cls.repo_data = {
+            "name": "ubuntu",
+            "suite": "trusty",
+            "section": "main restricted",
+            "type": "deb",
+            "uri": "http://localhost/ubuntu"
+        }
+
+    def test_get_repo_url(self):
+        self.assertEqual(
+            "http://localhost/ubuntu trusty main restricted",
+            self.builder.get_repo_url(self.repo_data)
+        )
+
+
+class TestYumUrlBuilder(base.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.builder = url_builder.get_url_builder("rpm")
+        cls.repo_data = {
+            "name": "centos",
+            "type": "rpm",
+            "uri": "http://localhost/os/x86_64"
+        }
+
+    def test_get_repo_url(self):
+        self.assertEqual(
+            "http://localhost/os",
+            self.builder.get_repo_url(self.repo_data)
+        )

contrib/fuel_mirror/fuel_mirror/tests/test_utils.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+import six
+
+from fuel_mirror.common import utils
+from fuel_mirror.tests import base
+
+
+class DictAsObj(object):
+    def __init__(self, d):
+        self.__dict__.update(d)
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__
+
+
+class TestUtils(base.TestCase):
+    def test_lists_merge(self):
+        main = [{"a": 1, "b": 2, "c": 0}, {"a": 2, "b": 3, "c": 1}]
+        patch = [{"a": 2, "b": 4}, {"a": 3, "b": 5}]
+        utils.lists_merge(
+            main,
+            patch,
+            key="a"
+        )
+        self.assertItemsEqual(
+            [{"a": 1, "b": 2, "c": 0},
+             {"a": 2, "b": 4, "c": 1},
+             {"a": 3, "b": 5}],
+            main
+        )
+
+    def test_first(self):
+        self.assertEqual(
+            1,
+            utils.first(0, 1, 0),
+        )
+        self.assertEqual(
+            1,
+            utils.first(None, [], '', 1),
+        )
+        self.assertIsNone(
+            utils.first(None, [], 0, ''),
+        )
+        self.assertIsNone(
+            utils.first(),
+        )
+
+    def test_is_subdict(self):
+        self.assertFalse(utils.is_subdict({"c": 1}, {"a": 1, "b": 1}))
+        self.assertFalse(utils.is_subdict({"a": 1, "b": 2}, {"a": 1, "b": 1}))
+        self.assertFalse(
+            utils.is_subdict({"a": 1, "b": 1, "c": 2}, {"a": 1, "b": 1})
+        )
+        self.assertFalse(
+            utils.is_subdict({"a": 1, "b": None}, {"a": 1})
+        )
+        self.assertTrue(utils.is_subdict({}, {"a": 1}))
+        self.assertTrue(utils.is_subdict({"a": 1}, {"a": 1, "b": 1}))
+        self.assertTrue(utils.is_subdict({"a": 1, "b": 1}, {"a": 1, "b": 1}))
+
+    @mock.patch("fuel_mirror.common.utils.open")
+    def test_get_fuel_settings(self, m_open):
+        m_open().__enter__.side_effect = [
+            six.StringIO(
+                'ADMIN_NETWORK:\n'
+                '  ipaddress: "10.20.0.4"\n'
+                'FUEL_ACCESS:\n'
+                '  user: "test"\n'
+                '  password: "test_pwd"\n',
+            ),
+            OSError
+        ]
+
+        self.assertEqual(
+            {
+                "server": "10.20.0.4",
+                "user": "test",
+                "password": "test_pwd",
+            },
+            utils.get_fuel_settings()
+        )
+
+        self.assertEqual(
+            {
+                "server": "10.20.0.2",
+                "user": None,
+                "password": None,
+            },
+            utils.get_fuel_settings()
+        )

contrib/fuel_mirror/fuel_mirror/tests/test_validate_config.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os.path
+import yaml
+
+from fuel_mirror.tests import base
+
+
+DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "data")
+
+
+class TestValidateConfigs(base.TestCase):
+    def test_validate_data_files(self):
+        for f in os.listdir(DATA_DIR):
+            with open(os.path.join(DATA_DIR, f), "r") as fd:
+                data = yaml.load(fd)
+                # TODO(add input data validation scheme)
+                self.assertIn("groups", data)
+                self.assertIn("fuel_release_match", data)

contrib/fuel_mirror/requirements.txt

@@ -0,0 +1,9 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+pbr>=0.8
+Babel>=1.3
+cliff>=1.7.0
+six>=1.5.2
+packetary

contrib/fuel_mirror/scripts/fuel-createmirror

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+echo "This script is DEPRECATED. Please use fuel-mirror utility!"
+
+# This shell script was wraps the fuel-mirror utility to provide backward compatibility
+# with previous version of tool.
+
+usage() {
+cat <<EOF
+Usage: `basename $0` [options]
+
+Create and update local mirrors of MOS and/or Ubuntu.
+
+IMPORTANT!
+If NO parameters specified, this script will:
+- Create/Update both MOS and Ubuntu local mirrors
+- Set them as repositories for existing NEW environments in Fuel UI
+- Set them as DEFAULT repositories for new environments
+
+Options:
+
+  -h| --help          This help screen.
+  -d| --no-default    Don't change default repositories for new environments
+  -a| --no-apply      Don't apply changes to Fuel environments
+  -M| --mos           Create/Update MOS local mirror only
+  -U| --ubuntu        Create/Update Ubuntu local mirror only
+  -p| --password      Fuel Master admin password (defaults to admin)
+
+EOF
+}
+
+# Parse options
+OPTS=`getopt -o hdaMUNp: -l help,no-default,no-apply,mos,ubuntu,password:,dry-run -- "$@"`
+if [ $? != 0 ]; then
+    usage
+    exit 1
+fi
+
+eval set -- "$OPTS"
+
+CMD_OPTS="--pattern=ubuntu"
+
+while true ; do
+    case "$1" in
+        -h| --help        ) usage ; exit 0;;
+        -d | --no-default ) OPT_NO_DEFAULT=1; shift;;
+        -a | --no-apply   ) OPT_NO_APPLY=1; shift;;
+        -N | --dry-run    ) EXEC_PREFIX="echo  EXEC  "; shift;;
+        -M | --mos        ) GROUPS="$GROUPS mos"; shift;;
+        -U | --ubuntu     ) GROUPS="$GROUPS ubuntu"; shift;;
+        -p | --password   ) CMD_OPTS="$CMD_OPTS --fuel-password=$2"; shift; shift;;
+        --                ) shift; break;;
+        *                 ) break;;
+    esac
+done
+
+if [[ "$@" != "" ]]; then
+    echo "Invalid option -- $@"
+    usage
+    exit 1
+fi
+
+if [[ "$GROUPS" == "" ]]; then
+    GROUPS="mos ubuntu"
+fi
+
+CMD_OPTS="CMD_OPTS --group $GROUPS"
+
+$EXEC_PREFIX fuel-mirror create ${CMD_OPTS}
+
+if [[ "$OPT_NO_DEFAULT" == "" ]]; then
+    CMD_OPTS="$CMD_OPTS --default"
+fi
+
+if [[ "OPT_NO_APPLY" == "" ]]; then
+    CMD_OPTS="$CMD_OPTS --apply"
+fi
+
+$EXEC_PREFIX fuel-mirror apply ${CMD_OPTS}

contrib/fuel_mirror/setup.cfg

@@ -0,0 +1,61 @@
+[metadata]
+name = fuel_mirror
+summary = The Utility to create local repositories with packages is
+          required for openstack deployment.
+description-file =
+    README.rst
+author = Mirantis Inc.
+author_email = product@mirantis.com
+url = http://mirantis.com
+home-page = http://mirantis.com
+classifier =
+    Development Status :: 4 - Beta
+    Environment :: OpenStack
+    Intended Audience :: Information Technology
+    Intended Audience :: System Administrators
+    License :: OSI Approved :: Apache Software License
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python
+    Programming Language :: Python :: 2
+    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.3
+    Programming Language :: Python :: 3.4
+    Topic :: Utilities
+
+[files]
+packages =
+    fuel_mirror
+data_files =
+    etc/fuel-mirror = etc/*
+    share/fuel-mirror = data/*
+
+[build_sphinx]
+source-dir = doc/source
+build-dir = doc/build
+all_files = 1
+
+[entry_points]
+console_scripts =
+    fuel-mirror=fuel_mirror.app:main
+
+fuel_mirror =
+    apply=fuel_mirror.commands.apply:ApplyCommand
+    create=fuel_mirror.commands.create:CreateCommand
+
+[upload_sphinx]
+upload-dir = doc/build/html
+
+[compile_catalog]
+directory = locale
+domain = fuel_mirror
+
+[update_catalog]
+domain = fuel_mirror
+output_dir = locale
+input_file = locale/fuel_mirror.pot
+
+[extract_messages]
+keywords = _ gettext ngettext l_ lazy_gettext
+mapping_file = babel.cfg
+output_file = locale/fuel_mirror.pot

contrib/fuel_mirror/setup.py

@@ -0,0 +1,28 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
+import setuptools
+
+# In python < 2.7.4, a lazy loading of package `pbr` will break
+# setuptools if some other modules registered functions in `atexit`.
+# solution from: http://bugs.python.org/issue15881#msg170215
+try:
+    import multiprocessing  # noqa
+except ImportError:
+    pass
+
+setuptools.setup(
+    setup_requires=['pbr'],
+    pbr=True)

contrib/fuel_mirror/test-requirements.txt

@@ -0,0 +1,17 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+hacking<0.11,>=0.10.0
+
+coverage>=3.6
+discover
+python-subunit>=0.0.18
+sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
+oslosphinx>=2.5.0 # Apache-2.0
+oslotest>=1.10.0 # Apache-2.0
+testrepository>=0.0.18
+testscenarios>=0.4
+testtools>=1.4.0
+cliff>=1.7.0
+six>=1.5.2

contrib/fuel_mirror/tox.ini

@@ -0,0 +1,35 @@
+[tox]
+minversion = 1.6
+envlist = py34,py27,py26,pep8
+skipsdist = True
+
+[testenv]
+usedevelop = True
+install_command = pip install -U {opts} {packages}
+setenv =
+   VIRTUAL_ENV={envdir}
+deps = -r{toxinidir}/test-requirements.txt
+commands = python setup.py test --slowest --testr-args='{posargs:fuel_mirror}'
+
+[testenv:pep8]
+commands = flake8
+
+[testenv:venv]
+commands = {posargs}
+
+[testenv:cover]
+commands = python setup.py test --coverage --testr-args='{posargs:fuel_mirror}'
+
+[testenv:docs]
+commands = python setup.py build_sphinx
+
+[testenv:debug]
+commands = oslo_debug_helper {posargs}
+
+[flake8]
+# E123, E125 skipped as they are invalid PEP-8.
+
+show-source = True
+ignore = E123,E125
+builtins = _
+exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build

packetary/drivers/deb_driver.py

@@ -180,21 +180,19 @@ class DebRepositoryDriver(RepositoryDriverBase):
 
     def fork_repository(self, connection, repository, destination,
                         source=False, locale=False):
-        """Overrides method of superclass."""
         # TODO(download gpk)
         # TODO(sources and locales)
-        if not destination.endswith(os.path.sep):
-            destination += os.path.sep
-
-        clone = copy.copy(repository)
-        clone.url = destination
+        new_repo = copy.copy(repository)
+        new_repo.url = utils.localize_repo_url(destination, repository.url)
         packages_file = utils.get_path_from_url(
-            self._get_url_of_metafile(clone, "Packages")
+            self._get_url_of_metafile(new_repo, "Packages")
         )
         release_file = utils.get_path_from_url(
-            self._get_url_of_metafile(clone, "Release")
+            self._get_url_of_metafile(new_repo, "Release")
+        )
+        self.logger.info(
+            "clone repository %s to %s", repository, new_repo.url
         )
-        self.logger.info("clone repository %s to %s", repository, destination)
         utils.ensure_dir_exist(os.path.dirname(release_file))
 
         release = deb822.Release()
@@ -208,7 +206,7 @@ class DebRepositoryDriver(RepositoryDriverBase):
 
         open(packages_file, "ab").close()
         gzip.open(packages_file + ".gz", "ab").close()
-        return clone
+        return new_repo
 
     def _update_suite_index(self, repository):
         """Updates the Release file in the suite."""

packetary/drivers/rpm_driver.py

@@ -72,15 +72,10 @@ class RpmRepositoryDriver(RepositoryDriverBase):
         return (url.rstrip("/") for url in urls)
 
     def get_repository(self, connection, url, arch, consumer):
-        """Overrides method of superclass."""
-        # Currently supported repositories, that has URL in following format:
-        # baseurl/{name}/{architecture}
-        # because the architecture is sentetic part of rpm repository URL
-        name = url.rsplit("/", 1)[-1]
-        baseurl = "/".join((url, arch, ""))
+        name = utils.get_path_from_url(url, False)
         consumer(Repository(
             name=name,
-            url=baseurl,
+            url=url + "/",
             architecture=arch,
             origin=""
         ))
@@ -165,17 +160,14 @@ class RpmRepositoryDriver(RepositoryDriverBase):
 
     def fork_repository(self, connection, repository, destination,
                         source=False, locale=False):
-        """Overrides method of superclass."""
         # TODO(download gpk)
         # TODO(sources and locales)
-        destination = os.path.join(
-            destination, repository.name,
-            repository.architecture, ""
-        )
         new_repo = copy.copy(repository)
-        new_repo.url = destination
-        self.logger.info("clone repository %s to %s", repository, destination)
-        utils.ensure_dir_exist(destination)
+        new_repo.url = utils.localize_repo_url(destination, repository.url)
+        self.logger.info(
+            "clone repository %s to %s", repository, new_repo.url
+        )
+        utils.ensure_dir_exist(new_repo.url)
         self.rebuild_repository(new_repo, set())
         return new_repo
 

packetary/library/utils.py

@@ -90,6 +90,16 @@ def get_path_from_url(url, ensure_file=True):
     return comps.path
 
 
+def localize_repo_url(localurl, repo_url):
+    """Gets local repository url.
+
+    :param localurl: the base local URL
+    :param repo_url: the origin URL of repository
+    :return: localurl + get_path_from_url(repo_url)
+    """
+    return localurl.rstrip("/") + urlparse(repo_url).path
+
+
 def ensure_dir_exist(path):
     """Creates directory if it does not exist.
 

packetary/tests/test_deb_driver.py

@@ -20,6 +20,7 @@ import six
 
 
 from packetary.drivers import deb_driver
+from packetary.library.utils import localize_repo_url
 from packetary.tests import base
 from packetary.tests.stubs.generator import gen_package
 from packetary.tests.stubs.generator import gen_repository
@@ -188,26 +189,29 @@ class TestDebDriver(base.TestCase):
         os.path.sep = "/"
         os.path.join = lambda *x: "/".join(x)
         utils.get_path_from_url = lambda x: x
-        repo = gen_repository(name=("trusty", "main"), url="http://localhost")
+        utils.localize_repo_url = localize_repo_url
+        repo = gen_repository(
+            name=("trusty", "main"), url="http://localhost/test/"
+        )
         files = [
             mock.MagicMock(),
             mock.MagicMock()
         ]
         open.side_effect = files
-        clone = self.driver.fork_repository(self.connection, repo, "/root")
-        self.assertEqual(repo.name, clone.name)
-        self.assertEqual(repo.architecture, clone.architecture)
-        self.assertEqual(repo.origin, clone.origin)
-        self.assertEqual("/root/", clone.url)
+        new_repo = self.driver.fork_repository(self.connection, repo, "/root")
+        self.assertEqual(repo.name, new_repo.name)
+        self.assertEqual(repo.architecture, new_repo.architecture)
+        self.assertEqual(repo.origin, new_repo.origin)
+        self.assertEqual("/root/test/", new_repo.url)
         utils.ensure_dir_exist.assert_called_once_with(os.path.dirname())
         open.assert_any_call(
-            "/root/dists/trusty/main/binary-amd64/Release", "wb"
+            "/root/test/dists/trusty/main/binary-amd64/Release", "wb"
         )
         open.assert_any_call(
-            "/root/dists/trusty/main/binary-amd64/Packages", "ab"
+            "/root/test/dists/trusty/main/binary-amd64/Packages", "ab"
         )
         gzip.open.assert_called_once_with(
-            "/root/dists/trusty/main/binary-amd64/Packages.gz", "ab"
+            "/root/test/dists/trusty/main/binary-amd64/Packages.gz", "ab"
         )
 
     @mock.patch.multiple(

packetary/tests/test_rpm_driver.py

@@ -20,6 +20,7 @@ import sys
 
 import six
 
+from packetary.library.utils import localize_repo_url
 from packetary.objects import FileChecksum
 from packetary.tests import base
 from packetary.tests.stubs.generator import gen_repository
@@ -65,12 +66,15 @@ class TestRpmDriver(base.TestCase):
         repos = []
 
         self.driver.get_repository(
-            self.connection, "http://host/centos/os", "x86_64", repos.append
+            self.connection,
+            "http://host/centos/os/x86_64",
+            "x86_64",
+            repos.append
         )
 
         self.assertEqual(1, len(repos))
         repo = repos[0]
-        self.assertEqual("os", repo.name)
+        self.assertEqual("/centos/os/x86_64", repo.name)
         self.assertEqual("", repo.origin)
         self.assertEqual("x86_64", repo.architecture)
         self.assertEqual("http://host/centos/os/x86_64/", repo.url)
@@ -189,16 +193,17 @@ class TestRpmDriver(base.TestCase):
 
     @mock.patch("packetary.drivers.rpm_driver.utils")
     def test_fork_repository(self, utils):
-        repo = gen_repository("os", url="http://localhost/os/x86_64")
-        clone = self.driver.fork_repository(
+        repo = gen_repository("os", url="http://localhost/os/x86_64/")
+        utils.localize_repo_url = localize_repo_url
+        new_repo = self.driver.fork_repository(
             self.connection,
             repo,
             "/repo"
         )
 
         utils.ensure_dir_exist.assert_called_once_with("/repo/os/x86_64/")
-        self.assertEqual(repo.name, clone.name)
-        self.assertEqual(repo.architecture, clone.architecture)
-        self.assertEqual("/repo/os/x86_64/", clone.url)
+        self.assertEqual(repo.name, new_repo.name)
+        self.assertEqual(repo.architecture, new_repo.architecture)
+        self.assertEqual("/repo/os/x86_64/", new_repo.url)
         self.createrepo.MetaDataGenerator()\
             .doFinalMove.assert_called_once_with()