From 671af8e61112cf8d90ba861364902c6b1c494b43 Mon Sep 17 00:00:00 2001 From: Bulat Gaifullin Date: Wed, 21 Oct 2015 12:58:54 +0300 Subject: [PATCH] [packetary] Repository class class Repository composes from: * RepositryDriver - low-level support for physical repository. deb, yum, etc. * RepositoryController - infrastcuture method to communicate with driver * RepositoryApi - high-level class, that provides methods to work with repository Change-Id: Iaf868fca982d91089e369d13a6fb381ff879ea73 Implements: blueprint refactor-local-mirror-scripts Partial-Bug: #1487077 --- packetary/__init__.py | 11 +- packetary/api.py | 217 +++++++++++++++++ packetary/controllers/__init__.py | 21 ++ packetary/controllers/repository.py | 169 ++++++++++++++ packetary/drivers/__init__.py | 0 packetary/drivers/base.py | 80 +++++++ packetary/objects/__init__.py | 2 + packetary/objects/packages_tree.py | 121 ++++++++++ packetary/objects/statistics.py | 45 ++++ packetary/tests/stubs/executor.py | 27 +++ packetary/tests/stubs/helpers.py | 39 ++++ packetary/tests/test_packages_tree.py | 110 +++++++++ packetary/tests/test_repository_api.py | 230 +++++++++++++++++++ packetary/tests/test_repository_contoller.py | 143 ++++++++++++ requirements.txt | 1 + 15 files changed, 1215 insertions(+), 1 deletion(-) create mode 100644 packetary/api.py create mode 100644 packetary/controllers/__init__.py create mode 100644 packetary/controllers/repository.py create mode 100644 packetary/drivers/__init__.py create mode 100644 packetary/drivers/base.py create mode 100644 packetary/objects/packages_tree.py create mode 100644 packetary/objects/statistics.py create mode 100644 packetary/tests/stubs/executor.py create mode 100644 packetary/tests/stubs/helpers.py create mode 100644 packetary/tests/test_packages_tree.py create mode 100644 packetary/tests/test_repository_api.py create mode 100644 packetary/tests/test_repository_contoller.py diff --git a/packetary/__init__.py b/packetary/__init__.py index 84dd670..432f8be 100644 --- a/packetary/__init__.py +++ b/packetary/__init__.py @@ -14,9 +14,18 @@ # License for the specific language governing permissions and limitations # under the License. - import pbr.version +from packetary.api import Configuration +from packetary.api import Context +from packetary.api import RepositoryApi + + +__all__ = [ + "Configuration", + "Context", + "RepositoryApi", +] __version__ = pbr.version.VersionInfo( 'packetary').version_string() diff --git a/packetary/api.py b/packetary/api.py new file mode 100644 index 0000000..c253714 --- /dev/null +++ b/packetary/api.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +import six + +from packetary.controllers import RepositoryController +from packetary.library.connections import ConnectionsManager +from packetary.library.executor import AsynchronousSection +from packetary.objects import Index +from packetary.objects import PackageRelation +from packetary.objects import PackagesTree +from packetary.objects.statistics import CopyStatistics + + +logger = logging.getLogger(__package__) + + +class Configuration(object): + """The configuration holder.""" + + def __init__(self, http_proxy=None, https_proxy=None, + retries_num=0, threads_num=0, + ignore_errors_num=0): + """Initialises. + + :param http_proxy: the url of proxy for connections over http, + no-proxy will be used if it is not specified + :param https_proxy: the url of proxy for connections over https, + no-proxy will be used if it is not specified + :param retries_num: the number of retries on errors + :param threads_num: the max number of active threads + :param ignore_errors_num: the number of errors that may occurs + before stop processing + """ + + self.http_proxy = http_proxy + self.https_proxy = https_proxy + self.ignore_errors_num = ignore_errors_num + self.retries_num = retries_num + self.threads_num = threads_num + + +class Context(object): + """The infra-objects holder.""" + + def __init__(self, config): + """Initialises. + + :param config: the configuration + """ + self._connection = ConnectionsManager( + proxy=config.http_proxy, + secure_proxy=config.https_proxy, + retries_num=config.retries_num + ) + self._threads_num = config.threads_num + self._ignore_errors_num = config.ignore_errors_num + + @property + def connection(self): + """Gets the connection.""" + return self._connection + + def async_section(self, ignore_errors_num=None): + """Gets the execution scope. + + :param ignore_errors_num: custom value for ignore_errors_num, + the class value is used if omitted. + """ + if ignore_errors_num is None: + ignore_errors_num = self._ignore_errors_num + + return AsynchronousSection(self._threads_num, ignore_errors_num) + + +class RepositoryApi(object): + """Provides high-level API to operate with repositories.""" + + def __init__(self, controller): + """Initialises. + + :param controller: the repository controller. + """ + self.controller = controller + + @classmethod + def create(cls, config, repotype, repoarch): + """Creates the repository API instance. + + :param config: the configuration + :param repotype: the kind of repository(deb, yum, etc) + :param repoarch: the architecture of repository (x86_64 or i386) + """ + context = config if isinstance(config, Context) else Context(config) + return cls(RepositoryController.load(context, repotype, repoarch)) + + def get_packages(self, origin, debs=None, requirements=None): + """Gets the list of packages from repository(es). + + :param origin: The list of repository`s URLs + :param debs: the list of repository`s URL to calculate list of + dependencies, that will be used to filter packages. + :param requirements: the list of package relations, + to resolve the list of mandatory packages. + :return: the set of packages + """ + repositories = self._get_repositories(origin) + return self._get_packages(repositories, debs, requirements) + + def clone_repositories(self, origin, destination, debs=None, + requirements=None, keep_existing=True, + include_source=False, include_locale=False): + """Creates the clones of specified repositories in local folder. + + :param origin: The list of repository`s URLs + :param destination: the destination folder path + :param debs: the list of repository`s URL to calculate list of + dependencies, that will be used to filter packages. + :param requirements: the list of package relations, + to resolve the list of mandatory packages. + :param keep_existing: If False - local packages that does not exist + in original repo will be removed. + :param include_source: if True, the source packages + will be copied as well. + :param include_locale: if True, the locales + will be copied as well. + :return: count of copied and total packages. + """ + repositories = self._get_repositories(origin) + packages = self._get_packages(repositories, debs, requirements) + mirrors = self.controller.clone_repositories( + repositories, destination, include_source, include_locale + ) + + package_groups = dict((x, set()) for x in repositories) + for pkg in packages: + package_groups[pkg.repository].add(pkg) + + stat = CopyStatistics() + for repo, packages in six.iteritems(package_groups): + mirror = mirrors[repo] + logger.info("copy packages from - %s", repo) + self.controller.copy_packages( + mirror, packages, keep_existing, stat.on_package_copied + ) + return stat + + def get_unresolved_dependencies(self, urls): + """Gets list of unresolved dependencies for repository(es). + + :param urls: The list of repository`s URLs + :return: list of unresolved dependencies + """ + packages = PackagesTree() + self.controller.load_packages( + self._get_repositories(urls), + packages.add + ) + return packages.get_unresolved_dependencies() + + def _get_repositories(self, urls): + """Gets the set of repositories by url.""" + repositories = set() + self.controller.load_repositories(urls, repositories.add) + return repositories + + def _get_packages(self, repositories, master, requirements): + """Gets the list of packages according to master and requirements.""" + if master is None and requirements is None: + packages = set() + self.controller.load_packages(repositories, packages.add) + return packages + + packages = PackagesTree() + self.controller.load_packages(repositories, packages.add) + if master is not None: + main_index = Index() + self.controller.load_packages( + self._get_repositories(master), + main_index.add + ) + else: + main_index = None + + return packages.get_minimal_subset( + main_index, + self._parse_requirements(requirements) + ) + + @staticmethod + def _parse_requirements(requirements): + """Gets the list of relations from requirements. + + :param requirements: the list of requirement in next format: + 'name [cmp version]|[alt [cmp version]]' + """ + if requirements is not None: + return set( + PackageRelation.from_args( + *(x.split() for x in r.split("|"))) for r in requirements + ) + return set() diff --git a/packetary/controllers/__init__.py b/packetary/controllers/__init__.py new file mode 100644 index 0000000..ce36712 --- /dev/null +++ b/packetary/controllers/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from packetary.controllers.repository import RepositoryController + +__all__ = [ + "RepositoryController" +] diff --git a/packetary/controllers/repository.py b/packetary/controllers/repository.py new file mode 100644 index 0000000..29ae993 --- /dev/null +++ b/packetary/controllers/repository.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os + +import six +import stevedore + + +logger = logging.getLogger(__package__) + +urljoin = six.moves.urllib.parse.urljoin + + +class RepositoryController(object): + """Implements low-level functionality to communicate with drivers.""" + + _drivers = None + + def __init__(self, context, driver, arch): + self.context = context + self.driver = driver + self.arch = arch + + @classmethod + def load(cls, context, driver_name, repoarch): + """Creates the repository manager. + + :param context: the context + :param driver_name: the name of required driver + :param repoarch: the architecture of repository (x86_64 or i386) + """ + if cls._drivers is None: + cls._drivers = stevedore.ExtensionManager( + "packetary.drivers", invoke_on_load=True + ) + try: + driver = cls._drivers[driver_name].obj + except KeyError: + raise NotImplementedError( + "The driver {0} is not supported yet.".format(driver_name) + ) + return cls(context, driver, repoarch) + + def load_repositories(self, urls, consumer): + """Loads the repository objects from url. + + :param urls: the list of repository urls. + :param consumer: the callback to consume objects + """ + if isinstance(urls, six.string_types): + urls = [urls] + + connection = self.context.connection + for parsed_url in self.driver.parse_urls(urls): + self.driver.get_repository( + connection, parsed_url, self.arch, consumer + ) + + def load_packages(self, repositories, consumer): + """Loads packages from repository. + + :param repositories: the repository object + :param consumer: the callback to consume objects + """ + connection = self.context.connection + for r in repositories: + self.driver.get_packages(connection, r, consumer) + + def assign_packages(self, repository, packages, keep_existing=True): + """Assigns new packages to the repository. + + It replaces the current repository`s packages. + + :param repository: the target repository + :param packages: the set of new packages + :param keep_existing: + if True, all existing packages will be kept as is. + if False, all existing packages, that are not included + to new packages will be removed. + """ + + if not isinstance(packages, set): + packages = set(packages) + else: + packages = packages.copy() + + if keep_existing: + consume_exist = packages.add + else: + def consume_exist(package): + if package not in packages: + filepath = os.path.join( + package.repository.url, package.filename + ) + logger.info("remove package - %s.", filepath) + os.remove(filepath) + + self.driver.get_packages( + self.context.connection, repository, consume_exist + ) + self.driver.rebuild_repository(repository, packages) + + def copy_packages(self, repository, packages, keep_existing, observer): + """Copies packages to repository. + + :param repository: the target repository + :param packages: the set of packages + :param keep_existing: see assign_packages for more details + :param observer: the package copying process observer + """ + with self.context.async_section() as section: + for package in packages: + section.execute( + self._copy_package, repository, package, observer + ) + self.assign_packages(repository, packages, keep_existing) + + def clone_repositories(self, repositories, destination, + source=False, locale=False): + """Creates copy of repositories. + + :param repositories: the origin repositories + :param destination: the target folder + :param source: If True, the source packages will be copied too. + :param locale: If True, the localisation will be copied too. + :return: the mapping origin to cloned repository. + """ + mirros = dict() + destination = os.path.abspath(destination) + with self.context.async_section(0) as section: + for r in repositories: + section.execute( + self._clone_repository, + r, destination, source, locale, mirros + ) + return mirros + + def _clone_repository(self, r, destination, source, locale, mirrors): + """Creates clone of repository and stores it in mirrors.""" + clone = self.driver.clone_repository( + self.context.connection, r, destination, source, locale + ) + mirrors[r] = clone + + def _copy_package(self, target, package, observer): + """Synchronises remote file to local fs.""" + dst_path = os.path.join(target.url, package.filename) + src_path = urljoin(package.repository.url, package.filename) + bytes_copied = self.context.connection.retrieve( + src_path, dst_path, size=package.filesize + ) + if package.filesize < 0: + package.filesize = bytes_copied + observer(bytes_copied) diff --git a/packetary/drivers/__init__.py b/packetary/drivers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packetary/drivers/base.py b/packetary/drivers/base.py new file mode 100644 index 0000000..f58340b --- /dev/null +++ b/packetary/drivers/base.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import logging + +import six + + +@six.add_metaclass(abc.ABCMeta) +class RepositoryDriverBase(object): + """The super class for Repository Drivers. + + For implementing support of new type of repository: + - inherit this class + - implement all abstract methods + - register implementation in 'packetary.drivers' namespace + """ + def __init__(self): + self.logger = logging.getLogger(__package__) + + @abc.abstractmethod + def parse_urls(self, urls): + """Parses the repository url. + + :return: the sequence of parsed urls + """ + + @abc.abstractmethod + def get_repository(self, connection, url, arch, consumer): + """Loads the repository meta information from URL. + + :param connection: the connection manager instance + :param url: the repository`s url + :param arch: the repository`s architecture + :param consumer: the callback to consume result + """ + + @abc.abstractmethod + def get_packages(self, connection, repository, consumer): + """Loads packages from repository. + + :param connection: the connection manager instance + :param repository: the repository object + :param consumer: the callback to consume result + """ + + @abc.abstractmethod + def clone_repository(self, connection, repository, destination, + source=False, locale=False): + """Creates copy of repository. + + :param connection: the connection manager instance + :param repository: the source repository + :param destination: the destination folder + :param source: copy source files + :param locale: copy localisation + :return: The copy of repository + """ + + @abc.abstractmethod + def rebuild_repository(self, repository, packages): + """Re-builds the repository. + + :param repository: the target repository + :param packages: the set of packages + """ diff --git a/packetary/objects/__init__.py b/packetary/objects/__init__.py index 1c7d351..234dc3e 100644 --- a/packetary/objects/__init__.py +++ b/packetary/objects/__init__.py @@ -20,6 +20,7 @@ from packetary.objects.package import FileChecksum from packetary.objects.package import Package from packetary.objects.package_relation import PackageRelation from packetary.objects.package_relation import VersionRange +from packetary.objects.packages_tree import PackagesTree from packetary.objects.repository import Repository @@ -28,6 +29,7 @@ __all__ = [ "Index", "Package", "PackageRelation", + "PackagesTree", "Repository", "VersionRange", ] diff --git a/packetary/objects/packages_tree.py b/packetary/objects/packages_tree.py new file mode 100644 index 0000000..3ffc18f --- /dev/null +++ b/packetary/objects/packages_tree.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import warnings + +from packetary.objects.index import Index + + +class UnresolvedWarning(UserWarning): + """Warning about unresolved depends.""" + pass + + +class PackagesTree(Index): + """Helper class to deal with dependency graph.""" + + def __init__(self): + super(PackagesTree, self).__init__() + self.mandatory_packages = [] + + def add(self, package): + super(PackagesTree, self).add(package) + # store all mandatory packages in separated list for quick access + if package.mandatory: + self.mandatory_packages.append(package) + + def get_unresolved_dependencies(self, unresolved=None): + """Gets the set of unresolved dependencies. + + :param unresolved: the known list of unresolved packages. + :return: the set of unresolved depends. + """ + return self.__get_unresolved_dependencies(self) + + def get_minimal_subset(self, main, requirements): + """Gets the minimal work subset. + + :param main: the main index, to complete requirements. + :param requirements: additional requirements. + :return: The set of resolved depends. + """ + + unresolved = set() + resolved = set() + if main is None: + def pkg_filter(*_): + pass + else: + pkg_filter = main.find + self.__get_unresolved_dependencies(main, requirements) + + stack = list() + stack.append((None, requirements)) + + # add all mandatory packages + for pkg in self.mandatory_packages: + stack.append((pkg, pkg.requires)) + + while len(stack) > 0: + pkg, required = stack.pop() + resolved.add(pkg) + for require in required: + for rel in require: + if rel not in unresolved: + if pkg_filter(rel.name, rel.version) is not None: + break + # use all packages that meets depends + candidates = self.find_all(rel.name, rel.version) + found = False + for cand in candidates: + if cand == pkg: + continue + found = True + if cand not in resolved: + stack.append((cand, cand.requires)) + + if found: + break + else: + unresolved.add(require) + msg = "Unresolved depends: {0}".format(require) + warnings.warn(UnresolvedWarning(msg)) + + resolved.remove(None) + return resolved + + @staticmethod + def __get_unresolved_dependencies(index, unresolved=None): + """Gets the set of unresolved dependencies. + + :param index: the search index. + :param unresolved: the known list of unresolved packages. + :return: the set of unresolved depends. + """ + + if unresolved is None: + unresolved = set() + + for pkg in index: + for require in pkg.requires: + for rel in require: + if rel not in unresolved: + candidate = index.find(rel.name, rel.version) + if candidate is not None and candidate != pkg: + break + else: + unresolved.add(require) + return unresolved diff --git a/packetary/objects/statistics.py b/packetary/objects/statistics.py new file mode 100644 index 0000000..408f7a5 --- /dev/null +++ b/packetary/objects/statistics.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + + +class CopyStatistics(object): + """The statistics of packages copying""" + def __init__(self): + # the number of copied packages + self.copied = 0 + # the number of total packages + self.total = 0 + + def on_package_copied(self, bytes_copied): + """Proceed next copied package.""" + if bytes_copied > 0: + self.copied += 1 + self.total += 1 + + def __iadd__(self, other): + if not isinstance(other, CopyStatistics): + raise TypeError + + self.copied += other.copied + self.total += other.total + return self + + def __add__(self, other): + result = copy.copy(self) + result += other + return result diff --git a/packetary/tests/stubs/executor.py b/packetary/tests/stubs/executor.py new file mode 100644 index 0000000..b517c3b --- /dev/null +++ b/packetary/tests/stubs/executor.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class Executor(object): + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + @staticmethod + def execute(f, *args, **kwargs): + return f(*args, **kwargs) diff --git a/packetary/tests/stubs/helpers.py b/packetary/tests/stubs/helpers.py new file mode 100644 index 0000000..2f26617 --- /dev/null +++ b/packetary/tests/stubs/helpers.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + + +class CallbacksAdapter(mock.MagicMock): + """Helper to return data through callback.""" + + def __call__(self, *args, **kwargs): + if len(args) > 0: + callback = args[-1] + else: + callback = None + + if not callable(callback): + return super(CallbacksAdapter, self).__call__(*args, **kwargs) + + args = args[:-1] + data = super(CallbacksAdapter, self).__call__(*args, **kwargs) + + if isinstance(data, list): + for d in data: + callback(d) + else: + callback(data) diff --git a/packetary/tests/test_packages_tree.py b/packetary/tests/test_packages_tree.py new file mode 100644 index 0000000..a1c7c87 --- /dev/null +++ b/packetary/tests/test_packages_tree.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import warnings + +from packetary.objects import Index +from packetary.objects import PackagesTree +from packetary.tests import base +from packetary.tests.stubs import generator + + +class TestPackagesTree(base.TestCase): + def setUp(self): + super(TestPackagesTree, self).setUp() + + def test_get_unresolved_dependencies(self): + ptree = PackagesTree() + ptree.add(generator.gen_package( + 1, requires=[generator.gen_relation("unresolved")])) + ptree.add(generator.gen_package(2, requires=None)) + ptree.add(generator.gen_package( + 3, requires=[generator.gen_relation("package1")] + )) + ptree.add(generator.gen_package( + 4, + requires=[generator.gen_relation("loop")], + obsoletes=[generator.gen_relation("loop", ["le", 1])] + )) + + unresolved = ptree.get_unresolved_dependencies() + self.assertItemsEqual( + ["loop", "unresolved"], + (x.name for x in unresolved) + ) + + def test_get_minimal_subset_with_master(self): + ptree = PackagesTree() + ptree.add(generator.gen_package(1, requires=None)) + ptree.add(generator.gen_package(2, requires=None)) + ptree.add(generator.gen_package(3, requires=None)) + ptree.add(generator.gen_package( + 4, requires=[generator.gen_relation("package1")] + )) + + master = Index() + master.add(generator.gen_package(1, requires=None)) + master.add(generator.gen_package( + 5, + requires=[generator.gen_relation( + "package10", + alternative=generator.gen_relation("package4") + )] + )) + + unresolved = set([generator.gen_relation("package3")]) + resolved = ptree.get_minimal_subset(master, unresolved) + self.assertItemsEqual( + ["package3", "package4"], + (x.name for x in resolved) + ) + + def test_get_minimal_subset_without_master(self): + ptree = PackagesTree() + ptree.add(generator.gen_package(1, requires=None)) + ptree.add(generator.gen_package(2, requires=None)) + ptree.add(generator.gen_package( + 3, requires=[generator.gen_relation("package1")] + )) + unresolved = set([generator.gen_relation("package3")]) + resolved = ptree.get_minimal_subset(None, unresolved) + self.assertItemsEqual( + ["package3", "package1"], + (x.name for x in resolved) + ) + + def test_mandatory_packages_always_included(self): + ptree = PackagesTree() + ptree.add(generator.gen_package(1, requires=None, mandatory=True)) + ptree.add(generator.gen_package(2, requires=None)) + ptree.add(generator.gen_package(3, requires=None)) + unresolved = set([generator.gen_relation("package3")]) + resolved = ptree.get_minimal_subset(None, unresolved) + self.assertItemsEqual( + ["package3", "package1"], + (x.name for x in resolved) + ) + + def test_warning_if_unresolved(self): + ptree = PackagesTree() + ptree.add(generator.gen_package( + 1, requires=None)) + + with warnings.catch_warnings(record=True) as log: + ptree.get_minimal_subset( + None, [generator.gen_relation("package2")] + ) + self.assertIn("package2", str(log[0])) diff --git a/packetary/tests/test_repository_api.py b/packetary/tests/test_repository_api.py new file mode 100644 index 0000000..aa9a01a --- /dev/null +++ b/packetary/tests/test_repository_api.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from packetary.api import Configuration +from packetary.api import Context +from packetary.api import RepositoryApi +from packetary.tests import base +from packetary.tests.stubs import generator +from packetary.tests.stubs.helpers import CallbacksAdapter + + +class TestRepositoryApi(base.TestCase): + def test_get_packages_as_is(self): + controller = CallbacksAdapter() + pkg = generator.gen_package(name="test") + controller.load_packages.side_effect = [ + pkg + ] + api = RepositoryApi(controller) + packages = api.get_packages("file:///repo1") + self.assertEqual(1, len(packages)) + package = packages.pop() + self.assertIs(pkg, package) + + def test_get_packages_with_depends_resolving(self): + controller = CallbacksAdapter() + controller.load_packages.side_effect = [ + [ + generator.gen_package(idx=1, requires=None), + generator.gen_package( + idx=2, requires=[generator.gen_relation("package1")] + ), + generator.gen_package( + idx=3, requires=[generator.gen_relation("package1")] + ), + generator.gen_package(idx=4, requires=None), + generator.gen_package(idx=5, requires=None), + ], + generator.gen_package( + idx=6, requires=[generator.gen_relation("package2")] + ), + ] + + api = RepositoryApi(controller) + packages = api.get_packages([ + "file:///repo1", "file:///repo2" + ], + "file:///repo3", ["package4"] + ) + + self.assertEqual(3, len(packages)) + self.assertItemsEqual( + ["package1", "package4", "package2"], + (x.name for x in packages) + ) + controller.load_repositories.assert_any_call( + ["file:///repo1", "file:///repo2"] + ) + controller.load_repositories.assert_any_call( + "file:///repo3" + ) + + def test_clone_repositories_as_is(self): + controller = CallbacksAdapter() + repo = generator.gen_repository(name="repo1") + packages = [ + generator.gen_package(name="test1", repository=repo), + generator.gen_package(name="test2", repository=repo) + ] + mirror = generator.gen_repository(name="mirror") + controller.load_repositories.return_value = repo + controller.load_packages.return_value = packages + controller.clone_repositories.return_value = {repo: mirror} + controller.copy_packages.return_value = [0, 1] + api = RepositoryApi(controller) + stats = api.clone_repositories( + ["file:///repo1"], "/mirror", keep_existing=True + ) + self.assertEqual(2, stats.total) + self.assertEqual(1, stats.copied) + controller.copy_packages.assert_called_once_with( + mirror, set(packages), True + ) + + def test_copy_minimal_subset_of_repository(self): + controller = CallbacksAdapter() + repo1 = generator.gen_repository(name="repo1") + repo2 = generator.gen_repository(name="repo2") + repo3 = generator.gen_repository(name="repo3") + mirror1 = generator.gen_repository(name="mirror1") + mirror2 = generator.gen_repository(name="mirror2") + pkg_group1 = [ + generator.gen_package( + idx=1, requires=None, repository=repo1 + ), + generator.gen_package( + idx=1, version=2, requires=None, repository=repo1 + ), + generator.gen_package( + idx=2, requires=None, repository=repo1 + ) + ] + pkg_group2 = [ + generator.gen_package( + idx=4, + requires=[generator.gen_relation("package1")], + repository=repo2, + mandatory=True, + ) + ] + pkg_group3 = [ + generator.gen_package( + idx=3, requires=None, repository=repo1 + ) + ] + controller.load_repositories.side_effect = [[repo1, repo2], repo3] + controller.load_packages.side_effect = [ + pkg_group1 + pkg_group2 + pkg_group3, + generator.gen_package( + idx=6, + repository=repo3, + requires=[generator.gen_relation("package2")] + ) + ] + controller.clone_repositories.return_value = { + repo1: mirror1, repo2: mirror2 + } + controller.copy_packages.return_value = 1 + api = RepositoryApi(controller) + api.clone_repositories( + ["file:///repo1", "file:///repo2"], "/mirror", + ["file:///repo3"], + keep_existing=True + ) + controller.copy_packages.assert_any_call( + mirror1, set(pkg_group1), True + ) + controller.copy_packages.assert_any_call( + mirror2, set(pkg_group2), True + ) + self.assertEqual(2, controller.copy_packages.call_count) + + def test_get_unresolved(self): + controller = CallbacksAdapter() + pkg = generator.gen_package( + name="test", requires=[generator.gen_relation("test2")] + ) + controller.load_packages.side_effect = [ + pkg + ] + api = RepositoryApi(controller) + r = api.get_unresolved_dependencies("file:///repo1") + controller.load_repositories.assert_called_once_with("file:///repo1") + self.assertItemsEqual( + ["test2"], + (x.name for x in r) + ) + + def test_parse_requirements(self): + requirements = RepositoryApi._parse_requirements( + ["p1 le 2 | p2 | p3 ge 2"] + ) + + expected = generator.gen_relation( + "p1", + ["le", '2'], + generator.gen_relation( + "p2", + None, + generator.gen_relation( + "p3", + ["ge", '2'] + ) + ) + ) + self.assertEqual(1, len(requirements)) + self.assertEqual( + list(expected), + list(requirements.pop()) + ) + + +class TestContext(base.TestCase): + @classmethod + def setUpClass(cls): + cls.config = Configuration( + threads_num=2, + ignore_errors_num=3, + retries_num=5, + http_proxy="http://localhost", + https_proxy="https://localhost" + ) + + @mock.patch("packetary.api.ConnectionsManager") + def test_initialise_connection_manager(self, conn_manager): + context = Context(self.config) + conn_manager.assert_called_once_with( + proxy="http://localhost", + secure_proxy="https://localhost", + retries_num=5 + ) + + self.assertIs( + conn_manager(), + context.connection + ) + + @mock.patch("packetary.api.AsynchronousSection") + def test_asynchronous_section(self, async_section): + context = Context(self.config) + s = context.async_section() + async_section.assert_called_with(2, 3) + self.assertIs(s, async_section()) + context.async_section(0) + async_section.assert_called_with(2, 0) diff --git a/packetary/tests/test_repository_contoller.py b/packetary/tests/test_repository_contoller.py new file mode 100644 index 0000000..38d4add --- /dev/null +++ b/packetary/tests/test_repository_contoller.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import mock +import six + +from packetary.controllers import RepositoryController +from packetary.tests import base +from packetary.tests.stubs.executor import Executor +from packetary.tests.stubs.generator import gen_package +from packetary.tests.stubs.generator import gen_repository +from packetary.tests.stubs.helpers import CallbacksAdapter + + +class TestRepositoryController(base.TestCase): + def setUp(self): + self.driver = mock.MagicMock() + self.context = mock.MagicMock() + self.context.async_section.return_value = Executor() + self.ctrl = RepositoryController(self.context, self.driver, "x86_64") + + def test_load_fail_if_unknown_driver(self): + with self.assertRaisesRegexp(NotImplementedError, "unknown_driver"): + RepositoryController.load( + self.context, + "unknown_driver", + "x86_64" + ) + + @mock.patch("packetary.controllers.repository.stevedore") + def test_load_driver(self, stevedore): + stevedore.ExtensionManager.return_value = { + "test": mock.MagicMock(obj=self.driver) + } + RepositoryController._drivers = None + controller = RepositoryController.load(self.context, "test", "x86_64") + self.assertIs(self.driver, controller.driver) + + def test_load_repositories(self): + self.driver.parse_urls.return_value = ["test1"] + consumer = mock.MagicMock() + self.ctrl.load_repositories("file:///test1", consumer) + self.driver.parse_urls.assert_called_once_with(["file:///test1"]) + self.driver.get_repository.assert_called_once_with( + self.context.connection, "test1", "x86_64", consumer + ) + for url in [six.u("file:///test1"), ["file:///test1"]]: + self.driver.reset_mock() + self.ctrl.load_repositories(url, consumer) + if not isinstance(url, list): + url = [url] + self.driver.parse_urls.assert_called_once_with(url) + + def test_load_packages(self): + repo = mock.MagicMock() + consumer = mock.MagicMock() + self.ctrl.load_packages([repo], consumer) + self.driver.get_packages.assert_called_once_with( + self.context.connection, repo, consumer + ) + + @mock.patch("packetary.controllers.repository.os") + def test_assign_packages(self, os): + repo = gen_repository(url="/test/repo") + packages = [ + gen_package(name="test1", repository=repo), + gen_package(name="test2", repository=repo) + ] + existed_packages = [ + gen_package(name="test3", repository=repo), + gen_package(name="test2", repository=repo) + ] + + os.path.join = lambda *x: "/".join(x) + self.driver.get_packages = CallbacksAdapter() + self.driver.get_packages.return_value = existed_packages + self.ctrl.assign_packages(repo, packages, True) + os.remove.assert_not_called() + all_packages = set(packages + existed_packages) + self.driver.rebuild_repository.assert_called_once_with( + repo, all_packages + ) + self.driver.rebuild_repository.reset_mock() + self.ctrl.assign_packages(repo, packages, False) + self.driver.rebuild_repository.assert_called_once_with( + repo, set(packages) + ) + os.remove.assert_called_once_with("/test/repo/test3.pkg") + + def test_copy_packages(self): + repo = gen_repository(url="file:///repo/") + packages = [ + gen_package(name="test1", repository=repo, filesize=10), + gen_package(name="test2", repository=repo, filesize=-1) + ] + target = gen_repository(url="/test/repo") + self.context.connection.retrieve.side_effect = [0, 10] + observer = mock.MagicMock() + self.ctrl.copy_packages(target, packages, True, observer) + observer.assert_has_calls([mock.call(0), mock.call(10)]) + self.context.connection.retrieve.assert_any_call( + "file:///repo/test1.pkg", + "/test/repo/test1.pkg", + size=10 + ) + self.context.connection.retrieve.assert_any_call( + "file:///repo/test2.pkg", + "/test/repo/test2.pkg", + size=-1 + ) + self.driver.rebuild_repository.assert_called_once_with( + target, set(packages) + ) + + @mock.patch("packetary.controllers.repository.os") + def test_clone_repository(self, os): + os.path.abspath.return_value = "/root/repo" + repos = [ + gen_repository(name="test1"), + gen_repository(name="test2") + ] + clones = [copy.copy(x) for x in repos] + self.driver.clone_repository.side_effect = clones + mirrors = self.ctrl.clone_repositories(repos, "./repo") + for r in repos: + self.driver.clone_repository.assert_any_call( + self.context.connection, r, "/root/repo", False, False + ) + self.assertEqual(mirrors, dict(zip(repos, clones))) diff --git a/requirements.txt b/requirements.txt index 07ceb85..8f8b744 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ Babel>=1.3 eventlet>=0.15 bintrees>=2.0.2 chardet>=2.3.0 +stevedore>=1.1.0 six>=1.5.2