x/packetary
Code Issues Proposed changes

Add package filtering feature

Browse Source 
Change-Id: Ia8bf716e9902d250c9fb26bbad325fbe2a204b54
This commit is contained in:
Vladimir Kozhukalov 2016-02-16 23:16:50 +03:00
parent e0059a7af3
commit 1238b33ebd
15 changed files with 219 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packetary/__init__.py
View File

@ -33,7 +33,4 @@ try:
__version__ = pbr.version.VersionInfo(
'packetary').version_string()
except Exception as e:
# when run tests without installing package
# pbr may raise exception.
print("ERROR:", e)
__version__ = "0.0.0"

74
packetary/api.py
View File

@ -18,6 +18,7 @@
from collections import defaultdict
import logging
import re
import jsonschema
import six
@ -30,6 +31,7 @@ from packetary.objects import PackagesForest
from packetary.objects import PackagesTree
from packetary.objects.statistics import CopyStatistics
from packetary.schemas import PACKAGE_FILES_SCHEMA
from packetary.schemas import PACKAGE_FILTER_SCHEMA
from packetary.schemas import PACKAGES_SCHEMA
logger = logging.getLogger(__package__)
@ -128,22 +130,27 @@ class RepositoryApi(object):
return self.controller.create_repository(repo_data, package_files)
def get_packages(self, repos_data, requirements_data=None,
include_mandatory=False):
include_mandatory=False, filter_data=None):
"""Gets the list of packages from repository(es).
:param repos_data: The list of repository descriptions
:param requirements_data: The list of package`s requirements
that should be included
:param include_mandatory: if True, all mandatory packages will be
included
:param filter_data: A set of filters that is used to exclude
those packages which match one of filters
:return: the set of packages
"""
repos = self._load_repositories(repos_data)
requirements = self._load_requirements(requirements_data)
return self._get_packages(repos, requirements, include_mandatory)
exclude_filter = self._load_filter(filter_data)
return self._get_packages(repos, requirements,
include_mandatory, exclude_filter)
def clone_repositories(self, repos_data, requirements_data, destination,
include_source=False, include_locale=False,
include_mandatory=False):
include_mandatory=False, filter_data=None):
"""Creates the clones of specified repositories in local folder.
:param repos_data: The list of repository descriptions
@ -155,12 +162,16 @@ class RepositoryApi(object):
:param include_locale: if True, the locales will be copied as well.
:param include_mandatory: if True, all mandatory packages will be
included
:param filter_data: A set of filters that is used to exclude
those packages which match one of filters
:return: count of copied and total packages.
"""
repos = self._load_repositories(repos_data)
reqs = self._load_requirements(requirements_data)
all_packages = self._get_packages(repos, reqs, include_mandatory)
exclude_filter = self._load_filter(filter_data)
all_packages = self._get_packages(
repos, reqs, include_mandatory, exclude_filter)
package_groups = defaultdict(set)
for pkg in all_packages:
package_groups[pkg.repository].add(pkg)
@ -191,7 +202,8 @@ class RepositoryApi(object):
self._load_packages(self._load_repositories(repos_data), packages.add)
return packages.get_unresolved_dependencies()
def _get_packages(self, repos, requirements, include_mandatory):
def _get_packages(self, repos, requirements, include_mandatory,
exclude_filter):
if requirements is not None:
forest = PackagesForest()
for repo in repos:
@ -199,7 +211,12 @@ class RepositoryApi(object):
return forest.get_packages(requirements, include_mandatory)
packages = set()
self._load_packages(repos, packages.add)
consumer = packages.add
if exclude_filter is not None:
def consumer(p):
if not exclude_filter(p):
packages.add(p)
self._load_packages(repos, consumer)
return packages
def _load_packages(self, repos, consumer):
@ -228,6 +245,51 @@ class RepositoryApi(object):
))
return result
def _load_filter(self, filter_data):
"""Loads filter from filter data.
Property value could be a string or a python regexp.
Example of filters data:
- name: full-package-name
section: section1
- name: /^.*substr/
:param filter_data: A list of filters
:return: Lambda that could match a particular package.
"""
if filter_data is None:
return
self._validate_filter_data(filter_data)
def get_pattern_match(pattern, key, value):
return lambda p: pattern.match(getattr(p, key))
def get_exact_match(key, value):
return lambda p: getattr(p, key) == value
def get_logical_and(filters):
return lambda p: all((f(p) for f in filters))
def get_logical_or(filters):
return lambda p: any((f(p) for f in filters))
filters = []
for fdata in filter_data:
matchers = []
for key, value in six.iteritems(fdata):
if value.startswith('/') and value.endswith('/'):
pattern = re.compile(value[1:-1])
matchers.append(get_pattern_match(pattern, key, value))
else:
matchers.append(get_exact_match(key, value))
filters.append(get_logical_and(matchers))
return get_logical_or(filters)
def _validate_filter_data(self, filter_data):
self._validate_data(filter_data, PACKAGE_FILTER_SCHEMA)
def _validate_repo_data(self, repo_data):
schema = self.controller.get_repository_data_schema()
self._validate_data(repo_data, schema)

13
packetary/cli/commands/base.py
View File

@ -111,7 +111,9 @@ class PackagesMixin(object):
help="Do not copy mandatory packages."
)
parser.add_argument(
group = parser.add_mutually_exclusive_group()
group.add_argument(
"-p", "--packages",
dest='requirements',
type=read_from_file,
@ -119,6 +121,15 @@ class PackagesMixin(object):
help="The path to file with list of packages."
"See documentation about format."
)
group.add_argument(
"-f", "--exclude-filter",
dest='exclude_filter_data',
type=read_from_file,
metavar='FILENAME',
help="The path to file with package exclude filter data."
"See documentation about format."
)
return parser

3
packetary/cli/commands/clone.py
View File

@ -54,7 +54,8 @@ class CloneCommand(PackagesMixin, RepositoriesMixin, BaseRepoCommand):
parsed_args.destination,
parsed_args.sources,
parsed_args.locales,
parsed_args.include_mandatory
parsed_args.include_mandatory,
filter_data=parsed_args.exclude_filter_data,
)
self.stdout.write(
"Packages copied: {0.copied}/{0.total}.\n".format(stat)

3
packetary/cli/commands/packages.py
View File

@ -41,7 +41,8 @@ class ListOfPackages(
return api.get_packages(
parsed_args.repositories,
parsed_args.requirements,
parsed_args.include_mandatory
parsed_args.include_mandatory,
filter_data=parsed_args.exclude_filter_data,
)

4
packetary/drivers/deb_driver.py
View File

@ -157,6 +157,7 @@ class DebRepositoryDriver(RepositoryDriverBase):
# The deb does not have obsoletes section
obsoletes=[],
provides=self._get_relations(dpkg, "provides"),
group=dpkg.get("section"),
))
except KeyError as e:
self.logger.error(
@ -255,7 +256,8 @@ class DebRepositoryDriver(RepositoryDriverBase):
"recommends"
),
provides=self._get_relations(debcontrol, "provides"),
obsoletes=[]
obsoletes=[],
group=debcontrol.get('section'),
)
def get_relative_path(self, repository, filename):

5
packetary/drivers/rpm_driver.py
View File

@ -144,7 +144,9 @@ class RpmRepositoryDriver(RepositoryDriverBase):
mandatory=name in mandatory,
requires=self._get_relations(tag, "requires"),
obsoletes=self._get_relations(tag, "obsoletes"),
provides=self._get_relations(tag, "provides")
provides=self._get_relations(tag, "provides"),
group=tag.find("./main:format/rpm:group",
_NAMESPACES).text,
))
except (ValueError, KeyError) as e:
self.logger.error(
@ -226,6 +228,7 @@ class RpmRepositoryDriver(RepositoryDriverBase):
requires=self._parse_package_relations(pkg.requires),
obsoletes=self._parse_package_relations(pkg.obsoletes),
provides=self._parse_package_relations(pkg.provides),
group=hdr["group"],
)
def get_relative_path(self, repository, filename):

5
packetary/objects/package.py
View File

@ -29,7 +29,8 @@ class Package(ComparableObject):
def __init__(self, repository, name, version, filename,
filesize, checksum, mandatory=False,
requires=None, provides=None, obsoletes=None):
requires=None, provides=None, obsoletes=None,
group=None):
"""Initialises.
:param name: the package`s name
@ -41,6 +42,7 @@ class Package(ComparableObject):
:param provides: the package`s provides(optional)
:param obsoletes: the package`s obsoletes(optional)
:param mandatory: indicates that package is mandatory
:param group: corresponds to rpm group and deb section
"""
self.repository = repository
@ -53,6 +55,7 @@ class Package(ComparableObject):
self.provides = provides or []
self.obsoletes = obsoletes or []
self.mandatory = mandatory
self.group = group
def __copy__(self):
"""Creates shallow copy of package."""

4
packetary/schemas/__init__.py
View File

@ -18,12 +18,14 @@
from packetary.schemas.deb_repo_schema import DEB_REPO_SCHEMA
from packetary.schemas.package_files_schema import PACKAGE_FILES_SCHEMA
from packetary.schemas.package_filter_schema import PACKAGE_FILTER_SCHEMA
from packetary.schemas.packages_schema import PACKAGES_SCHEMA
from packetary.schemas.rpm_repo_schema import RPM_REPO_SCHEMA
__all__ = [
"DEB_REPO_SCHEMA",
"PACKAGE_FILES_SCHEMA",
"PACKAGE_FILTER_SCHEMA",
"PACKAGES_SCHEMA",
"RPM_REPO_SCHEMA",
"PACKAGE_FILES_SCHEMA"
]

33
packetary/schemas/package_filter_schema.py Normal file
View File

@ -0,0 +1,33 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
PACKAGE_FILTER_SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"group": {
"type": "string"
}
}
}
}

7
packetary/tests/test_cli_commands.py
View File

@ -108,7 +108,10 @@ class TestCliCommands(base.TestCase):
read_file_mock.assert_any_call("packages.yaml")
api_instance.clone_repositories.assert_called_once_with(
[{"name": "repo"}], [{"name": "package"}], "/root",
False, False, False
False,
False,
False,
filter_data=None,
)
stdout_mock.write.assert_called_once_with(
"Packages copied: 0/0.\n"
@ -129,7 +132,7 @@ class TestCliCommands(base.TestCase):
)
self.check_common_config(api_mock.create.call_args[0][0])
api_instance.get_packages.assert_called_once_with(
[{"name": "repo"}], None, True
[{"name": "repo"}], None, True, filter_data=None
)
self.assertIn(
"test1; test1.pkg",

80
packetary/tests/test_repository_api.py
View File

@ -25,6 +25,7 @@ from packetary.api import Configuration
from packetary.api import Context
from packetary.api import RepositoryApi
from packetary.schemas import PACKAGE_FILES_SCHEMA
from packetary.schemas import PACKAGE_FILTER_SCHEMA
from packetary.schemas import PACKAGES_SCHEMA
from packetary.tests import base
from packetary.tests.stubs import generator
@ -119,7 +120,7 @@ class TestRepositoryApi(base.TestCase):
)
def test_get_packages_as_is(self, jsonschema_mock):
packages = self.api.get_packages([self.repo_data], None)
packages = self.api.get_packages([self.repo_data], None, False, None)
self.assertEqual(5, len(packages))
self.assertItemsEqual(
self.packages,
@ -133,7 +134,7 @@ class TestRepositoryApi(base.TestCase):
jsonschema_mock):
requirements = [{"name": "package1"}]
packages = self.api.get_packages(
[self.repo_data], requirements, True
[self.repo_data], requirements, True, None
)
self.assertEqual(3, len(packages))
self.assertItemsEqual(
@ -151,7 +152,7 @@ class TestRepositoryApi(base.TestCase):
jsonschema_mock):
requirements = [{"name": "package4"}]
packages = self.api.get_packages(
[self.repo_data], requirements, False
[self.repo_data], requirements, False, None
)
self.assertEqual(2, len(packages))
self.assertItemsEqual(
@ -239,6 +240,42 @@ class TestRepositoryApi(base.TestCase):
]
)
def test_clone_with_filter(self, jsonschema_mock):
repos_data = "repos_data"
requirements_data = "requirements_data"
filter_data = "filter_data"
repos = "repos"
requirements = "requirements"
exclude_filter = "exclude_filter"
self.api._load_repositories = mock.Mock(return_value=repos)
self.api._load_requirements = mock.Mock(return_value=requirements)
self.api._load_filter = mock.Mock(return_value=exclude_filter)
self.api._get_packages = mock.Mock(return_value=set())
self.api.controller = mock.Mock()
self.api.clone_repositories(repos_data, requirements_data,
"destination", filter_data=filter_data)
self.api._load_repositories.assert_called_once_with(repos_data)
self.api._load_requirements.assert_called_once_with(requirements_data)
self.api._load_filter.assert_called_once_with(filter_data)
self.api._get_packages.assert_called_once_with(
repos, requirements, False, exclude_filter)
def test_get_packages_with_exclude_filter(self, jsonschema_mock):
exclude_filter = lambda p: any([p == "p1", p == "p3"])
self.api._load_packages = CallbacksAdapter()
self.api._load_packages.return_value = ["p1", "p2", "p3", "p4"]
packages = self.api._get_packages("repos", None, False, exclude_filter)
self.assertSetEqual(packages, set(["p2", "p4"]))
def test_get_packages_without_exclude_filter(self, jsonschema_mock):
self.api._load_packages = CallbacksAdapter()
self.api._load_packages.return_value = ["p1", "p2"]
packages = self.api._get_packages("repos", None, False, None)
self.assertSetEqual(packages, set(["p1", "p2"]))
def test_get_unresolved(self, jsonschema_mock):
unresolved = self.api.get_unresolved_dependencies([self.repo_data])
self.assertItemsEqual(["package6"], (x.name for x in unresolved))
@ -246,6 +283,43 @@ class TestRepositoryApi(base.TestCase):
self.repo_data, self.schema
)
def test_load_filter_with_none(self, jsonschema_mock):
self.assertIsNone(self.api._load_filter(None))
def test_load_filter(self, jsonschema_mock):
self.api._validate_filter_data = mock.Mock()
filter_data = [
{"name": "p1", "group": "g1"},
{"name": "p2"},
{"group": "g3"},
{"name": "/^.5/", "group": "/^.*3/"},
{"group": "/^.*4/"},
]
exclude_filter = self.api._load_filter(filter_data)
p1 = generator.gen_package(name="p1", group="g1")
p2 = generator.gen_package(name="p2", group="g1")
p3 = generator.gen_package(name="p3", group="g2")
p4 = generator.gen_package(name="p4", group="g3")
p5 = generator.gen_package(name="p5", group="g3")
p6 = generator.gen_package(name="p6", group="g4")
cases = [
(True, (p1,)),
(True, (p2,)),
(False, (p3,)),
(True, (p4,)),
(True, (p5,)),
(True, (p6,)),
]
self._check_cases(self.assertEqual, cases, exclude_filter)
def test_validate_filter_data(self, jsonschema_mock):
self.api._validate_data = mock.Mock()
self.api._validate_filter_data("filter_data")
self.api._validate_data.assert_called_once_with("filter_data",
PACKAGE_FILTER_SCHEMA)
def test_load_requirements(self, jsonschema_mock):
expected = {
generator.gen_relation("test1"),

7
packetary/tests/test_rpm_driver.py
View File

@ -41,9 +41,9 @@ GROUPS_DB = path.join(path.dirname(__file__), "data", "groups.xml")
class TestRpmDriver(base.TestCase):
@classmethod
def setUpClass(cls):
cls.createrepo = sys.modules["createrepo"] = mock.MagicMock()
# import driver class after patching sys.modules
sys.modules["createrepo"] = mock.MagicMock()
from packetary.drivers import rpm_driver
cls.createrepo = rpm_driver.createrepo = mock.MagicMock()
super(TestRpmDriver, cls).setUpClass()
cls.driver = rpm_driver.RpmRepositoryDriver()
@ -243,7 +243,7 @@ class TestRpmDriver(base.TestCase):
self.createrepo.yumbased.YumLocalPackage.return_value = rpm_mock
rpm_mock.returnLocalHeader.return_value = {
"name": "Test", "epoch": 1, "version": "1.2.3", "release": "1",
"size": "10"
"size": "10", "group": "Group"
}
repo = gen_repository("Test", url="file:///repo/os/x86_64/")
pkg = self.driver.load_package_from_file(repo, "test.rpm")
@ -261,6 +261,7 @@ class TestRpmDriver(base.TestCase):
self.assertEqual("1-1.2.3-1", str(pkg.version))
self.assertEqual("test.rpm", pkg.filename)
self.assertEqual((3, 4, 5), pkg.checksum)
self.assertEqual("Group", pkg.group)
self.assertEqual(10, pkg.filesize)
self.assertItemsEqual(
['test1 (= 0-1.2.3-1.el5)'],

2
requirements.txt
View File

@ -9,7 +9,7 @@ eventlet>=0.15
bintrees>=2.0.2
chardet>=2.0.1
stevedore>=1.1.0
six>=1.5.2
six>=1.9.0 # MIT
python-debian>=0.1.21
lxml>=3.2
jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT

1
tox.ini
View File

@ -5,6 +5,7 @@ skipsdist = True
[testenv]
usedevelop = True
sitepackages = True
install_command = pip install -U {opts} {packages}
setenv =
VIRTUAL_ENV={envdir}