x/packetary
Code Issues Proposed changes

Fixed retrieve groups.xml on merge repositories

Browse Source 
Also add package name when warn about unresolved

Closes-Bug: 1551614
Change-Id: I9545b5c419f03fd9095a6a0abb5d5f21fb3a1ac0
This commit is contained in:
Bulat Gaifullin 2016-02-29 21:07:07 +03:00
parent f27bed00a5
commit d644789905
10 changed files with 121 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packetary/controllers/repository.py
View File

@ -130,7 +130,9 @@ class RepositoryController(object):
:param package_files: the list of paths of packages
:return : the new repository
"""
repo = self.driver.create_repository(repository_data, self.arch)
repo = self.driver.create_repository(
self.context.connection, repository_data, self.arch
)
packages = set()
with self.context.async_section() as section:
for url in package_files:

3
packetary/drivers/base.py
View File

@ -77,9 +77,10 @@ class RepositoryDriverBase(object):
"""
@abc.abstractmethod
def create_repository(self, repository_data, arch):
def create_repository(self, connection, repository_data, arch):
"""Create new repository.
:param connection: the connection manager instance
:param repository_data: repository input data
:param arch: the repository`s architecture
:return: new repository object

2
packetary/drivers/deb_driver.py
View File

@ -205,7 +205,7 @@ class DebRepositoryDriver(RepositoryDriverBase):
self._create_repository_structure(new_repo)
return new_repo
def create_repository(self, repository_data, arch):
def create_repository(self, connection, repository_data, arch):
url = utils.normalize_repository_url(repository_data['uri'])
suite = repository_data['suite']
component = repository_data.get('section')

32
packetary/drivers/rpm_driver.py
View File

@ -158,7 +158,7 @@ class RpmRepositoryDriver(RepositoryDriverBase):
def add_packages(self, connection, repository, packages):
groupstree = self._load_groups(connection, repository)
self._rebuild_repository(repository, packages, groupstree)
self._rebuild_repository(connection, repository, packages, groupstree)
def fork_repository(self, connection, repository, destination,
source=False, locale=False):
@ -168,10 +168,10 @@ class RpmRepositoryDriver(RepositoryDriverBase):
new_repo.url = utils.normalize_repository_url(destination)
utils.ensure_dir_exist(destination)
groupstree = self._load_groups(connection, repository)
self._rebuild_repository(new_repo, None, groupstree)
self._rebuild_repository(connection, new_repo, set(), groupstree)
return new_repo
def create_repository(self, repository_data, arch):
def create_repository(self, connection, repository_data, arch):
repository = Repository(
name=repository_data['name'],
url=utils.normalize_repository_url(repository_data["uri"]),
@ -180,7 +180,7 @@ class RpmRepositoryDriver(RepositoryDriverBase):
origin=repository_data.get('origin')
)
utils.ensure_dir_exist(utils.get_path_from_url(repository.url))
self._rebuild_repository(repository, None, None)
self._rebuild_repository(connection, repository, None, None)
return repository
def load_package_from_file(self, repository, filepath):
@ -208,24 +208,30 @@ class RpmRepositoryDriver(RepositoryDriverBase):
def get_relative_path(self, repository, filename):
return "packages/" + filename
def _rebuild_repository(self, repository, packages, groupstree=None):
basepath = utils.get_path_from_url(repository.url)
def _rebuild_repository(self, conn, repo, packages, groupstree=None):
basepath = utils.get_path_from_url(repo.url)
self.logger.info("rebuild repository in %s", basepath)
md_config = createrepo.MetaDataConfig()
update = packages is not None and \
os.path.exists(os.path.join(basepath, md_config.finaldir))
mdfile_path = os.path.join(
basepath, md_config.finaldir, md_config.repomdfile
)
update = packages is not None and os.path.exists(mdfile_path)
groupsfile = None
if groupstree is None and update:
# The createrepo lose the groups information on update
# to prevent this set group info manually
groupstree = self._load_groups(conn, repo)
if groupstree is not None:
with tempfile.NamedTemporaryFile(delete=False) as tmp:
groupstree.write(tmp)
groupsfile = tmp.name
groupsfile = os.path.join(tempfile.gettempdir(), "groups.xml")
with open(groupsfile, "w") as fd:
groupstree.write(fd)
try:
md_config.workers = multiprocessing.cpu_count()
md_config.directory = str(basepath)
md_config.groupfile = groupsfile
md_config.update = update
if packages is None:
if not packages:
# only generate meta-files, without packages info
md_config.excludes = ["*"]

11
packetary/objects/packages_forest.py
View File

@ -54,28 +54,29 @@ class PackagesForest(object):
resolved = set()
unresolved = set()
stack = [requirements]
stack = [(None, requirements)]
if include_mandatory:
for tree in self.trees:
for mandatory in tree.mandatory_packages:
resolved.add(mandatory)
stack.append(mandatory.requires)
stack.append((mandatory, mandatory.requires))
while stack:
requirements = stack.pop()
pkg, requirements = stack.pop()
for required in requirements:
for rel in required:
if rel not in unresolved:
candidate = self.find(rel)
if candidate is not None:
if candidate not in resolved:
stack.append(candidate.requires)
stack.append((candidate, candidate.requires))
resolved.add(candidate)
break
else:
unresolved.add(required)
logger.warning("Unresolved relation: %s", required)
logger.warning("Unresolved relation: %s from %s",
required, pkg and pkg.name)
return resolved
def find(self, relation):

5
packetary/tests/stubs/helpers.py
View File

@ -22,6 +22,11 @@ import mock
import six
class HTTPError(Exception):
def __init__(self, code):
self.code = code
class CallbacksAdapter(mock.MagicMock):
"""Helper to return data through callback."""

18
packetary/tests/test_deb_driver.py
View File

@ -26,16 +26,12 @@ from packetary.tests import base
from packetary.tests.stubs.generator import gen_package
from packetary.tests.stubs.generator import gen_repository
from packetary.tests.stubs.helpers import get_compressed
from packetary.tests.stubs.helpers import HTTPError
PACKAGES = path.join(path.dirname(__file__), "data", "Packages")
class HTTPError(Exception):
def __init__(self, code):
self.code = code
class TestDebDriver(base.TestCase):
@classmethod
def setUpClass(cls):
@ -318,7 +314,9 @@ class TestDebDriver(base.TestCase):
"section": "main", "type": "rpm", "priority": "100",
"origin": "Origin", "path": "/repo"
}
repo = self.driver.create_repository(repository_data, "x86_64")
repo = self.driver.create_repository(
self.connection, repository_data, "x86_64"
)
self.assertEqual(repository_data["name"], repo.name)
self.assertEqual("x86_64", repo.architecture)
self.assertEqual(repository_data["uri"] + "/", repo.url)
@ -346,10 +344,14 @@ class TestDebDriver(base.TestCase):
"origin": "Origin", "path": "/repo"
}
with self.assertRaisesRegexp(ValueError, "flat format"):
self.driver.create_repository(repository_data, "x86_64")
self.driver.create_repository(
self.connection, repository_data, "x86_64"
)
with self.assertRaisesRegexp(ValueError, "single component"):
repository_data["section"] = ["main", "universe"]
self.driver.create_repository(repository_data, "x86_64")
self.driver.create_repository(
self.connection, repository_data, "x86_64"
)
@mock.patch.multiple(
"packetary.drivers.deb_driver",

2
packetary/tests/test_repository_contoller.py
View File

@ -161,7 +161,7 @@ class TestRepositoryController(base.TestCase):
self.driver.get_relative_path.side_effect = ["pool/t/test1.pkg"]
self.ctrl.create_repository(repository_data, packages_list)
self.driver.create_repository.assert_called_once_with(
repository_data, self.ctrl.arch
self.context.connection, repository_data, self.ctrl.arch
)
self.driver.get_relative_path.assert_called_once_with(
repo, "test1.pkg"

89
packetary/tests/test_rpm_driver.py
View File

@ -171,12 +171,12 @@ class TestRpmDriver(base.TestCase):
self.assertTrue(package.mandatory)
@mock.patch("packetary.drivers.rpm_driver.os")
@mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
def test_add_packages_to_existing(self, tmp_mock, os_mock):
@mock.patch("packetary.drivers.rpm_driver.open")
def test_add_packages_to_existing(self, open_mock, os_mock):
self.configure_streams()
tmp_file = mock.MagicMock()
tmp_file.name = "/tmp/groups.gz"
tmp_mock.return_value.__enter__.return_value = tmp_file
open_mock.return_value.__enter__.return_value = tmp_file
repo = gen_repository("test", url="file:///repo/os/x86_64")
md_gen = mock.MagicMock()
self.createrepo.MetaDataGenerator.return_value = md_gen
@ -184,6 +184,11 @@ class TestRpmDriver(base.TestCase):
md_gen.tempdir = "tmp"
md_gen.finaldir = "repodata"
os_mock.path.exists.return_value = True
os_mock.path.join.side_effect = [
"/repo/os/x86_64/tmp",
tmp_file.name,
"/repo/os/x86_64/tmp"
]
self.driver.add_packages(self.connection, repo, set())
self.assertEqual(
"/repo/os/x86_64",
@ -202,14 +207,14 @@ class TestRpmDriver(base.TestCase):
@mock.patch("packetary.drivers.rpm_driver.os")
@mock.patch("packetary.drivers.rpm_driver.shutil")
@mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
@mock.patch("packetary.drivers.rpm_driver.open")
def test_add_packages_clean_metadata_on_error(
self, tmp_mock, shutil_mock, os_mock
self, open_mock, shutil_mock, os_mock
):
self.configure_streams()
tmp_file = mock.MagicMock()
tmp_file.name = "/tmp/groups.gz"
tmp_mock.return_value.__enter__.return_value = tmp_file
open_mock.return_value.__enter__.return_value = tmp_file
self.createrepo.MDError = ValueError
md_gen = mock.MagicMock()
self.createrepo.MetaDataGenerator.return_value = md_gen
@ -220,7 +225,11 @@ class TestRpmDriver(base.TestCase):
self.createrepo.MetaDataConfig().tempdir = "tmp"
self.createrepo.MetaDataConfig().finaldir = "repodata"
os_mock.path.exists.return_value = True
os_mock.path.join.side_effect = lambda *a: '/'.join(a)
os_mock.path.join.side_effect = [
"/repo/os/x86_64/tmp",
tmp_file.name,
"/repo/os/x86_64/tmp"
]
with self.assertRaises(RuntimeError):
self.driver.add_packages(self.connection, repo, set())
shutil_mock.rmtree.assert_called_once_with(
@ -229,15 +238,15 @@ class TestRpmDriver(base.TestCase):
os_mock.unlink.assert_called_once_with(tmp_file.name)
@mock.patch("packetary.drivers.rpm_driver.os")
@mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
@mock.patch("packetary.drivers.rpm_driver.open")
@mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist")
def test_fork_repository(
self, ensure_dir_exists_mock, tmp_mock, os_mock
def test_fork_repository_to_empty_destination(
self, m_ensure_dir_exists, m_open, m_os
):
self.configure_streams()
tmp_file = mock.MagicMock()
tmp_file.name = "/tmp/groups.gz"
tmp_mock.return_value.__enter__.return_value = tmp_file
m_open.return_value.__enter__.return_value = tmp_file
repo = gen_repository("os", url="http://localhost/os/x86_64/")
md_gen = mock.MagicMock()
self.createrepo.MetaDataGenerator.return_value = md_gen
@ -247,12 +256,18 @@ class TestRpmDriver(base.TestCase):
md_gen.finaldir = "repodata"
md_config = mock.MagicMock()
self.createrepo.MetaDataConfig.return_value = md_config
m_os.path.join.side_effect = [
"/repo/os/x86_64/repodata/repomd.xml",
tmp_file.name,
"/repo/os/x86_64/tmp"
]
m_os.path.exists.return_value = False
new_repo = self.driver.fork_repository(
self.connection,
repo,
"/repo/os/x86_64"
)
ensure_dir_exists_mock.assert_called_once_with("/repo/os/x86_64")
m_ensure_dir_exists.assert_called_once_with("/repo/os/x86_64")
self.assertEqual(repo.name, new_repo.name)
self.assertEqual(repo.architecture, new_repo.architecture)
self.assertEqual("file:///repo/os/x86_64/", new_repo.url)
@ -261,13 +276,51 @@ class TestRpmDriver(base.TestCase):
self.assertEqual(["*"], md_config.excludes)
self.assertFalse(md_config.update)
self.assertEqual(tmp_file.name, md_config.groupfile)
os_mock.unlink.assert_called_once_with(tmp_file.name)
m_open.assert_called_once_with(tmp_file.name, "w")
m_os.unlink.assert_called_once_with(tmp_file.name)
@mock.patch("packetary.drivers.rpm_driver.os")
@mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
@mock.patch("packetary.drivers.rpm_driver.open")
@mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist")
def test_fork_repository_to_existing_destination(
self, ensure_dir_exists_mock, open_mock, os_mock
):
self.configure_streams()
tmp_file = mock.MagicMock()
tmp_file.name = "/tmp/groups.gz"
open_mock.return_value.__enter__.return_value = tmp_file
repo = gen_repository("os", url="http://localhost/os/x86_64/")
md_gen = mock.MagicMock()
self.createrepo.MetaDataGenerator.return_value = md_gen
md_gen.doFinalMove.side_effect = [None]
md_gen.outputdir = "/repo/os/x86_64"
md_gen.tempdir = "tmp"
md_gen.finaldir = "repodata"
md_config = mock.MagicMock()
self.createrepo.MetaDataConfig.return_value = md_config
os_mock.path.join.side_effect = [
"/repo/os/x86_64/repodata/repomd.xml",
tmp_file.name,
"/repo/os/x86_64/tmp"
]
os_mock.path.exists.return_value = True
groups_xml = mock.MagicMock()
with mock.patch.object(self.driver, "_load_groups") as load_mock:
load_mock.side_effect = [None, groups_xml]
self.driver.fork_repository(
self.connection,
repo,
"/repo/os/x86_64"
)
self.assertEqual(tmp_file.name, md_config.groupfile)
os_mock.unlink.assert_called_once_with(tmp_file.name)
groups_xml.write.assert_called_once_with(tmp_file)
@mock.patch("packetary.drivers.rpm_driver.os")
@mock.patch("packetary.drivers.rpm_driver.open")
@mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist")
def test_create_repository(
self, ensure_dir_exists_mock, tmp_mock, os_mock
self, ensure_dir_exists_mock, open_mock, os_mock
):
repository_data = {
"name": "Test", "uri": "file:///repo/os/x86_64", "origin": "Test",
@ -283,7 +336,9 @@ class TestRpmDriver(base.TestCase):
md_config = mock.MagicMock()
self.createrepo.MetaDataConfig.return_value = md_config
repo = self.driver.create_repository(repository_data, "x86_64")
repo = self.driver.create_repository(
self.connection, repository_data, "x86_64"
)
ensure_dir_exists_mock.assert_called_once_with("/repo/os/x86_64/")
self.assertEqual(repository_data["name"], repo.name)
self.assertEqual("x86_64", repo.architecture)
@ -293,6 +348,8 @@ class TestRpmDriver(base.TestCase):
md_gen.doFinalMove.assert_called_once_with()
self.assertEqual(["*"], md_config.excludes)
self.assertFalse(md_config.update)
open_mock.assert_not_called()
os_mock.unlink.assert_not_called()
@mock.patch("packetary.drivers.rpm_driver.utils")
def test_load_package_from_file(self, utils):

1
setup.py
View File

@ -18,6 +18,7 @@
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools
import pkg_resources
# In python < 2.7.4, a lazy loading of package `pbr` will break
# setuptools if some other modules registered functions in `atexit`.