x/packetary
Code Issues Proposed changes

Copied groups index upon fork of repository

Browse Source 
Also keep original group index durring rebuild repository index.
Build only meta-files without package information durring create and fork
repository

Change-Id: I613ce2084750f02035a13c301aa5812e08a90482
Closes-Bug: 1549413
This commit is contained in:
Bulat Gaifullin 2016-02-24 21:58:26 +03:00
parent c0f04443b5
commit f27bed00a5
4 changed files with 173 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packetary/controllers/repository.py
View File

@ -170,7 +170,7 @@ class RepositoryController(object):
utils.get_path_from_url(urljoin(target.url, dst_path)),
size=package.filesize
)
if package.filesize < 0:
if package.filesize <= 0:
package.filesize = bytes_copied
if observer:
observer(bytes_copied)

90
packetary/drivers/rpm_driver.py
View File

@ -20,12 +20,12 @@ import copy
import multiprocessing
import os
import shutil
import tempfile
import createrepo
import lxml.etree as etree
import six
from packetary.drivers.base import RepositoryDriverBase
from packetary.library.checksum import composite as checksum_composite
from packetary.library.streams import GzipDecompress
@ -157,35 +157,8 @@ class RpmRepositoryDriver(RepositoryDriverBase):
self.logger.info("loaded: %d packages from %s.", counter, repository)
def add_packages(self, connection, repository, packages):
basepath = utils.get_path_from_url(repository.url)
self.logger.info("rebuild repository in %s", basepath)
md_config = createrepo.MetaDataConfig()
try:
md_config.workers = multiprocessing.cpu_count()
md_config.directory = str(basepath)
md_config.update = os.path.exists(
os.path.join(basepath, md_config.finaldir)
)
mdgen = createrepo.MetaDataGenerator(
config_obj=md_config, callback=CreaterepoCallBack(self.logger)
)
mdgen.doPkgMetadata()
mdgen.doRepoMetadata()
mdgen.doFinalMove()
except createrepo.MDError as e:
err_msg = six.text_type(e)
self.logger.exception(
"failed to create yum repository in %s: %s",
basepath,
err_msg
)
shutil.rmtree(
os.path.join(md_config.outputdir, md_config.tempdir),
ignore_errors=True
)
raise RuntimeError(
"Failed to create yum repository in {0}."
.format(err_msg))
groupstree = self._load_groups(connection, repository)
self._rebuild_repository(repository, packages, groupstree)
def fork_repository(self, connection, repository, destination,
source=False, locale=False):
@ -194,7 +167,8 @@ class RpmRepositoryDriver(RepositoryDriverBase):
new_repo = copy.copy(repository)
new_repo.url = utils.normalize_repository_url(destination)
utils.ensure_dir_exist(destination)
self.add_packages(connection, new_repo, set())
groupstree = self._load_groups(connection, repository)
self._rebuild_repository(new_repo, None, groupstree)
return new_repo
def create_repository(self, repository_data, arch):
@ -206,6 +180,7 @@ class RpmRepositoryDriver(RepositoryDriverBase):
origin=repository_data.get('origin')
)
utils.ensure_dir_exist(utils.get_path_from_url(repository.url))
self._rebuild_repository(repository, None, None)
return repository
def load_package_from_file(self, repository, filepath):
@ -233,6 +208,59 @@ class RpmRepositoryDriver(RepositoryDriverBase):
def get_relative_path(self, repository, filename):
return "packages/" + filename
def _rebuild_repository(self, repository, packages, groupstree=None):
basepath = utils.get_path_from_url(repository.url)
self.logger.info("rebuild repository in %s", basepath)
md_config = createrepo.MetaDataConfig()
update = packages is not None and \
os.path.exists(os.path.join(basepath, md_config.finaldir))
groupsfile = None
if groupstree is not None:
with tempfile.NamedTemporaryFile(delete=False) as tmp:
groupstree.write(tmp)
groupsfile = tmp.name
try:
md_config.workers = multiprocessing.cpu_count()
md_config.directory = str(basepath)
md_config.groupfile = groupsfile
md_config.update = update
if packages is None:
# only generate meta-files, without packages info
md_config.excludes = ["*"]
mdgen = createrepo.MetaDataGenerator(
config_obj=md_config, callback=CreaterepoCallBack(self.logger)
)
mdgen.doPkgMetadata()
mdgen.doRepoMetadata()
mdgen.doFinalMove()
except createrepo.MDError as e:
err_msg = six.text_type(e)
self.logger.exception(
"failed to create yum repository in %s: %s",
basepath,
err_msg
)
shutil.rmtree(
os.path.join(md_config.outputdir, md_config.tempdir),
ignore_errors=True
)
raise RuntimeError(
"Failed to create yum repository in {0}."
.format(err_msg))
finally:
if groupsfile is not None:
os.unlink(groupsfile)
def _load_groups(self, connection, repository):
repomd = urljoin(repository.url, "repodata/repomd.xml")
self.logger.debug("load repomd: %s", repomd)
repomd_tree = etree.parse(connection.open_stream(repomd))
return self._load_db(
connection, repository.url, repomd_tree, "group_gz", "group"
)
def _load_db(self, connection, baseurl, repomd, *aliases):
"""Loads database.

4
packetary/tests/stubs/helpers.py
View File

@ -64,3 +64,7 @@ def get_compressed(stream):
with closing(gzip.GzipFile(fileobj=compressed, mode="wb")) as gz:
gz.write(stream.read())
return Buffer(compressed)
def read_to_buffer(stream):
return six.BytesIO(stream.read())

160
packetary/tests/test_rpm_driver.py
View File

@ -27,6 +27,7 @@ from packetary.schemas import RPM_REPO_SCHEMA
from packetary.tests import base
from packetary.tests.stubs.generator import gen_repository
from packetary.tests.stubs.helpers import get_compressed
from packetary.tests.stubs.helpers import read_to_buffer
REPOMD = path.join(path.dirname(__file__), "data", "repomd.xml")
@ -53,6 +54,24 @@ class TestRpmDriver(base.TestCase):
self.createrepo.reset_mock()
self.connection = mock.MagicMock()
def configure_streams(self, groups_gzipped=True):
streams = []
if groups_gzipped:
groups_conv = get_compressed
md_file = REPOMD
else:
groups_conv = read_to_buffer
md_file = REPOMD2
for conv, fname in six.moves.zip(
(read_to_buffer, groups_conv, get_compressed),
(md_file, GROUPS_DB, PRIMARY_DB)
):
with open(fname, "rb") as s:
streams.append(conv(s))
self.connection.open_stream.side_effect = streams
def test_priority_sort(self):
repos = [
{"name": "repo0"},
@ -89,17 +108,8 @@ class TestRpmDriver(base.TestCase):
self.assertEqual("centos", repo.path)
def test_get_packages(self):
streams = []
for conv, fname in zip(
(lambda x: six.BytesIO(x.read()),
get_compressed, get_compressed),
(REPOMD, GROUPS_DB, PRIMARY_DB)
):
with open(fname, "rb") as s:
streams.append(conv(s))
self.configure_streams()
packages = []
self.connection.open_stream.side_effect = streams
self.driver.get_packages(
self.connection,
gen_repository("test", url="http://host/centos/os/x86_64/"),
@ -146,18 +156,8 @@ class TestRpmDriver(base.TestCase):
self.assertFalse(packages[1].mandatory)
def test_get_packages_if_group_not_gzipped(self):
streams = []
for conv, fname in zip(
(lambda x: six.BytesIO(x.read()),
lambda x: six.BytesIO(x.read()),
get_compressed),
(REPOMD2, GROUPS_DB, PRIMARY_DB)
):
with open(fname, "rb") as s:
streams.append(conv(s))
self.configure_streams(False)
packages = []
self.connection.open_stream.side_effect = streams
self.driver.get_packages(
self.connection,
gen_repository("test", url="http://host/centos/os/x86_64/"),
@ -170,46 +170,83 @@ class TestRpmDriver(base.TestCase):
package = packages[0]
self.assertTrue(package.mandatory)
@mock.patch("packetary.drivers.rpm_driver.os.path.exists")
@mock.patch("packetary.drivers.rpm_driver.shutil")
def test_add_packages(self, shutil, path_exists):
self.createrepo.MDError = ValueError
self.createrepo.MetaDataGenerator().doFinalMove.side_effect = [
None, self.createrepo.MDError()
]
@mock.patch("packetary.drivers.rpm_driver.os")
@mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
def test_add_packages_to_existing(self, tmp_mock, os_mock):
self.configure_streams()
tmp_file = mock.MagicMock()
tmp_file.name = "/tmp/groups.gz"
tmp_mock.return_value.__enter__.return_value = tmp_file
repo = gen_repository("test", url="file:///repo/os/x86_64")
self.createrepo.MetaDataConfig().outputdir = "/repo/os/x86_64"
self.createrepo.MetaDataConfig().tempdir = "tmp"
self.createrepo.MetaDataConfig().finaldir = "repodata"
path_exists.side_effect = [True, False]
md_gen = mock.MagicMock()
self.createrepo.MetaDataGenerator.return_value = md_gen
md_gen.outputdir = "/repo/os/x86_64"
md_gen.tempdir = "tmp"
md_gen.finaldir = "repodata"
os_mock.path.exists.return_value = True
self.driver.add_packages(self.connection, repo, set())
self.assertEqual(
"/repo/os/x86_64",
self.createrepo.MetaDataConfig().directory
)
self.assertTrue(self.createrepo.MetaDataConfig().update)
self.createrepo.MetaDataGenerator()\
.doPkgMetadata.assert_called_once_with()
self.createrepo.MetaDataGenerator()\
.doRepoMetadata.assert_called_once_with()
self.createrepo.MetaDataGenerator()\
.doFinalMove.assert_called_once_with()
md_gen.doPkgMetadata.assert_called_once_with()
md_gen.doRepoMetadata.assert_called_once_with()
md_gen.doFinalMove.assert_called_once_with()
with self.assertRaises(RuntimeError):
self.driver.add_packages(self.connection, repo, set())
self.assertFalse(self.createrepo.MetaDataConfig().update)
shutil.rmtree.assert_called_once_with(
"/repo/os/x86_64/tmp", ignore_errors=True
self.assertGreater(tmp_file.write.call_count, 0)
self.assertEqual(
tmp_file.name, self.createrepo.MetaDataConfig().groupfile
)
os_mock.unlink.assert_called_once_with(tmp_file.name)
@mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist")
def test_fork_repository(self, ensure_dir_exists_mock):
repo = gen_repository("os", url="http://localhost/os/x86_64/")
self.createrepo.MetaDataGenerator().doFinalMove.side_effect = [None]
@mock.patch("packetary.drivers.rpm_driver.os")
@mock.patch("packetary.drivers.rpm_driver.shutil")
@mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
def test_add_packages_clean_metadata_on_error(
self, tmp_mock, shutil_mock, os_mock
):
self.configure_streams()
tmp_file = mock.MagicMock()
tmp_file.name = "/tmp/groups.gz"
tmp_mock.return_value.__enter__.return_value = tmp_file
self.createrepo.MDError = ValueError
md_gen = mock.MagicMock()
self.createrepo.MetaDataGenerator.return_value = md_gen
md_gen.doFinalMove.side_effect = self.createrepo.MDError()
repo = gen_repository("test", url="file:///repo/os/x86_64")
self.createrepo.MetaDataConfig().outputdir = "/repo/os/x86_64"
self.createrepo.MetaDataConfig().tempdir = "tmp"
self.createrepo.MetaDataConfig().finaldir = "repodata"
os_mock.path.exists.return_value = True
os_mock.path.join.side_effect = lambda *a: '/'.join(a)
with self.assertRaises(RuntimeError):
self.driver.add_packages(self.connection, repo, set())
shutil_mock.rmtree.assert_called_once_with(
"/repo/os/x86_64/tmp", ignore_errors=True
)
os_mock.unlink.assert_called_once_with(tmp_file.name)
@mock.patch("packetary.drivers.rpm_driver.os")
@mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
@mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist")
def test_fork_repository(
self, ensure_dir_exists_mock, tmp_mock, os_mock
):
self.configure_streams()
tmp_file = mock.MagicMock()
tmp_file.name = "/tmp/groups.gz"
tmp_mock.return_value.__enter__.return_value = tmp_file
repo = gen_repository("os", url="http://localhost/os/x86_64/")
md_gen = mock.MagicMock()
self.createrepo.MetaDataGenerator.return_value = md_gen
md_gen.doFinalMove.side_effect = [None]
md_gen.outputdir = "/repo/os/x86_64"
md_gen.tempdir = "tmp"
md_gen.finaldir = "repodata"
md_config = mock.MagicMock()
self.createrepo.MetaDataConfig.return_value = md_config
new_repo = self.driver.fork_repository(
self.connection,
repo,
@ -219,15 +256,33 @@ class TestRpmDriver(base.TestCase):
self.assertEqual(repo.name, new_repo.name)
self.assertEqual(repo.architecture, new_repo.architecture)
self.assertEqual("file:///repo/os/x86_64/", new_repo.url)
self.createrepo.MetaDataGenerator()\
.doFinalMove.assert_called_once_with()
md_gen.doFinalMove.assert_called_once_with()
self.assertGreater(tmp_file.write.call_count, 0)
self.assertEqual(["*"], md_config.excludes)
self.assertFalse(md_config.update)
self.assertEqual(tmp_file.name, md_config.groupfile)
os_mock.unlink.assert_called_once_with(tmp_file.name)
@mock.patch("packetary.drivers.rpm_driver.os")
@mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
@mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist")
def test_create_repository(self, ensure_dir_exists_mock):
def test_create_repository(
self, ensure_dir_exists_mock, tmp_mock, os_mock
):
repository_data = {
"name": "Test", "uri": "file:///repo/os/x86_64", "origin": "Test",
"path": "centos"
}
self.configure_streams()
md_gen = mock.MagicMock()
self.createrepo.MetaDataGenerator.return_value = md_gen
md_gen.doFinalMove.side_effect = [None]
md_gen.outputdir = "/repo/os/x86_64"
md_gen.tempdir = "tmp"
md_gen.finaldir = "repodata"
md_config = mock.MagicMock()
self.createrepo.MetaDataConfig.return_value = md_config
repo = self.driver.create_repository(repository_data, "x86_64")
ensure_dir_exists_mock.assert_called_once_with("/repo/os/x86_64/")
self.assertEqual(repository_data["name"], repo.name)
@ -235,6 +290,9 @@ class TestRpmDriver(base.TestCase):
self.assertEqual(repository_data["uri"] + "/", repo.url)
self.assertEqual(repository_data["origin"], repo.origin)
self.assertEqual(repository_data["path"], repo.path)
md_gen.doFinalMove.assert_called_once_with()
self.assertEqual(["*"], md_config.excludes)
self.assertFalse(md_config.update)
@mock.patch("packetary.drivers.rpm_driver.utils")
def test_load_package_from_file(self, utils):