openstack/zun
Code Issues Proposed changes

Pull image from registry

Browse Source 
This commit complete the support of private docker registry.
Users can create a container with images from a specified
docker registry. The steps are as following:

1. Registry a docker registry in Zun (with options to specify
   the username/password to authenticate against the registry).
2. Run a container with a reference to the registry created in #1.

Closes-Bug: #1702830
Change-Id: I92f73bf0d759d9e770905debc6f40a5697ef0856
This commit is contained in:
Hongbin Lu 2019-01-10 04:06:00 +00:00
parent f608300050
commit bcf8455d8e
13 changed files with 89 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
zun/common/utils.py
View File

@ -162,7 +162,7 @@ def allow_all_content_types(f):
return _do_allow_certain_content_types(f, mimetypes.types_map.values())
def parse_image_name(image, driver=None):
def parse_image_name(image, driver=None, registry=None):
image_parts = docker_image.Reference.parse(image)
image_repo = image_parts['name']
@ -170,15 +170,18 @@ def parse_image_name(image, driver=None):
driver = CONF.default_image_driver
if driver == 'glance':
image_tag = ''
else:
image_tag = 'latest'
return image_repo, image_tag
image_tag = 'latest'
if image_parts['tag']:
image_tag = image_parts['tag']
registry, _ = image_parts.split_hostname()
if not registry and CONF.docker.default_registry:
image_repo = '%s/%s' % (CONF.docker.default_registry, image_repo)
domain, _ = image_parts.split_hostname()
if not domain:
if registry:
image_repo = '%s/%s' % (registry.domain, image_repo)
elif CONF.docker.default_registry:
image_repo = '%s/%s' % (CONF.docker.default_registry, image_repo)
return image_repo, image_tag

3
zun/compute/api.py
View File

@ -68,7 +68,8 @@ class API(object):
try:
images = self.rpcapi.image_search(
context, new_container.image,
new_container.image_driver, True, host_state['host'])
new_container.image_driver, True, new_container.registry,
host_state['host'])
if not images:
raise exception.ImageNotFound(image=new_container.image)
if len(images) > 1:

12
zun/compute/manager.py
View File

@ -293,12 +293,14 @@ class Manager(periodic_task.PeriodicTasks):
limits=None):
self._update_task_state(context, container, consts.IMAGE_PULLING)
image_driver_name = container.image_driver
repo, tag = utils.parse_image_name(container.image, image_driver_name)
repo, tag = utils.parse_image_name(container.image, image_driver_name,
registry=container.registry)
image_pull_policy = utils.get_image_pull_policy(
container.image_pull_policy, tag)
try:
image, image_loaded = self.driver.pull_image(
context, repo, tag, image_pull_policy, image_driver_name)
context, repo, tag, image_pull_policy, image_driver_name,
registry=container.registry)
image['repo'], image['tag'] = repo, tag
if not image_loaded:
self.driver.load_image(image['path'])
@ -1092,9 +1094,11 @@ class Manager(periodic_task.PeriodicTasks):
raise
@translate_exception
def image_search(self, context, image, image_driver_name, exact_match):
def image_search(self, context, image, image_driver_name, exact_match,
registry):
LOG.debug('Searching image...', image=image)
repo, tag = utils.parse_image_name(image, image_driver_name)
repo, tag = utils.parse_image_name(image, image_driver_name,
registry=registry)
try:
return self.driver.search_image(context, repo, tag,
image_driver_name, exact_match)

5
zun/compute/rpcapi.py
View File

@ -180,10 +180,11 @@ class API(rpc_service.API):
self._cast(host, 'image_pull', image=image)
def image_search(self, context, image, image_driver, exact_match,
host=None):
registry, host=None):
return self._call(host, 'image_search', image=image,
image_driver_name=image_driver,
exact_match=exact_match)
exact_match=exact_match,
registry=registry)
def capsule_create(self, context, host, capsule,
requested_networks, requested_volumes, limits):

4
zun/container/docker/driver.py
View File

@ -181,14 +181,14 @@ class DockerDriver(driver.ContainerDriver):
return docker.images(repo, quiet)
def pull_image(self, context, repo, tag, image_pull_policy='always',
driver_name=None):
driver_name=None, registry=None):
if driver_name is None:
driver_name = CONF.default_image_driver
try:
image_driver = self.image_drivers[driver_name]
image, image_loaded = image_driver.pull_image(
context, repo, tag, image_pull_policy)
context, repo, tag, image_pull_policy, registry)
if image:
image['driver'] = driver_name.split('.')[0]
except exception.ZunException:

14
zun/image/docker/driver.py
View File

@ -65,11 +65,15 @@ class DockerDriver(driver.ContainerImageDriver):
LOG.debug('Image %s not found locally', image)
return None
def _pull_image(self, repo, tag):
def _pull_image(self, repo, tag, registry):
auth_config = None
image_ref = docker_image.Reference.parse(repo)
registry, remainder = image_ref.split_hostname()
if (registry and registry == CONF.docker.default_registry and
registry_domain, remainder = image_ref.split_hostname()
if registry and registry.username:
auth_config = {'username': registry.username,
'password': registry.password}
elif (registry_domain and
registry_domain == CONF.docker.default_registry and
CONF.docker.default_registry_username):
auth_config = {'username': CONF.docker.default_registry_username,
'password': CONF.docker.default_registry_password}
@ -85,7 +89,7 @@ class DockerDriver(driver.ContainerImageDriver):
'repo': repo, 'tag': tag}
raise exception.ZunException(message)
def pull_image(self, context, repo, tag, image_pull_policy):
def pull_image(self, context, repo, tag, image_pull_policy, registry):
image_loaded = True
image = self._search_image_on_host(repo, tag)
if not utils.should_pull_image(image_pull_policy, bool(image)):
@ -101,7 +105,7 @@ class DockerDriver(driver.ContainerImageDriver):
LOG.debug('Pulling image from docker %(repo)s,'
' context %(context)s',
{'repo': repo, 'context': context})
self._pull_image(repo, tag)
self._pull_image(repo, tag, registry)
return {'image': repo, 'path': None}, image_loaded
except exception.ImageNotFound:
with excutils.save_and_reraise_exception():

2
zun/image/driver.py
View File

@ -61,7 +61,7 @@ def load_image_driver(image_driver=None):
class ContainerImageDriver(object):
"""Base class for container image driver."""
def pull_image(self, context, repo, tag, image_pull_policy):
def pull_image(self, context, repo, tag, image_pull_policy, registry):
"""Pull an image."""
raise NotImplementedError()

2
zun/image/glance/driver.py
View File

@ -71,7 +71,7 @@ class GlanceDriver(driver.ContainerImageDriver):
return True
return False
def pull_image(self, context, repo, tag, image_pull_policy):
def pull_image(self, context, repo, tag, image_pull_policy, registry):
image_loaded = False
image = self._search_image_on_host(context, repo, tag)

14
zun/tests/unit/common/test_utils.py
View File

@ -25,6 +25,7 @@ from zun import objects
from zun.objects.container import Container
from zun.tests import base
from zun.tests.unit.db import utils as db_utils
from zun.tests.unit.objects import utils as obj_utils
CONF = zun.conf.CONF
@ -64,7 +65,7 @@ class TestUtils(base.TestCase):
self.assertEqual(('test-test', 'test'),
utils.parse_image_name('test-test:test'))
def test_parse_image_name_with_registry(self):
def test_parse_image_name_with_default_registry(self):
CONF.set_override('default_registry', 'test.io', group='docker')
self.assertEqual(('test.io/test', 'latest'),
utils.parse_image_name('test'))
@ -73,6 +74,17 @@ class TestUtils(base.TestCase):
self.assertEqual(('other.com/test/test', 'latest'),
utils.parse_image_name('other.com/test/test'))
def test_parse_image_name_with_custom_registry(self):
registry = obj_utils.get_test_registry(self.context, domain='test.io')
self.assertEqual(('test.io/test', 'latest'),
utils.parse_image_name('test', registry=registry))
self.assertEqual(('test.io/test/test', 'latest'),
utils.parse_image_name('test/test',
registry=registry))
self.assertEqual(('other.com/test/test', 'latest'),
utils.parse_image_name('other.com/test/test',
registry=registry))
def test_get_image_pull_policy(self):
self.assertEqual('always',
utils.get_image_pull_policy('always',

4
zun/tests/unit/compute/test_compute_api.py
View File

@ -478,9 +478,9 @@ class TestAPI(base.TestCase):
@mock.patch('zun.compute.rpcapi.API._call')
def test_image_search(self, mock_call):
self.compute_api.image_search(
self.context, "ubuntu", "glance", True)
self.context, "ubuntu", "glance", True, None)
mock_call.assert_called_once_with(
None, "image_search", image="ubuntu",
None, "image_search", image="ubuntu", registry=None,
image_driver_name="glance", exact_match=True)
@mock.patch('zun.compute.rpcapi.API._cast')

15
zun/tests/unit/compute/test_compute_manager.py
View File

@ -200,7 +200,7 @@ class TestManager(base.TestCase):
networks, volumes)
mock_save.assert_called_with(self.context)
mock_pull.assert_any_call(self.context, container.image, '',
'always', 'glance')
'always', 'glance', registry=None)
mock_create.assert_called_once_with(self.context, container, image,
networks, volumes)
mock_event_start.assert_called_once()
@ -355,7 +355,7 @@ class TestManager(base.TestCase):
limits=None, run=True)
mock_save.assert_called_with(self.context)
mock_pull.assert_any_call(self.context, container.image, '',
'always', 'glance')
'always', 'glance', registry=None)
mock_create.assert_called_once_with(self.context, container, image,
networks, volumes)
mock_start.assert_called_once_with(self.context, container)
@ -463,7 +463,8 @@ class TestManager(base.TestCase):
self.assertEqual('Error', container.status)
self.assertEqual('Image Not Found', container.status_reason)
mock_pull.assert_called_once_with(self.context, 'test', 'latest',
'ifnotpresent', 'docker')
'ifnotpresent', 'docker',
registry=None)
mock_attach_volume.assert_called_once()
mock_detach_volume.assert_called_once()
mock_is_volume_available.assert_called_once()
@ -511,7 +512,8 @@ class TestManager(base.TestCase):
self.assertEqual('Error', container.status)
self.assertEqual('Image Not Found', container.status_reason)
mock_pull.assert_called_once_with(self.context, 'test', 'latest',
'ifnotpresent', 'docker')
'ifnotpresent', 'docker',
registry=None)
mock_attach_volume.assert_called_once()
mock_detach_volume.assert_called_once()
mock_is_volume_available.assert_called_once()
@ -559,7 +561,8 @@ class TestManager(base.TestCase):
self.assertEqual('Error', container.status)
self.assertEqual('Docker Error occurred', container.status_reason)
mock_pull.assert_called_once_with(self.context, 'test', 'latest',
'ifnotpresent', 'docker')
'ifnotpresent', 'docker',
registry=None)
mock_attach_volume.assert_called_once()
mock_detach_volume.assert_called_once()
mock_is_volume_available.assert_called_once()
@ -609,7 +612,7 @@ class TestManager(base.TestCase):
self.assertEqual('Error', container.status)
self.assertEqual('Docker Error occurred', container.status_reason)
mock_pull.assert_any_call(self.context, container.image, '',
'always', 'glance')
'always', 'glance', registry=None)
mock_create.assert_called_once_with(
self.context, container, image, networks, volumes)
mock_attach_volume.assert_called_once()

31
zun/tests/unit/image/docker/test_driver.py
View File

@ -20,6 +20,7 @@ from zun.common import exception
from zun.container.docker import utils
from zun.image.docker import driver
from zun.tests import base
from zun.tests.unit.objects import utils as obj_utils
class TempException(Exception):
@ -46,7 +47,7 @@ class TestDriver(base.BaseTestCase):
mock_should_pull_image.return_value = False
mock_search.return_value = None
self.assertRaises(exception.ImageNotFound, self.driver.pull_image,
None, 'nonexisting', 'tag', 'never')
None, 'nonexisting', 'tag', 'never', None)
@mock.patch.object(driver.DockerDriver,
'_search_image_on_host')
@ -57,7 +58,7 @@ class TestDriver(base.BaseTestCase):
mock_search.return_value = {'image': 'nginx', 'path': 'xyz'}
self.assertEqual(({'image': 'nginx', 'path': 'xyz'}, True),
self.driver.pull_image(None, 'nonexisting',
'tag', 'never'))
'tag', 'never', None))
@mock.patch.object(driver.DockerDriver,
'_search_image_on_host')
@ -65,11 +66,27 @@ class TestDriver(base.BaseTestCase):
def test_pull_image_success(self, mock_should_pull_image, mock_search):
mock_should_pull_image.return_value = True
mock_search.return_value = {'image': 'nginx', 'path': 'xyz'}
ret = self.driver.pull_image(None, 'test_image', 'latest', 'always')
ret = self.driver.pull_image(None, 'test_image', 'latest', 'always',
None)
self.assertEqual(({'image': 'test_image', 'path': None}, True), ret)
self.mock_docker.pull.assert_called_once_with(
'test_image', tag='latest', auth_config=None)
@mock.patch.object(driver.DockerDriver, '_search_image_on_host')
@mock.patch('zun.common.utils.should_pull_image')
def test_pull_image_with_registry(self, mock_should_pull_image,
mock_search):
mock_should_pull_image.return_value = True
mock_search.return_value = {'image': 'nginx', 'path': 'xyz'}
registry = obj_utils.get_test_registry(None)
ret = self.driver.pull_image(None, 'test_image', 'latest', 'always',
registry)
self.assertEqual(({'image': 'test_image', 'path': None}, True), ret)
expected_auth_config = {'username': registry.username,
'password': registry.password}
self.mock_docker.pull.assert_called_once_with(
'test_image', tag='latest', auth_config=expected_auth_config)
@mock.patch('zun.common.utils.parse_image_name')
@mock.patch.object(driver.DockerDriver,
'_search_image_on_host')
@ -82,7 +99,7 @@ class TestDriver(base.BaseTestCase):
self.mock_docker.pull = mock.Mock(
side_effect=errors.APIError('Error', '', ''))
self.assertRaises(exception.ZunException, self.driver.pull_image,
None, 'repo', 'tag', 'always')
None, 'repo', 'tag', 'always', None)
self.mock_docker.pull.assert_called_once_with(
'repo', tag='tag', auth_config=None)
@ -100,7 +117,7 @@ class TestDriver(base.BaseTestCase):
side_effect=exception.ImageNotFound('Error')
) as mock_pull:
self.assertRaises(exception.ImageNotFound, self.driver.pull_image,
None, 'repo', 'tag', 'always')
None, 'repo', 'tag', 'always', None)
self.mock_docker.pull.assert_called_once_with(
'repo', tag='tag', auth_config=None)
self.assertEqual(1, mock_pull.call_count)
@ -119,7 +136,7 @@ class TestDriver(base.BaseTestCase):
side_effect=exception.DockerError('Error')
) as mock_pull:
self.assertRaises(exception.DockerError, self.driver.pull_image,
None, 'repo', 'tag', 'always')
None, 'repo', 'tag', 'always', None)
self.mock_docker.pull.assert_called_once_with(
'repo', tag='tag', auth_config=None)
self.assertEqual(1, mock_pull.call_count)
@ -139,7 +156,7 @@ class TestDriver(base.BaseTestCase):
self.mock_docker.pull = mock.Mock(
side_effect=TempException('Error'))
self.assertRaises(exception.ZunException, self.driver.pull_image,
None, 'repo', 'tag', 'always')
None, 'repo', 'tag', 'always', None)
self.mock_docker.pull.assert_called_once_with(
'repo', tag='tag', auth_config=None)
self.assertEqual(1, mock_init.call_count)

11
zun/tests/unit/image/glance/test_driver.py
View File

@ -44,7 +44,7 @@ class TestDriver(base.BaseTestCase):
mock_should_pull_image.return_value = False
mock_search.return_value = None
self.assertRaises(exception.ImageNotFound, self.driver.pull_image,
None, 'nonexisting', 'tag', 'never')
None, 'nonexisting', 'tag', 'never', None)
@mock.patch.object(driver.GlanceDriver,
'_search_image_on_host')
@ -60,7 +60,7 @@ class TestDriver(base.BaseTestCase):
self.assertEqual(({'image': 'nginx', 'path': 'xyz',
'checksum': checksum}, True),
self.driver.pull_image(None, 'nonexisting',
'tag', 'never'))
'tag', 'never', None))
mock_open_file.assert_any_call('xyz', 'rb')
@mock.patch.object(driver.GlanceDriver,
@ -75,7 +75,7 @@ class TestDriver(base.BaseTestCase):
'checksum': 'xxx'}
mock_find_image.side_effect = Exception
self.assertRaises(exception.ZunException, self.driver.pull_image,
None, 'nonexisting', 'tag', 'always')
None, 'nonexisting', 'tag', 'always', None)
@mock.patch.object(driver.GlanceDriver,
'_search_image_on_host')
@ -97,7 +97,8 @@ class TestDriver(base.BaseTestCase):
out_path = os.path.join(self.test_dir, '1234' + '.tar')
mock_open_file = mock.mock_open()
with mock.patch('zun.image.glance.driver.open', mock_open_file):
ret = self.driver.pull_image(None, 'image', 'latest', 'always')
ret = self.driver.pull_image(None, 'image', 'latest', 'always',
None)
mock_open_file.assert_any_call(out_path, 'wb')
self.assertTrue(mock_search_on_host.called)
self.assertTrue(mock_should_pull_image.called)
@ -112,7 +113,7 @@ class TestDriver(base.BaseTestCase):
with mock.patch('zun.image.glance.utils.find_image') as mock_find:
mock_find.side_effect = exception.ImageNotFound
self.assertRaises(exception.ImageNotFound, self.driver.pull_image,
None, 'nonexisting', 'tag', 'always')
None, 'nonexisting', 'tag', 'always', None)
@mock.patch('zun.image.glance.utils.find_images')
def test_search_image_found(self, mock_find_images):