Pull image from registry

This commit complete the support of private docker registry.
Users can create a container with images from a specified
docker registry. The steps are as following:

1. Registry a docker registry in Zun (with options to specify
   the username/password to authenticate against the registry).
2. Run a container with a reference to the registry created in #1.

Closes-Bug: #1702830
Change-Id: I92f73bf0d759d9e770905debc6f40a5697ef0856

zun/common/utils.py

@@ -162,7 +162,7 @@ def allow_all_content_types(f):
     return _do_allow_certain_content_types(f, mimetypes.types_map.values())
 
 
-def parse_image_name(image, driver=None):
+def parse_image_name(image, driver=None, registry=None):
     image_parts = docker_image.Reference.parse(image)
 
     image_repo = image_parts['name']
@@ -170,14 +170,17 @@ def parse_image_name(image, driver=None):
         driver = CONF.default_image_driver
     if driver == 'glance':
         image_tag = ''
-    else:
+        return image_repo, image_tag
-        image_tag = 'latest'
 
+    image_tag = 'latest'
     if image_parts['tag']:
         image_tag = image_parts['tag']
 
-    registry, _ = image_parts.split_hostname()
+    domain, _ = image_parts.split_hostname()
-    if not registry and CONF.docker.default_registry:
+    if not domain:
+        if registry:
+            image_repo = '%s/%s' % (registry.domain, image_repo)
+        elif CONF.docker.default_registry:
             image_repo = '%s/%s' % (CONF.docker.default_registry, image_repo)
 
     return image_repo, image_tag

zun/compute/api.py

@@ -68,7 +68,8 @@ class API(object):
             try:
                 images = self.rpcapi.image_search(
                     context, new_container.image,
-                    new_container.image_driver, True, host_state['host'])
+                    new_container.image_driver, True, new_container.registry,
+                    host_state['host'])
                 if not images:
                     raise exception.ImageNotFound(image=new_container.image)
                 if len(images) > 1:

zun/compute/manager.py

@@ -293,12 +293,14 @@ class Manager(periodic_task.PeriodicTasks):
                                   limits=None):
         self._update_task_state(context, container, consts.IMAGE_PULLING)
         image_driver_name = container.image_driver
-        repo, tag = utils.parse_image_name(container.image, image_driver_name)
+        repo, tag = utils.parse_image_name(container.image, image_driver_name,
+                                           registry=container.registry)
         image_pull_policy = utils.get_image_pull_policy(
             container.image_pull_policy, tag)
         try:
             image, image_loaded = self.driver.pull_image(
-                context, repo, tag, image_pull_policy, image_driver_name)
+                context, repo, tag, image_pull_policy, image_driver_name,
+                registry=container.registry)
             image['repo'], image['tag'] = repo, tag
             if not image_loaded:
                 self.driver.load_image(image['path'])
@@ -1092,9 +1094,11 @@ class Manager(periodic_task.PeriodicTasks):
             raise
 
     @translate_exception
-    def image_search(self, context, image, image_driver_name, exact_match):
+    def image_search(self, context, image, image_driver_name, exact_match,
+                     registry):
         LOG.debug('Searching image...', image=image)
-        repo, tag = utils.parse_image_name(image, image_driver_name)
+        repo, tag = utils.parse_image_name(image, image_driver_name,
+                                           registry=registry)
         try:
             return self.driver.search_image(context, repo, tag,
                                             image_driver_name, exact_match)

zun/compute/rpcapi.py

@@ -180,10 +180,11 @@ class API(rpc_service.API):
         self._cast(host, 'image_pull', image=image)
 
     def image_search(self, context, image, image_driver, exact_match,
-                     host=None):
+                     registry, host=None):
         return self._call(host, 'image_search', image=image,
                           image_driver_name=image_driver,
-                          exact_match=exact_match)
+                          exact_match=exact_match,
+                          registry=registry)
 
     def capsule_create(self, context, host, capsule,
                        requested_networks, requested_volumes, limits):

zun/container/docker/driver.py

@@ -181,14 +181,14 @@ class DockerDriver(driver.ContainerDriver):
             return docker.images(repo, quiet)
 
     def pull_image(self, context, repo, tag, image_pull_policy='always',
-                   driver_name=None):
+                   driver_name=None, registry=None):
         if driver_name is None:
             driver_name = CONF.default_image_driver
 
         try:
             image_driver = self.image_drivers[driver_name]
             image, image_loaded = image_driver.pull_image(
-                context, repo, tag, image_pull_policy)
+                context, repo, tag, image_pull_policy, registry)
             if image:
                 image['driver'] = driver_name.split('.')[0]
         except exception.ZunException:

zun/image/docker/driver.py

@@ -65,11 +65,15 @@ class DockerDriver(driver.ContainerImageDriver):
                 LOG.debug('Image %s not found locally', image)
                 return None
 
-    def _pull_image(self, repo, tag):
+    def _pull_image(self, repo, tag, registry):
         auth_config = None
         image_ref = docker_image.Reference.parse(repo)
-        registry, remainder = image_ref.split_hostname()
+        registry_domain, remainder = image_ref.split_hostname()
-        if (registry and registry == CONF.docker.default_registry and
+        if registry and registry.username:
+            auth_config = {'username': registry.username,
+                           'password': registry.password}
+        elif (registry_domain and
+                registry_domain == CONF.docker.default_registry and
                 CONF.docker.default_registry_username):
             auth_config = {'username': CONF.docker.default_registry_username,
                            'password': CONF.docker.default_registry_password}
@@ -85,7 +89,7 @@ class DockerDriver(driver.ContainerImageDriver):
                     'repo': repo, 'tag': tag}
                 raise exception.ZunException(message)
 
-    def pull_image(self, context, repo, tag, image_pull_policy):
+    def pull_image(self, context, repo, tag, image_pull_policy, registry):
         image_loaded = True
         image = self._search_image_on_host(repo, tag)
         if not utils.should_pull_image(image_pull_policy, bool(image)):
@@ -101,7 +105,7 @@ class DockerDriver(driver.ContainerImageDriver):
             LOG.debug('Pulling image from docker %(repo)s,'
                       ' context %(context)s',
                       {'repo': repo, 'context': context})
-            self._pull_image(repo, tag)
+            self._pull_image(repo, tag, registry)
             return {'image': repo, 'path': None}, image_loaded
         except exception.ImageNotFound:
             with excutils.save_and_reraise_exception():

zun/image/driver.py

@@ -61,7 +61,7 @@ def load_image_driver(image_driver=None):
 class ContainerImageDriver(object):
     """Base class for container image driver."""
 
-    def pull_image(self, context, repo, tag, image_pull_policy):
+    def pull_image(self, context, repo, tag, image_pull_policy, registry):
         """Pull an image."""
         raise NotImplementedError()
 

zun/image/glance/driver.py

@@ -71,7 +71,7 @@ class GlanceDriver(driver.ContainerImageDriver):
             return True
         return False
 
-    def pull_image(self, context, repo, tag, image_pull_policy):
+    def pull_image(self, context, repo, tag, image_pull_policy, registry):
         image_loaded = False
         image = self._search_image_on_host(context, repo, tag)
 

zun/tests/unit/common/test_utils.py

@@ -25,6 +25,7 @@ from zun import objects
 from zun.objects.container import Container
 from zun.tests import base
 from zun.tests.unit.db import utils as db_utils
+from zun.tests.unit.objects import utils as obj_utils
 
 
 CONF = zun.conf.CONF
@@ -64,7 +65,7 @@ class TestUtils(base.TestCase):
         self.assertEqual(('test-test', 'test'),
                          utils.parse_image_name('test-test:test'))
 
-    def test_parse_image_name_with_registry(self):
+    def test_parse_image_name_with_default_registry(self):
         CONF.set_override('default_registry', 'test.io', group='docker')
         self.assertEqual(('test.io/test', 'latest'),
                          utils.parse_image_name('test'))
@@ -73,6 +74,17 @@ class TestUtils(base.TestCase):
         self.assertEqual(('other.com/test/test', 'latest'),
                          utils.parse_image_name('other.com/test/test'))
 
+    def test_parse_image_name_with_custom_registry(self):
+        registry = obj_utils.get_test_registry(self.context, domain='test.io')
+        self.assertEqual(('test.io/test', 'latest'),
+                         utils.parse_image_name('test', registry=registry))
+        self.assertEqual(('test.io/test/test', 'latest'),
+                         utils.parse_image_name('test/test',
+                                                registry=registry))
+        self.assertEqual(('other.com/test/test', 'latest'),
+                         utils.parse_image_name('other.com/test/test',
+                                                registry=registry))
+
     def test_get_image_pull_policy(self):
         self.assertEqual('always',
                          utils.get_image_pull_policy('always',

zun/tests/unit/compute/test_compute_api.py

@@ -478,9 +478,9 @@ class TestAPI(base.TestCase):
     @mock.patch('zun.compute.rpcapi.API._call')
     def test_image_search(self, mock_call):
         self.compute_api.image_search(
-            self.context, "ubuntu", "glance", True)
+            self.context, "ubuntu", "glance", True, None)
         mock_call.assert_called_once_with(
-            None, "image_search", image="ubuntu",
+            None, "image_search", image="ubuntu", registry=None,
             image_driver_name="glance", exact_match=True)
 
     @mock.patch('zun.compute.rpcapi.API._cast')

zun/tests/unit/compute/test_compute_manager.py

@@ -200,7 +200,7 @@ class TestManager(base.TestCase):
                                                   networks, volumes)
         mock_save.assert_called_with(self.context)
         mock_pull.assert_any_call(self.context, container.image, '',
-                                  'always', 'glance')
+                                  'always', 'glance', registry=None)
         mock_create.assert_called_once_with(self.context, container, image,
                                             networks, volumes)
         mock_event_start.assert_called_once()
@@ -355,7 +355,7 @@ class TestManager(base.TestCase):
             limits=None, run=True)
         mock_save.assert_called_with(self.context)
         mock_pull.assert_any_call(self.context, container.image, '',
-                                  'always', 'glance')
+                                  'always', 'glance', registry=None)
         mock_create.assert_called_once_with(self.context, container, image,
                                             networks, volumes)
         mock_start.assert_called_once_with(self.context, container)
@@ -463,7 +463,8 @@ class TestManager(base.TestCase):
         self.assertEqual('Error', container.status)
         self.assertEqual('Image Not Found', container.status_reason)
         mock_pull.assert_called_once_with(self.context, 'test', 'latest',
-                                          'ifnotpresent', 'docker')
+                                          'ifnotpresent', 'docker',
+                                          registry=None)
         mock_attach_volume.assert_called_once()
         mock_detach_volume.assert_called_once()
         mock_is_volume_available.assert_called_once()
@@ -511,7 +512,8 @@ class TestManager(base.TestCase):
         self.assertEqual('Error', container.status)
         self.assertEqual('Image Not Found', container.status_reason)
         mock_pull.assert_called_once_with(self.context, 'test', 'latest',
-                                          'ifnotpresent', 'docker')
+                                          'ifnotpresent', 'docker',
+                                          registry=None)
         mock_attach_volume.assert_called_once()
         mock_detach_volume.assert_called_once()
         mock_is_volume_available.assert_called_once()
@@ -559,7 +561,8 @@ class TestManager(base.TestCase):
         self.assertEqual('Error', container.status)
         self.assertEqual('Docker Error occurred', container.status_reason)
         mock_pull.assert_called_once_with(self.context, 'test', 'latest',
-                                          'ifnotpresent', 'docker')
+                                          'ifnotpresent', 'docker',
+                                          registry=None)
         mock_attach_volume.assert_called_once()
         mock_detach_volume.assert_called_once()
         mock_is_volume_available.assert_called_once()
@@ -609,7 +612,7 @@ class TestManager(base.TestCase):
         self.assertEqual('Error', container.status)
         self.assertEqual('Docker Error occurred', container.status_reason)
         mock_pull.assert_any_call(self.context, container.image, '',
-                                  'always', 'glance')
+                                  'always', 'glance', registry=None)
         mock_create.assert_called_once_with(
             self.context, container, image, networks, volumes)
         mock_attach_volume.assert_called_once()

zun/tests/unit/image/docker/test_driver.py

@@ -20,6 +20,7 @@ from zun.common import exception
 from zun.container.docker import utils
 from zun.image.docker import driver
 from zun.tests import base
+from zun.tests.unit.objects import utils as obj_utils
 
 
 class TempException(Exception):
@@ -46,7 +47,7 @@ class TestDriver(base.BaseTestCase):
         mock_should_pull_image.return_value = False
         mock_search.return_value = None
         self.assertRaises(exception.ImageNotFound, self.driver.pull_image,
-                          None, 'nonexisting', 'tag', 'never')
+                          None, 'nonexisting', 'tag', 'never', None)
 
     @mock.patch.object(driver.DockerDriver,
                        '_search_image_on_host')
@@ -57,7 +58,7 @@ class TestDriver(base.BaseTestCase):
         mock_search.return_value = {'image': 'nginx', 'path': 'xyz'}
         self.assertEqual(({'image': 'nginx', 'path': 'xyz'}, True),
                          self.driver.pull_image(None, 'nonexisting',
-                                                'tag', 'never'))
+                                                'tag', 'never', None))
 
     @mock.patch.object(driver.DockerDriver,
                        '_search_image_on_host')
@@ -65,11 +66,27 @@ class TestDriver(base.BaseTestCase):
     def test_pull_image_success(self, mock_should_pull_image, mock_search):
         mock_should_pull_image.return_value = True
         mock_search.return_value = {'image': 'nginx', 'path': 'xyz'}
-        ret = self.driver.pull_image(None, 'test_image', 'latest', 'always')
+        ret = self.driver.pull_image(None, 'test_image', 'latest', 'always',
+                                     None)
         self.assertEqual(({'image': 'test_image', 'path': None}, True), ret)
         self.mock_docker.pull.assert_called_once_with(
             'test_image', tag='latest', auth_config=None)
 
+    @mock.patch.object(driver.DockerDriver, '_search_image_on_host')
+    @mock.patch('zun.common.utils.should_pull_image')
+    def test_pull_image_with_registry(self, mock_should_pull_image,
+                                      mock_search):
+        mock_should_pull_image.return_value = True
+        mock_search.return_value = {'image': 'nginx', 'path': 'xyz'}
+        registry = obj_utils.get_test_registry(None)
+        ret = self.driver.pull_image(None, 'test_image', 'latest', 'always',
+                                     registry)
+        self.assertEqual(({'image': 'test_image', 'path': None}, True), ret)
+        expected_auth_config = {'username': registry.username,
+                                'password': registry.password}
+        self.mock_docker.pull.assert_called_once_with(
+            'test_image', tag='latest', auth_config=expected_auth_config)
+
     @mock.patch('zun.common.utils.parse_image_name')
     @mock.patch.object(driver.DockerDriver,
                        '_search_image_on_host')
@@ -82,7 +99,7 @@ class TestDriver(base.BaseTestCase):
         self.mock_docker.pull = mock.Mock(
             side_effect=errors.APIError('Error', '', ''))
         self.assertRaises(exception.ZunException, self.driver.pull_image,
-                          None, 'repo', 'tag', 'always')
+                          None, 'repo', 'tag', 'always', None)
         self.mock_docker.pull.assert_called_once_with(
             'repo', tag='tag', auth_config=None)
 
@@ -100,7 +117,7 @@ class TestDriver(base.BaseTestCase):
                                side_effect=exception.ImageNotFound('Error')
                                ) as mock_pull:
             self.assertRaises(exception.ImageNotFound, self.driver.pull_image,
-                              None, 'repo', 'tag', 'always')
+                              None, 'repo', 'tag', 'always', None)
             self.mock_docker.pull.assert_called_once_with(
                 'repo', tag='tag', auth_config=None)
             self.assertEqual(1, mock_pull.call_count)
@@ -119,7 +136,7 @@ class TestDriver(base.BaseTestCase):
                                side_effect=exception.DockerError('Error')
                                ) as mock_pull:
             self.assertRaises(exception.DockerError, self.driver.pull_image,
-                              None, 'repo', 'tag', 'always')
+                              None, 'repo', 'tag', 'always', None)
             self.mock_docker.pull.assert_called_once_with(
                 'repo', tag='tag', auth_config=None)
             self.assertEqual(1, mock_pull.call_count)
@@ -139,7 +156,7 @@ class TestDriver(base.BaseTestCase):
             self.mock_docker.pull = mock.Mock(
                 side_effect=TempException('Error'))
             self.assertRaises(exception.ZunException, self.driver.pull_image,
-                              None, 'repo', 'tag', 'always')
+                              None, 'repo', 'tag', 'always', None)
             self.mock_docker.pull.assert_called_once_with(
                 'repo', tag='tag', auth_config=None)
             self.assertEqual(1, mock_init.call_count)

zun/tests/unit/image/glance/test_driver.py

@@ -44,7 +44,7 @@ class TestDriver(base.BaseTestCase):
         mock_should_pull_image.return_value = False
         mock_search.return_value = None
         self.assertRaises(exception.ImageNotFound, self.driver.pull_image,
-                          None, 'nonexisting', 'tag', 'never')
+                          None, 'nonexisting', 'tag', 'never', None)
 
     @mock.patch.object(driver.GlanceDriver,
                        '_search_image_on_host')
@@ -60,7 +60,7 @@ class TestDriver(base.BaseTestCase):
             self.assertEqual(({'image': 'nginx', 'path': 'xyz',
                                'checksum': checksum}, True),
                              self.driver.pull_image(None, 'nonexisting',
-                                                    'tag', 'never'))
+                                                    'tag', 'never', None))
         mock_open_file.assert_any_call('xyz', 'rb')
 
     @mock.patch.object(driver.GlanceDriver,
@@ -75,7 +75,7 @@ class TestDriver(base.BaseTestCase):
                                     'checksum': 'xxx'}
         mock_find_image.side_effect = Exception
         self.assertRaises(exception.ZunException, self.driver.pull_image,
-                          None, 'nonexisting', 'tag', 'always')
+                          None, 'nonexisting', 'tag', 'always', None)
 
     @mock.patch.object(driver.GlanceDriver,
                        '_search_image_on_host')
@@ -97,7 +97,8 @@ class TestDriver(base.BaseTestCase):
         out_path = os.path.join(self.test_dir, '1234' + '.tar')
         mock_open_file = mock.mock_open()
         with mock.patch('zun.image.glance.driver.open', mock_open_file):
-            ret = self.driver.pull_image(None, 'image', 'latest', 'always')
+            ret = self.driver.pull_image(None, 'image', 'latest', 'always',
+                                         None)
         mock_open_file.assert_any_call(out_path, 'wb')
         self.assertTrue(mock_search_on_host.called)
         self.assertTrue(mock_should_pull_image.called)
@@ -112,7 +113,7 @@ class TestDriver(base.BaseTestCase):
         with mock.patch('zun.image.glance.utils.find_image') as mock_find:
             mock_find.side_effect = exception.ImageNotFound
             self.assertRaises(exception.ImageNotFound, self.driver.pull_image,
-                              None, 'nonexisting', 'tag', 'always')
+                              None, 'nonexisting', 'tag', 'always', None)
 
     @mock.patch('zun.image.glance.utils.find_images')
     def test_search_image_found(self, mock_find_images):