openstack/zun
Code Issues Proposed changes

Optimize docker image loading

Browse Source 
Loading image has significant overhead especially if the image is
very large. This commit optimized the image loading by checking
if there is a newer image pulled down. The container runtime
will only load the image if there is a newer version.

Change-Id: I481b4f204f0b95d91a72086c97140297ba481f26
This commit is contained in:
Hongbin Lu 2017-03-08 16:46:53 -06:00
parent 71817c79ab
commit 0b0db7fdc3
7 changed files with 32 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
zun/compute/manager.py
View File

@ -73,10 +73,11 @@ class Manager(object):
sandbox_image_pull_policy = CONF.sandbox_image_pull_policy
repo, tag = utils.parse_image_name(sandbox_image)
try:
image = image_driver.pull_image(context, repo, tag,
sandbox_image_pull_policy,
sandbox_image_driver)
self.driver.load_image(sandbox_image, image['path'])
image, image_loaded = image_driver.pull_image(
context, repo, tag, sandbox_image_pull_policy,
sandbox_image_driver)
if not image_loaded:
self.driver.load_image(sandbox_image, image['path'])
sandbox_id = self.driver.create_sandbox(context, container,
image=sandbox_image)
except Exception as e:
@ -94,10 +95,10 @@ class Manager(object):
container.image_pull_policy, tag)
image_driver_name = container.image_driver
try:
image = image_driver.pull_image(context, repo, tag,
image_pull_policy,
image_driver_name)
self.driver.load_image(container.image, image['path'])
image, image_loaded = image_driver.pull_image(
context, repo, tag, image_pull_policy, image_driver_name)
if not image_loaded:
self.driver.load_image(container.image, image['path'])
except exception.ImageNotFound as e:
with excutils.save_and_reraise_exception(reraise=reraise):
LOG.error(six.text_type(e))
@ -432,9 +433,10 @@ class Manager(object):
LOG.debug('Creating image...')
repo_tag = image.repo + ":" + image.tag
try:
pulled_image = image_driver.pull_image(context, image.repo,
image.tag)
self.driver.load_image(repo_tag, pulled_image['path'])
pulled_image, image_loaded = image_driver.pull_image(
context, image.repo, image.tag)
if not image_loaded:
self.driver.load_image(repo_tag, pulled_image['path'])
image_dict = self.driver.inspect_image(repo_tag)
image.image_id = image_dict['Id']
image.size = image_dict['Size']

5
zun/image/docker/driver.py
View File

@ -56,11 +56,12 @@ class DockerDriver(driver.ContainerImageDriver):
raise exception.DockerError(error['message'])
def pull_image(self, context, repo, tag, image_pull_policy):
image_loaded = True
image = self._search_image_on_host(repo, tag)
if not utils.should_pull_image(image_pull_policy, bool(image)):
if image:
LOG.debug('Image %s present locally' % repo)
return image
return image, image_loaded
else:
message = _('Image %s not present with pull policy of Never'
) % repo
@ -70,7 +71,7 @@ class DockerDriver(driver.ContainerImageDriver):
LOG.debug('Pulling image from docker %s,'
' context %s' % (repo, context))
self._pull_image(repo, tag)
return {'image': repo, 'path': None}
return {'image': repo, 'path': None}, image_loaded
except exception.ImageNotFound:
with excutils.save_and_reraise_exception():
LOG.error(

6
zun/image/driver.py
View File

@ -69,8 +69,8 @@ def pull_image(context, repo, tag, image_pull_policy, image_driver):
for driver in image_driver_list:
try:
image_driver = load_image_driver(driver)
image = image_driver.pull_image(context, repo,
tag, image_pull_policy)
image, image_loaded = image_driver.pull_image(
context, repo, tag, image_pull_policy)
if image:
image['driver'] = driver.split('.')[0]
break
@ -82,7 +82,7 @@ def pull_image(context, repo, tag, image_pull_policy, image_driver):
raise exception.ZunException(six.text_type(e))
if not image:
raise exception.ImageNotFound("Image %s not found" % repo)
return image
return image, image_loaded
def search_image(context, image_name, image_driver, exact_match):

6
zun/image/glance/driver.py
View File

@ -54,11 +54,13 @@ class GlanceDriver(driver.ContainerImageDriver):
def pull_image(self, context, repo, tag, image_pull_policy):
# TODO(shubhams): glance driver does not handle tags
# once metadata is stored in db then handle tags
image_loaded = False
image = self._search_image_on_host(context, repo)
if not common_utils.should_pull_image(image_pull_policy, bool(image)):
if image:
LOG.debug('Image %s present locally' % repo)
return image
image_loaded = True
return image, image_loaded
else:
message = _('Image %s not present with pull policy of Never'
) % repo
@ -89,7 +91,7 @@ class GlanceDriver(driver.ContainerImageDriver):
raise exception.ZunException(msg.format(e))
LOG.debug('Image %s was downloaded to path : %s'
% (repo, out_path))
return {'image': repo, 'path': out_path}
return {'image': repo, 'path': out_path}, image_loaded
def search_image(self, context, repo, tag, exact_match):
# TODO(mkrai): glance driver does not handle tags

8
zun/tests/unit/compute/test_compute_manager.py
View File

@ -51,7 +51,7 @@ class TestManager(base.TestCase):
mock_pull, mock_save):
container = Container(self.context, **utils.get_test_container())
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
mock_pull.return_value = image
mock_pull.return_value = image, False
mock_create_sandbox.return_value = 'fake_id'
self.compute_manager._do_container_create(self.context, container)
mock_save.assert_called_with(self.context)
@ -111,7 +111,7 @@ class TestManager(base.TestCase):
mock_save):
container = Container(self.context, **utils.get_test_container())
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
mock_pull.return_value = image
mock_pull.return_value = image, False
mock_create.side_effect = exception.DockerError("Creation Failed")
mock_create_sandbox.return_value = mock.MagicMock()
self.compute_manager._do_container_create(self.context, container)
@ -127,7 +127,7 @@ class TestManager(base.TestCase):
container = Container(self.context, **utils.get_test_container())
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
mock_create.return_value = container
mock_pull.return_value = image
mock_pull.return_value = image, False
container.status = 'Stopped'
self.compute_manager._do_container_run(self.context, container)
mock_save.assert_called_with(self.context)
@ -193,7 +193,7 @@ class TestManager(base.TestCase):
mock_fail,
mock_pull, mock_save):
container = Container(self.context, **utils.get_test_container())
mock_pull.return_value = {'name': 'nginx', 'path': None}
mock_pull.return_value = {'name': 'nginx', 'path': None}, True
mock_create.side_effect = exception.DockerError(
message="Docker Error occurred")
self.compute_manager._do_container_run(self.context,

4
zun/tests/unit/image/docker/test_driver.py
View File

@ -51,7 +51,7 @@ class TestDriver(base.BaseTestCase):
self, mock_should_pull_image, mock_search):
mock_should_pull_image.return_value = False
mock_search.return_value = {'image': 'nginx', 'path': 'xyz'}
self.assertEqual({'image': 'nginx', 'path': 'xyz'},
self.assertEqual(({'image': 'nginx', 'path': 'xyz'}, True),
self.driver.pull_image(None, 'nonexisting',
'tag', 'never'))
@ -62,7 +62,7 @@ class TestDriver(base.BaseTestCase):
mock_should_pull_image.return_value = True
mock_search.return_value = {'image': 'nginx', 'path': 'xyz'}
ret = self.driver.pull_image(None, 'test_image', 'latest', 'always')
self.assertEqual({'image': 'test_image', 'path': None}, ret)
self.assertEqual(({'image': 'test_image', 'path': None}, True), ret)
self.mock_docker.pull.assert_called_once_with(
'test_image',
tag='latest',

6
zun/tests/unit/image/glance/test_driver.py
View File

@ -53,7 +53,7 @@ class TestDriver(base.BaseTestCase):
self, mock_should_pull_image, mock_search):
mock_should_pull_image.return_value = False
mock_search.return_value = {'image': 'nginx', 'path': 'xyz'}
self.assertEqual({'image': 'nginx', 'path': 'xyz'},
self.assertEqual(({'image': 'nginx', 'path': 'xyz'}, True),
self.driver.pull_image(None, 'nonexisting',
'tag', 'never'))
@ -85,8 +85,8 @@ class TestDriver(base.BaseTestCase):
CONF.set_override('images_directory', self.test_dir, group='glance')
out_path = os.path.join(self.test_dir, '1234' + '.tar')
ret = self.driver.pull_image(None, 'image', 'latest', 'always')
self.assertEqual({'image': 'image', 'path': out_path}, ret)
self.assertTrue(os.path.isfile(ret['path']))
self.assertEqual(({'image': 'image', 'path': out_path}, False), ret)
self.assertTrue(os.path.isfile(ret[0]['path']))
@mock.patch('zun.image.glance.utils.create_glanceclient')
@mock.patch.object(driver.GlanceDriver,