Add copyimage do_action

Partially implements blueprint: pluggable-do-actions

Change-Id: I35dc0dcca286c7c9867aef417a72232f79b74ed4
This commit is contained in:
Alexander Gordeev 2016-04-26 17:29:05 +03:00
parent 9b4258a908
commit 981a3e9023
5 changed files with 310 additions and 242 deletions

100
bareon/actions/copyimage.py Normal file
View File

@ -0,0 +1,100 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from bareon.actions import base
from bareon import errors
from bareon.openstack.common import log as logging
from bareon.utils import artifact as au
from bareon.utils import fs as fu
from bareon.utils import hardware as hw
from bareon.utils import utils
LOG = logging.getLogger(__name__)
class CopyImageAction(base.BaseAction):
"""CopyImageAction
copies all necessary images on disks
"""
def validate(self):
# TODO(agordeev): implement validate for copyimage
pass
def execute(self):
self.do_copyimage()
def do_copyimage(self):
LOG.debug('--- Copying images (do_copyimage) ---')
for image in self.driver.image_scheme.images:
LOG.debug('Processing image: %s' % image.uri)
processing = au.Chain()
LOG.debug('Appending uri processor: %s' % image.uri)
processing.append(image.uri)
if image.uri.startswith('http://'):
LOG.debug('Appending HTTP processor')
processing.append(au.HttpUrl)
elif image.uri.startswith('file://'):
LOG.debug('Appending FILE processor')
processing.append(au.LocalFile)
if image.container == 'gzip':
LOG.debug('Appending GZIP processor')
processing.append(au.GunzipStream)
LOG.debug('Appending TARGET processor: %s' % image.target_device)
error = None
if not os.path.exists(image.target_device):
error = "TARGET processor '{0}' does not exist."
elif not hw.is_block_device(image.target_device):
error = "TARGET processor '{0}' is not a block device."
if error:
error = error.format(image.target_device)
LOG.error(error)
raise errors.WrongDeviceError(error)
processing.append(image.target_device)
LOG.debug('Launching image processing chain')
processing.process()
if image.size and image.md5:
LOG.debug('Trying to compare image checksum')
actual_md5 = utils.calculate_md5(image.target_device,
image.size)
if actual_md5 == image.md5:
LOG.debug('Checksum matches successfully: md5=%s' %
actual_md5)
else:
raise errors.ImageChecksumMismatchError(
'Actual checksum %s mismatches with expected %s for '
'file %s' % (actual_md5, image.md5,
image.target_device))
else:
LOG.debug('Skipping image checksum comparing. '
'Ether size or hash have been missed')
# TODO(agordeev): separate to another action?
LOG.debug('Extending image file systems')
if image.format in ('ext2', 'ext3', 'ext4', 'xfs'):
LOG.debug('Extending %s %s' %
(image.format, image.target_device))
fu.extend_fs(image.format, image.target_device)

View File

@ -22,15 +22,14 @@ import six
import yaml
from bareon.actions import configdrive
from bareon.actions import copyimage
from bareon.actions import partitioning
from bareon.drivers.deploy.base import BaseDeployDriver
from bareon import errors
from bareon.openstack.common import log as logging
from bareon.utils import artifact as au
from bareon.utils import build as bu
from bareon.utils import fs as fu
from bareon.utils import grub as gu
from bareon.utils import hardware as hw
from bareon.utils import utils
opts = [
@ -131,63 +130,7 @@ class Manager(BaseDeployDriver):
configdrive.ConfigDriveAction(self.driver).execute()
def do_copyimage(self):
LOG.debug('--- Copying images (do_copyimage) ---')
for image in self.driver.image_scheme.images:
LOG.debug('Processing image: %s' % image.uri)
processing = au.Chain()
LOG.debug('Appending uri processor: %s' % image.uri)
processing.append(image.uri)
if image.uri.startswith('http://'):
LOG.debug('Appending HTTP processor')
processing.append(au.HttpUrl)
elif image.uri.startswith('file://'):
LOG.debug('Appending FILE processor')
processing.append(au.LocalFile)
if image.container == 'gzip':
LOG.debug('Appending GZIP processor')
processing.append(au.GunzipStream)
LOG.debug('Appending TARGET processor: %s' % image.target_device)
error = None
if not os.path.exists(image.target_device):
error = "TARGET processor '{0}' does not exist."
elif not hw.is_block_device(image.target_device):
error = "TARGET processor '{0}' is not a block device."
if error:
error = error.format(image.target_device)
LOG.error(error)
raise errors.WrongDeviceError(error)
processing.append(image.target_device)
LOG.debug('Launching image processing chain')
processing.process()
if image.size and image.md5:
LOG.debug('Trying to compare image checksum')
actual_md5 = utils.calculate_md5(image.target_device,
image.size)
if actual_md5 == image.md5:
LOG.debug('Checksum matches successfully: md5=%s' %
actual_md5)
else:
raise errors.ImageChecksumMismatchError(
'Actual checksum %s mismatches with expected %s for '
'file %s' % (actual_md5, image.md5,
image.target_device))
else:
LOG.debug('Skipping image checksum comparing. '
'Ether size or hash have been missed')
LOG.debug('Extending image file systems')
if image.format in ('ext2', 'ext3', 'ext4', 'xfs'):
LOG.debug('Extending %s %s' %
(image.format, image.target_device))
fu.extend_fs(image.format, image.target_device)
copyimage.CopyImageAction(self.driver).execute()
@staticmethod
def _update_metadata_with_repos(metadata, repos):

View File

@ -0,0 +1,207 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
import six
import unittest2
from bareon.actions import copyimage
from bareon.drivers.data import nailgun
from bareon import errors
from bareon import objects
from bareon.utils import artifact as au
from bareon.utils import fs as fu
from bareon.utils import hardware as hu
from bareon.utils import utils
if six.PY2:
import mock
elif six.PY3:
import unittest.mock as mock
CONF = cfg.CONF
class FakeChain(object):
processors = []
def append(self, thing):
self.processors.append(thing)
def process(self):
pass
class TestCopyImageAction(unittest2.TestCase):
def setUp(self):
super(TestCopyImageAction, self).setUp()
self.drv = mock.MagicMock(spec=nailgun.Nailgun)
self.action = copyimage.CopyImageAction(self.drv)
self.drv.image_scheme.images = [
objects.Image('http://fake_uri', '/dev/mapper/os-root', 'ext4',
'gzip', size=1234),
objects.Image('file:///fake_uri', '/tmp/config-drive.img',
'iso9660', 'raw', size=123)
]
@mock.patch.object(copyimage.os.path, 'exists')
@mock.patch.object(hu, 'is_block_device')
@mock.patch.object(utils, 'calculate_md5')
@mock.patch('os.path.getsize')
@mock.patch.object(fu, 'extend_fs')
@mock.patch.object(au, 'GunzipStream')
@mock.patch.object(au, 'LocalFile')
@mock.patch.object(au, 'HttpUrl')
@mock.patch.object(au, 'Chain')
@mock.patch.object(utils, 'execute')
@mock.patch.object(utils, 'render_and_save')
@mock.patch.object(hu, 'list_block_devices')
def test_do_copyimage(self, mock_lbd, mock_u_ras, mock_u_e, mock_au_c,
mock_au_h, mock_au_l, mock_au_g, mock_fu_ef,
mock_get_size, mock_md5, mock_ibd, mock_os_path):
mock_os_path.return_value = True
mock_ibd.return_value = True
mock_au_c.return_value = FakeChain()
self.action.execute()
imgs = self.drv.image_scheme.images
self.assertEqual(2, len(imgs))
expected_processors_list = []
for img in imgs[:-1]:
expected_processors_list += [
img.uri,
au.HttpUrl,
au.GunzipStream,
img.target_device
]
expected_processors_list += [
imgs[-1].uri,
au.LocalFile,
imgs[-1].target_device
]
self.assertEqual(expected_processors_list,
mock_au_c.return_value.processors)
mock_fu_ef_expected_calls = [
mock.call('ext4', '/dev/mapper/os-root')]
self.assertEqual(mock_fu_ef_expected_calls, mock_fu_ef.call_args_list)
@mock.patch.object(copyimage.os.path, 'exists')
@mock.patch.object(hu, 'is_block_device')
@mock.patch.object(utils, 'calculate_md5')
@mock.patch('os.path.getsize')
@mock.patch.object(fu, 'extend_fs')
@mock.patch.object(au, 'GunzipStream')
@mock.patch.object(au, 'LocalFile')
@mock.patch.object(au, 'HttpUrl')
@mock.patch.object(au, 'Chain')
@mock.patch.object(utils, 'execute')
@mock.patch.object(utils, 'render_and_save')
@mock.patch.object(hu, 'list_block_devices')
def test_do_copyimage_target_doesnt_exist(self, mock_lbd, mock_u_ras,
mock_u_e, mock_au_c, mock_au_h,
mock_au_l, mock_au_g, mock_fu_ef,
mock_get_size, mock_md5,
mock_ibd, mock_os_path):
mock_os_path.return_value = False
mock_ibd.return_value = True
mock_au_c.return_value = FakeChain()
with self.assertRaisesRegexp(errors.WrongDeviceError,
'TARGET processor .* does not exist'):
self.action.execute()
@mock.patch.object(copyimage.os.path, 'exists')
@mock.patch.object(hu, 'is_block_device')
@mock.patch.object(utils, 'calculate_md5')
@mock.patch('os.path.getsize')
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(fu, 'extend_fs')
@mock.patch.object(au, 'GunzipStream')
@mock.patch.object(au, 'LocalFile')
@mock.patch.object(au, 'HttpUrl')
@mock.patch.object(au, 'Chain')
@mock.patch.object(utils, 'execute')
@mock.patch.object(utils, 'render_and_save')
@mock.patch.object(hu, 'list_block_devices')
def test_do_copyimage_target_not_block_device(self, mock_lbd, mock_u_ras,
mock_u_e, mock_au_c,
mock_au_h, mock_au_l,
mock_au_g, mock_fu_ef,
mock_http_req, mock_yaml,
mock_get_size, mock_md5,
mock_ibd, mock_os_path):
mock_os_path.return_value = True
mock_ibd.return_value = False
mock_au_c.return_value = FakeChain()
msg = 'TARGET processor .* is not a block device'
with self.assertRaisesRegexp(errors.WrongDeviceError, msg):
self.action.execute()
@mock.patch.object(copyimage.os.path, 'exists')
@mock.patch.object(hu, 'is_block_device')
@mock.patch.object(utils, 'calculate_md5')
@mock.patch('os.path.getsize')
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(fu, 'extend_fs')
@mock.patch.object(au, 'GunzipStream')
@mock.patch.object(au, 'LocalFile')
@mock.patch.object(au, 'HttpUrl')
@mock.patch.object(au, 'Chain')
@mock.patch.object(utils, 'execute')
@mock.patch.object(utils, 'render_and_save')
@mock.patch.object(hu, 'list_block_devices')
def test_do_copyimage_md5_matches(self, mock_lbd, mock_u_ras, mock_u_e,
mock_au_c, mock_au_h, mock_au_l,
mock_au_g, mock_fu_ef, mock_http_req,
mock_yaml, mock_get_size, mock_md5,
mock_ibd, mock_os_path):
mock_os_path.return_value = True
mock_ibd.return_value = True
mock_md5.side_effect = ['really_fakemd5']
mock_au_c.return_value = FakeChain()
self.drv.image_scheme.images[0].md5 = 'really_fakemd5'
self.assertEqual(2, len(self.drv.image_scheme.images))
self.action.execute()
expected_md5_calls = [mock.call('/dev/mapper/os-root', 1234)]
self.assertEqual(expected_md5_calls, mock_md5.call_args_list)
@mock.patch.object(hu, 'is_block_device')
@mock.patch.object(copyimage.os.path, 'exists')
@mock.patch.object(utils, 'calculate_md5')
@mock.patch('os.path.getsize')
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(fu, 'extend_fs')
@mock.patch.object(au, 'GunzipStream')
@mock.patch.object(au, 'LocalFile')
@mock.patch.object(au, 'HttpUrl')
@mock.patch.object(au, 'Chain')
@mock.patch.object(utils, 'execute')
@mock.patch.object(utils, 'render_and_save')
@mock.patch.object(hu, 'list_block_devices')
def test_do_copyimage_md5_mismatch(self, mock_lbd, mock_u_ras, mock_u_e,
mock_au_c, mock_au_h, mock_au_l,
mock_au_g, mock_fu_ef, mock_http_req,
mock_yaml, mock_get_size, mock_md5,
mock_os_path, mock_ibd):
mock_os_path.return_value = True
mock_ibd.return_value = True
mock_md5.side_effect = ['really_fakemd5']
mock_au_c.return_value = FakeChain()
self.drv.image_scheme.images[0].size = 1234
self.drv.image_scheme.images[0].md5 = 'fakemd5'
self.assertEqual(2, len(self.drv.image_scheme.images))
self.assertRaises(errors.ImageChecksumMismatchError,
self.action.execute)

View File

@ -25,8 +25,6 @@ from bareon.drivers.deploy import nailgun as nailgun_deploy
from bareon import errors
from bareon import objects
from bareon.tests import test_nailgun
from bareon.utils import artifact as au
from bareon.utils import fs as fu
from bareon.utils import hardware as hu
from bareon.utils import utils
@ -38,16 +36,6 @@ elif six.PY3:
CONF = cfg.CONF
class FakeChain(object):
processors = []
def append(self, thing):
self.processors.append(thing)
def process(self):
pass
@unittest2.skip("Fix after cray rebase")
class TestManager(unittest2.TestCase):
@ -280,177 +268,6 @@ class TestManager(unittest2.TestCase):
mock_utils.makedirs_if_not_exists.assert_called_once_with(
'/tmp/target/etc/nailgun-agent')
@mock.patch.object(nailgun_deploy.os.path, 'exists')
@mock.patch.object(hu, 'is_block_device')
@mock.patch.object(utils, 'calculate_md5')
@mock.patch('os.path.getsize')
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(fu, 'extend_fs')
@mock.patch.object(au, 'GunzipStream')
@mock.patch.object(au, 'LocalFile')
@mock.patch.object(au, 'HttpUrl')
@mock.patch.object(au, 'Chain')
@mock.patch.object(utils, 'execute')
@mock.patch.object(utils, 'render_and_save')
@mock.patch.object(hu, 'list_block_devices')
def test_do_copyimage(self, mock_lbd, mock_u_ras, mock_u_e, mock_au_c,
mock_au_h, mock_au_l, mock_au_g, mock_fu_ef,
mock_http_req, mock_yaml, mock_get_size, mock_md5,
mock_ibd, mock_os_path):
mock_os_path.return_value = True
mock_ibd.return_value = True
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
mock_au_c.return_value = FakeChain()
self.mgr.do_configdrive()
self.mgr.do_copyimage()
imgs = self.mgr.driver.image_scheme.images
self.assertEqual(2, len(imgs))
expected_processors_list = []
for img in imgs[:-1]:
expected_processors_list += [
img.uri,
au.HttpUrl,
au.GunzipStream,
img.target_device
]
expected_processors_list += [
imgs[-1].uri,
au.LocalFile,
imgs[-1].target_device
]
self.assertEqual(expected_processors_list,
mock_au_c.return_value.processors)
mock_fu_ef_expected_calls = [
mock.call('ext4', '/dev/mapper/os-root')]
self.assertEqual(mock_fu_ef_expected_calls, mock_fu_ef.call_args_list)
@mock.patch.object(nailgun_deploy.os.path, 'exists')
@mock.patch.object(hu, 'is_block_device')
@mock.patch.object(utils, 'calculate_md5')
@mock.patch('os.path.getsize')
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(fu, 'extend_fs')
@mock.patch.object(au, 'GunzipStream')
@mock.patch.object(au, 'LocalFile')
@mock.patch.object(au, 'HttpUrl')
@mock.patch.object(au, 'Chain')
@mock.patch.object(utils, 'execute')
@mock.patch.object(utils, 'render_and_save')
@mock.patch.object(hu, 'list_block_devices')
def test_do_copyimage_target_doesnt_exist(self, mock_lbd, mock_u_ras,
mock_u_e, mock_au_c, mock_au_h,
mock_au_l, mock_au_g, mock_fu_ef,
mock_http_req, mock_yaml,
mock_get_size, mock_md5,
mock_ibd, mock_os_path):
mock_os_path.return_value = False
mock_ibd.return_value = True
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
mock_au_c.return_value = FakeChain()
self.mgr.do_configdrive()
with self.assertRaisesRegexp(errors.WrongDeviceError,
'TARGET processor .* does not exist'):
self.mgr.do_copyimage()
@mock.patch.object(nailgun_deploy.os.path, 'exists')
@mock.patch.object(hu, 'is_block_device')
@mock.patch.object(utils, 'calculate_md5')
@mock.patch('os.path.getsize')
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(fu, 'extend_fs')
@mock.patch.object(au, 'GunzipStream')
@mock.patch.object(au, 'LocalFile')
@mock.patch.object(au, 'HttpUrl')
@mock.patch.object(au, 'Chain')
@mock.patch.object(utils, 'execute')
@mock.patch.object(utils, 'render_and_save')
@mock.patch.object(hu, 'list_block_devices')
def test_do_copyimage_target_not_block_device(self, mock_lbd, mock_u_ras,
mock_u_e, mock_au_c,
mock_au_h, mock_au_l,
mock_au_g, mock_fu_ef,
mock_http_req, mock_yaml,
mock_get_size, mock_md5,
mock_ibd, mock_os_path):
mock_os_path.return_value = True
mock_ibd.return_value = False
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
mock_au_c.return_value = FakeChain()
self.mgr.do_configdrive()
msg = 'TARGET processor .* is not a block device'
with self.assertRaisesRegexp(errors.WrongDeviceError, msg):
self.mgr.do_copyimage()
@mock.patch.object(nailgun_deploy.os.path, 'exists')
@mock.patch.object(hu, 'is_block_device')
@mock.patch.object(utils, 'calculate_md5')
@mock.patch('os.path.getsize')
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(fu, 'extend_fs')
@mock.patch.object(au, 'GunzipStream')
@mock.patch.object(au, 'LocalFile')
@mock.patch.object(au, 'HttpUrl')
@mock.patch.object(au, 'Chain')
@mock.patch.object(utils, 'execute')
@mock.patch.object(utils, 'render_and_save')
@mock.patch.object(hu, 'list_block_devices')
def test_do_copyimage_md5_matches(self, mock_lbd, mock_u_ras, mock_u_e,
mock_au_c, mock_au_h, mock_au_l,
mock_au_g, mock_fu_ef, mock_http_req,
mock_yaml, mock_get_size, mock_md5,
mock_ibd, mock_os_path):
mock_os_path.return_value = True
mock_ibd.return_value = True
mock_get_size.return_value = 123
mock_md5.side_effect = ['fakemd5', 'really_fakemd5', 'fakemd5']
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
mock_au_c.return_value = FakeChain()
self.mgr.driver.image_scheme.images[0].size = 1234
self.mgr.driver.image_scheme.images[0].md5 = 'really_fakemd5'
self.mgr.do_configdrive()
self.assertEqual(2, len(self.mgr.driver.image_scheme.images))
self.mgr.do_copyimage()
expected_md5_calls = [mock.call('/tmp/config-drive.img', 123),
mock.call('/dev/mapper/os-root', 1234),
mock.call('/dev/sda7', 123)]
self.assertEqual(expected_md5_calls, mock_md5.call_args_list)
@mock.patch.object(hu, 'is_block_device')
@mock.patch.object(nailgun_deploy.os.path, 'exists')
@mock.patch.object(utils, 'calculate_md5')
@mock.patch('os.path.getsize')
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(fu, 'extend_fs')
@mock.patch.object(au, 'GunzipStream')
@mock.patch.object(au, 'LocalFile')
@mock.patch.object(au, 'HttpUrl')
@mock.patch.object(au, 'Chain')
@mock.patch.object(utils, 'execute')
@mock.patch.object(utils, 'render_and_save')
@mock.patch.object(hu, 'list_block_devices')
def test_do_copyimage_md5_mismatch(self, mock_lbd, mock_u_ras, mock_u_e,
mock_au_c, mock_au_h, mock_au_l,
mock_au_g, mock_fu_ef, mock_http_req,
mock_yaml, mock_get_size, mock_md5,
mock_os_path, mock_ibd):
mock_os_path.return_value = True
mock_ibd.return_value = True
mock_get_size.return_value = 123
mock_md5.side_effect = ['fakemd5', 'really_fakemd5', 'fakemd5']
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
mock_au_c.return_value = FakeChain()
self.mgr.driver.image_scheme.images[0].size = 1234
self.mgr.driver.image_scheme.images[0].md5 = 'fakemd5'
self.mgr.do_configdrive()
self.assertEqual(2, len(self.mgr.driver.image_scheme.images))
self.assertRaises(errors.ImageChecksumMismatchError,
self.mgr.do_copyimage)
@mock.patch('bareon.drivers.deploy.nailgun.fu', create=True)
@mock.patch('bareon.drivers.deploy.nailgun.utils', create=True)
@mock.patch('bareon.drivers.deploy.nailgun.open',

View File

@ -42,6 +42,7 @@ bareon.drivers.deploy =
bareon.actions =
do_partitioning = bareon.actions.partitioning:PartitioningAction
do_configdrive = bareon.actions.configdrive:ConfigDriveAction
do_copyimage = bareon.actions.copyimage:CopyImageAction
oslo.config.opts =
bareon.manager = bareon.manager:list_opts