x/bareon
Code Issues Proposed changes

Add Python 3.4 support

Browse Source 
Python 3.4 env is added to tox config file
oslo.serialization is pinned to version smaller than 2.0

test_calculate_md5_ok had to be rewritten due to bug in mock_open in
unittest.mock library in python 3.4: https://bugs.python.org/issue23004

Change-Id: Idf2fa15ff4035b9fe92027acdcaf51e005b52541
Implements blueprint: volume-manager-refactoring
This commit is contained in:
Maciej Kwiek 2015-07-17 14:35:23 +02:00 committed by Igor Kalnitsky
parent d7f052ecc8
commit f78e4eba30
18 changed files with 158 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
fuel_agent/drivers/nailgun.py
View File

@ -475,11 +475,18 @@ class Nailgun(BaseDataDriver):
configdrive_scheme = objects.ConfigDriveScheme()
LOG.debug('Adding common parameters')
admin_interface = filter(
lambda x: (x['mac_address'] ==
data['kernel_options']['netcfg/choose_interface']),
[dict(name=name, **spec) for name, spec
in data['interfaces'].iteritems()])[0]
interface_dicts = [
dict(name=name, **spec)
for name, spec
in six.iteritems(data['interfaces'])
]
admin_interface = next(
x for x in interface_dicts
if (x['mac_address'] ==
data['kernel_options']['netcfg/choose_interface'])
)
ssh_auth_keys = data['ks_meta']['authorized_keys']
if data['ks_meta']['auth_key']:

35
fuel_agent/manager.py
View File

@ -12,13 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from io import open
import os
import shutil
import signal
import tempfile
import yaml
from oslo_config import cfg
import six
import yaml
from fuel_agent import errors
from fuel_agent.openstack.common import log as logging
@ -405,8 +407,8 @@ class Manager(object):
mtab_path = chroot + '/etc/mtab'
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, 'wb') as f:
f.write(mtab)
with open(mtab_path, 'wt', encoding='utf-8') as f:
f.write(six.text_type(mtab))
def umount_target(self, chroot, pseudo=True):
LOG.debug('Umounting target file systems: %s', chroot)
@ -470,26 +472,27 @@ class Manager(object):
# remapping in Ubuntu, so injecting files prior the first boot
# should work
with open(chroot + '/etc/udev/rules.d/70-persistent-net.rules',
'w') as f:
f.write('# Generated by fuel-agent during provisioning: '
'BEGIN\n')
'wt', encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning: '
u'BEGIN\n')
# pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1
for mapping in self.driver.configdrive_scheme.\
common.udevrules.split(','):
mac_addr, nic_name = mapping.split('_')
f.write('SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '
'ATTR{address}=="%s", ATTR{type}=="1", '
'KERNEL=="eth*", NAME="%s"\n' % (mac_addr,
f.write(u'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '
u'ATTR{address}=="%s", ATTR{type}=="1", '
u'KERNEL=="eth*", NAME="%s"\n' % (mac_addr,
nic_name))
f.write('# Generated by fuel-agent during provisioning: END\n')
f.write(
u'# Generated by fuel-agent during provisioning: END\n')
# FIXME(agordeev): Disable net-generator that will add new etries
# to 70-persistent-net.rules
with open(chroot +
'/etc/udev/rules.d/75-persistent-net-generator.rules',
'w') as f:
f.write('# Generated by fuel-agent during provisioning:\n'
'# DO NOT DELETE. It is needed to disable '
'net-generator\n')
'wt', encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning:\n'
u'# DO NOT DELETE. It is needed to disable '
u'net-generator\n')
# FIXME(kozhukalov): Prevent nailgun-agent from doing anything.
# This ugly hack is to be used together with the command removing
@ -518,7 +521,7 @@ class Manager(object):
# via updates.
utils.execute('mkdir', '-p', chroot + '/var/log/upstart')
with open(chroot + '/etc/fstab', 'wb') as f:
with open(chroot + '/etc/fstab', 'wt', encoding='utf-8') as f:
for fs in self.driver.partition_scheme.fss:
# TODO(kozhukalov): Think of improving the logic so as to
# insert a meaningful fsck order value which is last zero
@ -775,7 +778,7 @@ class Manager(object):
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Image metadata: %s', metadata)
with open(self.driver.metadata_uri.split('file://', 1)[1],
'w') as f:
'wt', encoding='utf-8') as f:
yaml.safe_dump(metadata, stream=f)
LOG.info('--- Building image END (do_build_image) ---')
except Exception as exc:

12
fuel_agent/objects/partition/parted.py
View File

@ -55,17 +55,15 @@ class Parted(base.Serializable):
@property
def logical(self):
return filter(lambda x: x.type == 'logical', self.partitions)
return [x for x in self.partitions if x.type == 'logical']
@property
def primary(self):
return filter(lambda x: x.type == 'primary', self.partitions)
return [x for x in self.partitions if x.type == 'primary']
@property
def extended(self):
found = filter(lambda x: x.type == 'extended', self.partitions)
if found:
return found[0]
return next((x for x in self.partitions if x.type == 'extended'), None)
def next_type(self):
if self.label == 'gpt':
@ -104,9 +102,7 @@ class Parted(base.Serializable):
return '%s%s%s' % (self.name, separator, self.next_count())
def partition_by_name(self, name):
found = filter(lambda x: (x.name == name), self.partitions)
if found:
return found[0]
return next((x for x in self.partitions if x.name == name), None)
def to_dict(self):
partitions = [partition.to_dict() for partition in self.partitions]

25
fuel_agent/objects/partition/scheme.py
View File

@ -73,9 +73,7 @@ class PartitionScheme(object):
return md
def md_by_name(self, name):
found = filter(lambda x: x.name == name, self.mds)
if found:
return found[0]
return next((x for x in self.mds if x.name == name), None)
def md_by_mount(self, mount):
fs = self.fs_by_mount(mount)
@ -115,14 +113,10 @@ class PartitionScheme(object):
if parted.partition_by_name(name)), None)
def vg_by_name(self, vgname):
found = filter(lambda x: (x.name == vgname), self.vgs)
if found:
return found[0]
return next((x for x in self.vgs if x.name == vgname), None)
def pv_by_name(self, pvname):
found = filter(lambda x: (x.name == pvname), self.pvs)
if found:
return found[0]
return next((x for x in self.pvs if x.name == pvname), None)
def vg_attach_by_name(self, pvname, vgname,
metadatasize=16, metadatacopies=2):
@ -133,14 +127,10 @@ class PartitionScheme(object):
vg.add_pv(pv.name)
def fs_by_mount(self, mount):
found = filter(lambda x: (x.mount and x.mount == mount), self.fss)
if found:
return found[0]
return next((x for x in self.fss if x.mount == mount), None)
def fs_by_device(self, device):
found = filter(lambda x: x.device == device, self.fss)
if found:
return found[0]
return next((x for x in self.fss if x.device == device), None)
def fs_sorted_by_depth(self, reverse=False):
"""Getting file systems sorted by path length.
@ -154,9 +144,8 @@ class PartitionScheme(object):
return sorted(self.fss, key=key, reverse=reverse)
def lv_by_device_name(self, device_name):
found = filter(lambda x: x.device_name == device_name, self.lvs)
if found:
return found[0]
return next((x for x in self.lvs if x.device_name == device_name),
None)
def root_device(self):
fs = self.fs_by_mount('/')

3
fuel_agent/tests/test_artifact_utils.py
View File

@ -17,7 +17,6 @@ from oslo_config import cfg
import unittest2
import zlib
from fuel_agent import errors
from fuel_agent.utils import artifact as au
from fuel_agent.utils import utils
@ -91,7 +90,7 @@ class TestHttpUrl(unittest2.TestCase):
class TestGunzipStream(unittest2.TestCase):
def test_gunzip_stream_next(self):
content = ['fake content #1']
content = [b'fake content #1']
compressed_stream = [zlib.compress(data) for data in content]
gunzip_stream = au.GunzipStream(compressed_stream)
for data in enumerate(gunzip_stream):

43
fuel_agent/tests/test_grub_utils.py
View File

@ -12,20 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import StringIO
import io
import mock
import six
from six import StringIO
import unittest2
from fuel_agent import errors
from fuel_agent.utils import grub as gu
if six.PY2:
OPEN_FUNCTION_NAME = '__builtin__.open'
else:
OPEN_FUNCTION_NAME = 'builtins.open'
class TestGrubUtils(unittest2.TestCase):
@ -313,12 +308,13 @@ class TestGrubUtils(unittest2.TestCase):
script = 'cat /tmp/grub.batch | /sbin/grub --no-floppy --batch'
mock_open = mock.mock_open()
with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
with mock.patch('fuel_agent.utils.grub.open', new=mock_open,
create=True):
gu.grub1_mbr('/dev/foo', '/dev/bar', '0', chroot='/target')
self.assertEqual(
mock_open.call_args_list,
[mock.call('/target/tmp/grub.batch', 'wb'),
mock.call('/target/tmp/grub.sh', 'wb')]
[mock.call('/target/tmp/grub.batch', 'wt', encoding='utf-8'),
mock.call('/target/tmp/grub.sh', 'wt', encoding='utf-8')]
)
mock_open_file = mock_open()
self.assertEqual(
@ -347,12 +343,13 @@ class TestGrubUtils(unittest2.TestCase):
script = 'cat /tmp/grub.batch | /sbin/grub --no-floppy --batch'
mock_open = mock.mock_open()
with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
with mock.patch('fuel_agent.utils.grub.open', new=mock_open,
create=True):
gu.grub1_mbr('/dev/foo', '/dev/foo', '0', chroot='/target')
self.assertEqual(
mock_open.call_args_list,
[mock.call('/target/tmp/grub.batch', 'wb'),
mock.call('/target/tmp/grub.sh', 'wb')]
[mock.call('/target/tmp/grub.batch', 'wt', encoding='utf-8'),
mock.call('/target/tmp/grub.sh', 'wt', encoding='utf-8')]
)
mock_open_file = mock_open()
self.assertEqual(
@ -408,9 +405,11 @@ title Default (kernel-version)
"""
mock_open = mock.mock_open()
with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
with mock.patch('fuel_agent.utils.grub.open', new=mock_open,
create=True):
gu.grub1_cfg(chroot='/target', kernel_params='kernel-params')
mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wb')
mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wt',
encoding='utf-8')
mock_open_file = mock_open()
mock_open_file.write.assert_called_once_with(config)
@ -424,12 +423,14 @@ title Default (kernel-version-set)
"""
mock_open = mock.mock_open()
with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
with mock.patch('fuel_agent.utils.grub.open', new=mock_open,
create=True):
gu.grub1_cfg(kernel='kernel-version-set',
initrd='initrd-version-set',
chroot='/target', kernel_params='kernel-params',
grub_timeout=10)
mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wb')
mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wt',
encoding='utf-8')
mock_open_file = mock_open()
mock_open_file.write.assert_called_once_with(config)
@ -463,19 +464,19 @@ bar
GRUB_RECORDFAIL_TIMEOUT=10
"""
with mock.patch(OPEN_FUNCTION_NAME,
with mock.patch('fuel_agent.utils.grub.open',
new=mock.mock_open(read_data=orig_content),
create=True) as mock_open:
mock_open.return_value = mock.MagicMock(spec=file)
mock_open.return_value = mock.MagicMock(spec=io.IOBase)
handle = mock_open.return_value.__enter__.return_value
handle.__iter__.return_value = StringIO.StringIO(orig_content)
handle.__iter__.return_value = StringIO(orig_content)
gu.grub2_cfg(kernel_params='kernel-params-new', chroot='/target',
grub_timeout=10)
self.assertEqual(
mock_open.call_args_list,
[mock.call('/target/etc/default/grub'),
mock.call('/target/etc/default/grub', 'wb')]
mock.call('/target/etc/default/grub', 'wt', encoding='utf-8')]
)
handle.write.assert_called_once_with(new_content)

7
fuel_agent/tests/test_hardware_utils.py
View File

@ -12,13 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import six
import unittest2
from fuel_agent import errors
from fuel_agent.utils import hardware as hu
from fuel_agent.utils import utils
if six.PY2:
import mock
elif six.PY3:
from unittest import mock
class TestHardwareUtils(unittest2.TestCase):

28
fuel_agent/tests/test_manager.py
View File

@ -16,8 +16,8 @@ import copy
import os
import signal
import mock
from oslo_config import cfg
import six
import unittest2
from fuel_agent.drivers import nailgun
@ -33,6 +33,11 @@ from fuel_agent.utils import md as mu
from fuel_agent.utils import partition as pu
from fuel_agent.utils import utils
if six.PY2:
import mock
elif six.PY3:
import unittest.mock as mock
CONF = cfg.CONF
@ -239,11 +244,11 @@ class TestManager(unittest2.TestCase):
self.mgr.do_bootloader()
expected_open_calls = [
mock.call('/tmp/target/etc/udev/rules.d/70-persistent-net.'
'rules', 'w'),
'rules', 'wt', encoding='utf-8'),
mock.call('/tmp/target/etc/udev/rules.d/75-persistent-net-'
'generator.rules', 'w'),
'generator.rules', 'wt', encoding='utf-8'),
mock.call('/tmp/target/etc/nailgun-agent/nodiscover', 'w'),
mock.call('/tmp/target/etc/fstab', 'wb')]
mock.call('/tmp/target/etc/fstab', 'wt', encoding='utf-8')]
self.assertEqual(expected_open_calls, mock_open.call_args_list)
expected_write_calls = [
mock.call('# Generated by fuel-agent during provisioning: '
@ -264,9 +269,11 @@ class TestManager(unittest2.TestCase):
'net-generator\n'),
mock.call('UUID=fake_UUID /boot ext2 defaults 0 0\n'),
mock.call('UUID=fake_UUID /tmp ext2 defaults 0 0\n'),
mock.call('UUID=fake_UUID / ext4 defaults,errors=panic 0 0\n'),
mock.call(
'UUID=fake_UUID / ext4 defaults,errors=panic 0 0\n'),
mock.call('UUID=fake_UUID swap swap defaults 0 0\n'),
mock.call('UUID=fake_UUID /var/lib/glance xfs defaults 0 0\n')]
mock.call('UUID=fake_UUID /var/lib/glance xfs defaults 0 0\n')
]
self.assertEqual(expected_write_calls,
file_handle_mock.write.call_args_list)
mock_umount.assert_called_once_with('/tmp/target')
@ -680,7 +687,8 @@ class TestManager(unittest2.TestCase):
mock_utils.execute.return_value = (None, None)
self.mgr.driver._partition_scheme = objects.PartitionScheme()
self.mgr.mount_target('fake_chroot')
mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wb')
mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wt',
encoding='utf-8')
mock_os.path.islink.assert_called_once_with('fake_chroot/etc/mtab')
mock_os.remove.assert_called_once_with('fake_chroot/etc/mtab')
@ -734,7 +742,8 @@ none /run/shm tmpfs rw,nosuid,nodev 0 0"""
mock_fu.mount_bind.call_args_list)
file_handle = mock_open.return_value.__enter__.return_value
file_handle.write.assert_called_once_with(fake_mtab)
mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wb')
mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wt',
encoding='utf-8')
mock_os.path.islink.assert_called_once_with('fake_chroot/etc/mtab')
self.assertFalse(mock_os.remove.called)
@ -965,7 +974,8 @@ class TestImageBuild(unittest2.TestCase):
mock.call('/tmp/img-boot', 'gzip',
chunk_size=CONF.data_chunk_size)],
mock_bu.containerize.call_args_list)
mock_open.assert_called_once_with('/fake/img.yaml', 'w')
mock_open.assert_called_once_with('/fake/img.yaml', 'wt',
encoding='utf-8')
self.assertEqual(
[mock.call('/tmp/img.gz', '/fake/img.img.gz'),
mock.call('/tmp/img-boot.gz', '/fake/img-boot.img.gz')],

3
fuel_agent/tests/test_md_utils.py
View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import six
import unittest2
@ -24,8 +23,10 @@ from fuel_agent.utils import utils
if six.PY2:
OPEN_FUNCTION_NAME = '__builtin__.open'
import mock
else:
OPEN_FUNCTION_NAME = 'builtins.open'
import unittest.mock as mock
class TestMdUtils(unittest2.TestCase):

5
fuel_agent/tests/test_nailgun.py
View File

@ -15,6 +15,7 @@
import copy
import mock
import six
import unittest2
import yaml
@ -867,7 +868,7 @@ class TestNailgunGetOSMethods(unittest2.TestCase):
'generic_os': {'obj': objects.OperatingSystem,
'minor': 'unknown', 'major': 'unknown'}}
drv = nailgun.Nailgun('fake_data')
for profile, obj in d.iteritems():
for profile, obj in six.iteritems(d):
os = drv.get_os_by_profile(profile)
self.assertIsInstance(os, obj['obj'])
self.assertEqual(obj['minor'], os.minor)
@ -878,7 +879,7 @@ class TestNailgunGetOSMethods(unittest2.TestCase):
'Ubuntu': objects.Ubuntu,
'unknown': None}
drv = nailgun.Nailgun('fake_data')
for os_name, obj in d.iteritems():
for os_name, obj in six.iteritems(d):
os = drv.get_os_by_image_meta(
{'name': os_name, 'minor': 1, 'major': 2})
if os:

13
fuel_agent/tests/test_utils.py
View File

@ -15,9 +15,9 @@
import socket
import mock
from oslo_config import cfg
import requests
import six
import stevedore
import unittest2
import urllib3
@ -25,6 +25,10 @@ import urllib3
from fuel_agent import errors
from fuel_agent.utils import utils
if six.PY2:
import mock
elif six.PY3:
from unittest import mock
CONF = cfg.CONF
@ -140,8 +144,13 @@ class ExecuteTestCase(unittest2.TestCase):
def test_calculate_md5_ok(self):
# calculated by 'printf %10000s | md5sum'
mock_open = mock.Mock()
mock_open.__enter__ = mock.Mock(
side_effect=(six.BytesIO(b' ' * 10000) for _ in range(6)))
mock_open.__exit__ = mock.Mock(return_value=False)
with mock.patch('six.moves.builtins.open',
mock.mock_open(read_data=' ' * 10000), create=True):
mock.Mock(return_value=mock_open), create=True):
self.assertEqual('f38898bb69bb02bccb9594dfe471c5c0',
utils.calculate_md5('fake', 10000))
self.assertEqual('6934d9d33cd2d0c005994e7d96d2e0d9',

9
fuel_agent/utils/artifact.py
View File

@ -62,6 +62,9 @@ class Target(object):
os.fsync(f.fileno())
LOG.debug('File is written: %s' % filename)
def __next__(self):
return self.next()
class LocalFile(Target):
def __init__(self, filename):
@ -123,7 +126,7 @@ class GunzipStream(Target):
def next(self):
try:
return self.decompressor.decompress(self.stream.next())
return self.decompressor.decompress(next(self.stream))
except StopIteration:
raise
@ -217,7 +220,7 @@ class Chain(object):
def jump(proc, next_proc):
# if next_proc is just a string we assume it is a filename
# and we save stream into a file
if isinstance(next_proc, (str, unicode)):
if isinstance(next_proc, six.string_types):
LOG.debug('Processor target: %s' % next_proc)
proc.target(next_proc)
return LocalFile(next_proc)
@ -225,4 +228,4 @@ class Chain(object):
# initialized with the previous one
else:
return next_proc(proc)
return reduce(jump, self.processors)
return six.moves.reduce(jump, self.processors)

19
fuel_agent/utils/grub.py
View File

@ -12,10 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from io import open
import os
import re
import shutil
import six
from fuel_agent import errors
from fuel_agent.openstack.common import log as logging
from fuel_agent.utils import utils
@ -161,15 +164,15 @@ def grub1_mbr(install_device, boot_disk, boot_part, chroot=''):
batch += 'setup (hd0)\n'
batch += 'quit\n'
with open(chroot + '/tmp/grub.batch', 'wb') as f:
with open(chroot + '/tmp/grub.batch', 'wt', encoding='utf-8') as f:
LOG.debug('Grub batch content: \n%s' % batch)
f.write(batch)
f.write(six.text_type(batch))
script = 'cat /tmp/grub.batch | {0} --no-floppy --batch'.format(
guess_grub(chroot=chroot))
with open(chroot + '/tmp/grub.sh', 'wb') as f:
with open(chroot + '/tmp/grub.sh', 'wt', encoding='utf-8') as f:
LOG.debug('Grub script content: \n%s' % script)
f.write(script)
f.write(six.text_type(script))
os.chmod(chroot + '/tmp/grub.sh', 0o755)
cmd = ['/tmp/grub.sh']
@ -211,8 +214,8 @@ title Default ({kernel})
""".format(kernel=kernel, initrd=initrd,
kernel_params=kernel_params,
grub_timeout=grub_timeout)
with open(chroot + '/boot/grub/grub.conf', 'wb') as f:
f.write(config)
with open(chroot + '/boot/grub/grub.conf', 'wt', encoding='utf-8') as f:
f.write(six.text_type(config))
def grub2_install(install_devices, chroot=''):
@ -241,8 +244,8 @@ def grub2_cfg(kernel_params='', chroot='', grub_timeout=5):
# prevent user confirmation appearing if unexpected reboot occured.
new_content += '\nGRUB_RECORDFAIL_TIMEOUT={grub_timeout}\n'.\
format(grub_timeout=grub_timeout)
with open(grub_defaults, 'wb') as f:
f.write(new_content)
with open(grub_defaults, 'wt', encoding='utf-8') as f:
f.write(six.text_type(new_content))
cmd = [guess_grub2_mkconfig(chroot), '-o', guess_grub2_conf(chroot)]
if chroot:
cmd[:0] = ['chroot', chroot]

34
fuel_agent/utils/lvm.py
View File

@ -53,7 +53,7 @@ def pvdisplay_parse(output):
def pvcreate(pvname, metadatasize=64, metadatacopies=2):
# check if pv already exists
if filter(lambda x: x['name'] == pvname, pvdisplay()):
if get_first_by_key_value(pvdisplay(), 'name', pvname, False):
raise errors.PVAlreadyExistsError(
'Error while creating pv: pv %s already exists' % pvname)
utils.execute('pvcreate',
@ -63,16 +63,16 @@ def pvcreate(pvname, metadatasize=64, metadatacopies=2):
def pvremove(pvname):
pv = filter(lambda x: x['name'] == pvname, pvdisplay())
pv = get_first_by_key_value(pvdisplay(), 'name', pvname)
# check if pv exists
if not pv:
raise errors.PVNotFoundError(
'Error while removing pv: pv %s not found' % pvname)
# check if pv is attached to some vg
if pv[0]['vg'] is not None:
if pv['vg'] is not None:
raise errors.PVBelongsToVGError('Error while removing pv: '
'pv belongs to vg %s' % pv[0]['vg'])
'pv belongs to vg %s' % pv['vg'])
utils.execute('pvremove', '-ff', '-y', pvname, check_exit_code=[0])
@ -121,7 +121,7 @@ def _vg_attach_validate(pvnames):
def vgcreate(vgname, pvname, *args):
# check if vg already exists
if filter(lambda x: x['name'] == vgname, vgdisplay()):
if get_first_by_key_value(vgdisplay(), 'name', vgname, False):
raise errors.VGAlreadyExistsError(
'Error while creating vg: vg %s already exists' % vgname)
pvnames = [pvname] + list(args)
@ -131,7 +131,7 @@ def vgcreate(vgname, pvname, *args):
def vgextend(vgname, pvname, *args):
# check if vg exists
if not filter(lambda x: x['name'] == vgname, vgdisplay()):
if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
raise errors.VGNotFoundError(
'Error while extending vg: vg %s not found' % vgname)
pvnames = [pvname] + list(args)
@ -141,7 +141,7 @@ def vgextend(vgname, pvname, *args):
def vgreduce(vgname, pvname, *args):
# check if vg exists
if not filter(lambda x: x['name'] == vgname, vgdisplay()):
if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
raise errors.VGNotFoundError(
'Error while reducing vg: vg %s not found' % vgname)
pvnames = [pvname] + list(args)
@ -156,7 +156,7 @@ def vgreduce(vgname, pvname, *args):
def vgremove(vgname):
# check if vg exists
if not filter(lambda x: x['name'] == vgname, vgdisplay()):
if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
raise errors.VGNotFoundError(
'Error while removing vg: vg %s not found' % vgname)
utils.execute('vgremove', '-f', vgname, check_exit_code=[0])
@ -196,20 +196,22 @@ def lvdisplay_parse(output):
def lvcreate(vgname, lvname, size):
vg = filter(lambda x: x['name'] == vgname, vgdisplay())
vg = get_first_by_key_value(vgdisplay(), 'name', vgname)
# check if vg exists
if not vg:
raise errors.VGNotFoundError(
'Error while creating vg: vg %s not found' % vgname)
# check if enough space is available
if vg[0]['free'] < size:
if vg['free'] < size:
raise errors.NotEnoughSpaceError(
'Error while creating lv: vg %s has only %s m of free space, '
'but at least %s m is needed' % (vgname, vg[0]['free'], size))
'but at least %s m is needed' % (vgname, vg['free'], size))
# check if lv already exists
if filter(lambda x: x['name'] == lvname and x['vg'] == vgname,
lvdisplay()):
if next(
(x for x in lvdisplay() if x['name'] == lvname and x['vg'] == vgname),
False
):
raise errors.LVAlreadyExistsError(
'Error while creating lv: lv %s already exists' % lvname)
# NOTE(agordeev): by default, lvcreate is configured to wipe signature
@ -228,7 +230,7 @@ def lvcreate(vgname, lvname, size):
def lvremove(lvpath):
# check if lv exists
if not filter(lambda x: x['path'] == lvpath, lvdisplay()):
if not get_first_by_key_value(lvdisplay(), 'path', lvpath):
raise errors.LVNotFoundError(
'Error while removing lv: lv %s not found' % lvpath)
utils.execute('lvremove', '-f', lvpath, check_exit_code=[0])
@ -247,3 +249,7 @@ def vgremove_all():
def pvremove_all():
for pv in pvdisplay():
pvremove(pv['name'])
def get_first_by_key_value(collection, key, value, default=None):
return next((x for x in collection if x[key] == value), default)

11
fuel_agent/utils/md.py
View File

@ -80,7 +80,7 @@ def mdcreate(mdname, level, devices, metadata='default'):
mds = mddisplay()
# check if md device already exists
if filter(lambda x: x['name'] == mdname, mds):
if next((x for x in mds if x['name'] == mdname), False):
raise errors.MDAlreadyExistsError(
'Error while creating md: md %s already exists' % mdname)
@ -98,15 +98,18 @@ def mdcreate(mdname, level, devices, metadata='default'):
'Error while creating md: at least one of devices is not found')
# check if devices are not parts of some md array
if set(devices) & \
set(itertools.chain(*[md.get('devices', []) for md in mds])):
if set(devices) & set(itertools.chain.from_iterable(
md.get('devices', []) for md in mds
)):
raise errors.MDDeviceDuplicationError(
'Error while creating md: at least one of devices is '
'already in belongs to some md')
# FIXME: mdadm will ask user to continue creating if any device appears to
# be a part of raid array. Superblock zeroing helps to avoid that.
map(mdclean, devices)
for device in devices:
mdclean(device)
utils.execute('mdadm', '--create', '--force', mdname, '-e', metadata,
'--level=%s' % level,
'--raid-devices=%s' % len(devices), *devices,

10
fuel_agent/utils/utils.py
View File

@ -26,10 +26,10 @@ import time
import jinja2
from oslo_config import cfg
import requests
import six
import stevedore.driver
import urllib3
from six.moves import zip_longest
from fuel_agent import errors
from fuel_agent.openstack.common import log as logging
@ -76,7 +76,7 @@ CONF.register_opts(u_opts)
def execute(*cmd, **kwargs):
command = ' '.join(cmd)
LOG.debug('Trying to execute command: %s', command)
commands = [c.strip() for c in re.split(ur'\|', command)]
commands = [c.strip() for c in re.split(r'\|', command)]
env = kwargs.pop('env_variables', copy.deepcopy(os.environ))
env['PATH'] = '/bin:/usr/bin:/sbin:/usr/sbin'
env['LC_ALL'] = env['LANG'] = env['LANGUAGE'] = kwargs.pop('language', 'C')
@ -96,7 +96,7 @@ def execute(*cmd, **kwargs):
if to_filename:
to_file = open(to_filename, 'wb')
for attempt in reversed(xrange(attempts)):
for attempt in reversed(six.moves.range(attempts)):
try:
process = []
for c in commands:
@ -104,7 +104,7 @@ def execute(*cmd, **kwargs):
# NOTE(eli): Python's shlex implementation doesn't like
# unicode. We have to convert to ascii before shlex'ing
# the command. http://bugs.python.org/issue6988
encoded_command = c.encode('ascii')
encoded_command = c.encode('ascii') if six.PY2 else c
process.append(subprocess.Popen(
shlex.split(encoded_command),
@ -241,7 +241,7 @@ def makedirs_if_not_exists(path, mode=0o755):
def grouper(iterable, n, fillvalue=None):
"""Collect data into fixed-length chunks or blocks"""
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)
return six.moves.zip_longest(*args, fillvalue=fillvalue)
def guess_filename(path, regexp, sort=True, reverse=True):

2
requirements.txt
View File

@ -3,7 +3,7 @@ eventlet>=0.13.0
iso8601>=0.1.9
jsonschema>=2.3.0
oslo.config>=1.6.0
oslo.serialization>=1.4.0
oslo.serialization>=1.4.0,<2.0
six>=1.5.2
pbr>=0.7.0
Jinja2

2
tox.ini
View File

@ -1,7 +1,7 @@
[tox]
minversion = 1.6
skipsdist = True
envlist = py26,py27,pep8
envlist = py26,py27,py34,pep8
[testenv]
usedevelop = True