diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py index be16945..34a3cca 100644 --- a/fuel_agent/drivers/nailgun.py +++ b/fuel_agent/drivers/nailgun.py @@ -475,11 +475,18 @@ class Nailgun(BaseDataDriver): configdrive_scheme = objects.ConfigDriveScheme() LOG.debug('Adding common parameters') - admin_interface = filter( - lambda x: (x['mac_address'] == - data['kernel_options']['netcfg/choose_interface']), - [dict(name=name, **spec) for name, spec - in data['interfaces'].iteritems()])[0] + + interface_dicts = [ + dict(name=name, **spec) + for name, spec + in six.iteritems(data['interfaces']) + ] + + admin_interface = next( + x for x in interface_dicts + if (x['mac_address'] == + data['kernel_options']['netcfg/choose_interface']) + ) ssh_auth_keys = data['ks_meta']['authorized_keys'] if data['ks_meta']['auth_key']: diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py index d37d063..10cdd8b 100644 --- a/fuel_agent/manager.py +++ b/fuel_agent/manager.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from io import open import os import shutil import signal import tempfile -import yaml from oslo_config import cfg +import six +import yaml from fuel_agent import errors from fuel_agent.openstack.common import log as logging @@ -405,8 +407,8 @@ class Manager(object): mtab_path = chroot + '/etc/mtab' if os.path.islink(mtab_path): os.remove(mtab_path) - with open(mtab_path, 'wb') as f: - f.write(mtab) + with open(mtab_path, 'wt', encoding='utf-8') as f: + f.write(six.text_type(mtab)) def umount_target(self, chroot, pseudo=True): LOG.debug('Umounting target file systems: %s', chroot) @@ -470,26 +472,27 @@ class Manager(object): # remapping in Ubuntu, so injecting files prior the first boot # should work with open(chroot + '/etc/udev/rules.d/70-persistent-net.rules', - 'w') as f: - f.write('# Generated by fuel-agent during provisioning: ' - 'BEGIN\n') + 'wt', encoding='utf-8') as f: + f.write(u'# Generated by fuel-agent during provisioning: ' + u'BEGIN\n') # pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1 for mapping in self.driver.configdrive_scheme.\ common.udevrules.split(','): mac_addr, nic_name = mapping.split('_') - f.write('SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ' - 'ATTR{address}=="%s", ATTR{type}=="1", ' - 'KERNEL=="eth*", NAME="%s"\n' % (mac_addr, - nic_name)) - f.write('# Generated by fuel-agent during provisioning: END\n') + f.write(u'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ' + u'ATTR{address}=="%s", ATTR{type}=="1", ' + u'KERNEL=="eth*", NAME="%s"\n' % (mac_addr, + nic_name)) + f.write( + u'# Generated by fuel-agent during provisioning: END\n') # FIXME(agordeev): Disable net-generator that will add new etries # to 70-persistent-net.rules with open(chroot + '/etc/udev/rules.d/75-persistent-net-generator.rules', - 'w') as f: - f.write('# Generated by fuel-agent during provisioning:\n' - '# DO NOT DELETE. It is needed to disable ' - 'net-generator\n') + 'wt', encoding='utf-8') as f: + f.write(u'# Generated by fuel-agent during provisioning:\n' + u'# DO NOT DELETE. It is needed to disable ' + u'net-generator\n') # FIXME(kozhukalov): Prevent nailgun-agent from doing anything. # This ugly hack is to be used together with the command removing @@ -518,7 +521,7 @@ class Manager(object): # via updates. utils.execute('mkdir', '-p', chroot + '/var/log/upstart') - with open(chroot + '/etc/fstab', 'wb') as f: + with open(chroot + '/etc/fstab', 'wt', encoding='utf-8') as f: for fs in self.driver.partition_scheme.fss: # TODO(kozhukalov): Think of improving the logic so as to # insert a meaningful fsck order value which is last zero @@ -775,7 +778,7 @@ class Manager(object): # NOTE(kozhukalov): implement abstract publisher LOG.debug('Image metadata: %s', metadata) with open(self.driver.metadata_uri.split('file://', 1)[1], - 'w') as f: + 'wt', encoding='utf-8') as f: yaml.safe_dump(metadata, stream=f) LOG.info('--- Building image END (do_build_image) ---') except Exception as exc: diff --git a/fuel_agent/objects/partition/parted.py b/fuel_agent/objects/partition/parted.py index 6fd4183..75758b4 100644 --- a/fuel_agent/objects/partition/parted.py +++ b/fuel_agent/objects/partition/parted.py @@ -55,17 +55,15 @@ class Parted(base.Serializable): @property def logical(self): - return filter(lambda x: x.type == 'logical', self.partitions) + return [x for x in self.partitions if x.type == 'logical'] @property def primary(self): - return filter(lambda x: x.type == 'primary', self.partitions) + return [x for x in self.partitions if x.type == 'primary'] @property def extended(self): - found = filter(lambda x: x.type == 'extended', self.partitions) - if found: - return found[0] + return next((x for x in self.partitions if x.type == 'extended'), None) def next_type(self): if self.label == 'gpt': @@ -104,9 +102,7 @@ class Parted(base.Serializable): return '%s%s%s' % (self.name, separator, self.next_count()) def partition_by_name(self, name): - found = filter(lambda x: (x.name == name), self.partitions) - if found: - return found[0] + return next((x for x in self.partitions if x.name == name), None) def to_dict(self): partitions = [partition.to_dict() for partition in self.partitions] diff --git a/fuel_agent/objects/partition/scheme.py b/fuel_agent/objects/partition/scheme.py index 6a765c7..8771377 100644 --- a/fuel_agent/objects/partition/scheme.py +++ b/fuel_agent/objects/partition/scheme.py @@ -73,9 +73,7 @@ class PartitionScheme(object): return md def md_by_name(self, name): - found = filter(lambda x: x.name == name, self.mds) - if found: - return found[0] + return next((x for x in self.mds if x.name == name), None) def md_by_mount(self, mount): fs = self.fs_by_mount(mount) @@ -115,14 +113,10 @@ class PartitionScheme(object): if parted.partition_by_name(name)), None) def vg_by_name(self, vgname): - found = filter(lambda x: (x.name == vgname), self.vgs) - if found: - return found[0] + return next((x for x in self.vgs if x.name == vgname), None) def pv_by_name(self, pvname): - found = filter(lambda x: (x.name == pvname), self.pvs) - if found: - return found[0] + return next((x for x in self.pvs if x.name == pvname), None) def vg_attach_by_name(self, pvname, vgname, metadatasize=16, metadatacopies=2): @@ -133,14 +127,10 @@ class PartitionScheme(object): vg.add_pv(pv.name) def fs_by_mount(self, mount): - found = filter(lambda x: (x.mount and x.mount == mount), self.fss) - if found: - return found[0] + return next((x for x in self.fss if x.mount == mount), None) def fs_by_device(self, device): - found = filter(lambda x: x.device == device, self.fss) - if found: - return found[0] + return next((x for x in self.fss if x.device == device), None) def fs_sorted_by_depth(self, reverse=False): """Getting file systems sorted by path length. @@ -154,9 +144,8 @@ class PartitionScheme(object): return sorted(self.fss, key=key, reverse=reverse) def lv_by_device_name(self, device_name): - found = filter(lambda x: x.device_name == device_name, self.lvs) - if found: - return found[0] + return next((x for x in self.lvs if x.device_name == device_name), + None) def root_device(self): fs = self.fs_by_mount('/') diff --git a/fuel_agent/tests/test_artifact_utils.py b/fuel_agent/tests/test_artifact_utils.py index c7d8c74..2cdde06 100644 --- a/fuel_agent/tests/test_artifact_utils.py +++ b/fuel_agent/tests/test_artifact_utils.py @@ -17,7 +17,6 @@ from oslo_config import cfg import unittest2 import zlib - from fuel_agent import errors from fuel_agent.utils import artifact as au from fuel_agent.utils import utils @@ -91,7 +90,7 @@ class TestHttpUrl(unittest2.TestCase): class TestGunzipStream(unittest2.TestCase): def test_gunzip_stream_next(self): - content = ['fake content #1'] + content = [b'fake content #1'] compressed_stream = [zlib.compress(data) for data in content] gunzip_stream = au.GunzipStream(compressed_stream) for data in enumerate(gunzip_stream): diff --git a/fuel_agent/tests/test_grub_utils.py b/fuel_agent/tests/test_grub_utils.py index 67ea812..35e4d8d 100644 --- a/fuel_agent/tests/test_grub_utils.py +++ b/fuel_agent/tests/test_grub_utils.py @@ -12,20 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import StringIO +import io import mock -import six +from six import StringIO import unittest2 from fuel_agent import errors from fuel_agent.utils import grub as gu -if six.PY2: - OPEN_FUNCTION_NAME = '__builtin__.open' -else: - OPEN_FUNCTION_NAME = 'builtins.open' - class TestGrubUtils(unittest2.TestCase): @@ -313,12 +308,13 @@ class TestGrubUtils(unittest2.TestCase): script = 'cat /tmp/grub.batch | /sbin/grub --no-floppy --batch' mock_open = mock.mock_open() - with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True): + with mock.patch('fuel_agent.utils.grub.open', new=mock_open, + create=True): gu.grub1_mbr('/dev/foo', '/dev/bar', '0', chroot='/target') self.assertEqual( mock_open.call_args_list, - [mock.call('/target/tmp/grub.batch', 'wb'), - mock.call('/target/tmp/grub.sh', 'wb')] + [mock.call('/target/tmp/grub.batch', 'wt', encoding='utf-8'), + mock.call('/target/tmp/grub.sh', 'wt', encoding='utf-8')] ) mock_open_file = mock_open() self.assertEqual( @@ -347,12 +343,13 @@ class TestGrubUtils(unittest2.TestCase): script = 'cat /tmp/grub.batch | /sbin/grub --no-floppy --batch' mock_open = mock.mock_open() - with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True): + with mock.patch('fuel_agent.utils.grub.open', new=mock_open, + create=True): gu.grub1_mbr('/dev/foo', '/dev/foo', '0', chroot='/target') self.assertEqual( mock_open.call_args_list, - [mock.call('/target/tmp/grub.batch', 'wb'), - mock.call('/target/tmp/grub.sh', 'wb')] + [mock.call('/target/tmp/grub.batch', 'wt', encoding='utf-8'), + mock.call('/target/tmp/grub.sh', 'wt', encoding='utf-8')] ) mock_open_file = mock_open() self.assertEqual( @@ -408,9 +405,11 @@ title Default (kernel-version) """ mock_open = mock.mock_open() - with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True): + with mock.patch('fuel_agent.utils.grub.open', new=mock_open, + create=True): gu.grub1_cfg(chroot='/target', kernel_params='kernel-params') - mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wb') + mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wt', + encoding='utf-8') mock_open_file = mock_open() mock_open_file.write.assert_called_once_with(config) @@ -424,12 +423,14 @@ title Default (kernel-version-set) """ mock_open = mock.mock_open() - with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True): + with mock.patch('fuel_agent.utils.grub.open', new=mock_open, + create=True): gu.grub1_cfg(kernel='kernel-version-set', initrd='initrd-version-set', chroot='/target', kernel_params='kernel-params', grub_timeout=10) - mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wb') + mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wt', + encoding='utf-8') mock_open_file = mock_open() mock_open_file.write.assert_called_once_with(config) @@ -463,19 +464,19 @@ bar GRUB_RECORDFAIL_TIMEOUT=10 """ - with mock.patch(OPEN_FUNCTION_NAME, + with mock.patch('fuel_agent.utils.grub.open', new=mock.mock_open(read_data=orig_content), create=True) as mock_open: - mock_open.return_value = mock.MagicMock(spec=file) + mock_open.return_value = mock.MagicMock(spec=io.IOBase) handle = mock_open.return_value.__enter__.return_value - handle.__iter__.return_value = StringIO.StringIO(orig_content) + handle.__iter__.return_value = StringIO(orig_content) gu.grub2_cfg(kernel_params='kernel-params-new', chroot='/target', grub_timeout=10) self.assertEqual( mock_open.call_args_list, [mock.call('/target/etc/default/grub'), - mock.call('/target/etc/default/grub', 'wb')] + mock.call('/target/etc/default/grub', 'wt', encoding='utf-8')] ) handle.write.assert_called_once_with(new_content) diff --git a/fuel_agent/tests/test_hardware_utils.py b/fuel_agent/tests/test_hardware_utils.py index b6a7bde..d1bb7fd 100644 --- a/fuel_agent/tests/test_hardware_utils.py +++ b/fuel_agent/tests/test_hardware_utils.py @@ -12,13 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +import six import unittest2 from fuel_agent import errors from fuel_agent.utils import hardware as hu from fuel_agent.utils import utils +if six.PY2: + import mock +elif six.PY3: + from unittest import mock + class TestHardwareUtils(unittest2.TestCase): diff --git a/fuel_agent/tests/test_manager.py b/fuel_agent/tests/test_manager.py index 03a5f2a..cb84585 100644 --- a/fuel_agent/tests/test_manager.py +++ b/fuel_agent/tests/test_manager.py @@ -16,8 +16,8 @@ import copy import os import signal -import mock from oslo_config import cfg +import six import unittest2 from fuel_agent.drivers import nailgun @@ -33,6 +33,11 @@ from fuel_agent.utils import md as mu from fuel_agent.utils import partition as pu from fuel_agent.utils import utils +if six.PY2: + import mock +elif six.PY3: + import unittest.mock as mock + CONF = cfg.CONF @@ -239,11 +244,11 @@ class TestManager(unittest2.TestCase): self.mgr.do_bootloader() expected_open_calls = [ mock.call('/tmp/target/etc/udev/rules.d/70-persistent-net.' - 'rules', 'w'), + 'rules', 'wt', encoding='utf-8'), mock.call('/tmp/target/etc/udev/rules.d/75-persistent-net-' - 'generator.rules', 'w'), + 'generator.rules', 'wt', encoding='utf-8'), mock.call('/tmp/target/etc/nailgun-agent/nodiscover', 'w'), - mock.call('/tmp/target/etc/fstab', 'wb')] + mock.call('/tmp/target/etc/fstab', 'wt', encoding='utf-8')] self.assertEqual(expected_open_calls, mock_open.call_args_list) expected_write_calls = [ mock.call('# Generated by fuel-agent during provisioning: ' @@ -264,9 +269,11 @@ class TestManager(unittest2.TestCase): 'net-generator\n'), mock.call('UUID=fake_UUID /boot ext2 defaults 0 0\n'), mock.call('UUID=fake_UUID /tmp ext2 defaults 0 0\n'), - mock.call('UUID=fake_UUID / ext4 defaults,errors=panic 0 0\n'), + mock.call( + 'UUID=fake_UUID / ext4 defaults,errors=panic 0 0\n'), mock.call('UUID=fake_UUID swap swap defaults 0 0\n'), - mock.call('UUID=fake_UUID /var/lib/glance xfs defaults 0 0\n')] + mock.call('UUID=fake_UUID /var/lib/glance xfs defaults 0 0\n') + ] self.assertEqual(expected_write_calls, file_handle_mock.write.call_args_list) mock_umount.assert_called_once_with('/tmp/target') @@ -680,7 +687,8 @@ class TestManager(unittest2.TestCase): mock_utils.execute.return_value = (None, None) self.mgr.driver._partition_scheme = objects.PartitionScheme() self.mgr.mount_target('fake_chroot') - mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wb') + mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wt', + encoding='utf-8') mock_os.path.islink.assert_called_once_with('fake_chroot/etc/mtab') mock_os.remove.assert_called_once_with('fake_chroot/etc/mtab') @@ -734,7 +742,8 @@ none /run/shm tmpfs rw,nosuid,nodev 0 0""" mock_fu.mount_bind.call_args_list) file_handle = mock_open.return_value.__enter__.return_value file_handle.write.assert_called_once_with(fake_mtab) - mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wb') + mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wt', + encoding='utf-8') mock_os.path.islink.assert_called_once_with('fake_chroot/etc/mtab') self.assertFalse(mock_os.remove.called) @@ -965,7 +974,8 @@ class TestImageBuild(unittest2.TestCase): mock.call('/tmp/img-boot', 'gzip', chunk_size=CONF.data_chunk_size)], mock_bu.containerize.call_args_list) - mock_open.assert_called_once_with('/fake/img.yaml', 'w') + mock_open.assert_called_once_with('/fake/img.yaml', 'wt', + encoding='utf-8') self.assertEqual( [mock.call('/tmp/img.gz', '/fake/img.img.gz'), mock.call('/tmp/img-boot.gz', '/fake/img-boot.img.gz')], diff --git a/fuel_agent/tests/test_md_utils.py b/fuel_agent/tests/test_md_utils.py index 8c1b258..beda74b 100644 --- a/fuel_agent/tests/test_md_utils.py +++ b/fuel_agent/tests/test_md_utils.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock import six import unittest2 @@ -24,8 +23,10 @@ from fuel_agent.utils import utils if six.PY2: OPEN_FUNCTION_NAME = '__builtin__.open' + import mock else: OPEN_FUNCTION_NAME = 'builtins.open' + import unittest.mock as mock class TestMdUtils(unittest2.TestCase): diff --git a/fuel_agent/tests/test_nailgun.py b/fuel_agent/tests/test_nailgun.py index 0910e85..71153b2 100644 --- a/fuel_agent/tests/test_nailgun.py +++ b/fuel_agent/tests/test_nailgun.py @@ -15,6 +15,7 @@ import copy import mock +import six import unittest2 import yaml @@ -867,7 +868,7 @@ class TestNailgunGetOSMethods(unittest2.TestCase): 'generic_os': {'obj': objects.OperatingSystem, 'minor': 'unknown', 'major': 'unknown'}} drv = nailgun.Nailgun('fake_data') - for profile, obj in d.iteritems(): + for profile, obj in six.iteritems(d): os = drv.get_os_by_profile(profile) self.assertIsInstance(os, obj['obj']) self.assertEqual(obj['minor'], os.minor) @@ -878,7 +879,7 @@ class TestNailgunGetOSMethods(unittest2.TestCase): 'Ubuntu': objects.Ubuntu, 'unknown': None} drv = nailgun.Nailgun('fake_data') - for os_name, obj in d.iteritems(): + for os_name, obj in six.iteritems(d): os = drv.get_os_by_image_meta( {'name': os_name, 'minor': 1, 'major': 2}) if os: diff --git a/fuel_agent/tests/test_utils.py b/fuel_agent/tests/test_utils.py index 9019f7d..cb221c8 100644 --- a/fuel_agent/tests/test_utils.py +++ b/fuel_agent/tests/test_utils.py @@ -15,9 +15,9 @@ import socket -import mock from oslo_config import cfg import requests +import six import stevedore import unittest2 import urllib3 @@ -25,6 +25,10 @@ import urllib3 from fuel_agent import errors from fuel_agent.utils import utils +if six.PY2: + import mock +elif six.PY3: + from unittest import mock CONF = cfg.CONF @@ -140,8 +144,13 @@ class ExecuteTestCase(unittest2.TestCase): def test_calculate_md5_ok(self): # calculated by 'printf %10000s | md5sum' + mock_open = mock.Mock() + mock_open.__enter__ = mock.Mock( + side_effect=(six.BytesIO(b' ' * 10000) for _ in range(6))) + + mock_open.__exit__ = mock.Mock(return_value=False) with mock.patch('six.moves.builtins.open', - mock.mock_open(read_data=' ' * 10000), create=True): + mock.Mock(return_value=mock_open), create=True): self.assertEqual('f38898bb69bb02bccb9594dfe471c5c0', utils.calculate_md5('fake', 10000)) self.assertEqual('6934d9d33cd2d0c005994e7d96d2e0d9', diff --git a/fuel_agent/utils/artifact.py b/fuel_agent/utils/artifact.py index d98fd77..56c5fed 100644 --- a/fuel_agent/utils/artifact.py +++ b/fuel_agent/utils/artifact.py @@ -62,6 +62,9 @@ class Target(object): os.fsync(f.fileno()) LOG.debug('File is written: %s' % filename) + def __next__(self): + return self.next() + class LocalFile(Target): def __init__(self, filename): @@ -123,7 +126,7 @@ class GunzipStream(Target): def next(self): try: - return self.decompressor.decompress(self.stream.next()) + return self.decompressor.decompress(next(self.stream)) except StopIteration: raise @@ -217,7 +220,7 @@ class Chain(object): def jump(proc, next_proc): # if next_proc is just a string we assume it is a filename # and we save stream into a file - if isinstance(next_proc, (str, unicode)): + if isinstance(next_proc, six.string_types): LOG.debug('Processor target: %s' % next_proc) proc.target(next_proc) return LocalFile(next_proc) @@ -225,4 +228,4 @@ class Chain(object): # initialized with the previous one else: return next_proc(proc) - return reduce(jump, self.processors) + return six.moves.reduce(jump, self.processors) diff --git a/fuel_agent/utils/grub.py b/fuel_agent/utils/grub.py index eced27a..9746cd8 100644 --- a/fuel_agent/utils/grub.py +++ b/fuel_agent/utils/grub.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from io import open import os import re import shutil +import six + from fuel_agent import errors from fuel_agent.openstack.common import log as logging from fuel_agent.utils import utils @@ -161,15 +164,15 @@ def grub1_mbr(install_device, boot_disk, boot_part, chroot=''): batch += 'setup (hd0)\n' batch += 'quit\n' - with open(chroot + '/tmp/grub.batch', 'wb') as f: + with open(chroot + '/tmp/grub.batch', 'wt', encoding='utf-8') as f: LOG.debug('Grub batch content: \n%s' % batch) - f.write(batch) + f.write(six.text_type(batch)) script = 'cat /tmp/grub.batch | {0} --no-floppy --batch'.format( guess_grub(chroot=chroot)) - with open(chroot + '/tmp/grub.sh', 'wb') as f: + with open(chroot + '/tmp/grub.sh', 'wt', encoding='utf-8') as f: LOG.debug('Grub script content: \n%s' % script) - f.write(script) + f.write(six.text_type(script)) os.chmod(chroot + '/tmp/grub.sh', 0o755) cmd = ['/tmp/grub.sh'] @@ -211,8 +214,8 @@ title Default ({kernel}) """.format(kernel=kernel, initrd=initrd, kernel_params=kernel_params, grub_timeout=grub_timeout) - with open(chroot + '/boot/grub/grub.conf', 'wb') as f: - f.write(config) + with open(chroot + '/boot/grub/grub.conf', 'wt', encoding='utf-8') as f: + f.write(six.text_type(config)) def grub2_install(install_devices, chroot=''): @@ -241,8 +244,8 @@ def grub2_cfg(kernel_params='', chroot='', grub_timeout=5): # prevent user confirmation appearing if unexpected reboot occured. new_content += '\nGRUB_RECORDFAIL_TIMEOUT={grub_timeout}\n'.\ format(grub_timeout=grub_timeout) - with open(grub_defaults, 'wb') as f: - f.write(new_content) + with open(grub_defaults, 'wt', encoding='utf-8') as f: + f.write(six.text_type(new_content)) cmd = [guess_grub2_mkconfig(chroot), '-o', guess_grub2_conf(chroot)] if chroot: cmd[:0] = ['chroot', chroot] diff --git a/fuel_agent/utils/lvm.py b/fuel_agent/utils/lvm.py index 69ef5e4..7beab6d 100644 --- a/fuel_agent/utils/lvm.py +++ b/fuel_agent/utils/lvm.py @@ -53,7 +53,7 @@ def pvdisplay_parse(output): def pvcreate(pvname, metadatasize=64, metadatacopies=2): # check if pv already exists - if filter(lambda x: x['name'] == pvname, pvdisplay()): + if get_first_by_key_value(pvdisplay(), 'name', pvname, False): raise errors.PVAlreadyExistsError( 'Error while creating pv: pv %s already exists' % pvname) utils.execute('pvcreate', @@ -63,16 +63,16 @@ def pvcreate(pvname, metadatasize=64, metadatacopies=2): def pvremove(pvname): - pv = filter(lambda x: x['name'] == pvname, pvdisplay()) + pv = get_first_by_key_value(pvdisplay(), 'name', pvname) # check if pv exists if not pv: raise errors.PVNotFoundError( 'Error while removing pv: pv %s not found' % pvname) # check if pv is attached to some vg - if pv[0]['vg'] is not None: + if pv['vg'] is not None: raise errors.PVBelongsToVGError('Error while removing pv: ' - 'pv belongs to vg %s' % pv[0]['vg']) + 'pv belongs to vg %s' % pv['vg']) utils.execute('pvremove', '-ff', '-y', pvname, check_exit_code=[0]) @@ -121,7 +121,7 @@ def _vg_attach_validate(pvnames): def vgcreate(vgname, pvname, *args): # check if vg already exists - if filter(lambda x: x['name'] == vgname, vgdisplay()): + if get_first_by_key_value(vgdisplay(), 'name', vgname, False): raise errors.VGAlreadyExistsError( 'Error while creating vg: vg %s already exists' % vgname) pvnames = [pvname] + list(args) @@ -131,7 +131,7 @@ def vgcreate(vgname, pvname, *args): def vgextend(vgname, pvname, *args): # check if vg exists - if not filter(lambda x: x['name'] == vgname, vgdisplay()): + if not get_first_by_key_value(vgdisplay(), 'name', vgname, False): raise errors.VGNotFoundError( 'Error while extending vg: vg %s not found' % vgname) pvnames = [pvname] + list(args) @@ -141,7 +141,7 @@ def vgextend(vgname, pvname, *args): def vgreduce(vgname, pvname, *args): # check if vg exists - if not filter(lambda x: x['name'] == vgname, vgdisplay()): + if not get_first_by_key_value(vgdisplay(), 'name', vgname, False): raise errors.VGNotFoundError( 'Error while reducing vg: vg %s not found' % vgname) pvnames = [pvname] + list(args) @@ -156,7 +156,7 @@ def vgreduce(vgname, pvname, *args): def vgremove(vgname): # check if vg exists - if not filter(lambda x: x['name'] == vgname, vgdisplay()): + if not get_first_by_key_value(vgdisplay(), 'name', vgname, False): raise errors.VGNotFoundError( 'Error while removing vg: vg %s not found' % vgname) utils.execute('vgremove', '-f', vgname, check_exit_code=[0]) @@ -196,20 +196,22 @@ def lvdisplay_parse(output): def lvcreate(vgname, lvname, size): - vg = filter(lambda x: x['name'] == vgname, vgdisplay()) + vg = get_first_by_key_value(vgdisplay(), 'name', vgname) # check if vg exists if not vg: raise errors.VGNotFoundError( 'Error while creating vg: vg %s not found' % vgname) # check if enough space is available - if vg[0]['free'] < size: + if vg['free'] < size: raise errors.NotEnoughSpaceError( 'Error while creating lv: vg %s has only %s m of free space, ' - 'but at least %s m is needed' % (vgname, vg[0]['free'], size)) + 'but at least %s m is needed' % (vgname, vg['free'], size)) # check if lv already exists - if filter(lambda x: x['name'] == lvname and x['vg'] == vgname, - lvdisplay()): + if next( + (x for x in lvdisplay() if x['name'] == lvname and x['vg'] == vgname), + False + ): raise errors.LVAlreadyExistsError( 'Error while creating lv: lv %s already exists' % lvname) # NOTE(agordeev): by default, lvcreate is configured to wipe signature @@ -228,7 +230,7 @@ def lvcreate(vgname, lvname, size): def lvremove(lvpath): # check if lv exists - if not filter(lambda x: x['path'] == lvpath, lvdisplay()): + if not get_first_by_key_value(lvdisplay(), 'path', lvpath): raise errors.LVNotFoundError( 'Error while removing lv: lv %s not found' % lvpath) utils.execute('lvremove', '-f', lvpath, check_exit_code=[0]) @@ -247,3 +249,7 @@ def vgremove_all(): def pvremove_all(): for pv in pvdisplay(): pvremove(pv['name']) + + +def get_first_by_key_value(collection, key, value, default=None): + return next((x for x in collection if x[key] == value), default) diff --git a/fuel_agent/utils/md.py b/fuel_agent/utils/md.py index b5b413c..82c3945 100644 --- a/fuel_agent/utils/md.py +++ b/fuel_agent/utils/md.py @@ -80,7 +80,7 @@ def mdcreate(mdname, level, devices, metadata='default'): mds = mddisplay() # check if md device already exists - if filter(lambda x: x['name'] == mdname, mds): + if next((x for x in mds if x['name'] == mdname), False): raise errors.MDAlreadyExistsError( 'Error while creating md: md %s already exists' % mdname) @@ -98,15 +98,18 @@ def mdcreate(mdname, level, devices, metadata='default'): 'Error while creating md: at least one of devices is not found') # check if devices are not parts of some md array - if set(devices) & \ - set(itertools.chain(*[md.get('devices', []) for md in mds])): + if set(devices) & set(itertools.chain.from_iterable( + md.get('devices', []) for md in mds + )): raise errors.MDDeviceDuplicationError( 'Error while creating md: at least one of devices is ' 'already in belongs to some md') # FIXME: mdadm will ask user to continue creating if any device appears to # be a part of raid array. Superblock zeroing helps to avoid that. - map(mdclean, devices) + for device in devices: + mdclean(device) + utils.execute('mdadm', '--create', '--force', mdname, '-e', metadata, '--level=%s' % level, '--raid-devices=%s' % len(devices), *devices, diff --git a/fuel_agent/utils/utils.py b/fuel_agent/utils/utils.py index 11bebce..b2d163c 100644 --- a/fuel_agent/utils/utils.py +++ b/fuel_agent/utils/utils.py @@ -26,10 +26,10 @@ import time import jinja2 from oslo_config import cfg import requests +import six import stevedore.driver import urllib3 -from six.moves import zip_longest from fuel_agent import errors from fuel_agent.openstack.common import log as logging @@ -76,7 +76,7 @@ CONF.register_opts(u_opts) def execute(*cmd, **kwargs): command = ' '.join(cmd) LOG.debug('Trying to execute command: %s', command) - commands = [c.strip() for c in re.split(ur'\|', command)] + commands = [c.strip() for c in re.split(r'\|', command)] env = kwargs.pop('env_variables', copy.deepcopy(os.environ)) env['PATH'] = '/bin:/usr/bin:/sbin:/usr/sbin' env['LC_ALL'] = env['LANG'] = env['LANGUAGE'] = kwargs.pop('language', 'C') @@ -96,7 +96,7 @@ def execute(*cmd, **kwargs): if to_filename: to_file = open(to_filename, 'wb') - for attempt in reversed(xrange(attempts)): + for attempt in reversed(six.moves.range(attempts)): try: process = [] for c in commands: @@ -104,7 +104,7 @@ def execute(*cmd, **kwargs): # NOTE(eli): Python's shlex implementation doesn't like # unicode. We have to convert to ascii before shlex'ing # the command. http://bugs.python.org/issue6988 - encoded_command = c.encode('ascii') + encoded_command = c.encode('ascii') if six.PY2 else c process.append(subprocess.Popen( shlex.split(encoded_command), @@ -241,7 +241,7 @@ def makedirs_if_not_exists(path, mode=0o755): def grouper(iterable, n, fillvalue=None): """Collect data into fixed-length chunks or blocks""" args = [iter(iterable)] * n - return zip_longest(*args, fillvalue=fillvalue) + return six.moves.zip_longest(*args, fillvalue=fillvalue) def guess_filename(path, regexp, sort=True, reverse=True): diff --git a/requirements.txt b/requirements.txt index 17b45ae..704c1c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ eventlet>=0.13.0 iso8601>=0.1.9 jsonschema>=2.3.0 oslo.config>=1.6.0 -oslo.serialization>=1.4.0 +oslo.serialization>=1.4.0,<2.0 six>=1.5.2 pbr>=0.7.0 Jinja2 diff --git a/tox.ini b/tox.ini index 7c849fe..14f3ea8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] minversion = 1.6 skipsdist = True -envlist = py26,py27,pep8 +envlist = py26,py27,py34,pep8 [testenv] usedevelop = True