Add Python 3.4 support

Python 3.4 env is added to tox config file
oslo.serialization is pinned to version smaller than 2.0

test_calculate_md5_ok had to be rewritten due to bug in mock_open in
unittest.mock library in python 3.4: https://bugs.python.org/issue23004

Change-Id: Idf2fa15ff4035b9fe92027acdcaf51e005b52541
Implements blueprint: volume-manager-refactoring

fuel_agent/drivers/nailgun.py

@@ -475,11 +475,18 @@ class Nailgun(BaseDataDriver):
         configdrive_scheme = objects.ConfigDriveScheme()
 
         LOG.debug('Adding common parameters')
-        admin_interface = filter(
+
-            lambda x: (x['mac_address'] ==
+        interface_dicts = [
-                       data['kernel_options']['netcfg/choose_interface']),
+            dict(name=name, **spec)
-            [dict(name=name, **spec) for name, spec
+            for name, spec
-             in data['interfaces'].iteritems()])[0]
+            in six.iteritems(data['interfaces'])
+        ]
+
+        admin_interface = next(
+            x for x in interface_dicts
+            if (x['mac_address'] ==
+                data['kernel_options']['netcfg/choose_interface'])
+        )
 
         ssh_auth_keys = data['ks_meta']['authorized_keys']
         if data['ks_meta']['auth_key']:

fuel_agent/manager.py

@@ -12,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from io import open
 import os
 import shutil
 import signal
 import tempfile
-import yaml
 
 from oslo_config import cfg
+import six
+import yaml
 
 from fuel_agent import errors
 from fuel_agent.openstack.common import log as logging
@@ -405,8 +407,8 @@ class Manager(object):
             mtab_path = chroot + '/etc/mtab'
             if os.path.islink(mtab_path):
                 os.remove(mtab_path)
-            with open(mtab_path, 'wb') as f:
+            with open(mtab_path, 'wt', encoding='utf-8') as f:
-                f.write(mtab)
+                f.write(six.text_type(mtab))
 
     def umount_target(self, chroot, pseudo=True):
         LOG.debug('Umounting target file systems: %s', chroot)
@@ -470,26 +472,27 @@ class Manager(object):
             # remapping in Ubuntu, so injecting files prior the first boot
             # should work
             with open(chroot + '/etc/udev/rules.d/70-persistent-net.rules',
-                      'w') as f:
+                      'wt', encoding='utf-8') as f:
-                f.write('# Generated by fuel-agent during provisioning: '
+                f.write(u'# Generated by fuel-agent during provisioning: '
-                        'BEGIN\n')
+                        u'BEGIN\n')
                 # pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1
                 for mapping in self.driver.configdrive_scheme.\
                         common.udevrules.split(','):
                     mac_addr, nic_name = mapping.split('_')
-                    f.write('SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '
+                    f.write(u'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '
-                            'ATTR{address}=="%s", ATTR{type}=="1", '
+                            u'ATTR{address}=="%s", ATTR{type}=="1", '
-                            'KERNEL=="eth*", NAME="%s"\n' % (mac_addr,
+                            u'KERNEL=="eth*", NAME="%s"\n' % (mac_addr,
-                                                             nic_name))
+                                                              nic_name))
-                f.write('# Generated by fuel-agent during provisioning: END\n')
+                f.write(
+                    u'# Generated by fuel-agent during provisioning: END\n')
             # FIXME(agordeev): Disable net-generator that will add new etries
             # to 70-persistent-net.rules
             with open(chroot +
                       '/etc/udev/rules.d/75-persistent-net-generator.rules',
-                      'w') as f:
+                      'wt', encoding='utf-8') as f:
-                f.write('# Generated by fuel-agent during provisioning:\n'
+                f.write(u'# Generated by fuel-agent during provisioning:\n'
-                        '# DO NOT DELETE. It is needed to disable '
+                        u'# DO NOT DELETE. It is needed to disable '
-                        'net-generator\n')
+                        u'net-generator\n')
 
         # FIXME(kozhukalov): Prevent nailgun-agent from doing anything.
         # This ugly hack is to be used together with the command removing
@@ -518,7 +521,7 @@ class Manager(object):
         # via updates.
         utils.execute('mkdir', '-p', chroot + '/var/log/upstart')
 
-        with open(chroot + '/etc/fstab', 'wb') as f:
+        with open(chroot + '/etc/fstab', 'wt', encoding='utf-8') as f:
             for fs in self.driver.partition_scheme.fss:
                 # TODO(kozhukalov): Think of improving the logic so as to
                 # insert a meaningful fsck order value which is last zero
@@ -775,7 +778,7 @@ class Manager(object):
             # NOTE(kozhukalov): implement abstract publisher
             LOG.debug('Image metadata: %s', metadata)
             with open(self.driver.metadata_uri.split('file://', 1)[1],
-                      'w') as f:
+                      'wt', encoding='utf-8') as f:
                 yaml.safe_dump(metadata, stream=f)
             LOG.info('--- Building image END (do_build_image) ---')
         except Exception as exc:

fuel_agent/objects/partition/parted.py

@@ -55,17 +55,15 @@ class Parted(base.Serializable):
 
     @property
     def logical(self):
-        return filter(lambda x: x.type == 'logical', self.partitions)
+        return [x for x in self.partitions if x.type == 'logical']
 
     @property
     def primary(self):
-        return filter(lambda x: x.type == 'primary', self.partitions)
+        return [x for x in self.partitions if x.type == 'primary']
 
     @property
     def extended(self):
-        found = filter(lambda x: x.type == 'extended', self.partitions)
+        return next((x for x in self.partitions if x.type == 'extended'), None)
-        if found:
-            return found[0]
 
     def next_type(self):
         if self.label == 'gpt':
@@ -104,9 +102,7 @@ class Parted(base.Serializable):
         return '%s%s%s' % (self.name, separator, self.next_count())
 
     def partition_by_name(self, name):
-        found = filter(lambda x: (x.name == name), self.partitions)
+        return next((x for x in self.partitions if x.name == name), None)
-        if found:
-            return found[0]
 
     def to_dict(self):
         partitions = [partition.to_dict() for partition in self.partitions]

fuel_agent/objects/partition/scheme.py

@@ -73,9 +73,7 @@ class PartitionScheme(object):
         return md
 
     def md_by_name(self, name):
-        found = filter(lambda x: x.name == name, self.mds)
+        return next((x for x in self.mds if x.name == name), None)
-        if found:
-            return found[0]
 
     def md_by_mount(self, mount):
         fs = self.fs_by_mount(mount)
@@ -115,14 +113,10 @@ class PartitionScheme(object):
                     if parted.partition_by_name(name)), None)
 
     def vg_by_name(self, vgname):
-        found = filter(lambda x: (x.name == vgname), self.vgs)
+        return next((x for x in self.vgs if x.name == vgname), None)
-        if found:
-            return found[0]
 
     def pv_by_name(self, pvname):
-        found = filter(lambda x: (x.name == pvname), self.pvs)
+        return next((x for x in self.pvs if x.name == pvname), None)
-        if found:
-            return found[0]
 
     def vg_attach_by_name(self, pvname, vgname,
                           metadatasize=16, metadatacopies=2):
@@ -133,14 +127,10 @@ class PartitionScheme(object):
         vg.add_pv(pv.name)
 
     def fs_by_mount(self, mount):
-        found = filter(lambda x: (x.mount and x.mount == mount), self.fss)
+        return next((x for x in self.fss if x.mount == mount), None)
-        if found:
-            return found[0]
 
     def fs_by_device(self, device):
-        found = filter(lambda x: x.device == device, self.fss)
+        return next((x for x in self.fss if x.device == device), None)
-        if found:
-            return found[0]
 
     def fs_sorted_by_depth(self, reverse=False):
         """Getting file systems sorted by path length.
@@ -154,9 +144,8 @@ class PartitionScheme(object):
         return sorted(self.fss, key=key, reverse=reverse)
 
     def lv_by_device_name(self, device_name):
-        found = filter(lambda x: x.device_name == device_name, self.lvs)
+        return next((x for x in self.lvs if x.device_name == device_name),
-        if found:
+                    None)
-            return found[0]
 
     def root_device(self):
         fs = self.fs_by_mount('/')

fuel_agent/tests/test_artifact_utils.py

@@ -17,7 +17,6 @@ from oslo_config import cfg
 import unittest2
 import zlib
 
-
 from fuel_agent import errors
 from fuel_agent.utils import artifact as au
 from fuel_agent.utils import utils
@@ -91,7 +90,7 @@ class TestHttpUrl(unittest2.TestCase):
 
 class TestGunzipStream(unittest2.TestCase):
     def test_gunzip_stream_next(self):
-        content = ['fake content #1']
+        content = [b'fake content #1']
         compressed_stream = [zlib.compress(data) for data in content]
         gunzip_stream = au.GunzipStream(compressed_stream)
         for data in enumerate(gunzip_stream):

fuel_agent/tests/test_grub_utils.py

@@ -12,20 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import StringIO
+import io
 
 import mock
-import six
+from six import StringIO
 import unittest2
 
 from fuel_agent import errors
 from fuel_agent.utils import grub as gu
 
-if six.PY2:
-    OPEN_FUNCTION_NAME = '__builtin__.open'
-else:
-    OPEN_FUNCTION_NAME = 'builtins.open'
-
 
 class TestGrubUtils(unittest2.TestCase):
 
@@ -313,12 +308,13 @@ class TestGrubUtils(unittest2.TestCase):
         script = 'cat /tmp/grub.batch | /sbin/grub --no-floppy --batch'
 
         mock_open = mock.mock_open()
-        with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
+        with mock.patch('fuel_agent.utils.grub.open', new=mock_open,
+                        create=True):
             gu.grub1_mbr('/dev/foo', '/dev/bar', '0', chroot='/target')
         self.assertEqual(
             mock_open.call_args_list,
-            [mock.call('/target/tmp/grub.batch', 'wb'),
+            [mock.call('/target/tmp/grub.batch', 'wt', encoding='utf-8'),
-             mock.call('/target/tmp/grub.sh', 'wb')]
+             mock.call('/target/tmp/grub.sh', 'wt', encoding='utf-8')]
         )
         mock_open_file = mock_open()
         self.assertEqual(
@@ -347,12 +343,13 @@ class TestGrubUtils(unittest2.TestCase):
         script = 'cat /tmp/grub.batch | /sbin/grub --no-floppy --batch'
 
         mock_open = mock.mock_open()
-        with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
+        with mock.patch('fuel_agent.utils.grub.open', new=mock_open,
+                        create=True):
             gu.grub1_mbr('/dev/foo', '/dev/foo', '0', chroot='/target')
         self.assertEqual(
             mock_open.call_args_list,
-            [mock.call('/target/tmp/grub.batch', 'wb'),
+            [mock.call('/target/tmp/grub.batch', 'wt', encoding='utf-8'),
-             mock.call('/target/tmp/grub.sh', 'wb')]
+             mock.call('/target/tmp/grub.sh', 'wt', encoding='utf-8')]
         )
         mock_open_file = mock_open()
         self.assertEqual(
@@ -408,9 +405,11 @@ title Default (kernel-version)
     """
 
         mock_open = mock.mock_open()
-        with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
+        with mock.patch('fuel_agent.utils.grub.open', new=mock_open,
+                        create=True):
             gu.grub1_cfg(chroot='/target', kernel_params='kernel-params')
-        mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wb')
+        mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wt',
+                                          encoding='utf-8')
         mock_open_file = mock_open()
         mock_open_file.write.assert_called_once_with(config)
 
@@ -424,12 +423,14 @@ title Default (kernel-version-set)
     """
 
         mock_open = mock.mock_open()
-        with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
+        with mock.patch('fuel_agent.utils.grub.open', new=mock_open,
+                        create=True):
             gu.grub1_cfg(kernel='kernel-version-set',
                          initrd='initrd-version-set',
                          chroot='/target', kernel_params='kernel-params',
                          grub_timeout=10)
-        mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wb')
+        mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wt',
+                                          encoding='utf-8')
         mock_open_file = mock_open()
         mock_open_file.write.assert_called_once_with(config)
 
@@ -463,19 +464,19 @@ bar
 GRUB_RECORDFAIL_TIMEOUT=10
 """
 
-        with mock.patch(OPEN_FUNCTION_NAME,
+        with mock.patch('fuel_agent.utils.grub.open',
                         new=mock.mock_open(read_data=orig_content),
                         create=True) as mock_open:
-            mock_open.return_value = mock.MagicMock(spec=file)
+            mock_open.return_value = mock.MagicMock(spec=io.IOBase)
             handle = mock_open.return_value.__enter__.return_value
-            handle.__iter__.return_value = StringIO.StringIO(orig_content)
+            handle.__iter__.return_value = StringIO(orig_content)
             gu.grub2_cfg(kernel_params='kernel-params-new', chroot='/target',
                          grub_timeout=10)
 
             self.assertEqual(
                 mock_open.call_args_list,
                 [mock.call('/target/etc/default/grub'),
-                 mock.call('/target/etc/default/grub', 'wb')]
+                 mock.call('/target/etc/default/grub', 'wt', encoding='utf-8')]
             )
 
             handle.write.assert_called_once_with(new_content)

fuel_agent/tests/test_hardware_utils.py

@@ -12,13 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
+import six
 import unittest2
 
 from fuel_agent import errors
 from fuel_agent.utils import hardware as hu
 from fuel_agent.utils import utils
 
+if six.PY2:
+    import mock
+elif six.PY3:
+    from unittest import mock
+
 
 class TestHardwareUtils(unittest2.TestCase):
 

fuel_agent/tests/test_manager.py

@@ -16,8 +16,8 @@ import copy
 import os
 import signal
 
-import mock
 from oslo_config import cfg
+import six
 import unittest2
 
 from fuel_agent.drivers import nailgun
@@ -33,6 +33,11 @@ from fuel_agent.utils import md as mu
 from fuel_agent.utils import partition as pu
 from fuel_agent.utils import utils
 
+if six.PY2:
+    import mock
+elif six.PY3:
+    import unittest.mock as mock
+
 CONF = cfg.CONF
 
 
@@ -239,11 +244,11 @@ class TestManager(unittest2.TestCase):
             self.mgr.do_bootloader()
             expected_open_calls = [
                 mock.call('/tmp/target/etc/udev/rules.d/70-persistent-net.'
-                          'rules', 'w'),
+                          'rules', 'wt', encoding='utf-8'),
                 mock.call('/tmp/target/etc/udev/rules.d/75-persistent-net-'
-                          'generator.rules', 'w'),
+                          'generator.rules', 'wt', encoding='utf-8'),
                 mock.call('/tmp/target/etc/nailgun-agent/nodiscover', 'w'),
-                mock.call('/tmp/target/etc/fstab', 'wb')]
+                mock.call('/tmp/target/etc/fstab', 'wt', encoding='utf-8')]
             self.assertEqual(expected_open_calls, mock_open.call_args_list)
             expected_write_calls = [
                 mock.call('# Generated by fuel-agent during provisioning: '
@@ -264,9 +269,11 @@ class TestManager(unittest2.TestCase):
                           'net-generator\n'),
                 mock.call('UUID=fake_UUID /boot ext2 defaults 0 0\n'),
                 mock.call('UUID=fake_UUID /tmp ext2 defaults 0 0\n'),
-                mock.call('UUID=fake_UUID / ext4 defaults,errors=panic 0 0\n'),
+                mock.call(
+                    'UUID=fake_UUID / ext4 defaults,errors=panic 0 0\n'),
                 mock.call('UUID=fake_UUID swap swap defaults 0 0\n'),
-                mock.call('UUID=fake_UUID /var/lib/glance xfs defaults 0 0\n')]
+                mock.call('UUID=fake_UUID /var/lib/glance xfs defaults 0 0\n')
+            ]
             self.assertEqual(expected_write_calls,
                              file_handle_mock.write.call_args_list)
         mock_umount.assert_called_once_with('/tmp/target')
@@ -680,7 +687,8 @@ class TestManager(unittest2.TestCase):
         mock_utils.execute.return_value = (None, None)
         self.mgr.driver._partition_scheme = objects.PartitionScheme()
         self.mgr.mount_target('fake_chroot')
-        mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wb')
+        mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wt',
+                                          encoding='utf-8')
         mock_os.path.islink.assert_called_once_with('fake_chroot/etc/mtab')
         mock_os.remove.assert_called_once_with('fake_chroot/etc/mtab')
 
@@ -734,7 +742,8 @@ none /run/shm tmpfs rw,nosuid,nodev 0 0"""
                          mock_fu.mount_bind.call_args_list)
         file_handle = mock_open.return_value.__enter__.return_value
         file_handle.write.assert_called_once_with(fake_mtab)
-        mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wb')
+        mock_open.assert_called_once_with('fake_chroot/etc/mtab', 'wt',
+                                          encoding='utf-8')
         mock_os.path.islink.assert_called_once_with('fake_chroot/etc/mtab')
         self.assertFalse(mock_os.remove.called)
 
@@ -965,7 +974,8 @@ class TestImageBuild(unittest2.TestCase):
                           mock.call('/tmp/img-boot', 'gzip',
                                     chunk_size=CONF.data_chunk_size)],
                          mock_bu.containerize.call_args_list)
-        mock_open.assert_called_once_with('/fake/img.yaml', 'w')
+        mock_open.assert_called_once_with('/fake/img.yaml', 'wt',
+                                          encoding='utf-8')
         self.assertEqual(
             [mock.call('/tmp/img.gz', '/fake/img.img.gz'),
              mock.call('/tmp/img-boot.gz', '/fake/img-boot.img.gz')],

fuel_agent/tests/test_md_utils.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
 import six
 import unittest2
 
@@ -24,8 +23,10 @@ from fuel_agent.utils import utils
 
 if six.PY2:
     OPEN_FUNCTION_NAME = '__builtin__.open'
+    import mock
 else:
     OPEN_FUNCTION_NAME = 'builtins.open'
+    import unittest.mock as mock
 
 
 class TestMdUtils(unittest2.TestCase):

fuel_agent/tests/test_nailgun.py

@@ -15,6 +15,7 @@
 import copy
 
 import mock
+import six
 import unittest2
 import yaml
 
@@ -867,7 +868,7 @@ class TestNailgunGetOSMethods(unittest2.TestCase):
              'generic_os': {'obj': objects.OperatingSystem,
                             'minor': 'unknown', 'major': 'unknown'}}
         drv = nailgun.Nailgun('fake_data')
-        for profile, obj in d.iteritems():
+        for profile, obj in six.iteritems(d):
             os = drv.get_os_by_profile(profile)
             self.assertIsInstance(os, obj['obj'])
             self.assertEqual(obj['minor'], os.minor)
@@ -878,7 +879,7 @@ class TestNailgunGetOSMethods(unittest2.TestCase):
              'Ubuntu': objects.Ubuntu,
              'unknown': None}
         drv = nailgun.Nailgun('fake_data')
-        for os_name, obj in d.iteritems():
+        for os_name, obj in six.iteritems(d):
             os = drv.get_os_by_image_meta(
                 {'name': os_name, 'minor': 1, 'major': 2})
             if os:

fuel_agent/tests/test_utils.py

@@ -15,9 +15,9 @@
 
 import socket
 
-import mock
 from oslo_config import cfg
 import requests
+import six
 import stevedore
 import unittest2
 import urllib3
@@ -25,6 +25,10 @@ import urllib3
 from fuel_agent import errors
 from fuel_agent.utils import utils
 
+if six.PY2:
+    import mock
+elif six.PY3:
+    from unittest import mock
 
 CONF = cfg.CONF
 
@@ -140,8 +144,13 @@ class ExecuteTestCase(unittest2.TestCase):
 
     def test_calculate_md5_ok(self):
         # calculated by 'printf %10000s | md5sum'
+        mock_open = mock.Mock()
+        mock_open.__enter__ = mock.Mock(
+            side_effect=(six.BytesIO(b' ' * 10000) for _ in range(6)))
+
+        mock_open.__exit__ = mock.Mock(return_value=False)
         with mock.patch('six.moves.builtins.open',
-                        mock.mock_open(read_data=' ' * 10000), create=True):
+                        mock.Mock(return_value=mock_open), create=True):
             self.assertEqual('f38898bb69bb02bccb9594dfe471c5c0',
                              utils.calculate_md5('fake', 10000))
             self.assertEqual('6934d9d33cd2d0c005994e7d96d2e0d9',

fuel_agent/utils/artifact.py

@@ -62,6 +62,9 @@ class Target(object):
             os.fsync(f.fileno())
         LOG.debug('File is written: %s' % filename)
 
+    def __next__(self):
+        return self.next()
+
 
 class LocalFile(Target):
     def __init__(self, filename):
@@ -123,7 +126,7 @@ class GunzipStream(Target):
 
     def next(self):
         try:
-            return self.decompressor.decompress(self.stream.next())
+            return self.decompressor.decompress(next(self.stream))
         except StopIteration:
             raise
 
@@ -217,7 +220,7 @@ class Chain(object):
         def jump(proc, next_proc):
             # if next_proc is just a string we assume it is a filename
             # and we save stream into a file
-            if isinstance(next_proc, (str, unicode)):
+            if isinstance(next_proc, six.string_types):
                 LOG.debug('Processor target: %s' % next_proc)
                 proc.target(next_proc)
                 return LocalFile(next_proc)
@@ -225,4 +228,4 @@ class Chain(object):
             # initialized with the previous one
             else:
                 return next_proc(proc)
-        return reduce(jump, self.processors)
+        return six.moves.reduce(jump, self.processors)

fuel_agent/utils/grub.py

@@ -12,10 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from io import open
 import os
 import re
 import shutil
 
+import six
+
 from fuel_agent import errors
 from fuel_agent.openstack.common import log as logging
 from fuel_agent.utils import utils
@@ -161,15 +164,15 @@ def grub1_mbr(install_device, boot_disk, boot_part, chroot=''):
     batch += 'setup (hd0)\n'
     batch += 'quit\n'
 
-    with open(chroot + '/tmp/grub.batch', 'wb') as f:
+    with open(chroot + '/tmp/grub.batch', 'wt', encoding='utf-8') as f:
         LOG.debug('Grub batch content: \n%s' % batch)
-        f.write(batch)
+        f.write(six.text_type(batch))
 
     script = 'cat /tmp/grub.batch | {0} --no-floppy --batch'.format(
         guess_grub(chroot=chroot))
-    with open(chroot + '/tmp/grub.sh', 'wb') as f:
+    with open(chroot + '/tmp/grub.sh', 'wt', encoding='utf-8') as f:
         LOG.debug('Grub script content: \n%s' % script)
-        f.write(script)
+        f.write(six.text_type(script))
 
     os.chmod(chroot + '/tmp/grub.sh', 0o755)
     cmd = ['/tmp/grub.sh']
@@ -211,8 +214,8 @@ title Default ({kernel})
     """.format(kernel=kernel, initrd=initrd,
                kernel_params=kernel_params,
                grub_timeout=grub_timeout)
-    with open(chroot + '/boot/grub/grub.conf', 'wb') as f:
+    with open(chroot + '/boot/grub/grub.conf', 'wt', encoding='utf-8') as f:
-        f.write(config)
+        f.write(six.text_type(config))
 
 
 def grub2_install(install_devices, chroot=''):
@@ -241,8 +244,8 @@ def grub2_cfg(kernel_params='', chroot='', grub_timeout=5):
     # prevent user confirmation appearing if unexpected reboot occured.
     new_content += '\nGRUB_RECORDFAIL_TIMEOUT={grub_timeout}\n'.\
                    format(grub_timeout=grub_timeout)
-    with open(grub_defaults, 'wb') as f:
+    with open(grub_defaults, 'wt', encoding='utf-8') as f:
-        f.write(new_content)
+        f.write(six.text_type(new_content))
     cmd = [guess_grub2_mkconfig(chroot), '-o', guess_grub2_conf(chroot)]
     if chroot:
         cmd[:0] = ['chroot', chroot]

fuel_agent/utils/lvm.py

@@ -53,7 +53,7 @@ def pvdisplay_parse(output):
 
 def pvcreate(pvname, metadatasize=64, metadatacopies=2):
     # check if pv already exists
-    if filter(lambda x: x['name'] == pvname, pvdisplay()):
+    if get_first_by_key_value(pvdisplay(), 'name', pvname, False):
         raise errors.PVAlreadyExistsError(
             'Error while creating pv: pv %s already exists' % pvname)
     utils.execute('pvcreate',
@@ -63,16 +63,16 @@ def pvcreate(pvname, metadatasize=64, metadatacopies=2):
 
 
 def pvremove(pvname):
-    pv = filter(lambda x: x['name'] == pvname, pvdisplay())
+    pv = get_first_by_key_value(pvdisplay(), 'name', pvname)
 
     # check if pv exists
     if not pv:
         raise errors.PVNotFoundError(
             'Error while removing pv: pv %s not found' % pvname)
     # check if pv is attached to some vg
-    if pv[0]['vg'] is not None:
+    if pv['vg'] is not None:
         raise errors.PVBelongsToVGError('Error while removing pv: '
-                                        'pv belongs to vg %s' % pv[0]['vg'])
+                                        'pv belongs to vg %s' % pv['vg'])
     utils.execute('pvremove', '-ff', '-y', pvname, check_exit_code=[0])
 
 
@@ -121,7 +121,7 @@ def _vg_attach_validate(pvnames):
 
 def vgcreate(vgname, pvname, *args):
     # check if vg already exists
-    if filter(lambda x: x['name'] == vgname, vgdisplay()):
+    if get_first_by_key_value(vgdisplay(), 'name', vgname, False):
         raise errors.VGAlreadyExistsError(
             'Error while creating vg: vg %s already exists' % vgname)
     pvnames = [pvname] + list(args)
@@ -131,7 +131,7 @@ def vgcreate(vgname, pvname, *args):
 
 def vgextend(vgname, pvname, *args):
     # check if vg exists
-    if not filter(lambda x: x['name'] == vgname, vgdisplay()):
+    if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
         raise errors.VGNotFoundError(
             'Error while extending vg: vg %s not found' % vgname)
     pvnames = [pvname] + list(args)
@@ -141,7 +141,7 @@ def vgextend(vgname, pvname, *args):
 
 def vgreduce(vgname, pvname, *args):
     # check if vg exists
-    if not filter(lambda x: x['name'] == vgname, vgdisplay()):
+    if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
         raise errors.VGNotFoundError(
             'Error while reducing vg: vg %s not found' % vgname)
     pvnames = [pvname] + list(args)
@@ -156,7 +156,7 @@ def vgreduce(vgname, pvname, *args):
 
 def vgremove(vgname):
     # check if vg exists
-    if not filter(lambda x: x['name'] == vgname, vgdisplay()):
+    if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
         raise errors.VGNotFoundError(
             'Error while removing vg: vg %s not found' % vgname)
     utils.execute('vgremove', '-f', vgname, check_exit_code=[0])
@@ -196,20 +196,22 @@ def lvdisplay_parse(output):
 
 
 def lvcreate(vgname, lvname, size):
-    vg = filter(lambda x: x['name'] == vgname, vgdisplay())
+    vg = get_first_by_key_value(vgdisplay(), 'name', vgname)
 
     # check if vg exists
     if not vg:
         raise errors.VGNotFoundError(
             'Error while creating vg: vg %s not found' % vgname)
     # check if enough space is available
-    if vg[0]['free'] < size:
+    if vg['free'] < size:
         raise errors.NotEnoughSpaceError(
             'Error while creating lv: vg %s has only %s m of free space, '
-            'but at least %s m is needed' % (vgname, vg[0]['free'], size))
+            'but at least %s m is needed' % (vgname, vg['free'], size))
     # check if lv already exists
-    if filter(lambda x: x['name'] == lvname and x['vg'] == vgname,
+    if next(
-              lvdisplay()):
+        (x for x in lvdisplay() if x['name'] == lvname and x['vg'] == vgname),
+        False
+    ):
         raise errors.LVAlreadyExistsError(
             'Error while creating lv: lv %s already exists' % lvname)
     # NOTE(agordeev): by default, lvcreate is configured to wipe signature
@@ -228,7 +230,7 @@ def lvcreate(vgname, lvname, size):
 
 def lvremove(lvpath):
     # check if lv exists
-    if not filter(lambda x: x['path'] == lvpath, lvdisplay()):
+    if not get_first_by_key_value(lvdisplay(), 'path', lvpath):
         raise errors.LVNotFoundError(
             'Error while removing lv: lv %s not found' % lvpath)
     utils.execute('lvremove', '-f', lvpath, check_exit_code=[0])
@@ -247,3 +249,7 @@ def vgremove_all():
 def pvremove_all():
     for pv in pvdisplay():
         pvremove(pv['name'])
+
+
+def get_first_by_key_value(collection, key, value, default=None):
+    return next((x for x in collection if x[key] == value), default)

fuel_agent/utils/md.py

@@ -80,7 +80,7 @@ def mdcreate(mdname, level, devices, metadata='default'):
     mds = mddisplay()
 
     # check if md device already exists
-    if filter(lambda x: x['name'] == mdname, mds):
+    if next((x for x in mds if x['name'] == mdname), False):
         raise errors.MDAlreadyExistsError(
             'Error while creating md: md %s already exists' % mdname)
 
@@ -98,15 +98,18 @@ def mdcreate(mdname, level, devices, metadata='default'):
             'Error while creating md: at least one of devices is not found')
 
     # check if devices are not parts of some md array
-    if set(devices) & \
+    if set(devices) & set(itertools.chain.from_iterable(
-            set(itertools.chain(*[md.get('devices', []) for md in mds])):
+        md.get('devices', []) for md in mds
+    )):
         raise errors.MDDeviceDuplicationError(
             'Error while creating md: at least one of devices is '
             'already in belongs to some md')
 
     # FIXME: mdadm will ask user to continue creating if any device appears to
     #       be a part of raid array. Superblock zeroing helps to avoid that.
-    map(mdclean, devices)
+    for device in devices:
+        mdclean(device)
+
     utils.execute('mdadm', '--create', '--force', mdname, '-e', metadata,
                   '--level=%s' % level,
                   '--raid-devices=%s' % len(devices), *devices,

fuel_agent/utils/utils.py

@@ -26,10 +26,10 @@ import time
 import jinja2
 from oslo_config import cfg
 import requests
+import six
 import stevedore.driver
 import urllib3
 
-from six.moves import zip_longest
 
 from fuel_agent import errors
 from fuel_agent.openstack.common import log as logging
@@ -76,7 +76,7 @@ CONF.register_opts(u_opts)
 def execute(*cmd, **kwargs):
     command = ' '.join(cmd)
     LOG.debug('Trying to execute command: %s', command)
-    commands = [c.strip() for c in re.split(ur'\|', command)]
+    commands = [c.strip() for c in re.split(r'\|', command)]
     env = kwargs.pop('env_variables', copy.deepcopy(os.environ))
     env['PATH'] = '/bin:/usr/bin:/sbin:/usr/sbin'
     env['LC_ALL'] = env['LANG'] = env['LANGUAGE'] = kwargs.pop('language', 'C')
@@ -96,7 +96,7 @@ def execute(*cmd, **kwargs):
     if to_filename:
         to_file = open(to_filename, 'wb')
 
-    for attempt in reversed(xrange(attempts)):
+    for attempt in reversed(six.moves.range(attempts)):
         try:
             process = []
             for c in commands:
@@ -104,7 +104,7 @@ def execute(*cmd, **kwargs):
                     # NOTE(eli): Python's shlex implementation doesn't like
                     # unicode. We have to convert to ascii before shlex'ing
                     # the command. http://bugs.python.org/issue6988
-                    encoded_command = c.encode('ascii')
+                    encoded_command = c.encode('ascii') if six.PY2 else c
 
                     process.append(subprocess.Popen(
                         shlex.split(encoded_command),
@@ -241,7 +241,7 @@ def makedirs_if_not_exists(path, mode=0o755):
 def grouper(iterable, n, fillvalue=None):
     """Collect data into fixed-length chunks or blocks"""
     args = [iter(iterable)] * n
-    return zip_longest(*args, fillvalue=fillvalue)
+    return six.moves.zip_longest(*args, fillvalue=fillvalue)
 
 
 def guess_filename(path, regexp, sort=True, reverse=True):

requirements.txt

@@ -3,7 +3,7 @@ eventlet>=0.13.0
 iso8601>=0.1.9
 jsonschema>=2.3.0
 oslo.config>=1.6.0
-oslo.serialization>=1.4.0
+oslo.serialization>=1.4.0,<2.0
 six>=1.5.2
 pbr>=0.7.0
 Jinja2

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 minversion = 1.6
 skipsdist = True
-envlist = py26,py27,pep8
+envlist = py26,py27,py34,pep8
 
 [testenv]
 usedevelop = True