Update to run under Python 3

Refactoring of charm to support execution under Python 3.

Change-Id: I8d48614f38abf31159311d82b0cbb54e60f710c1

.gitignore

@@ -10,4 +10,5 @@ tags
 .pydevproject
 func-results.json
 .stestr
+__pycache__
 joined-string

actions/git_reinstall.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #
 # Copyright 2016 Canonical Ltd
 #

actions/pause_resume.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #
 # Copyright 2016 Canonical Ltd
 #

hooks/neutron_ovs_context.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import glob
 import os
 import uuid
@@ -248,12 +249,12 @@ def resolve_dpdk_ports():
     '''
     ports = config('data-port')
     devices = PCINetDevices()
-    resolved_devices = {}
+    resolved_devices = collections.OrderedDict()
     db = kv()
     if ports:
         # NOTE: ordered dict of format {[mac]: bridge}
         portmap = parse_data_port_mappings(ports)
-        for mac, bridge in portmap.iteritems():
+        for mac, bridge in portmap.items():
             pcidev = devices.get_device_from_mac(mac)
             if pcidev:
                 # NOTE: store mac->pci allocation as post binding
@@ -311,7 +312,7 @@ class OVSDPDKDeviceContext(OSContextGenerator):
         '''
         num_cores = config('dpdk-socket-cores')
         mask = 0
-        for cores in numa_node_cores().itervalues():
+        for cores in numa_node_cores().values():
             for core in cores[:num_cores]:
                 mask = mask | 1 << core
         return format(mask, '#04x')

hooks/neutron_ovs_hooks.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #
 # Copyright 2016 Canonical Ltd
 #

hooks/neutron_ovs_utils.py

@@ -318,7 +318,7 @@ def register_configs(release=None):
     release = release or os_release('neutron-common', base='icehouse')
     configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
                                           openstack_release=release)
-    for cfg, rscs in resource_map().iteritems():
+    for cfg, rscs in resource_map().items():
         configs.register(cfg, rscs['contexts'])
     return configs
 
@@ -389,7 +389,7 @@ def restart_map():
     Constructs a restart map based on charm config settings and relation
     state.
     '''
-    return {k: v['services'] for k, v in resource_map().iteritems()}
+    return {k: v['services'] for k, v in resource_map().items()}
 
 
 def get_topics():
@@ -455,12 +455,12 @@ def configure_ovs():
     if not use_dpdk():
         portmaps = DataPortContext()()
         bridgemaps = parse_bridge_mappings(config('bridge-mappings'))
-        for br in bridgemaps.itervalues():
+        for br in bridgemaps.values():
             add_bridge(br, datapath_type)
             if not portmaps:
                 continue
 
-            for port, _br in portmaps.iteritems():
+            for port, _br in portmaps.items():
                 if _br == br:
                     if not is_linuxbridge_interface(port):
                         add_bridge_port(br, port, promisc=True)
@@ -471,7 +471,7 @@ def configure_ovs():
         #       with type 'dpdk'
         dpdk_bridgemaps = neutron_ovs_context.resolve_dpdk_ports()
         device_index = 0
-        for br in dpdk_bridgemaps.itervalues():
+        for br in dpdk_bridgemaps.values():
             add_bridge(br, datapath_type)
             dpdk_add_bridge_port(br, 'dpdk{}'.format(device_index),
                                  port_type='dpdk')
@@ -656,10 +656,10 @@ def git_pre_install():
     add_user_to_group('neutron', 'neutron')
 
     for d in dirs:
-        mkdir(d, owner='neutron', group='neutron', perms=0755, force=False)
+        mkdir(d, owner='neutron', group='neutron', perms=0o755, force=False)
 
     for l in logs:
-        write_file(l, '', owner='neutron', group='neutron', perms=0600)
+        write_file(l, '', owner='neutron', group='neutron', perms=0o600)
 
 
 def git_post_install(projects_yaml):

hooks/pci.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #
 # Copyright 2016 Canonical Ltd
 #
@@ -196,7 +196,7 @@ class PCINetDevices(object):
 
     def get_pci_ethernet_addresses(self):
         cmd = ['lspci', '-m', '-D']
-        lspci_output = subprocess.check_output(cmd)
+        lspci_output = subprocess.check_output(cmd).decode('UTF-8')
         pci_addresses = []
         for line in lspci_output.split('\n'):
             columns = shlex.split(line)

test-requirements.txt

@@ -5,12 +5,12 @@ coverage>=3.6
 mock>=1.2
 flake8>=2.2.4,<=2.4.1
 os-testr>=0.4.1
-charm-tools>=2.0.0
+charm-tools>=2.0.0;python_version=='2.7'
 requests==2.6.0
 # BEGIN: Amulet OpenStack Charm Helper Requirements
 # Liberty client lower constraints
-amulet>=1.14.3,<2.0
+amulet>=1.14.3,<2.0;python_version=='2.7'
-bundletester>=0.6.1,<1.0
+bundletester>=0.6.1,<1.0;python_version=='2.7'
 python-ceilometerclient>=1.5.0
 python-cinderclient>=1.4.0
 python-glanceclient>=1.1.0

tox.ini

@@ -2,7 +2,7 @@
 # This file is managed centrally by release-tools and should not be modified
 # within individual charm repos.
 [tox]
-envlist = pep8,py27
+envlist = pep8,py27,py35
 skipsdist = True
 
 [testenv]
@@ -20,6 +20,7 @@ passenv = HOME TERM AMULET_* CS_API_*
 basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
+commands = /bin/true
 
 [testenv:py35]
 basepython = python3.5

unit_tests/__init__.py

@@ -16,3 +16,4 @@ import sys
 
 sys.path.append('actions/')
 sys.path.append('hooks/')
+sys.path.append('unit_tests')

unit_tests/pci_responses.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # flake8: noqa
-LSPCI = """
+LSPCI = b"""
 0000:00:00.0 "Host bridge" "Intel Corporation" "Haswell-E DMI2" -r02 "Intel Corporation" "Device 0000"
 0000:00:03.0 "PCI bridge" "Intel Corporation" "Haswell-E PCI Express Root Port 3" -r02 "" ""
 0000:00:03.2 "PCI bridge" "Intel Corporation" "Haswell-E PCI Express Root Port 3" -r02 "" ""

unit_tests/test_neutron_ovs_hooks.py

@@ -106,7 +106,7 @@ class NeutronOVSHooksTests(CharmTestCase):
             with open('unit_tests/%s' % sample, 'r') as f:
                 content = f.read()
 
-            with patch('__builtin__.open', mock_open(read_data=content),
+            with patch('builtins.open', mock_open(read_data=content),
                        create=True):
                 self._call_hook('upgrade-charm')
                 if should_migrate:

unit_tests/test_neutron_ovs_utils.py

@@ -143,8 +143,11 @@ class TestNeutronOVSUtils(CharmTestCase):
         self.os_release.return_value = 'icehouse'
         _head_pkgs.return_value = head_pkg
         pkg_list = nutils.determine_packages()
-        expect = ['neutron-plugin-openvswitch-agent', head_pkg]
+        expect = [
-        self.assertItemsEqual(pkg_list, expect)
+            head_pkg,
+            'neutron-plugin-openvswitch-agent'
+        ]
+        self.assertEqual(pkg_list, expect)
 
     @patch.object(nutils, 'use_dvr')
     @patch.object(nutils, 'git_install_requested')
@@ -159,8 +162,11 @@ class TestNeutronOVSUtils(CharmTestCase):
         self.os_release.return_value = 'mitaka'
         _head_pkgs.return_value = head_pkg
         pkg_list = nutils.determine_packages()
-        expect = ['neutron-openvswitch-agent', head_pkg]
+        expect = [
-        self.assertItemsEqual(pkg_list, expect)
+            head_pkg,
+            'neutron-openvswitch-agent',
+        ]
+        self.assertEqual(pkg_list, expect)
 
     @patch.object(nutils, 'use_dvr')
     @patch.object(nutils, 'git_install_requested')
@@ -175,9 +181,13 @@ class TestNeutronOVSUtils(CharmTestCase):
         self.os_release.return_value = 'icehouse'
         _head_pkgs.return_value = head_pkg
         pkg_list = nutils.determine_packages()
-        expect = ['neutron-plugin-openvswitch-agent', head_pkg,
+        expect = [
-                  'neutron-metadata-agent', 'neutron-dhcp-agent']
+            head_pkg,
-        self.assertItemsEqual(pkg_list, expect)
+            'neutron-plugin-openvswitch-agent',
+            'neutron-dhcp-agent',
+            'neutron-metadata-agent',
+        ]
+        self.assertEqual(pkg_list, expect)
 
     @patch.object(nutils, 'use_dvr')
     @patch.object(nutils, 'git_install_requested')
@@ -209,11 +219,11 @@ class TestNeutronOVSUtils(CharmTestCase):
         _head_pkgs.return_value = head_pkg
         pkg_list = nutils.determine_packages()
         expect = [
-            'neutron-plugin-sriov-agent',
-            'neutron-plugin-openvswitch-agent',
             head_pkg,
+            'neutron-plugin-openvswitch-agent',
+            'neutron-plugin-sriov-agent',
         ]
-        self.assertItemsEqual(pkg_list, expect)
+        self.assertEqual(pkg_list, expect)
 
     @patch.object(nutils, 'use_dvr')
     @patch.object(nutils, 'git_install_requested')
@@ -230,11 +240,11 @@ class TestNeutronOVSUtils(CharmTestCase):
         _head_pkgs.return_value = head_pkg
         pkg_list = nutils.determine_packages()
         expect = [
-            'neutron-sriov-agent',
+            head_pkg,
             'neutron-openvswitch-agent',
-            head_pkg
+            'neutron-sriov-agent',
         ]
-        self.assertItemsEqual(pkg_list, expect)
+        self.assertEqual(pkg_list, expect)
 
     @patch.object(nutils, 'use_dvr')
     def test_register_configs(self, _use_dvr):
@@ -256,7 +266,7 @@ class TestNeutronOVSUtils(CharmTestCase):
                  '/etc/neutron/plugins/ml2/ml2_conf.ini',
                  '/etc/default/openvswitch-switch',
                  '/etc/init/os-charm-phy-nic-mtu.conf']
-        self.assertItemsEqual(_regconfs.configs, confs)
+        self.assertEqual(_regconfs.configs, confs)
 
     @patch.object(nutils, 'use_dvr')
     def test_register_configs_mitaka(self, _use_dvr):
@@ -278,7 +288,7 @@ class TestNeutronOVSUtils(CharmTestCase):
                  '/etc/neutron/plugins/ml2/openvswitch_agent.ini',
                  '/etc/default/openvswitch-switch',
                  '/etc/init/os-charm-phy-nic-mtu.conf']
-        self.assertItemsEqual(_regconfs.configs, confs)
+        self.assertEqual(_regconfs.configs, confs)
 
     @patch.object(nutils, 'use_dvr')
     def test_resource_map(self, _use_dvr):
@@ -499,11 +509,11 @@ class TestNeutronOVSUtils(CharmTestCase):
     @patch('charmhelpers.contrib.openstack.context.config')
     def test_configure_ovs_dpdk(self, mock_config, _use_dvr,
                                 _resolve_dpdk_ports):
-        _resolve_dpdk_ports.return_value = {
+        _resolve_dpdk_ports.return_value = OrderedDict([
-            '0000:001c.01': 'br-phynet1',
+            ('0000:001c.01', 'br-phynet1'),
-            '0000:001c.02': 'br-phynet2',
+            ('0000:001c.02', 'br-phynet2'),
-            '0000:001c.03': 'br-phynet3',
+            ('0000:001c.03', 'br-phynet3'),
-        }
+        ])
         _use_dvr.return_value = True
         self.use_dpdk.return_value = True
         mock_config.side_effect = self.test_config.get
@@ -515,13 +525,15 @@ class TestNeutronOVSUtils(CharmTestCase):
             call('br-ex', 'netdev'),
             call('br-phynet1', 'netdev'),
             call('br-phynet2', 'netdev'),
-            call('br-phynet3', 'netdev'),
+            call('br-phynet3', 'netdev')],
-        ])
+            any_order=True
+        )
         self.dpdk_add_bridge_port.assert_has_calls([
             call('br-phynet1', 'dpdk0', port_type='dpdk'),
             call('br-phynet2', 'dpdk1', port_type='dpdk'),
-            call('br-phynet3', 'dpdk2', port_type='dpdk'),
+            call('br-phynet3', 'dpdk2', port_type='dpdk')],
-        ])
+            any_order=True
+        )
 
     @patch.object(neutron_ovs_context, 'SharedSecretContext')
     def test_get_shared_secret(self, _dvr_secret_ctxt):
@@ -559,16 +571,16 @@ class TestNeutronOVSUtils(CharmTestCase):
         add_user_to_group.assert_called_with('neutron', 'neutron')
         expected = [
             call('/var/lib/neutron', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
             call('/var/lib/neutron/lock', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
             call('/var/log/neutron', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
         ]
         self.assertEqual(mkdir.call_args_list, expected)
         expected = [
             call('/var/log/neutron/server.log', '', owner='neutron',
-                 group='neutron', perms=0600),
+                 group='neutron', perms=0o600),
         ]
         self.assertEqual(write_file.call_args_list, expected)
 

unit_tests/test_utils.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import io
 import logging
 import unittest
 import os
@@ -50,7 +51,7 @@ def get_default_config():
     '''
     default_config = {}
     config = load_config()
-    for k, v in config.iteritems():
+    for k, v in config.items():
         if 'default' in v:
             default_config[k] = v['default']
         else:
@@ -124,12 +125,12 @@ def patch_open():
 
     Yields the mock for "open" and "file", respectively.'''
     mock_open = MagicMock(spec=open)
-    mock_file = MagicMock(spec=file)
+    mock_file = MagicMock(spec=io.FileIO)
 
     @contextmanager
     def stub_open(*args, **kwargs):
         mock_open(*args, **kwargs)
         yield mock_file
 
-    with patch('__builtin__.open', stub_open):
+    with patch('builtins.open', stub_open):
         yield mock_open, mock_file