Move virtualpdu to Python 3

Python 2 is long gone

Also move all tests in the project and use proper templates

Change-Id: I352074a3a0cc2e1342fe2534153599dcb0360cbe

.gitignore

@@ -27,8 +27,9 @@ cover/
 !.coveragerc
 .tox
 nosetests.xml
-.testrepository
+.stestr/
 .venv
+ .testrepository/
 
 # Translations
 *.mo
@@ -56,3 +57,4 @@ ChangeLog
 
 # Files created by releasenotes build
 releasenotes/build
+

.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=./virtualpdu/tests
+top_dir=.

bindep.txt

@@ -0,0 +1,9 @@
+# these are needed to compile Python dependencies from sources
+python3-all-dev [platform:dpkg !platform:ubuntu-precise test compile]
+python3-devel [platform:rpm test compile]
+build-essential [platform:dpkg test compile]
+libssl-dev [platform:dpkg test compile]
+libvirt-dev [platform:dpkg test compile]
+libvirt-devel [platform:rpm test compile]
+libzmq5 [platform:dpkg test compile]
+pkg-config [compile test]

doc/requirements.txt

@@ -0,0 +1,4 @@
+reno>=3.1.0 # Apache-2.0
+sphinx>=2.0.0,!=2.1.0 # BSD
+openstackdocstheme>=2.2.1 # Apache-2.0
+sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD

doc/source/conf.py

@@ -21,8 +21,6 @@ sys.path.insert(0, os.path.abspath('../..'))
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     'sphinx.ext.autodoc',
-    #'sphinx.ext.intersphinx',
-    'oslosphinx'
 ]
 
 # autodoc generation is a bit aggressive and a nuisance when doing heavy

releasenotes/source/conf.py

@@ -38,7 +38,6 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'oslosphinx',
     'reno.sphinxext',
 ]
 

requirements.txt

@@ -2,8 +2,6 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-pbr>=1.6 # Apache-2.0
-libvirt-python>=1.2.5 # LGPLv2+
-pyasn1 # BSD
-pysnmp<5.0.0,>=4.2.3 # BSD
-futures==3.0.5 # BSD
+libvirt-python>=6.0.0 # LGPLv2+
+pyasn1>=0.4.8 # BSD
+pysnmp>=4.2.3 # BSD

setup.cfg

@@ -1,11 +1,12 @@
 [metadata]
 name = virtualpdu
 summary = VirtualPDU is a service for simulating virtual power distribution outlets (PDUs).
-description-file =
+description_file =
     README.rst
 author = OpenStack
-author-email = openstack-dev@lists.openstack.org
-home-page = http://www.openstack.org/
+author_email = openstack-discuss@lists.openstack.org
+home_page = https://docs.openstack.org/virtualpdu/latest/
+python_requires = >=3.6
 classifier =
     Environment :: OpenStack
     Intended Audience :: Information Technology
@@ -13,11 +14,14 @@ classifier =
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
     Programming Language :: Python
-    Programming Language :: Python :: 2
-    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: Implementation :: CPython
+    Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.3
-    Programming Language :: Python :: 3.4
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
 
 [files]
 packages =

setup.py

@@ -13,17 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
 import setuptools
 
-# In python < 2.7.4, a lazy loading of package `pbr` will break
-# setuptools if some other modules registered functions in `atexit`.
-# solution from: http://bugs.python.org/issue15881#msg170215
-try:
-    import multiprocessing  # noqa
-except ImportError:
-    pass
-
 setuptools.setup(
-    setup_requires=['pbr'],
+    setup_requires=['pbr>=2.0.0'],
     pbr=True)

test-requirements.txt

@@ -2,18 +2,8 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-hacking<0.11,>=0.10.0
-
-coverage>=3.6
-python-subunit>=0.0.18
-retrying>=1.2.3,!=1.3.0
-sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
-oslosphinx>=2.5.0 # Apache-2.0
-oslotest>=1.10.0 # Apache-2.0
-testrepository>=0.0.18
-testscenarios>=0.4
-testtools>=1.4.0
-cached-property>=1.3.0
-
-# releasenotes
-reno>=1.6.2 # Apache2
+# unit tests
+coverage!=4.4,>=4.0 # Apache-2.0
+oslotest>=3.2.0 # Apache-2.0
+stestr>=2.0.0 # Apache-2.0
+retrying>=1.3.3 # Apache-2.0

tox.ini

@@ -1,38 +1,91 @@
 [tox]
-minversion = 2.0
-envlist = py34,py35,py27,pep8
-skipsdist = True
+minversion = 3.18.0
+envlist = py3,pep8
+ignore_basepython_conflict=true
 
 [testenv]
+basepython = python3
 usedevelop = True
-install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
-deps = -r{toxinidir}/test-requirements.txt
-commands = python setup.py test --slowest --testr-args='{posargs}'
+   PYTHONWARNINGS=default::DeprecationWarning
+deps =
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/requirements.txt
+commands = stestr run --slowest {posargs}
 
 [testenv:pep8]
+deps=
+    hacking>=4.1.0,<5.0.0 # Apache-2.0
+    flake8-import-order>=0.17.1 # LGPLv3
+    pycodestyle>=2.0.0,<3.0.0 # MIT
 commands = flake8 {posargs}
 
 [testenv:venv]
+setenv = PYTHONHASHSEED=0
+deps =
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -r{toxinidir}/requirements.txt
+  -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/doc/requirements.txt
 commands = {posargs}
 
 [testenv:cover]
-commands = python setup.py test --coverage --testr-args='{posargs}'
+setenv =
+   {[testenv]setenv}
+   PYTHON=coverage run --parallel-mode
+# After running this target, visit virtualpdu/cover/index.html
+# in your browser, to see a nicer presentation report with annotated
+# HTML listings detailing missed lines.
+commands = coverage erase
+           stestr run {posargs}
+           coverage combine
+           coverage report
+           coverage html
+           coverage xml -o cover/coverage.xml
 
 [testenv:docs]
-commands = python setup.py build_sphinx
+deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+       -r{toxinidir}/requirements.txt
+       -r{toxinidir}/doc/requirements.txt
+commands = sphinx-build -W -b html doc/source doc/build/html
+
+[testenv:pdf-docs]
+usedevelop = False
+allowlist_externals = make
+deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+       -r{toxinidir}/doc/requirements.txt
+commands = sphinx-build -b latex doc/source doc/build/pdf
+           make -C doc/build/pdf
 
 [testenv:releasenotes]
-commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
+usedevelop = False
+deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+       -r{toxinidir}/doc/requirements.txt
+commands =
+  sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
 
 [testenv:debug]
-commands = oslo_debug_helper {posargs}
+commands = oslo_debug_helper -t virtualpdu/tests {posargs}
 
 [flake8]
-# E123, E125 skipped as they are invalid PEP-8.
-
 show-source = True
-ignore = E123,E125
+# E123, E125 skipped as they are invalid PEP-8.
+# E741 ambiguous variable name.
+# note(rpittau): we must fix this ASAP
+# F811 Redefinition of unused name from line n.
+# W503 Line break occurred before a binary operator. Conflicts with W504.
+ignore = E123,E125,E741,F811,W503
+# [H106] Don't put vim configuration in source files.
+# [H203] Use assertIs(Not)None to check for None.
+# [H204] Use assert(Not)Equal to check for equality.
+# [H205] Use assert(Greater|Less)(Equal) for comparison.
+# [H210] Require 'autospec', 'spec', or 'spec_set' in mock.patch/mock.patch.object calls
+# [H904] Delay string interpolations at logging calls.
+enable-extensions=H106,H203,H204,H205,H210,H904
 builtins = _
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build
+import-order-style = pep8
+application-import-names = virtualpdu
+filename = *.py

virtualpdu/main.py

@@ -11,15 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import logging
+import sys
 
 try:
     import configparser
 except ImportError:
     import ConfigParser as configparser
 
-import sys
-
 import virtualpdu.core
 from virtualpdu.drivers import libvirt_driver
 from virtualpdu.pdu import apc_rackpdu
@@ -114,7 +114,7 @@ def main():
 
         try:
             for t in pdu_threads:
-                while t.isAlive():
+                while t.is_alive():
                     t.join(1)
 
         except KeyboardInterrupt:

virtualpdu/pdu/pysnmp_handler.py

@@ -213,8 +213,8 @@ def create_snmp_engine(power_unit,
             priv_protocol, priv_key
         )
 
-        if (auth_protocol != config.usmNoAuthProtocol and
-                priv_protocol != config.usmNoPrivProtocol):
+        if (auth_protocol != config.usmNoAuthProtocol
+                and priv_protocol != config.usmNoPrivProtocol):
             sec_level = 'authPriv'
         elif priv_protocol != config.usmNoAuthProtocol:
             sec_level = 'authNoPriv'

virtualpdu/tests/integration/pdu/__init__.py

@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import mock
-from pysnmp.entity.rfc3413.oneliner import cmdgen
 import random
+from unittest import mock
+
+from pysnmp.entity.rfc3413.oneliner import cmdgen
 
 from virtualpdu.pdu import pysnmp_handler
 from virtualpdu.tests import base

virtualpdu/tests/integration/pdu/test_apc_rackpdu.py

@@ -11,10 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from virtualpdu import core
 from virtualpdu.pdu.apc_rackpdu import APCRackPDU
 from virtualpdu.pdu.apc_rackpdu import APCRackPDUOutletControl
-
 from virtualpdu.tests.integration.pdu import PDUTestCase
 
 

virtualpdu/tests/integration/pdu/test_baytech_mrp27.py

@@ -11,10 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from virtualpdu import core
 from virtualpdu.pdu.baytech_mrp27 import BaytechMRP27PDU
 from virtualpdu.pdu.baytech_mrp27 import BaytechMRP27PDUOutletControl
-
 from virtualpdu.tests.integration.pdu import PDUTestCase
 
 

virtualpdu/tests/integration/pdu/test_pysnmp_handler.py

@@ -12,10 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
+from random import randint
+from unittest import mock
+
 from pyasn1.type import univ
 from pysnmp.entity.rfc3413.oneliner import cmdgen
-from random import randint
+
 from virtualpdu.pdu import pysnmp_handler
 from virtualpdu.tests import base
 from virtualpdu.tests import snmp_client
@@ -123,7 +125,7 @@ class TestSNMPPDUHarness(base.TestCase):
         harness.stop()
         harness.join(timeout=5)
 
-        self.assertFalse(harness.isAlive())
+        self.assertFalse(harness.is_alive())
 
 
 class TestSNMPv3Operations(base.TestCase):

virtualpdu/tests/integration/test_core_integration.py

@@ -16,6 +16,7 @@ import random
 import time
 
 from pysnmp.entity.rfc3413.oneliner import cmdgen
+
 from virtualpdu import core
 from virtualpdu import drivers
 from virtualpdu.drivers import libvirt_driver

virtualpdu/tests/integration/test_entrypoint.py

@@ -11,13 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import os
 import signal
 import subprocess
 import sys
 import tempfile
 import threading
 
-import os
 from pysnmp.entity.rfc3413.oneliner import cmdgen
 from retrying import retry
 
@@ -52,7 +53,7 @@ class TestEntryPointIntegration(base.TestCase):
             stderr=subprocess.PIPE
         )
         stdout, stderr = p.communicate()
-        self.assertEqual(
+        self.assertIn(
             b'Missing configuration file as first parameter.\n',
             stderr)
         self.assertEqual(1, p.returncode)

virtualpdu/tests/snmp_client.py

@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from pysnmp.proto.api import v2c
+
 from virtualpdu.pdu.pysnmp_handler import auth_protocols
 from virtualpdu.pdu.pysnmp_handler import priv_protocols
 from virtualpdu.tests import snmp_error_indications
 
-from pysnmp.proto.api import v2c
-
 
 class SnmpClient(object):
     def __init__(self, oneliner_cmdgen, host, port, **snmp_options):

virtualpdu/tests/unit/drivers/test_libvirt_driver.py

@@ -12,8 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from unittest import mock
+
 import libvirt
-from mock import mock
 
 from virtualpdu import drivers
 from virtualpdu.drivers import libvirt_driver
@@ -117,8 +118,7 @@ class TestLibvirtDriver(base.TestCase):
     def test_get_power_state_domain_not_found(self, mock_open):
         connection_mock = mock.Mock()
         connection_mock.lookupByName.side_effect = \
-            libvirt.libvirtError('virDomainLookupByName() failed',
-                                 conn=connection_mock)
+            libvirt.libvirtError('virDomainLookupByName() failed')
         mock_open.return_value = connection_mock
 
         self.assertRaises(drivers.DeviceNotFound,
@@ -129,8 +129,7 @@ class TestLibvirtDriver(base.TestCase):
     def test_power_on_domain_not_found(self, mock_open):
         connection_mock = mock.Mock()
         connection_mock.lookupByName.side_effect = \
-            libvirt.libvirtError('virDomainLookupByName() failed',
-                                 conn=connection_mock)
+            libvirt.libvirtError('virDomainLookupByName() failed')
         mock_open.return_value = connection_mock
 
         self.assertRaises(drivers.DeviceNotFound,
@@ -141,8 +140,7 @@ class TestLibvirtDriver(base.TestCase):
     def test_power_off_domain_not_found(self, mock_open):
         connection_mock = mock.Mock()
         connection_mock.lookupByName.side_effect = \
-            libvirt.libvirtError('virDomainLookupByName() failed',
-                                 conn=connection_mock)
+            libvirt.libvirtError('virDomainLookupByName() failed')
         mock_open.return_value = connection_mock
 
         self.assertRaises(drivers.DeviceNotFound,

virtualpdu/tests/unit/pdu/base_pdu_test_cases.py

@@ -11,8 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from cached_property import cached_property
-from mock import mock
+
+import functools
+from unittest import mock
+
 from virtualpdu import core
 
 
@@ -20,11 +22,11 @@ class BasePDUTests(object):
     pdu_class = None
     outlet_control_oid = None
 
-    @cached_property
+    @functools.cached_property
     def core_mock(self):
         return mock.Mock()
 
-    @cached_property
+    @functools.cached_property
     def pdu(self):
         return self.pdu_class(name='my_pdu', core=self.core_mock)
 

virtualpdu/tests/unit/pdu/test_apc_rackpdu.py

@@ -11,7 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from pyasn1.type import univ
+
 from virtualpdu import core
 from virtualpdu.pdu import apc_rackpdu
 from virtualpdu.pdu import sysDescr

virtualpdu/tests/unit/pdu/test_baytech_mrp27.py

@@ -11,7 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from pyasn1.type import univ
+
 from virtualpdu.pdu import baytech_mrp27
 from virtualpdu.pdu import sysDescr
 from virtualpdu.pdu import sysObjectID

virtualpdu/tests/unit/test_core.py

@@ -11,9 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import time
 
-from mock import mock
+import time
+from unittest import mock
+
 from virtualpdu import core
 from virtualpdu import drivers
 from virtualpdu.tests import base

virtualpdu/tests/unit/test_pysnmp_handler.py

@@ -13,9 +13,10 @@
 # limitations under the License.
 
 import unittest
-
-from mock import Mock
-from mock import patch
+# note(rpittau): related to F811, see tox.ini
+# from unittest import mock
+from unittest.mock import Mock
+from unittest.mock import patch
 
 from pysnmp.proto.errind import UnknownPDUHandler
 from pysnmp.proto.rfc1902 import Integer
@@ -23,7 +24,6 @@ from pysnmp.proto.rfc1902 import ObjectName
 from pysnmp.proto.rfc1902 import ObjectSyntax
 from pysnmp.proto.rfc1902 import OctetString
 from pysnmp.proto.rfc1902 import SimpleSyntax
-
 from pysnmp.proto.rfc1905 import _BindValue
 from pysnmp.proto.rfc1905 import NoSuchInstance
 from pysnmp.proto.rfc1905 import VarBindList

virtualpdu/tests/unit/test_snmp_client.py

@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import mock
-from mock import sentinel
+from unittest import mock
+from unittest.mock import sentinel
+
 from pysnmp.proto import errind
 from pysnmp.proto.errind import ErrorIndication
 from pysnmp.proto.rfc1905 import NoSuchInstance
-
 import testtools
 
 from virtualpdu.tests import base

virtualpdu/tests/unit/test_transitive_state.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 import threading
+from unittest import mock
 
-from mock import mock
 from virtualpdu import core
 from virtualpdu import drivers
 from virtualpdu.tests import base

zuul.d/project.yaml

@@ -0,0 +1,13 @@
+- project:
+    templates:
+      - check-requirements
+      - openstack-cover-jobs
+      - openstack-python3-jobs
+      - publish-openstack-docs-pti
+      - release-notes-jobs-python3
+    check:
+      jobs:
+        - ironic-tempest-wholedisk-bios-snmp-pxe-virtualpdu-src
+    gate:
+      jobs:
+        - ironic-tempest-wholedisk-bios-snmp-pxe-virtualpdu-src

zuul.d/virtualpdu-jobs.yaml

@@ -0,0 +1,15 @@
+- job:
+    name: ironic-tempest-wholedisk-bios-snmp-pxe-virtualpdu-src
+    description: SNMP power, no-op management and whole disk images.
+    parent: ironic-base
+    required-projects:
+      - x/virtualpdu
+    vars:
+      devstack_localrc:
+        IRONIC_ENABLED_HARDWARE_TYPES: snmp
+        IRONIC_DEPLOY_DRIVER: snmp
+        IRONIC_TEMPEST_WHOLE_DISK_IMAGE: True
+        IRONIC_VM_EPHEMERAL_DISK: 0
+        IRONIC_AUTOMATED_CLEAN_ENABLED: False
+        IRONIC_ENFORCE_SCOPE: True
+        IRONIC_BOOT_MODE: bios