Port to Python3

Brief summary of the modifications:

* Use six for compatibility with both Python 2 and 3;
* Replace UserDict.DictMixin with collections.MutableMapping;
* Fix relative imports;
* Use test-requirements.txt for requirements that are common to both Python 2
  and 3, and test-requirements-py{2,3}.txt for version-specific requirements;
* Miscellaneous fixes.
* Use a specific test_db_py3.cfg file for Python 3, that only runs tests on
  sqlite.

Thanks to Victor Stinner who co-wrote this patch.

Change-Id: Ia6dc536c39d274924c21fd5bb619e8e5721e04c4
Co-Authored-By: Victor Stinner <victor.stinner@enovance.com>

migrate/changeset/ansisql.py

@@ -4,7 +4,6 @@
    At the moment, this isn't so much based off of ANSI as much as
    things that just happen to work with multiple databases.
 """
-import StringIO
 
 import sqlalchemy as sa
 from sqlalchemy.schema import SchemaVisitor
@@ -20,6 +19,7 @@ from migrate import exceptions
 import sqlalchemy.sql.compiler
 from migrate.changeset import constraint
 from migrate.changeset import util
+from six.moves import StringIO
 
 from sqlalchemy.schema import AddConstraint, DropConstraint
 from sqlalchemy.sql.compiler import DDLCompiler
@@ -43,11 +43,12 @@ class AlterTableVisitor(SchemaVisitor):
         try:
             return self.connection.execute(self.buffer.getvalue())
         finally:
-            self.buffer.truncate(0)
+            self.buffer.seek(0)
+            self.buffer.truncate()
 
     def __init__(self, dialect, connection, **kw):
         self.connection = connection
-        self.buffer = StringIO.StringIO()
+        self.buffer = StringIO()
         self.preparer = dialect.identifier_preparer
         self.dialect = dialect
 

migrate/changeset/databases/sqlite.py

@@ -3,7 +3,10 @@
 
    .. _`SQLite`: http://www.sqlite.org/
 """
-from UserDict import DictMixin
+try:  # Python 3
+    from collections import MutableMapping as DictMixin
+except ImportError:  # Python 2
+    from UserDict import DictMixin
 from copy import copy
 
 from sqlalchemy.databases import sqlite as sa_base

migrate/changeset/schema.py

@@ -1,10 +1,14 @@
 """
    Schema module providing common schema operations.
 """
+import abc
+try:  # Python 3
+    from collections import MutableMapping as DictMixin
+except ImportError:  # Python 2
+    from UserDict import DictMixin
 import warnings
 
-from UserDict import DictMixin
+import six
-
 import sqlalchemy
 
 from sqlalchemy.schema import ForeignKeyConstraint
@@ -163,7 +167,39 @@ def _to_index(index, table=None, engine=None):
     return ret
 
 
-class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
+
+# Python3: if we just use:
+#
+#     class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
+#         ...
+#
+# We get the following error:
+# TypeError: metaclass conflict: the metaclass of a derived class must be a
+# (non-strict) subclass of the metaclasses of all its bases.
+#
+# The complete inheritance/metaclass relationship list of ColumnDelta can be
+# summarized by this following dot file:
+#
+# digraph test123 {
+#     ColumnDelta -> MutableMapping;
+#     MutableMapping -> Mapping;
+#     Mapping -> {Sized Iterable Container};
+#     {Sized Iterable Container} -> ABCMeta[style=dashed];
+#
+#     ColumnDelta -> SchemaItem;
+#     SchemaItem -> {SchemaEventTarget Visitable};
+#     SchemaEventTarget -> object;
+#     Visitable -> {VisitableType object} [style=dashed];
+#     VisitableType -> type;
+# }
+#
+# We need to use a metaclass that inherits from all the metaclasses of
+# DictMixin and sqlalchemy.schema.SchemaItem. Let's call it "MyMeta".
+class MyMeta(sqlalchemy.sql.visitors.VisitableType, abc.ABCMeta, object):
+    pass
+
+
+class ColumnDelta(six.with_metaclass(MyMeta, DictMixin, sqlalchemy.schema.SchemaItem)):
     """Extracts the differences between two columns/column-parameters
 
         May receive parameters arranged in several different ways:
@@ -229,7 +265,7 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
                 diffs = self.compare_1_column(*p, **kw)
         else:
             # Zero columns specified
-            if not len(p) or not isinstance(p[0], basestring):
+            if not len(p) or not isinstance(p[0], six.string_types):
                 raise ValueError("First argument must be column name")
             diffs = self.compare_parameters(*p, **kw)
 
@@ -254,6 +290,12 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
     def __delitem__(self, key):
         raise NotImplementedError
 
+    def __len__(self):
+        raise NotImplementedError
+
+    def __iter__(self):
+        raise NotImplementedError
+
     def keys(self):
         return self.diffs.keys()
 
@@ -332,7 +374,7 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
         """Extracts data from p and modifies diffs"""
         p = list(p)
         while len(p):
-            if isinstance(p[0], basestring):
+            if isinstance(p[0], six.string_types):
                 k.setdefault('name', p.pop(0))
             elif isinstance(p[0], sqlalchemy.types.TypeEngine):
                 k.setdefault('type', p.pop(0))
@@ -370,7 +412,7 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
         return getattr(self, '_table', None)
 
     def _set_table(self, table):
-        if isinstance(table, basestring):
+        if isinstance(table, six.string_types):
             if self.alter_metadata:
                 if not self.meta:
                     raise ValueError("metadata must be specified for table"
@@ -587,7 +629,7 @@ populated with defaults
             if isinstance(cons,(ForeignKeyConstraint,
                                 UniqueConstraint)):
                 for col_name in cons.columns:
-                    if not isinstance(col_name,basestring):
+                    if not isinstance(col_name,six.string_types):
                         col_name = col_name.name
                     if self.name==col_name:
                         to_drop.add(cons)
@@ -622,7 +664,7 @@ populated with defaults
         if (getattr(self, name[:-5]) and not obj):
             raise InvalidConstraintError("Column.create() accepts index_name,"
             " primary_key_name and unique_name to generate constraints")
-        if not isinstance(obj, basestring) and obj is not None:
+        if not isinstance(obj, six.string_types) and obj is not None:
             raise InvalidConstraintError(
             "%s argument for column must be constraint name" % name)
 

migrate/tests/__init__.py

@@ -6,10 +6,11 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
 from unittest import TestCase
 import migrate
+import six
 
 
 class TestVersionDefined(TestCase):
     def test_version(self):
         """Test for migrate.__version__"""
-        self.assertTrue(isinstance(migrate.__version__, basestring))
+        self.assertTrue(isinstance(migrate.__version__, six.string_types))
         self.assertTrue(len(migrate.__version__) > 0)

migrate/tests/changeset/test_changeset.py

@@ -11,6 +11,7 @@ from migrate.changeset import constraint
 from migrate.changeset.schema import ColumnDelta
 from migrate.tests import fixture
 from migrate.tests.fixture.warnings import catch_warnings
+import six
 
 class TestAddDropColumn(fixture.DB):
     """Test add/drop column through all possible interfaces
@@ -400,7 +401,7 @@ class TestAddDropColumn(fixture.DB):
             if isinstance(cons,ForeignKeyConstraint):
                 col_names = []
                 for col_name in cons.columns:
-                    if not isinstance(col_name,basestring):
+                    if not isinstance(col_name,six.string_types):
                         col_name = col_name.name
                     col_names.append(col_name)
                 result.append(col_names)
@@ -612,7 +613,7 @@ class TestColumnChange(fixture.DB):
             self.table.drop()
         try:
             self.table.create()
-        except sqlalchemy.exc.SQLError, e:
+        except sqlalchemy.exc.SQLError:
             # SQLite: database schema has changed
             if not self.url.startswith('sqlite://'):
                 raise
@@ -621,7 +622,7 @@ class TestColumnChange(fixture.DB):
         if self.table.exists():
             try:
                 self.table.drop(self.engine)
-            except sqlalchemy.exc.SQLError,e:
+            except sqlalchemy.exc.SQLError:
                 # SQLite: database schema has changed
                 if not self.url.startswith('sqlite://'):
                     raise
@@ -843,7 +844,7 @@ class TestColumnDelta(fixture.DB):
 
     def verify(self, expected, original, *p, **k):
         self.delta = ColumnDelta(original, *p, **k)
-        result = self.delta.keys()
+        result = list(self.delta.keys())
         result.sort()
         self.assertEqual(expected, result)
         return self.delta

migrate/tests/fixture/__init__.py

@@ -12,7 +12,7 @@ def main(imports=None):
         defaultTest=None
     return testtools.TestProgram(defaultTest=defaultTest)
 
-from base import Base
+from .base import Base
-from migrate.tests.fixture.pathed import Pathed
+from .pathed import Pathed
-from shell import Shell
+from .shell import Shell
-from database import DB,usedb
+from .database import DB,usedb

migrate/tests/fixture/database.py

@@ -3,6 +3,9 @@
 
 import os
 import logging
+import sys
+
+import six
 from decorator import decorator
 
 from sqlalchemy import create_engine, Table, MetaData
@@ -23,7 +26,7 @@ log = logging.getLogger(__name__)
 def readurls():
     """read URLs from config file return a list"""
     # TODO: remove tmpfile since sqlite can store db in memory
-    filename = 'test_db.cfg'
+    filename = 'test_db.cfg' if six.PY2 else "test_db_py3.cfg"
     ret = list()
     tmpfile = Pathed.tmp()
     fullpath = os.path.join(os.curdir, filename)
@@ -46,12 +49,12 @@ def is_supported(url, supported, not_supported):
     db = url.split(':', 1)[0]
 
     if supported is not None:
-        if isinstance(supported, basestring):
+        if isinstance(supported, six.string_types):
             return supported == db
         else:
             return db in supported
     elif not_supported is not None:
-        if isinstance(not_supported, basestring):
+        if isinstance(not_supported, six.string_types):
             return not_supported != db
         else:
             return not (db in not_supported)
@@ -96,7 +99,7 @@ def usedb(supported=None, not_supported=None):
                 finally:
                     try:
                         self._teardown()
-                    except Exception,e:
+                    except Exception as e:
                         teardown_exception=e
                     else:
                         teardown_exception=None
@@ -106,14 +109,14 @@ def usedb(supported=None, not_supported=None):
                         'setup: %r\n'
                         'teardown: %r\n'
                         )%(setup_exception,teardown_exception))
-            except Exception,e:
+            except Exception:
                 failed_for.append(url)
-                fail = True
+                fail = sys.exc_info()
         for url in failed_for:
             log.error('Failed for %s', url)
         if fail:
             # cause the failure :-)
-            raise
+            six.reraise(*fail)
     return dec
 
 

migrate/tests/versioning/test_api.py

@@ -1,6 +1,8 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
+import six
+
 from migrate.exceptions import *
 from migrate.versioning import api
 
@@ -12,7 +14,7 @@ from migrate.tests import fixture
 class TestAPI(Pathed):
 
     def test_help(self):
-        self.assertTrue(isinstance(api.help('help'), basestring))
+        self.assertTrue(isinstance(api.help('help'), six.string_types))
         self.assertRaises(UsageError, api.help)
         self.assertRaises(UsageError, api.help, 'foobar')
         self.assertTrue(isinstance(api.help('create'), str))
@@ -48,7 +50,7 @@ class TestAPI(Pathed):
         repo = self.tmp_repos()
         api.create(repo, 'temp')
         api.version_control('sqlite:///', repo)
-        api.version_control('sqlite:///', unicode(repo))
+        api.version_control('sqlite:///', six.text_type(repo))
 
     def test_source(self):
         repo = self.tmp_repos()

migrate/tests/versioning/test_genmodel.py

@@ -2,6 +2,7 @@
 
 import os
 
+import six
 import sqlalchemy
 from sqlalchemy import *
 
@@ -43,13 +44,12 @@ class TestSchemaDiff(fixture.DB):
     # so the schema diffs on the columns don't work with this test.
     @fixture.usedb(not_supported='ibm_db_sa')
     def test_functional(self):
-
         def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff):
             diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
             self.assertEqual(
                 (diff.tables_missing_from_B,
                  diff.tables_missing_from_A,
-                 diff.tables_different.keys(),
+                 list(diff.tables_different.keys()),
                  bool(diff)),
                 (tablesMissingInDatabase,
                  tablesMissingInModel,
@@ -97,10 +97,11 @@ class TestSchemaDiff(fixture.DB):
         diff = schemadiff.getDiffOfModelAgainstDatabase(MetaData(), self.engine, excludeTables=['migrate_version'])
         src = genmodel.ModelGenerator(diff,self.engine).genBDefinition()
 
-        exec src in locals()
+        namespace = {}
+        six.exec_(src, namespace)
 
         c1 = Table('tmp_schemadiff', self.meta, autoload=True).c
-        c2 = tmp_schemadiff.c
+        c2 = namespace['tmp_schemadiff'].c
         self.compare_columns_equal(c1, c2, ['type'])
         # TODO: get rid of ignoring type
 
@@ -139,19 +140,19 @@ class TestSchemaDiff(fixture.DB):
         decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(diff,self.engine).genB2AMigration(indent='')
 
         # decls have changed since genBDefinition
-        exec decls in locals()
+        six.exec_(decls, namespace)
         # migration commands expect a namespace containing migrate_engine
-        migrate_engine = self.engine
+        namespace['migrate_engine'] = self.engine
         # run the migration up and down
-        exec upgradeCommands in locals()
+        six.exec_(upgradeCommands, namespace)
         assertDiff(False, [], [], [])
 
-        exec decls in locals()
+        six.exec_(decls, namespace)
-        exec downgradeCommands in locals()
+        six.exec_(downgradeCommands, namespace)
         assertDiff(True, [], [], [self.table_name])
 
-        exec decls in locals()
+        six.exec_(decls, namespace)
-        exec upgradeCommands in locals()
+        six.exec_(upgradeCommands, namespace)
         assertDiff(False, [], [], [])
 
         if not self.engine.name == 'oracle':

migrate/tests/versioning/test_repository.py

@@ -111,7 +111,6 @@ class TestVersionedRepository(fixture.Pathed):
         # Create a script and test again
         now = int(datetime.utcnow().strftime('%Y%m%d%H%M%S'))
         repos.create_script('')
-        print repos.latest
         self.assertEqual(repos.latest, now)
 
     def test_source(self):

migrate/tests/versioning/test_schema.py

@@ -4,6 +4,8 @@
 import os
 import shutil
 
+import six
+
 from migrate import exceptions
 from migrate.versioning.schema import *
 from migrate.versioning import script, schemadiff
@@ -163,10 +165,10 @@ class TestControlledSchema(fixture.Pathed, fixture.DB):
     def test_create_model(self):
         """Test workflow to generate create_model"""
         model = ControlledSchema.create_model(self.engine, self.repos, declarative=False)
-        self.assertTrue(isinstance(model, basestring))
+        self.assertTrue(isinstance(model, six.string_types))
 
         model = ControlledSchema.create_model(self.engine, self.repos.path, declarative=True)
-        self.assertTrue(isinstance(model, basestring))
+        self.assertTrue(isinstance(model, six.string_types))
 
     @fixture.usedb()
     def test_compare_model_to_db(self):

migrate/tests/versioning/test_schemadiff.py

@@ -27,9 +27,9 @@ class SchemaDiffBase(fixture.DB):
         # print diff
         self.assertTrue(diff)
         self.assertEqual(1,len(diff.tables_different))
-        td = diff.tables_different.values()[0]
+        td = list(diff.tables_different.values())[0]
         self.assertEqual(1,len(td.columns_different))
-        cd = td.columns_different.values()[0]
+        cd = list(td.columns_different.values())[0]
         label_width = max(len(self.name1), len(self.name2))
         self.assertEqual(('Schema diffs:\n'
              '  table with differences: xtable\n'

migrate/tests/versioning/test_script.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import imp
 import os
 import sys
 import shutil
 
+import six
 from migrate import exceptions
 from migrate.versioning import version, repository
 from migrate.versioning.script import *
@@ -51,6 +53,9 @@ class TestPyScript(fixture.Pathed, fixture.DB):
         self.assertRaises(exceptions.ScriptError, pyscript._func, 'foobar')
 
         # clean pyc file
+        if six.PY3:
+            os.remove(imp.cache_from_source(script_path))
+        else:
             os.remove(script_path + 'c')
 
         # test deprecated upgrade/downgrade with no arguments
@@ -94,7 +99,7 @@ class TestPyScript(fixture.Pathed, fixture.DB):
         path = self.tmp_py()
         # Create empty file
         f = open(path, 'w')
-        f.write("def zergling():\n\tprint 'rush'")
+        f.write("def zergling():\n\tprint('rush')")
         f.close()
         self.assertRaises(exceptions.InvalidScriptError, self.cls.verify_module, path)
         # script isn't verified on creation, but on module reference

migrate/tests/versioning/test_shell.py

@@ -5,7 +5,8 @@ import os
 import sys
 import tempfile
 
-from cStringIO import StringIO
+import six
+from six.moves import cStringIO
 from sqlalchemy import MetaData, Table
 
 from migrate.exceptions import *
@@ -29,7 +30,7 @@ class TestShellCommands(Shell):
         # we can only test that we get some output
         for cmd in api.__all__:
             result = self.env.run('migrate help %s' % cmd)
-            self.assertTrue(isinstance(result.stdout, basestring))
+            self.assertTrue(isinstance(result.stdout, six.string_types))
             self.assertTrue(result.stdout)
             self.assertFalse(result.stderr)
 
@@ -61,11 +62,11 @@ class TestShellCommands(Shell):
     def _check_error(self,args,code,expected,**kw):
         original = sys.stderr
         try:
-            actual = StringIO()
+            actual = cStringIO()
             sys.stderr = actual
             try:
                 shell.main(args,**kw)
-            except SystemExit, e:
+            except SystemExit as e:
                 self.assertEqual(code,e.args[0])
             else:
                 self.fail('No exception raised')
@@ -502,7 +503,7 @@ class TestShellDatabase(Shell, DB):
 
         result = self.env.run('migrate create_model %s %s' % (self.url, repos_path))
         temp_dict = dict()
-        exec result.stdout in temp_dict
+        six.exec_(result.stdout, temp_dict)
 
         # TODO: breaks on SA06 and SA05 - in need of total refactor - use different approach
 

migrate/versioning/cfgparse.py

@@ -2,7 +2,7 @@
    Configuration parser module.
 """
 
-from ConfigParser import ConfigParser
+from six.moves.configparser import ConfigParser
 
 from migrate.versioning.config import *
 from migrate.versioning import pathed

migrate/versioning/genmodel.py

@@ -9,6 +9,7 @@ http://code.google.com/p/sqlautocode/
 import sys
 import logging
 
+import six
 import sqlalchemy
 
 import migrate
@@ -68,6 +69,9 @@ class ModelGenerator(object):
 
         # crs: not sure if this is good idea, but it gets rid of extra
         # u''
+        if six.PY3:
+            name = col.name
+        else:
             name = col.name.encode('utf8')
 
         type_ = col.type
@@ -192,7 +196,7 @@ class ModelGenerator(object):
             downgradeCommands.append(
                 "post_meta.tables[%(table)r].drop()" % {'table': tn})
 
-        for (tn, td) in self.diff.tables_different.iteritems():
+        for (tn, td) in six.iteritems(self.diff.tables_different):
             if td.columns_missing_from_A or td.columns_different:
                 pre_table = self.diff.metadataB.tables[tn]
                 decls.extend(self._getTableDefn(

migrate/versioning/repository.py

@@ -43,7 +43,7 @@ class Changeset(dict):
         """
         In a series of upgrades x -> y, keys are version x. Sorted.
         """
-        ret = super(Changeset, self).keys()
+        ret = list(super(Changeset, self).keys())
         # Reverse order if downgrading
         ret.sort(reverse=(self.step < 1))
         return ret
@@ -94,7 +94,7 @@ class Repository(pathed.Pathed):
             cls.require_found(path)
             cls.require_found(os.path.join(path, cls._config))
             cls.require_found(os.path.join(path, cls._versions))
-        except exceptions.PathNotFoundError, e:
+        except exceptions.PathNotFoundError:
             raise exceptions.InvalidRepositoryError(path)
 
     @classmethod
@@ -221,7 +221,7 @@ class Repository(pathed.Pathed):
             range_mod = 0
             op = 'downgrade'
 
-        versions = range(start + range_mod, end + range_mod, step)
+        versions = range(int(start) + range_mod, int(end) + range_mod, step)
         changes = [self.version(v).script(database, op) for v in versions]
         ret = Changeset(start, step=step, *changes)
         return ret

migrate/versioning/schema.py

@@ -4,6 +4,7 @@
 import sys
 import logging
 
+import six
 from sqlalchemy import (Table, Column, MetaData, String, Text, Integer,
     create_engine)
 from sqlalchemy.sql import and_
@@ -24,7 +25,7 @@ class ControlledSchema(object):
     """A database under version control"""
 
     def __init__(self, engine, repository):
-        if isinstance(repository, basestring):
+        if isinstance(repository, six.string_types):
             repository = Repository(repository)
         self.engine = engine
         self.repository = repository
@@ -49,7 +50,8 @@ class ControlledSchema(object):
             data = list(result)[0]
         except:
             cls, exc, tb = sys.exc_info()
-            raise exceptions.DatabaseNotControlledError, exc.__str__(), tb
+            six.reraise(exceptions.DatabaseNotControlledError,
+                        exceptions.DatabaseNotControlledError(str(exc)), tb)
 
         self.version = data['version']
         return data
@@ -133,7 +135,7 @@ class ControlledSchema(object):
         """
         # Confirm that the version # is valid: positive, integer,
         # exists in repos
-        if isinstance(repository, basestring):
+        if isinstance(repository, six.string_types):
             repository = Repository(repository)
         version = cls._validate_version(repository, version)
         table = cls._create_table_version(engine, repository, version)
@@ -198,7 +200,7 @@ class ControlledSchema(object):
         """
         Compare the current model against the current database.
         """
-        if isinstance(repository, basestring):
+        if isinstance(repository, six.string_types):
             repository = Repository(repository)
         model = load_model(model)
 
@@ -211,7 +213,7 @@ class ControlledSchema(object):
         """
         Dump the current database as a Python model.
         """
-        if isinstance(repository, basestring):
+        if isinstance(repository, six.string_types):
             repository = Repository(repository)
 
         diff = schemadiff.getDiffOfModelAgainstDatabase(

migrate/versioning/schemadiff.py

@@ -99,6 +99,9 @@ class ColDiff(object):
     def __nonzero__(self):
         return self.diff
 
+    __bool__ = __nonzero__
+
+
 class TableDiff(object):
     """
     Container for differences in one :class:`~sqlalchemy.schema.Table`
@@ -135,6 +138,8 @@ class TableDiff(object):
             self.columns_different
             )
 
+    __bool__ = __nonzero__
+
 class SchemaDiff(object):
     """
     Compute the difference between two :class:`~sqlalchemy.schema.MetaData`

migrate/versioning/script/py.py

@@ -5,7 +5,6 @@ import shutil
 import warnings
 import logging
 import inspect
-from StringIO import StringIO
 
 import migrate
 from migrate.versioning import genmodel, schemadiff
@@ -14,6 +13,8 @@ from migrate.versioning.template import Template
 from migrate.versioning.script import base
 from migrate.versioning.util import import_path, load_model, with_engine
 from migrate.exceptions import MigrateDeprecationWarning, InvalidScriptError, ScriptError
+import six
+from six.moves import StringIO
 
 log = logging.getLogger(__name__)
 __all__ = ['PythonScript']
@@ -51,7 +52,7 @@ class PythonScript(base.BaseScript):
         :rtype: string
         """
 
-        if isinstance(repository, basestring):
+        if isinstance(repository, six.string_types):
             # oh dear, an import cycle!
             from migrate.versioning.repository import Repository
             repository = Repository(repository)
@@ -96,7 +97,7 @@ class PythonScript(base.BaseScript):
         module = import_path(path)
         try:
             assert callable(module.upgrade)
-        except Exception, e:
+        except Exception as e:
             raise InvalidScriptError(path + ': %s' % str(e))
         return module
 
@@ -127,7 +128,9 @@ class PythonScript(base.BaseScript):
         :type engine: string
         :type step: int
         """
-        if step > 0:
+        if step in ('downgrade', 'upgrade'):
+            op = step
+        elif step > 0:
             op = 'upgrade'
         elif step < 0:
             op = 'downgrade'

migrate/versioning/shell.py

@@ -12,6 +12,7 @@ from migrate import exceptions
 from migrate.versioning import api
 from migrate.versioning.config import *
 from migrate.versioning.util import asbool
+import six
 
 
 alias = dict(
@@ -23,7 +24,7 @@ alias = dict(
 
 def alias_setup():
     global alias
-    for key, val in alias.iteritems():
+    for key, val in six.iteritems(alias):
         setattr(api, key, val)
 alias_setup()
 
@@ -135,7 +136,7 @@ def main(argv=None, **kwargs):
             override_kwargs[opt] = value
 
     # override kwargs with options if user is overwriting
-    for key, value in options.__dict__.iteritems():
+    for key, value in six.iteritems(options.__dict__):
         if value is not None:
             override_kwargs[key] = value
 
@@ -143,7 +144,7 @@ def main(argv=None, **kwargs):
     f_required = list(f_args)
     candidates = dict(kwargs)
     candidates.update(override_kwargs)
-    for key, value in candidates.iteritems():
+    for key, value in six.iteritems(candidates):
         if key in f_args:
             f_required.remove(key)
 
@@ -160,7 +161,7 @@ def main(argv=None, **kwargs):
     kwargs.update(override_kwargs)
 
     # configure options
-    for key, value in options.__dict__.iteritems():
+    for key, value in six.iteritems(options.__dict__):
         kwargs.setdefault(key, value)
 
     # configure logging
@@ -198,6 +199,7 @@ def main(argv=None, **kwargs):
         num_defaults = 0
     f_args_default = f_args[len(f_args) - num_defaults:]
     required = list(set(f_required) - set(f_args_default))
+    required.sort()
     if required:
         parser.error("Not enough arguments for command %s: %s not specified" \
             % (command, ', '.join(required)))
@@ -207,7 +209,7 @@ def main(argv=None, **kwargs):
         ret = command_func(**kwargs)
         if ret is not None:
             log.info(ret)
-    except (exceptions.UsageError, exceptions.KnownError), e:
+    except (exceptions.UsageError, exceptions.KnownError) as e:
         parser.error(e.args[0])
 
 if __name__ == "__main__":

migrate/versioning/templates/manage/default.py_tmpl

@@ -2,10 +2,11 @@
 from migrate.versioning.shell import main
 
 {{py:
+import six
 _vars = locals().copy()
 del _vars['__template_name__']
 _vars.pop('repository_name', None)
-defaults = ", ".join(["%s='%s'" % var for var in _vars.iteritems()])
+defaults = ", ".join(["%s='%s'" % var for var in six.iteritems(_vars)])
 }}
 
 if __name__ == '__main__':

migrate/versioning/templates/manage/pylons.py_tmpl

@@ -17,9 +17,10 @@ else:
     conf_path = 'development.ini'
 
 {{py:
+import six
 _vars = locals().copy()
 del _vars['__template_name__']
-defaults = ", ".join(["%s='%s'" % var for var in _vars.iteritems()])
+defaults = ", ".join(["%s='%s'" % var for var in six.iteritems(_vars)])
 }}
 
 conf_dict = ConfigLoader(conf_path).parser._sections['app:main']

migrate/versioning/util/__init__.py

@@ -7,6 +7,7 @@ import logging
 from decorator import decorator
 from pkg_resources import EntryPoint
 
+import six
 from sqlalchemy import create_engine
 from sqlalchemy.engine import Engine
 from sqlalchemy.pool import StaticPool
@@ -26,7 +27,7 @@ def load_model(dotted_name):
     .. versionchanged:: 0.5.4
 
     """
-    if isinstance(dotted_name, basestring):
+    if isinstance(dotted_name, six.string_types):
         if ':' not in dotted_name:
             # backwards compatibility
             warnings.warn('model should be in form of module.model:User '
@@ -39,7 +40,7 @@ def load_model(dotted_name):
 
 def asbool(obj):
     """Do everything to use object as bool"""
-    if isinstance(obj, basestring):
+    if isinstance(obj, six.string_types):
         obj = obj.strip().lower()
         if obj in ['true', 'yes', 'on', 'y', 't', '1']:
             return True
@@ -87,7 +88,7 @@ def catch_known_errors(f, *a, **kw):
 
     try:
         return f(*a, **kw)
-    except exceptions.PathFoundError, e:
+    except exceptions.PathFoundError as e:
         raise exceptions.KnownError("The path %s already exists" % e.args[0])
 
 def construct_engine(engine, **opts):
@@ -112,7 +113,7 @@ def construct_engine(engine, **opts):
     """
     if isinstance(engine, Engine):
         return engine
-    elif not isinstance(engine, basestring):
+    elif not isinstance(engine, six.string_types):
         raise ValueError("you need to pass either an existing engine or a database uri")
 
     # get options for create_engine
@@ -130,7 +131,7 @@ def construct_engine(engine, **opts):
         kwargs['echo'] = echo
 
     # parse keyword arguments
-    for key, value in opts.iteritems():
+    for key, value in six.iteritems(opts):
         if key.startswith('engine_arg_'):
             kwargs[key[11:]] = guess_obj_type(value)
 
@@ -174,6 +175,6 @@ class Memoize:
         self.memo = {}
 
     def __call__(self, *args):
-        if not self.memo.has_key(args):
+        if args not in self.memo:
             self.memo[args] = self.fn(*args)
         return self.memo[args]

migrate/versioning/util/importpath.py

@@ -1,6 +1,8 @@
 import os
 import sys
 
+from six.moves import reload_module as reload
+
 def import_path(fullpath):
     """ Import a file with full path specification. Allows one to
         import from anywhere, something __import__ does not do.

migrate/versioning/version.py

@@ -9,6 +9,7 @@ import logging
 from migrate import exceptions
 from migrate.versioning import pathed, script
 from datetime import datetime
+import six
 
 
 log = logging.getLogger(__name__)
@@ -64,6 +65,10 @@ class VerNum(object):
     def __int__(self):
         return int(self.value)
 
+    if six.PY3:
+        def __hash__(self):
+            return hash(self.value)
+
 
 class Collection(pathed.Pathed):
     """A collection of versioning scripts in a repository"""
@@ -102,7 +107,7 @@ class Collection(pathed.Pathed):
     @property
     def latest(self):
         """:returns: Latest version in Collection"""
-        return max([VerNum(0)] + self.versions.keys())
+        return max([VerNum(0)] + list(self.versions.keys()))
 
     def _next_ver_num(self, use_timestamp_numbering):
         if use_timestamp_numbering == True:

test-requirements-py2.txt

@@ -0,0 +1,2 @@
+ibm_db_sa>=0.3.0
+MySQL-python

test-requirements-py3.txt

@@ -0,0 +1 @@
+ibm-db-sa-py3

test-requirements.txt

@@ -8,21 +8,17 @@ coverage>=3.6
 discover
 feedparser
 fixtures>=0.3.14
-ibm_db_sa>=0.3.0
 mock>=1.0
 mox>=0.5.3
-MySQL-python
 psycopg2
-pylint==0.25.2
 python-subunit>=0.0.18
 sphinx>=1.1.2,<1.2
 sphinxcontrib_issuetracker
 testrepository>=0.0.17
 testtools>=0.9.34
 
-# NOTE: scripttest 1.0.1 removes base_path argument to ScriptTest
+scripttest
-scripttest==1.0
 # NOTE(rpodolyaka): This version identifier is currently necessary as
 #                   pytz otherwise does not install on pip 1.4 or higher
+pylint
 pytz>=2010h
-pysqlite

test_db_py3.cfg

@@ -0,0 +1,15 @@
+# test_db.cfg
+#
+# This file contains a list of connection strings which will be used by
+# database tests. Tests will be executed once for each string in this file.
+# You should be sure that the database used for the test doesn't contain any
+# important data. See README for more information.
+#
+# The string '__tmp__' is substituted for a temporary file in each connection
+# string. This is useful for sqlite tests.
+sqlite:///__tmp__
+#postgresql://openstack_citest:openstack_citest@localhost/openstack_citest
+#mysql://openstack_citest:openstack_citest@localhost/openstack_citest
+#oracle://scott:tiger@localhost
+#firebird://scott:tiger@localhost//var/lib/firebird/databases/test_migrate
+#ibm_db_sa://migrate:migrate@localhost:50000/migrate

tox.ini

@@ -15,40 +15,53 @@ commands =
 [testenv:py26]
 deps = sqlalchemy>=0.9
        -r{toxinidir}/test-requirements.txt
+       -r{toxinidir}/test-requirements-py2.txt
 
 [testenv:py27]
 deps = sqlalchemy>=0.9
        -r{toxinidir}/test-requirements.txt
+       -r{toxinidir}/test-requirements-py2.txt
 
 [testenv:py26sa07]
 basepython = python2.6
 deps = sqlalchemy>=0.7,<=0.7.99
        -r{toxinidir}/test-requirements.txt
+       -r{toxinidir}/test-requirements-py2.txt
 
 [testenv:py26sa08]
 basepython = python2.6
 deps = sqlalchemy>=0.8,<=0.8.99
        -r{toxinidir}/test-requirements.txt
+       -r{toxinidir}/test-requirements-py2.txt
 
 [testenv:py26sa09]
 basepython = python2.6
 deps = sqlalchemy>=0.9,<=0.9.99
        -r{toxinidir}/test-requirements.txt
+       -r{toxinidir}/test-requirements-py2.txt
 
 [testenv:py27sa07]
 basepython = python2.7
 deps = sqlalchemy>=0.7,<=0.7.99
        -r{toxinidir}/test-requirements.txt
+       -r{toxinidir}/test-requirements-py2.txt
 
 [testenv:py27sa08]
 basepython = python2.7
 deps = sqlalchemy>=0.8,<=0.8.99
        -r{toxinidir}/test-requirements.txt
+       -r{toxinidir}/test-requirements-py2.txt
 
 [testenv:py27sa09]
 basepython = python2.7
 deps = sqlalchemy>=0.9,<=0.9.99
        -r{toxinidir}/test-requirements.txt
+       -r{toxinidir}/test-requirements-py2.txt
+
+[testenv:py33]
+deps = sqlalchemy>=0.9
+       -r{toxinidir}/test-requirements.txt
+       -r{toxinidir}/test-requirements-py3.txt
 
 [testenv:pep8]
 commands = flake8