From 5289c4df3bb18ff9f1da09faa812fcfced2eae8f Mon Sep 17 00:00:00 2001 From: "jan.dittberner" Date: Sun, 25 Jan 2009 21:08:33 +0000 Subject: [PATCH] migrate.versioning PEP-8 improvements, more documentation - made api.py, cfgparse.py, exceptions.py, genmodel.py, migrate_repository.py, and pathed.py PEP-8 clean - add tools.rst documenting the usage of migrate_repository.py - add more modules to api.rst - reference tools.rst from index.rst --- docs/api.rst | 27 +++ docs/index.rst | 1 + docs/tools.rst | 19 ++ migrate/versioning/__init__.py | 8 +- migrate/versioning/api.py | 234 ++++++++++++++--------- migrate/versioning/cfgparse.py | 23 ++- migrate/versioning/exceptions.py | 79 ++++++-- migrate/versioning/genmodel.py | 142 ++++++++------ migrate/versioning/migrate_repository.py | 28 +-- migrate/versioning/pathed.py | 54 ++++-- 10 files changed, 406 insertions(+), 209 deletions(-) create mode 100644 docs/tools.rst diff --git a/docs/api.rst b/docs/api.rst index b5803d6..9d2a83a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -84,3 +84,30 @@ Module :mod:`migrate.versioning` .. automodule:: migrate.versioning :members: :synopsis: Database version and repository management + +Module :mod:`api ` +------------------------------------------ + +.. automodule:: migrate.versioning.api + :synopsis: External API for :mod:`migrate.versioning` + +Module :mod:`exceptions ` +-------------------------------------------------------- + +.. automodule:: migrate.versioning.exceptions + :members: + :synopsis: Exception classes for :mod:`migrate.versioning` + +Module :mod:`genmodel ` +---------------------------------------------------- + +.. automodule:: migrate.versioning.genmodel + :members: + :synopsis: Python database model generator and differencer + +Module :mod:`pathed ` +------------------------------------------------ + +.. automodule:: migrate.versioning.pathed + :members: + :synopsis: File/Directory handling class diff --git a/docs/index.rst b/docs/index.rst index 3a88da6..f7c4b5a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,6 +28,7 @@ versioning API is available as the :command:`migrate` command. versioning changeset + tools api .. _`google's summer of code`: http://code.google.com/soc diff --git a/docs/tools.rst b/docs/tools.rst new file mode 100644 index 0000000..db6e1de --- /dev/null +++ b/docs/tools.rst @@ -0,0 +1,19 @@ +SQLAlchemy migrate tools +======================== + +The most commonly used tool is the :command:`migrate` script that is +discussed in depth in the :ref:`Database schema versioning +` part of the documentation. + +.. index:: repository migration + +There is a second tool :command:`migrate_repository.py` that may be +used to migrate your repository from a version before 0.4.5 of +SQLAlchemy migrate to the current version. + +.. module:: migrate.versioning.migrate_repository + :synopsis: Tool for migrating pre 0.4.5 repositories to current layout + +Running :command:`migrate_repository.py` is as easy as: + + :samp:`migrate_repository.py {repository_directory}` diff --git a/migrate/versioning/__init__.py b/migrate/versioning/__init__.py index 810ee92..8b5a736 100644 --- a/migrate/versioning/__init__.py +++ b/migrate/versioning/__init__.py @@ -1,7 +1,5 @@ """ -Module migrate.versioning -------------------------- - -This package provides functionality to create and manage repositories of -database schema changesets and to apply these changesets to databases. + This package provides functionality to create and manage + repositories of database schema changesets and to apply these + changesets to databases. """ diff --git a/migrate/versioning/api.py b/migrate/versioning/api.py index fe44871..9802bee 100644 --- a/migrate/versioning/api.py +++ b/migrate/versioning/api.py @@ -1,10 +1,21 @@ -"""An external API to the versioning system -Used by the shell utility; could also be used by other scripts """ + This module provides an external API to the versioning system. + + Used by the shell utility; could also be used by other scripts +""" +# Dear migrate developers, +# +# please do not comment this module using sphinx syntax because its +# docstrings are presented as user help and most users cannot +# interpret sphinx annotated ReStructuredText. +# +# Thanks, +# Jan Dittberner + import sys import inspect from sqlalchemy import create_engine -from migrate.versioning import exceptions,repository,schema,version +from migrate.versioning import exceptions, repository, schema, version import script as script_ #command name conflict __all__=[ @@ -32,7 +43,8 @@ cls_schema = schema.ControlledSchema cls_vernum = version.VerNum cls_script_python = script_.PythonScript -def help(cmd=None,**opts): + +def help(cmd=None, **opts): """%prog help COMMAND Displays help on a given command. @@ -42,57 +54,74 @@ def help(cmd=None,**opts): try: func = globals()[cmd] except: - raise exceptions.UsageError("'%s' isn't a valid command. Try 'help COMMAND'"%cmd) + raise exceptions.UsageError( + "'%s' isn't a valid command. Try 'help COMMAND'" % cmd) ret = func.__doc__ if sys.argv[0]: - ret = ret.replace('%prog',sys.argv[0]) + ret = ret.replace('%prog', sys.argv[0]) return ret -def create(repository,name,**opts): + +def create(repository, name, **opts): """%prog create REPOSITORY_PATH NAME [--table=TABLE] Create an empty repository at the specified path. - You can specify the version_table to be used; by default, it is '_version'. - This table is created in all version-controlled databases. + You can specify the version_table to be used; by default, it is + '_version'. This table is created in all version-controlled + databases. """ try: - rep=cls_repository.create(repository,name,**opts) - except exceptions.PathFoundError,e: - raise exceptions.KnownError("The path %s already exists"%e.args[0]) + rep=cls_repository.create(repository, name, **opts) + except exceptions.PathFoundError, e: + raise exceptions.KnownError("The path %s already exists" % e.args[0]) -def script(description,repository=None,**opts): + +def script(description, repository=None, **opts): """%prog script [--repository=REPOSITORY_PATH] DESCRIPTION - Create an empty change script using the next unused version number appended with the given description. - For instance, manage.py script "Add initial tables" creates: repository/versions/001_Add_initial_tables.py + Create an empty change script using the next unused version number + appended with the given description. + + For instance, manage.py script "Add initial tables" creates: + repository/versions/001_Add_initial_tables.py """ try: if repository is None: raise exceptions.UsageError("A repository must be specified") repos = cls_repository(repository) - repos.create_script(description,**opts) - except exceptions.PathFoundError,e: + repos.create_script(description, **opts) + except exceptions.PathFoundError, e: raise exceptions.KnownError("The path %s already exists"%e.args[0]) -def script_sql(database,repository=None,**opts): + +def script_sql(database, repository=None, **opts): """%prog script_sql [--repository=REPOSITORY_PATH] DATABASE - Create empty change SQL scripts for given DATABASE, where DATABASE is either specific ('postgres', 'mysql', - 'oracle', 'sqlite', etc.) or generic ('default'). + Create empty change SQL scripts for given DATABASE, where DATABASE + is either specific ('postgres', 'mysql', 'oracle', 'sqlite', etc.) + or generic ('default'). + For instance, manage.py script_sql postgres creates: - repository/versions/001_upgrade_postgres.py and repository/versions/001_downgrade_postgres.py + repository/versions/001_upgrade_postgres.py and + repository/versions/001_downgrade_postgres.py """ try: if repository is None: raise exceptions.UsageError("A repository must be specified") repos = cls_repository(repository) - repos.create_script_sql(database,**opts) - except exceptions.PathFoundError,e: + repos.create_script_sql(database, **opts) + except exceptions.PathFoundError, e: raise exceptions.KnownError("The path %s already exists"%e.args[0]) -def test(repository,url=None,**opts): + +def test(repository, url=None, **opts): """%prog test REPOSITORY_PATH URL [VERSION] + + Performs the upgrade and downgrade option on the given + database. This is not a real test and may leave the database in a + bad state. You should therefore better run the test on a copy of + your database. """ engine=create_engine(url) repos=cls_repository(repository) @@ -100,7 +129,7 @@ def test(repository,url=None,**opts): # Upgrade print "Upgrading...", try: - script.run(engine,1) + script.run(engine, 1) except: print "ERROR" raise @@ -108,14 +137,15 @@ def test(repository,url=None,**opts): print "Downgrading...", try: - script.run(engine,-1) + script.run(engine, -1) except: print "ERROR" raise print "done" print "Success" -def version(repository,**opts): + +def version(repository, **opts): """%prog version REPOSITORY_PATH Display the latest version available in a repository. @@ -123,111 +153,124 @@ def version(repository,**opts): repos=cls_repository(repository) return repos.latest -def source(version,dest=None,repository=None,**opts): + +def source(version, dest=None, repository=None, **opts): """%prog source VERSION [DESTINATION] --repository=REPOSITORY_PATH - Display the Python code for a particular version in this repository. - Save it to the file at DESTINATION or, if omitted, send to stdout. + Display the Python code for a particular version in this + repository. Save it to the file at DESTINATION or, if omitted, + send to stdout. """ if repository is None: raise exceptions.UsageError("A repository must be specified") repos=cls_repository(repository) ret=repos.version(version).script().source() if dest is not None: - dest=open(dest,'w') + dest=open(dest, 'w') dest.write(ret) ret=None return ret -def version_control(url,repository,version=None,**opts): + +def version_control(url, repository, version=None, **opts): """%prog version_control URL REPOSITORY_PATH [VERSION] Mark a database as under this repository's version control. - Once a database is under version control, schema changes should only be - done via change scripts in this repository. + + Once a database is under version control, schema changes should + only be done via change scripts in this repository. This creates the table version_table in the database. The url should be any valid SQLAlchemy connection string. - By default, the database begins at version 0 and is assumed to be empty. - If the database is not empty, you may specify a version at which to begin - instead. No attempt is made to verify this version's correctness - the - database schema is expected to be identical to what it would be if the - database were created from scratch. + By default, the database begins at version 0 and is assumed to be + empty. If the database is not empty, you may specify a version at + which to begin instead. No attempt is made to verify this + version's correctness - the database schema is expected to be + identical to what it would be if the database were created from + scratch. """ echo = 'True' == opts.get('echo', False) engine = create_engine(url, echo=echo) - cls_schema.create(engine,repository,version) + cls_schema.create(engine, repository, version) -def db_version(url,repository,**opts): + +def db_version(url, repository, **opts): """%prog db_version URL REPOSITORY_PATH - Show the current version of the repository with the given connection - string, under version control of the specified repository. + Show the current version of the repository with the given + connection string, under version control of the specified + repository. The url should be any valid SQLAlchemy connection string. """ echo = 'True' == opts.get('echo', False) engine = create_engine(url, echo=echo) - schema = cls_schema(engine,repository) + schema = cls_schema(engine, repository) return schema.version -def upgrade(url,repository,version=None,**opts): + +def upgrade(url, repository, version=None, **opts): """%prog upgrade URL REPOSITORY_PATH [VERSION] [--preview_py|--preview_sql] - Upgrade a database to a later version. + Upgrade a database to a later version. + This runs the upgrade() function defined in your change scripts. - By default, the database is updated to the latest available version. You - may specify a version instead, if you wish. + By default, the database is updated to the latest available + version. You may specify a version instead, if you wish. - You may preview the Python or SQL code to be executed, rather than actually - executing it, using the appropriate 'preview' option. + You may preview the Python or SQL code to be executed, rather than + actually executing it, using the appropriate 'preview' option. """ err = "Cannot upgrade a database of version %s to version %s. "\ "Try 'downgrade' instead." - return _migrate(url,repository,version,upgrade=True,err=err,**opts) + return _migrate(url, repository, version, upgrade=True, err=err, **opts) -def downgrade(url,repository,version,**opts): + +def downgrade(url, repository, version, **opts): """%prog downgrade URL REPOSITORY_PATH VERSION [--preview_py|--preview_sql] Downgrade a database to an earlier version. - This is the reverse of upgrade; this runs the downgrade() function defined - in your change scripts. - You may preview the Python or SQL code to be executed, rather than actually - executing it, using the appropriate 'preview' option. + This is the reverse of upgrade; this runs the downgrade() function + defined in your change scripts. + + You may preview the Python or SQL code to be executed, rather than + actually executing it, using the appropriate 'preview' option. """ err = "Cannot downgrade a database of version %s to version %s. "\ "Try 'upgrade' instead." - return _migrate(url,repository,version,upgrade=False,err=err,**opts) + return _migrate(url, repository, version, upgrade=False, err=err, **opts) -def _migrate(url,repository,version,upgrade,err,**opts): + +def _migrate(url, repository, version, upgrade, err, **opts): echo = 'True' == opts.get('echo', False) engine = create_engine(url, echo=echo) - schema = cls_schema(engine,repository) - version = _migrate_version(schema,version,upgrade,err) + schema = cls_schema(engine, repository) + version = _migrate_version(schema, version, upgrade, err) changeset = schema.changeset(version) - for ver,change in changeset: + for ver, change in changeset: nextver = ver + changeset.step - print '%s -> %s... '%(ver,nextver), + print '%s -> %s... '%(ver, nextver), if opts.get('preview_sql'): print print change.log elif opts.get('preview_py'): - source_ver = max(ver,nextver) + source_ver = max(ver, nextver) module = schema.repository.version(source_ver).script().module funcname = upgrade and "upgrade" or "downgrade" - func = getattr(module,funcname) + func = getattr(module, funcname) print print inspect.getsource(module.upgrade) else: - schema.runchange(ver,change,changeset.step) + schema.runchange(ver, change, changeset.step) print 'done' -def _migrate_version(schema,version,upgrade,err): + +def _migrate_version(schema, version, upgrade, err): if version is None: return version # Version is specified: ensure we're upgrading in the right direction @@ -240,48 +283,54 @@ def _migrate_version(schema,version,upgrade,err): else: direction = cur >= version if not direction: - raise exceptions.KnownError(err%(cur,version)) + raise exceptions.KnownError(err%(cur, version)) return version -def drop_version_control(url,repository,**opts): + +def drop_version_control(url, repository, **opts): """%prog drop_version_control URL REPOSITORY_PATH Removes version control from a database. """ echo = 'True' == opts.get('echo', False) engine = create_engine(url, echo=echo) - schema=cls_schema(engine,repository) + schema=cls_schema(engine, repository) schema.drop() -def manage(file,**opts): + +def manage(file, **opts): """%prog manage FILENAME VARIABLES... - Creates a script that runs Migrate with a set of default values. + Creates a script that runs Migrate with a set of default values. For example:: - %prog manage manage.py --repository=/path/to/repository --url=sqlite:///project.db + %prog manage manage.py --repository=/path/to/repository \ +--url=sqlite:///project.db - would create the script manage.py. The following two commands would then - have exactly the same results:: + would create the script manage.py. The following two commands + would then have exactly the same results:: python manage.py version %prog version --repository=/path/to/repository """ - return repository.manage(file,**opts) + return repository.manage(file, **opts) -def compare_model_to_db(url,model,repository,**opts): + +def compare_model_to_db(url, model, repository, **opts): """%prog compare_model_to_db URL MODEL REPOSITORY_PATH - Compare the current model (assumed to be a module level variable of type sqlalchemy.MetaData) against the current database. + Compare the current model (assumed to be a module level variable + of type sqlalchemy.MetaData) against the current database. NOTE: This is EXPERIMENTAL. """ # TODO: get rid of EXPERIMENTAL label echo = 'True' == opts.get('echo', False) engine = create_engine(url, echo=echo) - print cls_schema.compare_model_to_db(engine,model,repository) + print cls_schema.compare_model_to_db(engine, model, repository) -def create_model(url,repository,**opts): + +def create_model(url, repository, **opts): """%prog create_model URL REPOSITORY_PATH Dump the current database as a Python model to stdout. @@ -291,32 +340,37 @@ def create_model(url,repository,**opts): echo = 'True' == opts.get('echo', False) engine = create_engine(url, echo=echo) declarative = opts.get('declarative', False) - print cls_schema.create_model(engine,repository,declarative) + print cls_schema.create_model(engine, repository, declarative) -def make_update_script_for_model(url,oldmodel,model,repository,**opts): + +def make_update_script_for_model(url, oldmodel, model, repository, **opts): """%prog make_update_script_for_model URL OLDMODEL MODEL REPOSITORY_PATH - Create a script changing the old Python model to the new (current) Python model, sending to stdout. + Create a script changing the old Python model to the new (current) + Python model, sending to stdout. NOTE: This is EXPERIMENTAL. """ # TODO: get rid of EXPERIMENTAL label echo = 'True' == opts.get('echo', False) engine = create_engine(url, echo=echo) try: - print cls_script_python.make_update_script_for_model(engine,oldmodel,model,repository,**opts) - except exceptions.PathFoundError,e: - raise exceptions.KnownError("The path %s already exists"%e.args[0]) # TODO: get rid of this? if we don't add back path param + print cls_script_python.make_update_script_for_model( + engine, oldmodel, model, repository, **opts) + except exceptions.PathFoundError, e: + # TODO: get rid of this? if we don't add back path param + raise exceptions.KnownError("The path %s already exists" % e.args[0]) -def update_db_from_model(url,model,repository,**opts): + +def update_db_from_model(url, model, repository, **opts): """%prog update_db_from_model URL MODEL REPOSITORY_PATH - Modify the database to match the structure of the current Python model. - This also sets the db_version number to the latest in the repository. + Modify the database to match the structure of the current Python + model. This also sets the db_version number to the latest in the + repository. NOTE: This is EXPERIMENTAL. """ # TODO: get rid of EXPERIMENTAL label echo = 'True' == opts.get('echo', False) engine = create_engine(url, echo=echo) - schema = cls_schema(engine,repository) + schema = cls_schema(engine, repository) schema.update_db_from_model(model) - diff --git a/migrate/versioning/cfgparse.py b/migrate/versioning/cfgparse.py index 9c850b4..e8f9a85 100644 --- a/migrate/versioning/cfgparse.py +++ b/migrate/versioning/cfgparse.py @@ -1,19 +1,26 @@ +""" + Configuration parser module. +""" + from migrate.versioning.base import * from migrate.versioning import pathed from ConfigParser import ConfigParser -#__all__=['MigrateConfigParser'] class Parser(ConfigParser): - """A project configuration file""" - def to_dict(self,sections=None): + """A project configuration file.""" + + def to_dict(self, sections=None): """It's easier to access config values like dictionaries""" return self._sections -class Config(pathed.Pathed,Parser): - def __init__(self,path,*p,**k): - """Confirm the config file exists; read it""" + +class Config(pathed.Pathed, Parser): + """Configuration class.""" + + def __init__(self, path, *p, **k): + """Confirm the config file exists; read it.""" self.require_found(path) - pathed.Pathed.__init__(self,path) - Parser.__init__(self,*p,**k) + pathed.Pathed.__init__(self, path) + Parser.__init__(self, *p, **k) self.read(path) diff --git a/migrate/versioning/exceptions.py b/migrate/versioning/exceptions.py index 35b7144..6cbf309 100644 --- a/migrate/versioning/exceptions.py +++ b/migrate/versioning/exceptions.py @@ -1,32 +1,60 @@ +""" + Provide exception classes for :mod:`migrate.versioning` +""" + class Error(Exception): + """Error base class.""" pass + + class ApiError(Error): + """Base class for API errors.""" pass + + class KnownError(ApiError): - """A known error condition""" + """A known error condition.""" + + class UsageError(ApiError): - """A known error condition where help should be displayed""" + """A known error condition where help should be displayed.""" + class ControlledSchemaError(Error): - pass -class InvalidVersionError(ControlledSchemaError): - """Invalid version number""" -class DatabaseNotControlledError(ControlledSchemaError): - """Database shouldn't be under vc, but it is""" -class DatabaseAlreadyControlledError(ControlledSchemaError): - """Database should be under vc, but it's not""" -class WrongRepositoryError(ControlledSchemaError): - """This database is under version control by another repository""" -class NoSuchTableError(ControlledSchemaError): + """Base class for controlled schema errors.""" pass + +class InvalidVersionError(ControlledSchemaError): + """Invalid version number.""" + + +class DatabaseNotControlledError(ControlledSchemaError): + """Database should be under version control, but it's not.""" + + +class DatabaseAlreadyControlledError(ControlledSchemaError): + """Database shouldn't be under version control, but it is""" + + +class WrongRepositoryError(ControlledSchemaError): + """This database is under version control by another repository.""" + + +class NoSuchTableError(ControlledSchemaError): + """The table does not exist.""" + pass + + class LogSqlError(Error): - """A SQLError, with a traceback of where that statement was logged""" - def __init__(self,sqlerror,entry): + """A SQLError, with a traceback of where that statement was logged.""" + + def __init__(self, sqlerror, entry): Exception.__init__(self) self.sqlerror = sqlerror self.entry = entry + def __str__(self): ret = "SQL error in statement: \n%s\n"%(str(self.entry)) ret += "Traceback from change script:\n" @@ -34,25 +62,42 @@ class LogSqlError(Error): ret += str(self.sqlerror) return ret + class PathError(Error): + """Base class for path errors.""" pass + + class PathNotFoundError(PathError): - """A path with no file was required; found a file""" + """A path with no file was required; found a file.""" pass + + class PathFoundError(PathError): - """A path with a file was required; found no file""" + """A path with a file was required; found no file.""" pass + class RepositoryError(Error): + """Base class for repository errors.""" pass + + class InvalidRepositoryError(RepositoryError): + """Invalid repository error.""" pass + class ScriptError(Error): + """Base class for script errors.""" pass + + class InvalidScriptError(ScriptError): + """Invalid script error.""" pass + class InvalidVersionError(Error): + """Invalid version error.""" pass - diff --git a/migrate/versioning/genmodel.py b/migrate/versioning/genmodel.py index 384025a..b62cb65 100644 --- a/migrate/versioning/genmodel.py +++ b/migrate/versioning/genmodel.py @@ -1,9 +1,14 @@ +""" + Code to generate a Python model from a database or differences + between a model and database. -# Code to generate a Python model from a database or differences between a model and database. -# Some of this is borrowed heavily from the AutoCode project at: http://code.google.com/p/sqlautocode/ + Some of this is borrowed heavily from the AutoCode project at: + http://code.google.com/p/sqlautocode/ +""" import sys -import migrate, sqlalchemy +import migrate +import sqlalchemy HEADER = """ @@ -22,61 +27,73 @@ from sqlalchemy.ext import declarative Base = declarative.declarative_base() """ + class ModelGenerator(object): def __init__(self, diff, declarative=False): self.diff = diff self.declarative = declarative - dialectModule = sys.modules[self.diff.conn.dialect.__module__] # is there an easier way to get this? - self.colTypeMappings = dict( (v,k) for k,v in dialectModule.colspecs.items() ) - + # is there an easier way to get this? + dialectModule = sys.modules[self.diff.conn.dialect.__module__] + self.colTypeMappings = dict((v, k) for k, v in \ + dialectModule.colspecs.items()) + def column_repr(self, col): kwarg = [] - if col.key != col.name: kwarg.append('key') + if col.key != col.name: + kwarg.append('key') if col.primary_key: col.primary_key = True # otherwise it dumps it as 1 kwarg.append('primary_key') - if not col.nullable: kwarg.append('nullable') - if col.onupdate: kwarg.append('onupdate') + if not col.nullable: + kwarg.append('nullable') + if col.onupdate: + kwarg.append('onupdate') if col.default: if col.primary_key: - # I found that Postgres automatically creates a default value for the sequence, but let's not show that. + # I found that PostgreSQL automatically creates a + # default value for the sequence, but let's not show + # that. pass else: kwarg.append('default') - ks = ', '.join('%s=%r' % (k, getattr(col, k)) for k in kwarg ) - - name = col.name.encode('utf8') # crs: not sure if this is good idea, but it gets rid of extra u'' + ks = ', '.join('%s=%r' % (k, getattr(col, k)) for k in kwarg) + + # crs: not sure if this is good idea, but it gets rid of extra + # u'' + name = col.name.encode('utf8') type = self.colTypeMappings.get(col.type.__class__, None) if type: # Make the column type be an instance of this type. type = type() else: - # We must already be a model type, no need to map from the database-specific types. + # We must already be a model type, no need to map from the + # database-specific types. type = col.type - data = {'name' : name, - 'type' : type, - 'constraints' : ', '.join([repr(cn) for cn in col.constraints]), - 'args' : ks and ks or '' - } + data = { + 'name': name, + 'type': type, + 'constraints': ', '.join([repr(cn) for cn in col.constraints]), + 'args': ks and ks or ''} if data['constraints']: - if data['args']: data['args'] = ',' + data['args'] - + if data['args']: + data['args'] = ',' + data['args'] + if data['constraints'] or data['args']: data['maybeComma'] = ',' else: data['maybeComma'] = '' - commonStuff = " %(maybeComma)s %(constraints)s %(args)s)""" % data + commonStuff = """ %(maybeComma)s %(constraints)s %(args)s)""" % data commonStuff = commonStuff.strip() data['commonStuff'] = commonStuff if self.declarative: return """%(name)s = Column(%(type)r%(commonStuff)s""" % data else: return """Column(%(name)r, %(type)r%(commonStuff)s""" % data - + def getTableDefn(self, table): out = [] tableName = table.name @@ -86,14 +103,15 @@ class ModelGenerator(object): for col in table.columns: out.append(" %s" % self.column_repr(col)) else: - out.append("%(table)s = Table('%(table)s', meta," % {'table': tableName}) + out.append("%(table)s = Table('%(table)s', meta," % \ + {'table': tableName}) for col in table.columns: out.append(" %s," % self.column_repr(col)) out.append(")") return out - + def toPython(self): - ''' Assume database is current and model is empty. ''' + """Assume database is current and model is empty.""" out = [] if self.declarative: out.append(DECLARATIVE_HEADER) @@ -104,44 +122,46 @@ class ModelGenerator(object): out.extend(self.getTableDefn(table)) out.append("") return '\n'.join(out) - + def toUpgradeDowngradePython(self, indent=' '): ''' Assume model is most current and database is out-of-date. ''' - + decls = ['meta = MetaData(migrate_engine)'] - for table in self.diff.tablesMissingInModel + self.diff.tablesMissingInDatabase: + for table in self.diff.tablesMissingInModel + \ + self.diff.tablesMissingInDatabase: decls.extend(self.getTableDefn(table)) - + upgradeCommands, downgradeCommands = [], [] for table in self.diff.tablesMissingInModel: tableName = table.name upgradeCommands.append("%(table)s.drop()" % {'table': tableName}) - downgradeCommands.append("%(table)s.create()" % {'table': tableName}) + downgradeCommands.append("%(table)s.create()" % \ + {'table': tableName}) for table in self.diff.tablesMissingInDatabase: tableName = table.name upgradeCommands.append("%(table)s.create()" % {'table': tableName}) downgradeCommands.append("%(table)s.drop()" % {'table': tableName}) - - return ('\n'.join(decls), - '\n'.join(['%s%s' % (indent, line) for line in upgradeCommands]), - '\n'.join(['%s%s' % (indent, line) for line in downgradeCommands]) - ) - + + return ( + '\n'.join(decls), + '\n'.join(['%s%s' % (indent, line) for line in upgradeCommands]), + '\n'.join(['%s%s' % (indent, line) for line in downgradeCommands])) + def applyModel(self): - ''' Apply model to current database. ''' - - # Yuck! We have to import from changeset to apply the monkey-patch to allow column adding/dropping. + """Apply model to current database.""" + # Yuck! We have to import from changeset to apply the + # monkey-patch to allow column adding/dropping. from migrate.changeset import schema - + def dbCanHandleThisChange(missingInDatabase, missingInModel, diffDecl): if missingInDatabase and not missingInModel and not diffDecl: # Even sqlite can handle this. return True else: return not self.diff.conn.url.drivername.startswith('sqlite') - + meta = sqlalchemy.MetaData(self.diff.conn.engine) - + for table in self.diff.tablesMissingInModel: table = table.tometadata(meta) table.drop() @@ -152,8 +172,10 @@ class ModelGenerator(object): modelTable = modelTable.tometadata(meta) dbTable = self.diff.reflected_model.tables[modelTable.name] tableName = modelTable.name - missingInDatabase, missingInModel, diffDecl = self.diff.colDiffs[tableName] - if dbCanHandleThisChange(missingInDatabase, missingInModel, diffDecl): + missingInDatabase, missingInModel, diffDecl = \ + self.diff.colDiffs[tableName] + if dbCanHandleThisChange(missingInDatabase, missingInModel, + diffDecl): for col in missingInDatabase: modelTable.columns[col.name].create() for col in missingInModel: @@ -161,25 +183,34 @@ class ModelGenerator(object): for modelCol, databaseCol, modelDecl, databaseDecl in diffDecl: databaseCol.alter(modelCol) else: - # Sqlite doesn't support drop column, so you have to do more: - # create temp table, copy data to it, drop old table, create new table, copy data back. - - tempName = '_temp_%s' % modelTable.name # I wonder if this is guaranteed to be unique? + # Sqlite doesn't support drop column, so you have to + # do more: create temp table, copy data to it, drop + # old table, create new table, copy data back. + # + # I wonder if this is guaranteed to be unique? + tempName = '_temp_%s' % modelTable.name + def getCopyStatement(): preparer = self.diff.conn.engine.dialect.preparer commonCols = [] for modelCol in modelTable.columns: - if dbTable.columns.has_key(modelCol.name): + if modelCol.name in dbTable.columns: commonCols.append(modelCol.name) commonColsStr = ', '.join(commonCols) - return 'INSERT INTO %s (%s) SELECT %s FROM %s' % (tableName, commonColsStr, commonColsStr, tempName) - - # Move the data in one transaction, so that we don't leave the database in a nasty state. + return 'INSERT INTO %s (%s) SELECT %s FROM %s' % \ + (tableName, commonColsStr, commonColsStr, tempName) + + # Move the data in one transaction, so that we don't + # leave the database in a nasty state. connection = self.diff.conn.connect() trans = connection.begin() try: - connection.execute('CREATE TEMPORARY TABLE %s as SELECT * from %s' % (tempName, modelTable.name)) - modelTable.drop(bind=connection) # make sure the drop takes place inside our transaction with the bind parameter + connection.execute( + 'CREATE TEMPORARY TABLE %s as SELECT * from %s' % \ + (tempName, modelTable.name)) + # make sure the drop takes place inside our + # transaction with the bind parameter + modelTable.drop(bind=connection) modelTable.create(bind=connection) connection.execute(getCopyStatement()) connection.execute('DROP TABLE %s' % tempName) @@ -187,4 +218,3 @@ class ModelGenerator(object): except: trans.rollback() raise - diff --git a/migrate/versioning/migrate_repository.py b/migrate/versioning/migrate_repository.py index 7ab1e3c..acc728e 100644 --- a/migrate/versioning/migrate_repository.py +++ b/migrate/versioning/migrate_repository.py @@ -1,17 +1,22 @@ -""" Script to migrate repository. This shouldn't use any other migrate -modules, so that it can work in any version. """ +""" + Script to migrate repository from sqlalchemy <= 0.4.4 to the new + repository schema. This shouldn't use any other migrate modules, so + that it can work in any version. +""" -import os, os.path, sys +import os +import os.path +import sys def usage(): """Gives usage information.""" - print '''Usage: %(prog)s repository-to-migrate + print """Usage: %(prog)s repository-to-migrate -Upgrade your repository to the new flat format. + Upgrade your repository to the new flat format. -NOTE: You should probably make a backup before running this. -''' % {'prog': sys.argv[0]} + NOTE: You should probably make a backup before running this. + """ % {'prog': sys.argv[0]} sys.exit(1) @@ -27,7 +32,8 @@ def move_file(src, tgt): print ' Moving file %s to %s' % (src, tgt) if os.path.exists(tgt): raise Exception( - 'Cannot move file %s because target %s already exists' % (src, tgt)) + 'Cannot move file %s because target %s already exists' % \ + (src, tgt)) os.rename(src, tgt) @@ -35,7 +41,7 @@ def delete_directory(dirpath): """Delete a directory and print a message.""" print ' Deleting directory: %s' % dirpath os.rmdir(dirpath) - + def migrate_repository(repos): """Does the actual migration to the new repository format.""" @@ -43,7 +49,7 @@ def migrate_repository(repos): versions = '%s/versions' % repos dirs = os.listdir(versions) # Only use int's in list. - numdirs = [ int(dirname) for dirname in dirs if dirname.isdigit() ] + numdirs = [int(dirname) for dirname in dirs if dirname.isdigit()] numdirs.sort() # Sort list. for dirname in numdirs: origdir = '%s/%s' % (versions, dirname) @@ -51,7 +57,6 @@ def migrate_repository(repos): files = os.listdir(origdir) files.sort() for filename in files: - # Delete compiled Python files. if filename.endswith('.pyc') or filename.endswith('.pyo'): delete_file('%s/%s' % (origdir, filename)) @@ -91,4 +96,3 @@ def main(): if __name__ == '__main__': main() - diff --git a/migrate/versioning/pathed.py b/migrate/versioning/pathed.py index 15242dd..a316d90 100644 --- a/migrate/versioning/pathed.py +++ b/migrate/versioning/pathed.py @@ -1,60 +1,72 @@ +""" + A path/directory class. +""" + from migrate.versioning.base import * from migrate.versioning.util import KeyedInstance -import os,shutil from migrate.versioning import exceptions +import os +import shutil + class Pathed(KeyedInstance): - """A class associated with a path/directory tree - Only one instance of this class may exist for a particular file; + """ + A class associated with a path/directory tree. + + Only one instance of this class may exist for a particular file; __new__ will return an existing instance if possible """ parent=None @classmethod - def _key(cls,path): + def _key(cls, path): return str(path) - def __init__(self,path): + def __init__(self, path): self.path=path if self.__class__.parent is not None: self._init_parent(path) - def _init_parent(self,path): + def _init_parent(self, path): """Try to initialize this object's parent, if it has one""" parent_path=self.__class__._parent_path(path) self.parent=self.__class__.parent(parent_path) - log.info("Getting parent %r:%r"%(self.__class__.parent,parent_path)) - self.parent._init_child(path,self) - - def _init_child(self,child,path): - """Run when a child of this object is initialized - Parameters: the child object; the path to this object (its parent) + log.info("Getting parent %r:%r" % (self.__class__.parent, parent_path)) + self.parent._init_child(path, self) + + def _init_child(self, child, path): + """Run when a child of this object is initialized. + + Parameters: the child object; the path to this object (its + parent) """ pass - + @classmethod - def _parent_path(cls,path): - """Fetch the path of this object's parent from this object's path + def _parent_path(cls, path): """ - # os.path.dirname(), but strip directories like files (like unix basename) + Fetch the path of this object's parent from this object's path. + """ + # os.path.dirname(), but strip directories like files (like + # unix basename) + # # Treat directories like files... - if path[-1]=='/': + if path[-1] == '/': path=path[:-1] ret = os.path.dirname(path) return ret @classmethod - def require_notfound(cls,path): + def require_notfound(cls, path): """Ensures a given path does not already exist""" if os.path.exists(path): raise exceptions.PathFoundError(path) - + @classmethod - def require_found(cls,path): + def require_found(cls, path): """Ensures a given path already exists""" if not os.path.exists(path): raise exceptions.PathNotFoundError(path) def __str__(self): return self.path -