x/sqlalchemy-migrate
Code Issues Proposed changes

migrate.versioning PEP-8 improvements, more documentation

Browse Source 
- made api.py, cfgparse.py, exceptions.py, genmodel.py,
  migrate_repository.py, and pathed.py PEP-8 clean
- add tools.rst documenting the usage of migrate_repository.py
- add more modules to api.rst
- reference tools.rst from index.rst
This commit is contained in:
jan.dittberner 2009-01-25 21:08:33 +00:00
parent 5be16f226f
commit 5289c4df3b
10 changed files with 406 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
docs/api.rst
View File

@ -84,3 +84,30 @@ Module :mod:`migrate.versioning`
.. automodule:: migrate.versioning
:members:
:synopsis: Database version and repository management
Module :mod:`api <migrate.versioning.api>`
------------------------------------------
.. automodule:: migrate.versioning.api
:synopsis: External API for :mod:`migrate.versioning`
Module :mod:`exceptions <migrate.versioning.exceptions>`
--------------------------------------------------------
.. automodule:: migrate.versioning.exceptions
:members:
:synopsis: Exception classes for :mod:`migrate.versioning`
Module :mod:`genmodel <migrate.versioning.genmodel>`
----------------------------------------------------
.. automodule:: migrate.versioning.genmodel
:members:
:synopsis: Python database model generator and differencer
Module :mod:`pathed <migrate.versioning.pathed>`
------------------------------------------------
.. automodule:: migrate.versioning.pathed
:members:
:synopsis: File/Directory handling class

1
docs/index.rst
View File

@ -28,6 +28,7 @@ versioning API is available as the :command:`migrate` command.
versioning
changeset
tools
api
.. _`google's summer of code`: http://code.google.com/soc

19
docs/tools.rst Normal file
View File

@ -0,0 +1,19 @@
SQLAlchemy migrate tools
========================
The most commonly used tool is the :command:`migrate` script that is
discussed in depth in the :ref:`Database schema versioning
<versioning-system>` part of the documentation.
.. index:: repository migration
There is a second tool :command:`migrate_repository.py` that may be
used to migrate your repository from a version before 0.4.5 of
SQLAlchemy migrate to the current version.
.. module:: migrate.versioning.migrate_repository
:synopsis: Tool for migrating pre 0.4.5 repositories to current layout
Running :command:`migrate_repository.py` is as easy as:
:samp:`migrate_repository.py {repository_directory}`

8
migrate/versioning/__init__.py
View File

@ -1,7 +1,5 @@
"""
Module migrate.versioning
-------------------------
This package provides functionality to create and manage repositories of
database schema changesets and to apply these changesets to databases.
This package provides functionality to create and manage
repositories of database schema changesets and to apply these
changesets to databases.
"""

234
migrate/versioning/api.py
View File

@ -1,10 +1,21 @@
"""An external API to the versioning system
Used by the shell utility; could also be used by other scripts
"""
This module provides an external API to the versioning system.
Used by the shell utility; could also be used by other scripts
"""
# Dear migrate developers,
#
# please do not comment this module using sphinx syntax because its
# docstrings are presented as user help and most users cannot
# interpret sphinx annotated ReStructuredText.
#
# Thanks,
# Jan Dittberner
import sys
import inspect
from sqlalchemy import create_engine
from migrate.versioning import exceptions,repository,schema,version
from migrate.versioning import exceptions, repository, schema, version
import script as script_ #command name conflict
__all__=[
@ -32,7 +43,8 @@ cls_schema = schema.ControlledSchema
cls_vernum = version.VerNum
cls_script_python = script_.PythonScript
def help(cmd=None,**opts):
def help(cmd=None, **opts):
"""%prog help COMMAND
Displays help on a given command.
@ -42,57 +54,74 @@ def help(cmd=None,**opts):
try:
func = globals()[cmd]
except:
raise exceptions.UsageError("'%s' isn't a valid command. Try 'help COMMAND'"%cmd)
raise exceptions.UsageError(
"'%s' isn't a valid command. Try 'help COMMAND'" % cmd)
ret = func.__doc__
if sys.argv[0]:
ret = ret.replace('%prog',sys.argv[0])
ret = ret.replace('%prog', sys.argv[0])
return ret
def create(repository,name,**opts):
def create(repository, name, **opts):
"""%prog create REPOSITORY_PATH NAME [--table=TABLE]
Create an empty repository at the specified path.
You can specify the version_table to be used; by default, it is '_version'.
This table is created in all version-controlled databases.
You can specify the version_table to be used; by default, it is
'_version'. This table is created in all version-controlled
databases.
"""
try:
rep=cls_repository.create(repository,name,**opts)
except exceptions.PathFoundError,e:
raise exceptions.KnownError("The path %s already exists"%e.args[0])
rep=cls_repository.create(repository, name, **opts)
except exceptions.PathFoundError, e:
raise exceptions.KnownError("The path %s already exists" % e.args[0])
def script(description,repository=None,**opts):
def script(description, repository=None, **opts):
"""%prog script [--repository=REPOSITORY_PATH] DESCRIPTION
Create an empty change script using the next unused version number appended with the given description.
For instance, manage.py script "Add initial tables" creates: repository/versions/001_Add_initial_tables.py
Create an empty change script using the next unused version number
appended with the given description.
For instance, manage.py script "Add initial tables" creates:
repository/versions/001_Add_initial_tables.py
"""
try:
if repository is None:
raise exceptions.UsageError("A repository must be specified")
repos = cls_repository(repository)
repos.create_script(description,**opts)
except exceptions.PathFoundError,e:
repos.create_script(description, **opts)
except exceptions.PathFoundError, e:
raise exceptions.KnownError("The path %s already exists"%e.args[0])
def script_sql(database,repository=None,**opts):
def script_sql(database, repository=None, **opts):
"""%prog script_sql [--repository=REPOSITORY_PATH] DATABASE
Create empty change SQL scripts for given DATABASE, where DATABASE is either specific ('postgres', 'mysql',
'oracle', 'sqlite', etc.) or generic ('default').
Create empty change SQL scripts for given DATABASE, where DATABASE
is either specific ('postgres', 'mysql', 'oracle', 'sqlite', etc.)
or generic ('default').
For instance, manage.py script_sql postgres creates:
repository/versions/001_upgrade_postgres.py and repository/versions/001_downgrade_postgres.py
repository/versions/001_upgrade_postgres.py and
repository/versions/001_downgrade_postgres.py
"""
try:
if repository is None:
raise exceptions.UsageError("A repository must be specified")
repos = cls_repository(repository)
repos.create_script_sql(database,**opts)
except exceptions.PathFoundError,e:
repos.create_script_sql(database, **opts)
except exceptions.PathFoundError, e:
raise exceptions.KnownError("The path %s already exists"%e.args[0])
def test(repository,url=None,**opts):
def test(repository, url=None, **opts):
"""%prog test REPOSITORY_PATH URL [VERSION]
Performs the upgrade and downgrade option on the given
database. This is not a real test and may leave the database in a
bad state. You should therefore better run the test on a copy of
your database.
"""
engine=create_engine(url)
repos=cls_repository(repository)
@ -100,7 +129,7 @@ def test(repository,url=None,**opts):
# Upgrade
print "Upgrading...",
try:
script.run(engine,1)
script.run(engine, 1)
except:
print "ERROR"
raise
@ -108,14 +137,15 @@ def test(repository,url=None,**opts):
print "Downgrading...",
try:
script.run(engine,-1)
script.run(engine, -1)
except:
print "ERROR"
raise
print "done"
print "Success"
def version(repository,**opts):
def version(repository, **opts):
"""%prog version REPOSITORY_PATH
Display the latest version available in a repository.
@ -123,111 +153,124 @@ def version(repository,**opts):
repos=cls_repository(repository)
return repos.latest
def source(version,dest=None,repository=None,**opts):
def source(version, dest=None, repository=None, **opts):
"""%prog source VERSION [DESTINATION] --repository=REPOSITORY_PATH
Display the Python code for a particular version in this repository.
Save it to the file at DESTINATION or, if omitted, send to stdout.
Display the Python code for a particular version in this
repository. Save it to the file at DESTINATION or, if omitted,
send to stdout.
"""
if repository is None:
raise exceptions.UsageError("A repository must be specified")
repos=cls_repository(repository)
ret=repos.version(version).script().source()
if dest is not None:
dest=open(dest,'w')
dest=open(dest, 'w')
dest.write(ret)
ret=None
return ret
def version_control(url,repository,version=None,**opts):
def version_control(url, repository, version=None, **opts):
"""%prog version_control URL REPOSITORY_PATH [VERSION]
Mark a database as under this repository's version control.
Once a database is under version control, schema changes should only be
done via change scripts in this repository.
Once a database is under version control, schema changes should
only be done via change scripts in this repository.
This creates the table version_table in the database.
The url should be any valid SQLAlchemy connection string.
By default, the database begins at version 0 and is assumed to be empty.
If the database is not empty, you may specify a version at which to begin
instead. No attempt is made to verify this version's correctness - the
database schema is expected to be identical to what it would be if the
database were created from scratch.
By default, the database begins at version 0 and is assumed to be
empty. If the database is not empty, you may specify a version at
which to begin instead. No attempt is made to verify this
version's correctness - the database schema is expected to be
identical to what it would be if the database were created from
scratch.
"""
echo = 'True' == opts.get('echo', False)
engine = create_engine(url, echo=echo)
cls_schema.create(engine,repository,version)
cls_schema.create(engine, repository, version)
def db_version(url,repository,**opts):
def db_version(url, repository, **opts):
"""%prog db_version URL REPOSITORY_PATH
Show the current version of the repository with the given connection
string, under version control of the specified repository.
Show the current version of the repository with the given
connection string, under version control of the specified
repository.
The url should be any valid SQLAlchemy connection string.
"""
echo = 'True' == opts.get('echo', False)
engine = create_engine(url, echo=echo)
schema = cls_schema(engine,repository)
schema = cls_schema(engine, repository)
return schema.version
def upgrade(url,repository,version=None,**opts):
def upgrade(url, repository, version=None, **opts):
"""%prog upgrade URL REPOSITORY_PATH [VERSION] [--preview_py|--preview_sql]
Upgrade a database to a later version.
Upgrade a database to a later version.
This runs the upgrade() function defined in your change scripts.
By default, the database is updated to the latest available version. You
may specify a version instead, if you wish.
By default, the database is updated to the latest available
version. You may specify a version instead, if you wish.
You may preview the Python or SQL code to be executed, rather than actually
executing it, using the appropriate 'preview' option.
You may preview the Python or SQL code to be executed, rather than
actually executing it, using the appropriate 'preview' option.
"""
err = "Cannot upgrade a database of version %s to version %s. "\
"Try 'downgrade' instead."
return _migrate(url,repository,version,upgrade=True,err=err,**opts)
return _migrate(url, repository, version, upgrade=True, err=err, **opts)
def downgrade(url,repository,version,**opts):
def downgrade(url, repository, version, **opts):
"""%prog downgrade URL REPOSITORY_PATH VERSION [--preview_py|--preview_sql]
Downgrade a database to an earlier version.
This is the reverse of upgrade; this runs the downgrade() function defined
in your change scripts.
You may preview the Python or SQL code to be executed, rather than actually
executing it, using the appropriate 'preview' option.
This is the reverse of upgrade; this runs the downgrade() function
defined in your change scripts.
You may preview the Python or SQL code to be executed, rather than
actually executing it, using the appropriate 'preview' option.
"""
err = "Cannot downgrade a database of version %s to version %s. "\
"Try 'upgrade' instead."
return _migrate(url,repository,version,upgrade=False,err=err,**opts)
return _migrate(url, repository, version, upgrade=False, err=err, **opts)
def _migrate(url,repository,version,upgrade,err,**opts):
def _migrate(url, repository, version, upgrade, err, **opts):
echo = 'True' == opts.get('echo', False)
engine = create_engine(url, echo=echo)
schema = cls_schema(engine,repository)
version = _migrate_version(schema,version,upgrade,err)
schema = cls_schema(engine, repository)
version = _migrate_version(schema, version, upgrade, err)
changeset = schema.changeset(version)
for ver,change in changeset:
for ver, change in changeset:
nextver = ver + changeset.step
print '%s -> %s... '%(ver,nextver),
print '%s -> %s... '%(ver, nextver),
if opts.get('preview_sql'):
print
print change.log
elif opts.get('preview_py'):
source_ver = max(ver,nextver)
source_ver = max(ver, nextver)
module = schema.repository.version(source_ver).script().module
funcname = upgrade and "upgrade" or "downgrade"
func = getattr(module,funcname)
func = getattr(module, funcname)
print
print inspect.getsource(module.upgrade)
else:
schema.runchange(ver,change,changeset.step)
schema.runchange(ver, change, changeset.step)
print 'done'
def _migrate_version(schema,version,upgrade,err):
def _migrate_version(schema, version, upgrade, err):
if version is None:
return version
# Version is specified: ensure we're upgrading in the right direction
@ -240,48 +283,54 @@ def _migrate_version(schema,version,upgrade,err):
else:
direction = cur >= version
if not direction:
raise exceptions.KnownError(err%(cur,version))
raise exceptions.KnownError(err%(cur, version))
return version
def drop_version_control(url,repository,**opts):
def drop_version_control(url, repository, **opts):
"""%prog drop_version_control URL REPOSITORY_PATH
Removes version control from a database.
"""
echo = 'True' == opts.get('echo', False)
engine = create_engine(url, echo=echo)
schema=cls_schema(engine,repository)
schema=cls_schema(engine, repository)
schema.drop()
def manage(file,**opts):
def manage(file, **opts):
"""%prog manage FILENAME VARIABLES...
Creates a script that runs Migrate with a set of default values.
Creates a script that runs Migrate with a set of default values.
For example::
%prog manage manage.py --repository=/path/to/repository --url=sqlite:///project.db
%prog manage manage.py --repository=/path/to/repository \
--url=sqlite:///project.db
would create the script manage.py. The following two commands would then
have exactly the same results::
would create the script manage.py. The following two commands
would then have exactly the same results::
python manage.py version
%prog version --repository=/path/to/repository
"""
return repository.manage(file,**opts)
return repository.manage(file, **opts)
def compare_model_to_db(url,model,repository,**opts):
def compare_model_to_db(url, model, repository, **opts):
"""%prog compare_model_to_db URL MODEL REPOSITORY_PATH
Compare the current model (assumed to be a module level variable of type sqlalchemy.MetaData) against the current database.
Compare the current model (assumed to be a module level variable
of type sqlalchemy.MetaData) against the current database.
NOTE: This is EXPERIMENTAL.
""" # TODO: get rid of EXPERIMENTAL label
echo = 'True' == opts.get('echo', False)
engine = create_engine(url, echo=echo)
print cls_schema.compare_model_to_db(engine,model,repository)
print cls_schema.compare_model_to_db(engine, model, repository)
def create_model(url,repository,**opts):
def create_model(url, repository, **opts):
"""%prog create_model URL REPOSITORY_PATH
Dump the current database as a Python model to stdout.
@ -291,32 +340,37 @@ def create_model(url,repository,**opts):
echo = 'True' == opts.get('echo', False)
engine = create_engine(url, echo=echo)
declarative = opts.get('declarative', False)
print cls_schema.create_model(engine,repository,declarative)
print cls_schema.create_model(engine, repository, declarative)
def make_update_script_for_model(url,oldmodel,model,repository,**opts):
def make_update_script_for_model(url, oldmodel, model, repository, **opts):
"""%prog make_update_script_for_model URL OLDMODEL MODEL REPOSITORY_PATH
Create a script changing the old Python model to the new (current) Python model, sending to stdout.
Create a script changing the old Python model to the new (current)
Python model, sending to stdout.
NOTE: This is EXPERIMENTAL.
""" # TODO: get rid of EXPERIMENTAL label
echo = 'True' == opts.get('echo', False)
engine = create_engine(url, echo=echo)
try:
print cls_script_python.make_update_script_for_model(engine,oldmodel,model,repository,**opts)
except exceptions.PathFoundError,e:
raise exceptions.KnownError("The path %s already exists"%e.args[0]) # TODO: get rid of this? if we don't add back path param
print cls_script_python.make_update_script_for_model(
engine, oldmodel, model, repository, **opts)
except exceptions.PathFoundError, e:
# TODO: get rid of this? if we don't add back path param
raise exceptions.KnownError("The path %s already exists" % e.args[0])
def update_db_from_model(url,model,repository,**opts):
def update_db_from_model(url, model, repository, **opts):
"""%prog update_db_from_model URL MODEL REPOSITORY_PATH
Modify the database to match the structure of the current Python model.
This also sets the db_version number to the latest in the repository.
Modify the database to match the structure of the current Python
model. This also sets the db_version number to the latest in the
repository.
NOTE: This is EXPERIMENTAL.
""" # TODO: get rid of EXPERIMENTAL label
echo = 'True' == opts.get('echo', False)
engine = create_engine(url, echo=echo)
schema = cls_schema(engine,repository)
schema = cls_schema(engine, repository)
schema.update_db_from_model(model)

23
migrate/versioning/cfgparse.py
View File

@ -1,19 +1,26 @@
"""
Configuration parser module.
"""
from migrate.versioning.base import *
from migrate.versioning import pathed
from ConfigParser import ConfigParser
#__all__=['MigrateConfigParser']
class Parser(ConfigParser):
"""A project configuration file"""
def to_dict(self,sections=None):
"""A project configuration file."""
def to_dict(self, sections=None):
"""It's easier to access config values like dictionaries"""
return self._sections
class Config(pathed.Pathed,Parser):
def __init__(self,path,*p,**k):
"""Confirm the config file exists; read it"""
class Config(pathed.Pathed, Parser):
"""Configuration class."""
def __init__(self, path, *p, **k):
"""Confirm the config file exists; read it."""
self.require_found(path)
pathed.Pathed.__init__(self,path)
Parser.__init__(self,*p,**k)
pathed.Pathed.__init__(self, path)
Parser.__init__(self, *p, **k)
self.read(path)

79
migrate/versioning/exceptions.py
View File

@ -1,32 +1,60 @@
"""
Provide exception classes for :mod:`migrate.versioning`
"""
class Error(Exception):
"""Error base class."""
pass
class ApiError(Error):
"""Base class for API errors."""
pass
class KnownError(ApiError):
"""A known error condition"""
"""A known error condition."""
class UsageError(ApiError):
"""A known error condition where help should be displayed"""
"""A known error condition where help should be displayed."""
class ControlledSchemaError(Error):
pass
class InvalidVersionError(ControlledSchemaError):
"""Invalid version number"""
class DatabaseNotControlledError(ControlledSchemaError):
"""Database shouldn't be under vc, but it is"""
class DatabaseAlreadyControlledError(ControlledSchemaError):
"""Database should be under vc, but it's not"""
class WrongRepositoryError(ControlledSchemaError):
"""This database is under version control by another repository"""
class NoSuchTableError(ControlledSchemaError):
"""Base class for controlled schema errors."""
pass
class InvalidVersionError(ControlledSchemaError):
"""Invalid version number."""
class DatabaseNotControlledError(ControlledSchemaError):
"""Database should be under version control, but it's not."""
class DatabaseAlreadyControlledError(ControlledSchemaError):
"""Database shouldn't be under version control, but it is"""
class WrongRepositoryError(ControlledSchemaError):
"""This database is under version control by another repository."""
class NoSuchTableError(ControlledSchemaError):
"""The table does not exist."""
pass
class LogSqlError(Error):
"""A SQLError, with a traceback of where that statement was logged"""
def __init__(self,sqlerror,entry):
"""A SQLError, with a traceback of where that statement was logged."""
def __init__(self, sqlerror, entry):
Exception.__init__(self)
self.sqlerror = sqlerror
self.entry = entry
def __str__(self):
ret = "SQL error in statement: \n%s\n"%(str(self.entry))
ret += "Traceback from change script:\n"
@ -34,25 +62,42 @@ class LogSqlError(Error):
ret += str(self.sqlerror)
return ret
class PathError(Error):
"""Base class for path errors."""
pass
class PathNotFoundError(PathError):
"""A path with no file was required; found a file"""
"""A path with no file was required; found a file."""
pass
class PathFoundError(PathError):
"""A path with a file was required; found no file"""
"""A path with a file was required; found no file."""
pass
class RepositoryError(Error):
"""Base class for repository errors."""
pass
class InvalidRepositoryError(RepositoryError):
"""Invalid repository error."""
pass
class ScriptError(Error):
"""Base class for script errors."""
pass
class InvalidScriptError(ScriptError):
"""Invalid script error."""
pass
class InvalidVersionError(Error):
"""Invalid version error."""
pass

142
migrate/versioning/genmodel.py
View File

@ -1,9 +1,14 @@
"""
Code to generate a Python model from a database or differences
between a model and database.
# Code to generate a Python model from a database or differences between a model and database.
# Some of this is borrowed heavily from the AutoCode project at: http://code.google.com/p/sqlautocode/
Some of this is borrowed heavily from the AutoCode project at:
http://code.google.com/p/sqlautocode/
"""
import sys
import migrate, sqlalchemy
import migrate
import sqlalchemy
HEADER = """
@ -22,61 +27,73 @@ from sqlalchemy.ext import declarative
Base = declarative.declarative_base()
"""
class ModelGenerator(object):
def __init__(self, diff, declarative=False):
self.diff = diff
self.declarative = declarative
dialectModule = sys.modules[self.diff.conn.dialect.__module__] # is there an easier way to get this?
self.colTypeMappings = dict( (v,k) for k,v in dialectModule.colspecs.items() )
# is there an easier way to get this?
dialectModule = sys.modules[self.diff.conn.dialect.__module__]
self.colTypeMappings = dict((v, k) for k, v in \
dialectModule.colspecs.items())
def column_repr(self, col):
kwarg = []
if col.key != col.name: kwarg.append('key')
if col.key != col.name:
kwarg.append('key')
if col.primary_key:
col.primary_key = True # otherwise it dumps it as 1
kwarg.append('primary_key')
if not col.nullable: kwarg.append('nullable')
if col.onupdate: kwarg.append('onupdate')
if not col.nullable:
kwarg.append('nullable')
if col.onupdate:
kwarg.append('onupdate')
if col.default:
if col.primary_key:
# I found that Postgres automatically creates a default value for the sequence, but let's not show that.
# I found that PostgreSQL automatically creates a
# default value for the sequence, but let's not show
# that.
pass
else:
kwarg.append('default')
ks = ', '.join('%s=%r' % (k, getattr(col, k)) for k in kwarg )
name = col.name.encode('utf8') # crs: not sure if this is good idea, but it gets rid of extra u''
ks = ', '.join('%s=%r' % (k, getattr(col, k)) for k in kwarg)
# crs: not sure if this is good idea, but it gets rid of extra
# u''
name = col.name.encode('utf8')
type = self.colTypeMappings.get(col.type.__class__, None)
if type:
# Make the column type be an instance of this type.
type = type()
else:
# We must already be a model type, no need to map from the database-specific types.
# We must already be a model type, no need to map from the
# database-specific types.
type = col.type
data = {'name' : name,
'type' : type,
'constraints' : ', '.join([repr(cn) for cn in col.constraints]),
'args' : ks and ks or ''
}
data = {
'name': name,
'type': type,
'constraints': ', '.join([repr(cn) for cn in col.constraints]),
'args': ks and ks or ''}
if data['constraints']:
if data['args']: data['args'] = ',' + data['args']
if data['args']:
data['args'] = ',' + data['args']
if data['constraints'] or data['args']:
data['maybeComma'] = ','
else:
data['maybeComma'] = ''
commonStuff = " %(maybeComma)s %(constraints)s %(args)s)""" % data
commonStuff = """ %(maybeComma)s %(constraints)s %(args)s)""" % data
commonStuff = commonStuff.strip()
data['commonStuff'] = commonStuff
if self.declarative:
return """%(name)s = Column(%(type)r%(commonStuff)s""" % data
else:
return """Column(%(name)r, %(type)r%(commonStuff)s""" % data
def getTableDefn(self, table):
out = []
tableName = table.name
@ -86,14 +103,15 @@ class ModelGenerator(object):
for col in table.columns:
out.append(" %s" % self.column_repr(col))
else:
out.append("%(table)s = Table('%(table)s', meta," % {'table': tableName})
out.append("%(table)s = Table('%(table)s', meta," % \
{'table': tableName})
for col in table.columns:
out.append(" %s," % self.column_repr(col))
out.append(")")
return out
def toPython(self):
''' Assume database is current and model is empty. '''
"""Assume database is current and model is empty."""
out = []
if self.declarative:
out.append(DECLARATIVE_HEADER)
@ -104,44 +122,46 @@ class ModelGenerator(object):
out.extend(self.getTableDefn(table))
out.append("")
return '\n'.join(out)
def toUpgradeDowngradePython(self, indent=' '):
''' Assume model is most current and database is out-of-date. '''
decls = ['meta = MetaData(migrate_engine)']
for table in self.diff.tablesMissingInModel + self.diff.tablesMissingInDatabase:
for table in self.diff.tablesMissingInModel + \
self.diff.tablesMissingInDatabase:
decls.extend(self.getTableDefn(table))
upgradeCommands, downgradeCommands = [], []
for table in self.diff.tablesMissingInModel:
tableName = table.name
upgradeCommands.append("%(table)s.drop()" % {'table': tableName})
downgradeCommands.append("%(table)s.create()" % {'table': tableName})
downgradeCommands.append("%(table)s.create()" % \
{'table': tableName})
for table in self.diff.tablesMissingInDatabase:
tableName = table.name
upgradeCommands.append("%(table)s.create()" % {'table': tableName})
downgradeCommands.append("%(table)s.drop()" % {'table': tableName})
return ('\n'.join(decls),
'\n'.join(['%s%s' % (indent, line) for line in upgradeCommands]),
'\n'.join(['%s%s' % (indent, line) for line in downgradeCommands])
)
return (
'\n'.join(decls),
'\n'.join(['%s%s' % (indent, line) for line in upgradeCommands]),
'\n'.join(['%s%s' % (indent, line) for line in downgradeCommands]))
def applyModel(self):
''' Apply model to current database. '''
# Yuck! We have to import from changeset to apply the monkey-patch to allow column adding/dropping.
"""Apply model to current database."""
# Yuck! We have to import from changeset to apply the
# monkey-patch to allow column adding/dropping.
from migrate.changeset import schema
def dbCanHandleThisChange(missingInDatabase, missingInModel, diffDecl):
if missingInDatabase and not missingInModel and not diffDecl:
# Even sqlite can handle this.
return True
else:
return not self.diff.conn.url.drivername.startswith('sqlite')
meta = sqlalchemy.MetaData(self.diff.conn.engine)
for table in self.diff.tablesMissingInModel:
table = table.tometadata(meta)
table.drop()
@ -152,8 +172,10 @@ class ModelGenerator(object):
modelTable = modelTable.tometadata(meta)
dbTable = self.diff.reflected_model.tables[modelTable.name]
tableName = modelTable.name
missingInDatabase, missingInModel, diffDecl = self.diff.colDiffs[tableName]
if dbCanHandleThisChange(missingInDatabase, missingInModel, diffDecl):
missingInDatabase, missingInModel, diffDecl = \
self.diff.colDiffs[tableName]
if dbCanHandleThisChange(missingInDatabase, missingInModel,
diffDecl):
for col in missingInDatabase:
modelTable.columns[col.name].create()
for col in missingInModel:
@ -161,25 +183,34 @@ class ModelGenerator(object):
for modelCol, databaseCol, modelDecl, databaseDecl in diffDecl:
databaseCol.alter(modelCol)
else:
# Sqlite doesn't support drop column, so you have to do more:
# create temp table, copy data to it, drop old table, create new table, copy data back.
tempName = '_temp_%s' % modelTable.name # I wonder if this is guaranteed to be unique?
# Sqlite doesn't support drop column, so you have to
# do more: create temp table, copy data to it, drop
# old table, create new table, copy data back.
#
# I wonder if this is guaranteed to be unique?
tempName = '_temp_%s' % modelTable.name
def getCopyStatement():
preparer = self.diff.conn.engine.dialect.preparer
commonCols = []
for modelCol in modelTable.columns:
if dbTable.columns.has_key(modelCol.name):
if modelCol.name in dbTable.columns:
commonCols.append(modelCol.name)
commonColsStr = ', '.join(commonCols)
return 'INSERT INTO %s (%s) SELECT %s FROM %s' % (tableName, commonColsStr, commonColsStr, tempName)
# Move the data in one transaction, so that we don't leave the database in a nasty state.
return 'INSERT INTO %s (%s) SELECT %s FROM %s' % \
(tableName, commonColsStr, commonColsStr, tempName)
# Move the data in one transaction, so that we don't
# leave the database in a nasty state.
connection = self.diff.conn.connect()
trans = connection.begin()
try:
connection.execute('CREATE TEMPORARY TABLE %s as SELECT * from %s' % (tempName, modelTable.name))
modelTable.drop(bind=connection) # make sure the drop takes place inside our transaction with the bind parameter
connection.execute(
'CREATE TEMPORARY TABLE %s as SELECT * from %s' % \
(tempName, modelTable.name))
# make sure the drop takes place inside our
# transaction with the bind parameter
modelTable.drop(bind=connection)
modelTable.create(bind=connection)
connection.execute(getCopyStatement())
connection.execute('DROP TABLE %s' % tempName)
@ -187,4 +218,3 @@ class ModelGenerator(object):
except:
trans.rollback()
raise

28
migrate/versioning/migrate_repository.py
View File

@ -1,17 +1,22 @@
""" Script to migrate repository. This shouldn't use any other migrate
modules, so that it can work in any version. """
"""
Script to migrate repository from sqlalchemy <= 0.4.4 to the new
repository schema. This shouldn't use any other migrate modules, so
that it can work in any version.
"""
import os, os.path, sys
import os
import os.path
import sys
def usage():
"""Gives usage information."""
print '''Usage: %(prog)s repository-to-migrate
print """Usage: %(prog)s repository-to-migrate
Upgrade your repository to the new flat format.
Upgrade your repository to the new flat format.
NOTE: You should probably make a backup before running this.
''' % {'prog': sys.argv[0]}
NOTE: You should probably make a backup before running this.
""" % {'prog': sys.argv[0]}
sys.exit(1)
@ -27,7 +32,8 @@ def move_file(src, tgt):
print ' Moving file %s to %s' % (src, tgt)
if os.path.exists(tgt):
raise Exception(
'Cannot move file %s because target %s already exists' % (src, tgt))
'Cannot move file %s because target %s already exists' % \
(src, tgt))
os.rename(src, tgt)
@ -35,7 +41,7 @@ def delete_directory(dirpath):
"""Delete a directory and print a message."""
print ' Deleting directory: %s' % dirpath
os.rmdir(dirpath)
def migrate_repository(repos):
"""Does the actual migration to the new repository format."""
@ -43,7 +49,7 @@ def migrate_repository(repos):
versions = '%s/versions' % repos
dirs = os.listdir(versions)
# Only use int's in list.
numdirs = [ int(dirname) for dirname in dirs if dirname.isdigit() ]
numdirs = [int(dirname) for dirname in dirs if dirname.isdigit()]
numdirs.sort() # Sort list.
for dirname in numdirs:
origdir = '%s/%s' % (versions, dirname)
@ -51,7 +57,6 @@ def migrate_repository(repos):
files = os.listdir(origdir)
files.sort()
for filename in files:
# Delete compiled Python files.
if filename.endswith('.pyc') or filename.endswith('.pyo'):
delete_file('%s/%s' % (origdir, filename))
@ -91,4 +96,3 @@ def main():
if __name__ == '__main__':
main()

54
migrate/versioning/pathed.py
View File

@ -1,60 +1,72 @@
"""
A path/directory class.
"""
from migrate.versioning.base import *
from migrate.versioning.util import KeyedInstance
import os,shutil
from migrate.versioning import exceptions
import os
import shutil
class Pathed(KeyedInstance):
"""A class associated with a path/directory tree
Only one instance of this class may exist for a particular file;
"""
A class associated with a path/directory tree.
Only one instance of this class may exist for a particular file;
__new__ will return an existing instance if possible
"""
parent=None
@classmethod
def _key(cls,path):
def _key(cls, path):
return str(path)
def __init__(self,path):
def __init__(self, path):
self.path=path
if self.__class__.parent is not None:
self._init_parent(path)
def _init_parent(self,path):
def _init_parent(self, path):
"""Try to initialize this object's parent, if it has one"""
parent_path=self.__class__._parent_path(path)
self.parent=self.__class__.parent(parent_path)
log.info("Getting parent %r:%r"%(self.__class__.parent,parent_path))
self.parent._init_child(path,self)
def _init_child(self,child,path):
"""Run when a child of this object is initialized
Parameters: the child object; the path to this object (its parent)
log.info("Getting parent %r:%r" % (self.__class__.parent, parent_path))
self.parent._init_child(path, self)
def _init_child(self, child, path):
"""Run when a child of this object is initialized.
Parameters: the child object; the path to this object (its
parent)
"""
pass
@classmethod
def _parent_path(cls,path):
"""Fetch the path of this object's parent from this object's path
def _parent_path(cls, path):
"""
# os.path.dirname(), but strip directories like files (like unix basename)
Fetch the path of this object's parent from this object's path.
"""
# os.path.dirname(), but strip directories like files (like
# unix basename)
#
# Treat directories like files...
if path[-1]=='/':
if path[-1] == '/':
path=path[:-1]
ret = os.path.dirname(path)
return ret
@classmethod
def require_notfound(cls,path):
def require_notfound(cls, path):
"""Ensures a given path does not already exist"""
if os.path.exists(path):
raise exceptions.PathFoundError(path)
@classmethod
def require_found(cls,path):
def require_found(cls, path):
"""Ensures a given path already exists"""
if not os.path.exists(path):
raise exceptions.PathNotFoundError(path)
def __str__(self):
return self.path