openstack/ironic
Code Issues Proposed changes

Merge "Phase 3 - SQLAlchemy 2.0 Compatability"

Browse Source
This commit is contained in:
Zuul 2022-10-17 23:27:14 +00:00 committed by Gerrit Code Review
commit fcd29982f2
1 changed files with 206 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

309
ironic/tests/unit/db/sqlalchemy/test_migrations.py
View File

@ -36,6 +36,7 @@ For postgres on Ubuntu this can be done with the following commands:
import collections import collections
import contextlib import contextlib
import json
from unittest import mock from unittest import mock
from alembic import script from alembic import script
@ -114,6 +115,7 @@ class WalkVersionsMixin(object):
check = getattr(self, "_check_%s" % version, None) check = getattr(self, "_check_%s" % version, None)
if check: if check:
check(engine, data) check(engine, data)
except Exception: except Exception:
LOG.error("Failed to migrate to version %(version)s on engine " LOG.error("Failed to migrate to version %(version)s on engine "
"%(engine)s", "%(engine)s",
@ -248,17 +250,24 @@ class MigrationCheckersMixin(object):
with engine.begin() as connection: with engine.begin() as connection:
insert_conductor = conductors.insert().values(data_conductor) insert_conductor = conductors.insert().values(data_conductor)
connection.execute(insert_conductor) connection.execute(insert_conductor)
conductor_stmt = conductors.select( conductor_stmt = sqlalchemy.select(
conductors.c.hostname == data_conductor['hostname']) models.Conductor.id
).where(
models.Conductor.hostname == 'test_host'
)
conductor = connection.execute(conductor_stmt).first() conductor = connection.execute(conductor_stmt).first()
data_node = {'uuid': uuidutils.generate_uuid(), data_node = {'uuid': uuidutils.generate_uuid(),
'conductor_affinity': conductor['id']} 'conductor_affinity': conductor.id}
insert_node = nodes.insert().values(data_node) insert_node = nodes.insert().values(data_node)
connection.execute(insert_node) connection.execute(insert_node)
node_stmt = nodes.select(nodes.c.uuid == data_node['uuid']) node_stmt = sqlalchemy.select(
models.Node.conductor_affinity
).where(
models.Node.uuid == data_node['uuid']
)
node = connection.execute(node_stmt).first() node = connection.execute(node_stmt).first()
self.assertEqual(conductor['id'], node['conductor_affinity']) self.assertEqual(conductor.id, node.conductor_affinity)
def _check_242cc6a923b3(self, engine, data): def _check_242cc6a923b3(self, engine, data):
nodes = db_utils.get_table(engine, 'nodes') nodes = db_utils.get_table(engine, 'nodes')
@ -285,21 +294,26 @@ class MigrationCheckersMixin(object):
return data return data
def _check_5674c57409b9(self, engine, data): def _check_5674c57409b9(self, engine, data):
nodes = db_utils.get_table(engine, 'nodes') with engine.begin() as connection:
result = engine.execute(nodes.select()) result = connection.execute(
sqlalchemy.select(
models.Node.uuid,
models.Node.provision_state
)
)
def _get_state(uuid): def _get_state(uuid):
for row in data: for row in data:
if row['uuid'] == uuid: if row['uuid'] == uuid:
return row['provision_state'] return row['provision_state']
for row in result: for row in result:
old = _get_state(row['uuid']) old = _get_state(row.uuid)
new = row['provision_state'] new = row['provision_state']
if old is None: if old is None:
self.assertEqual('available', new) self.assertEqual('available', new)
else: else:
self.assertEqual(old, new) self.assertEqual(old, new)
def _check_bb59b63f55a(self, engine, data): def _check_bb59b63f55a(self, engine, data):
nodes = db_utils.get_table(engine, 'nodes') nodes = db_utils.get_table(engine, 'nodes')
@ -360,9 +374,13 @@ class MigrationCheckersMixin(object):
with engine.begin() as connection: with engine.begin() as connection:
insert_node = nodes.insert().values(data) insert_node = nodes.insert().values(data)
connection.execute(insert_node) connection.execute(insert_node)
node_stmt = nodes.select(nodes.c.uuid == uuid) node_stmt = sqlalchemy.select(
models.Node.name
).where(
models.Node.uuid == uuid
)
node = connection.execute(node_stmt).first() node = connection.execute(node_stmt).first()
self.assertEqual(bigstring, node['name']) self.assertEqual(bigstring, node.name)
def _check_516faf1bb9b1(self, engine, data): def _check_516faf1bb9b1(self, engine, data):
nodes = db_utils.get_table(engine, 'nodes') nodes = db_utils.get_table(engine, 'nodes')
@ -372,9 +390,13 @@ class MigrationCheckersMixin(object):
with engine.begin() as connection: with engine.begin() as connection:
insert_node = nodes.insert().values(data) insert_node = nodes.insert().values(data)
connection.execute(insert_node) connection.execute(insert_node)
node_stmt = nodes.select(nodes.c.uuid == uuid) node_stmt = sqlalchemy.select(
models.Node.driver
).where(
models.Node.uuid == uuid
)
node = connection.execute(node_stmt).first() node = connection.execute(node_stmt).first()
self.assertEqual(bigstring, node['driver']) self.assertEqual(bigstring, node.driver)
def _check_48d6c242bb9b(self, engine, data): def _check_48d6c242bb9b(self, engine, data):
node_tags = db_utils.get_table(engine, 'node_tags') node_tags = db_utils.get_table(engine, 'node_tags')
@ -390,9 +412,13 @@ class MigrationCheckersMixin(object):
data = {'node_id': '123', 'tag': 'tag1'} data = {'node_id': '123', 'tag': 'tag1'}
insert_node_tag = node_tags.insert().values(data) insert_node_tag = node_tags.insert().values(data)
connection.execute(insert_node_tag) connection.execute(insert_node_tag)
tag_stmt = node_tags.select(node_tags.c.node_id == '123') tag_stmt = sqlalchemy.select(
models.NodeTag.tag
).where(
models.NodeTag.node_id == '123'
)
tag = connection.execute(tag_stmt).first() tag = connection.execute(tag_stmt).first()
self.assertEqual('tag1', tag['tag']) self.assertEqual('tag1', tag.tag)
def _check_5ea1b0d310e(self, engine, data): def _check_5ea1b0d310e(self, engine, data):
portgroup = db_utils.get_table(engine, 'portgroups') portgroup = db_utils.get_table(engine, 'portgroups')
@ -441,17 +467,22 @@ class MigrationCheckersMixin(object):
return data return data
def _check_f6fdb920c182(self, engine, data): def _check_f6fdb920c182(self, engine, data):
ports = db_utils.get_table(engine, 'ports')
result = engine.execute(ports.select())
def _was_inserted(uuid): with engine.begin() as connection:
for row in data: port_stmt = sqlalchemy.select(
if row['uuid'] == uuid: models.Port.uuid,
return True models.Port.pxe_enabled
)
result = connection.execute(port_stmt)
for row in result: def _was_inserted(uuid):
if _was_inserted(row['uuid']): for row in data:
self.assertTrue(row['pxe_enabled']) if row['uuid'] == uuid:
return True
for row in result:
if _was_inserted(row['uuid']):
self.assertTrue(row['pxe_enabled'])
def _check_e294876e8028(self, engine, data): def _check_e294876e8028(self, engine, data):
nodes = db_utils.get_table(engine, 'nodes') nodes = db_utils.get_table(engine, 'nodes')
@ -494,18 +525,21 @@ class MigrationCheckersMixin(object):
return data return data
def _check_c14cef6dfedf(self, engine, data): def _check_c14cef6dfedf(self, engine, data):
nodes = db_utils.get_table(engine, 'nodes')
result = engine.execute(nodes.select())
counts = collections.defaultdict(int) counts = collections.defaultdict(int)
with engine.begin() as connection:
result = connection.execute(
sqlalchemy.select(
models.Node.uuid,
models.Node.network_interface))
def _was_inserted(uuid): def _was_inserted(uuid):
for row in data: for row in data:
if row['uuid'] == uuid: if row['uuid'] == uuid:
return True return True
for row in result: for row in result:
if _was_inserted(row['uuid']): if _was_inserted(row['uuid']):
counts[row['network_interface']] += 1 counts[row['network_interface']] += 1
# using default config values, we should have 2 flat and one neutron # using default config values, we should have 2 flat and one neutron
self.assertEqual(2, counts['flat']) self.assertEqual(2, counts['flat'])
@ -602,8 +636,10 @@ class MigrationCheckersMixin(object):
self.assertIn('mode', col_names) self.assertIn('mode', col_names)
self.assertIsInstance(portgroups.c.mode.type, self.assertIsInstance(portgroups.c.mode.type,
sqlalchemy.types.String) sqlalchemy.types.String)
with engine.begin() as connection:
result = engine.execute(portgroups.select()) result = connection.execute(
sqlalchemy.select(models.Portgroup.mode)
)
for row in result: for row in result:
self.assertEqual(CONF.default_portgroup_mode, row['mode']) self.assertEqual(CONF.default_portgroup_mode, row['mode'])
@ -675,9 +711,13 @@ class MigrationCheckersMixin(object):
with engine.begin() as connection: with engine.begin() as connection:
insert_node = nodes.insert().values(data) insert_node = nodes.insert().values(data)
connection.execute(insert_node) connection.execute(insert_node)
node_stmt = nodes.select(nodes.c.uuid == data['uuid']) node_stmt = sqlalchemy.select(
models.Node.id
).where(
models.Node.uuid == data['uuid']
)
node = connection.execute(node_stmt).first() node = connection.execute(node_stmt).first()
data['id'] = node['id'] data['id'] = node.id
return data return data
def _check_b4130a7fc904(self, engine, data): def _check_b4130a7fc904(self, engine, data):
@ -694,10 +734,13 @@ class MigrationCheckersMixin(object):
with engine.begin() as connection: with engine.begin() as connection:
insert_trait = node_traits.insert().values(trait) insert_trait = node_traits.insert().values(trait)
connection.execute(insert_trait) connection.execute(insert_trait)
trait_stmt = node_traits.select( trait_stmt = sqlalchemy.select(
node_traits.c.node_id == data['id']) models.NodeTrait.trait
).where(
models.NodeTrait.node_id == data['id']
)
trait = connection.execute(trait_stmt).first() trait = connection.execute(trait_stmt).first()
self.assertEqual('trait1', trait['trait']) self.assertEqual('trait1', trait.trait)
def _pre_upgrade_82c315d60161(self, engine): def _pre_upgrade_82c315d60161(self, engine):
# Create a node to which bios setting can be added. # Create a node to which bios setting can be added.
@ -706,9 +749,11 @@ class MigrationCheckersMixin(object):
with engine.begin() as connection: with engine.begin() as connection:
insert_node = nodes.insert().values(data) insert_node = nodes.insert().values(data)
connection.execute(insert_node) connection.execute(insert_node)
node_stmt = nodes.select(nodes.c.uuid == data['uuid']) node_stmt = sqlalchemy.select(
models.Node.id
).where(models.Node.uuid == data['uuid'])
node = connection.execute(node_stmt).first() node = connection.execute(node_stmt).first()
data['id'] = node['id'] data['id'] = node.id
return data return data
def _check_82c315d60161(self, engine, data): def _check_82c315d60161(self, engine, data):
@ -736,10 +781,12 @@ class MigrationCheckersMixin(object):
with engine.begin() as connection: with engine.begin() as connection:
insert_bios_settings = bios_settings.insert().values(setting) insert_bios_settings = bios_settings.insert().values(setting)
connection.execute(insert_bios_settings) connection.execute(insert_bios_settings)
setting_stmt = bios_settings.select( setting_stmt = sqlalchemy.select(
sqlalchemy.sql.and_( models.BIOSSetting.value
bios_settings.c.node_id == data['id'], ).where(
bios_settings.c.name == setting['name'])) models.BIOSSetting.node_id == data['id'],
models.BIOSSetting.name == setting['name']
)
setting = connection.execute(setting_stmt).first() setting = connection.execute(setting_stmt).first()
self.assertEqual('on', setting['value']) self.assertEqual('on', setting['value'])
@ -826,15 +873,21 @@ class MigrationCheckersMixin(object):
self.assertIsInstance(tbl.c.conductor_group.type, self.assertIsInstance(tbl.c.conductor_group.type,
sqlalchemy.types.String) sqlalchemy.types.String)
with engine.begin() as connection: with engine.begin() as connection:
node_stmt = nodes_tbl.select( node_stmt = sqlalchemy.select(
nodes_tbl.c.uuid == data['node_uuid']) models.Node.uuid,
models.Node.conductor_group,
).where(
models.Node.uuid == data['node_uuid'])
node = connection.execute(node_stmt).first() node = connection.execute(node_stmt).first()
self.assertEqual(node['conductor_group'], "") self.assertEqual(node.conductor_group, "")
conductor_stmt = conductors_tbl.select( conductor_stmt = sqlalchemy.select(
conductors_tbl.c.id == data['conductor_id']) models.Conductor.conductor_group,
).where(
models.Conductor.id == data['conductor_id'],
)
conductor = connection.execute(conductor_stmt).first() conductor = connection.execute(conductor_stmt).first()
self.assertEqual(conductor['conductor_group'], "") self.assertEqual(conductor.conductor_group, "")
def _check_d2b036ae9378(self, engine, data): def _check_d2b036ae9378(self, engine, data):
nodes = db_utils.get_table(engine, 'nodes') nodes = db_utils.get_table(engine, 'nodes')
@ -859,11 +912,15 @@ class MigrationCheckersMixin(object):
self.assertIn('protected_reason', col_names) self.assertIn('protected_reason', col_names)
with engine.begin() as connection: with engine.begin() as connection:
node_stmt = nodes.select( node_stmt = sqlalchemy.select(
nodes.c.uuid == data['node_uuid']) models.Node.uuid,
models.Node.protected,
models.Node.protected_reason
).where(
models.Node.uuid == data['node_uuid'])
node = connection.execute(node_stmt).first() node = connection.execute(node_stmt).first()
self.assertFalse(node['protected']) self.assertFalse(node.protected)
self.assertIsNone(node['protected_reason']) self.assertIsNone(node.protected_reason)
def _check_f190f9d00a11(self, engine, data): def _check_f190f9d00a11(self, engine, data):
nodes = db_utils.get_table(engine, 'nodes') nodes = db_utils.get_table(engine, 'nodes')
@ -887,10 +944,13 @@ class MigrationCheckersMixin(object):
self.assertIn('allocation_id', col_names) self.assertIn('allocation_id', col_names)
with engine.begin() as connection: with engine.begin() as connection:
node_stmt = nodes.select( node_stmt = sqlalchemy.select(
nodes.c.uuid == data['node_uuid']) models.Node.allocation_id
).where(
models.Node.uuid == data['node_uuid']
)
node = connection.execute(node_stmt).first() node = connection.execute(node_stmt).first()
self.assertIsNone(node['allocation_id']) self.assertIsNone(node.allocation_id)
allocations = db_utils.get_table(engine, 'allocations') allocations = db_utils.get_table(engine, 'allocations')
col_names = [column.name for column in allocations.c] col_names = [column.name for column in allocations.c]
@ -996,22 +1056,33 @@ class MigrationCheckersMixin(object):
insert_dpt = deploy_templates.insert().values(template) insert_dpt = deploy_templates.insert().values(template)
connection.execute(insert_dpt) connection.execute(insert_dpt)
# Query by UUID. # Query by UUID.
dpt_uuid_stmt = deploy_templates.select( dpt_uuid_stmt = sqlalchemy.select(
deploy_templates.c.uuid == uuid) models.DeployTemplate.id,
models.DeployTemplate.name,
).where(
models.DeployTemplate.uuid == uuid
)
result = connection.execute(dpt_uuid_stmt).first() result = connection.execute(dpt_uuid_stmt).first()
template_id = result['id'] template_id = result.id
self.assertEqual(name, result['name']) self.assertEqual(name, result.name)
# Query by name. # Query by name.
dpt_name_stmt = deploy_templates.select( dpt_name_stmt = sqlalchemy.select(
deploy_templates.c.name == name) models.DeployTemplate.id
).where(
models.DeployTemplate.name == name
)
result = connection.execute(dpt_name_stmt).first() result = connection.execute(dpt_name_stmt).first()
self.assertEqual(template_id, result['id']) self.assertEqual(template_id, result.id)
# Query by ID. # Query by ID.
dpt_id_stmt = deploy_templates.select( dpt_id_stmt = sqlalchemy.select(
deploy_templates.c.id == template_id) models.DeployTemplate.uuid,
models.DeployTemplate.name
).where(
models.DeployTemplate.id == template_id
)
result = connection.execute(dpt_id_stmt).first() result = connection.execute(dpt_id_stmt).first()
self.assertEqual(uuid, result['uuid']) self.assertEqual(uuid, result.uuid)
self.assertEqual(name, result['name']) self.assertEqual(name, result.name)
savepoint_uuid = connection.begin_nested() savepoint_uuid = connection.begin_nested()
# UUID is unique. # UUID is unique.
template = {'name': 'CUSTOM_DT2', 'uuid': uuid} template = {'name': 'CUSTOM_DT2', 'uuid': uuid}
@ -1030,6 +1101,7 @@ class MigrationCheckersMixin(object):
# Insert a deploy template step. # Insert a deploy template step.
interface = 'raid' interface = 'raid'
step_name = 'create_configuration' step_name = 'create_configuration'
# The line below is JSON.
args = '{"logical_disks": []}' args = '{"logical_disks": []}'
priority = 10 priority = 10
step = {'deploy_template_id': template_id, 'interface': interface, step = {'deploy_template_id': template_id, 'interface': interface,
@ -1037,15 +1109,30 @@ class MigrationCheckersMixin(object):
insert_dpts = deploy_template_steps.insert().values(step) insert_dpts = deploy_template_steps.insert().values(step)
connection.execute(insert_dpts) connection.execute(insert_dpts)
# Query by deploy template ID. # Query by deploy template ID.
query_id_stmt = deploy_template_steps.select( query_id_stmt = sqlalchemy.select(
deploy_template_steps.c.deploy_template_id models.DeployTemplateStep.deploy_template_id,
== template_id) models.DeployTemplateStep.interface,
models.DeployTemplateStep.step,
models.DeployTemplateStep.args,
models.DeployTemplateStep.priority,
).where(
models.DeployTemplateStep.deploy_template_id == template_id
)
result = connection.execute(query_id_stmt).first() result = connection.execute(query_id_stmt).first()
self.assertEqual(template_id, result['deploy_template_id']) self.assertEqual(template_id, result.deploy_template_id)
self.assertEqual(interface, result['interface']) self.assertEqual(interface, result.interface)
self.assertEqual(step_name, result['step']) self.assertEqual(step_name, result.step)
self.assertEqual(args, result['args']) if isinstance(result.args, dict):
self.assertEqual(priority, result['priority']) # Postgres testing results in a dict being returned
# at this level which if you str() it, you get a dict,
# so comparing string to string fails.
result_args = json.dumps(result.args)
else:
# Mysql/MariaDB appears to be actually hand us
# a string back so we should be able to compare it.
result_args = result.args
self.assertEqual(args, result_args)
self.assertEqual(priority, result.priority)
# Insert another step for the same template. # Insert another step for the same template.
insert_step = deploy_template_steps.insert().values(step) insert_step = deploy_template_steps.insert().values(step)
connection.execute(insert_step) connection.execute(insert_step)
@ -1103,11 +1190,15 @@ class MigrationCheckersMixin(object):
self.assertIn('retired_reason', col_names) self.assertIn('retired_reason', col_names)
with engine.begin() as connection: with engine.begin() as connection:
node_stmt = nodes.select( node_stmt = sqlalchemy.select(
nodes.c.uuid == data['node_uuid']) models.Node.retired,
models.Node.retired_reason,
).where(
models.Node.uuid == data['node_uuid']
)
node = connection.execute(node_stmt).first() node = connection.execute(node_stmt).first()
self.assertFalse(node['retired']) self.assertFalse(node.retired)
self.assertIsNone(node['retired_reason']) self.assertIsNone(node.retired_reason)
def _check_b2ad35726bb0(self, engine, data): def _check_b2ad35726bb0(self, engine, data):
nodes = db_utils.get_table(engine, 'nodes') nodes = db_utils.get_table(engine, 'nodes')
@ -1186,12 +1277,16 @@ class TestMigrationsMySQL(MigrationCheckersMixin,
# this should always fail pre-upgrade # this should always fail pre-upgrade
mediumtext = 'a' * (pow(2, 16) + 1) mediumtext = 'a' * (pow(2, 16) + 1)
json_text = str({'key': mediumtext})
uuid = uuidutils.generate_uuid() uuid = uuidutils.generate_uuid()
expected_to_fail_data = {'uuid': uuid, 'instance_info': mediumtext} expected_to_fail_data = {'uuid': uuid, 'instance_info': json_text}
# this should always work pre-upgrade # this should always work pre-upgrade
text = 'a' * (pow(2, 16) - 1) text = 'a' * (pow(2, 16) - 13)
# The field needs to contain JSON for the decoder to work against
# the field.
json_text = str({'key': text})
uuid2 = uuidutils.generate_uuid() uuid2 = uuidutils.generate_uuid()
valid_pre_upgrade_data = {'uuid': uuid2, 'instance_info': text} valid_pre_upgrade_data = {'uuid': uuid2, 'instance_info': json_text}
with engine.begin() as connection: with engine.begin() as connection:
self.assertRaises(db_exc.DBError, connection.execute, self.assertRaises(db_exc.DBError, connection.execute,
nodes.insert(), expected_to_fail_data) nodes.insert(), expected_to_fail_data)
@ -1207,21 +1302,29 @@ class TestMigrationsMySQL(MigrationCheckersMixin,
with engine.begin() as connection: with engine.begin() as connection:
# check that the data for the successful pre-upgrade # check that the data for the successful pre-upgrade
# entry didn't change # entry didn't change
node_stmt = nodes.select(nodes.c.uuid == data['uuid']) # NOTE(TheJulia): Directly select the field to bypass
node = connection.execute(node_stmt).first() # field decoding
self.assertIsNotNone(node) i_info = connection.execute(
self.assertEqual(data['instance_info'], node['instance_info']) sqlalchemy.text(
"SELECT instance_info from nodes WHERE uuid = "
"'%s'" % data['uuid'])).one()
self.assertIsNotNone(i_info[0])
self.assertEqual(data['instance_info'], i_info[0])
# now this should pass post-upgrade # now this should pass post-upgrade
test = 'b' * (pow(2, 16) + 1) test = 'b' * (pow(2, 16) + 1)
test_text = str({'a': test})
uuid = uuidutils.generate_uuid() uuid = uuidutils.generate_uuid()
data = {'uuid': uuid, 'instance_info': test} data = {'uuid': uuid, 'instance_info': test_text}
insert_node = nodes.insert().values(data) insert_node = nodes.insert().values(data)
connection.execute(insert_node) connection.execute(insert_node)
node_stmt = nodes.select(nodes.c.uuid == uuid) # Re-uses the same query to fetch current results
node = connection.execute(node_stmt).first() i_info = connection.execute(
self.assertEqual(test, node['instance_info']) sqlalchemy.text(
"SELECT instance_info from nodes WHERE uuid = "
"'%s'" % data['uuid'])).one()
self.assertEqual(test_text, i_info[0])
class TestMigrationsPostgreSQL(MigrationCheckersMixin, class TestMigrationsPostgreSQL(MigrationCheckersMixin,