From d0e39ba3698599a38df7c3037237b34670728f36 Mon Sep 17 00:00:00 2001 From: Rajaram Mallya and Gavri Fernandez Date: Tue, 20 Dec 2011 18:16:39 +0530 Subject: [PATCH] Bug fixes accomodating mysql and postgresql Change-Id: I73e114e78410e9f9cf46b6ac3d3000c0fb0dad00 --- etc/melange/melange.conf.sample | 2 + .../migrate_repo/versions/001_base_schema.py | 10 +- melange/ipam/models.py | 5 +- melange/tests/__init__.py | 1 + melange/tests/unit/test_ipam_models.py | 97 ++++++++++--------- melange/tests/unit/test_ipam_service.py | 13 ++- melange/tests/unit/test_ipam_views.py | 33 ++++--- 7 files changed, 90 insertions(+), 71 deletions(-) diff --git a/etc/melange/melange.conf.sample b/etc/melange/melange.conf.sample index b0b8e9e6..90741c21 100644 --- a/etc/melange/melange.conf.sample +++ b/etc/melange/melange.conf.sample @@ -15,6 +15,8 @@ bind_port = 9898 # registry server. Any valid SQLAlchemy connection string is fine. # See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine sql_connection = sqlite:///melange_test.sqlite +#sql_connection = mysql://root:root@localhost/melange +#sql_connection = postgresql://melange:melange@localhost/melange # Period in seconds after which SQLAlchemy should reestablish its connection # to the database. diff --git a/melange/db/sqlalchemy/migrate_repo/versions/001_base_schema.py b/melange/db/sqlalchemy/migrate_repo/versions/001_base_schema.py index e21f981f..21b8ea04 100644 --- a/melange/db/sqlalchemy/migrate_repo/versions/001_base_schema.py +++ b/melange/db/sqlalchemy/migrate_repo/versions/001_base_schema.py @@ -25,6 +25,7 @@ from melange.db.sqlalchemy.migrate_repo.schema import create_tables from melange.db.sqlalchemy.migrate_repo.schema import DateTime from melange.db.sqlalchemy.migrate_repo.schema import drop_tables from melange.db.sqlalchemy.migrate_repo.schema import Integer +from melange.db.sqlalchemy.migrate_repo.schema import BigInteger from melange.db.sqlalchemy.migrate_repo.schema import String from melange.db.sqlalchemy.migrate_repo.schema import Table @@ -42,10 +43,11 @@ ip_blocks = Table('ip_blocks', meta, Column('gateway', String(255)), Column('dns1', String(255)), Column('dns2', String(255)), - Column('allocatable_ip_counter', Integer()), + Column('allocatable_ip_counter', BigInteger()), Column('is_full', Boolean()), Column('policy_id', String(36), ForeignKey('policies.id')), - Column('parent_id', String(36), ForeignKey('ip_blocks.id'))) + Column('parent_id', String(36), ForeignKey('ip_blocks.id', + ondelete="CASCADE"))) ip_addresses = Table('ip_addresses', meta, @@ -119,13 +121,13 @@ allocatable_ips = Table('allocatable_ips', meta, mac_address_ranges = Table('mac_address_ranges', meta, Column('id', String(36), primary_key=True, nullable=False), Column('cidr', String(255), nullable=False), - Column('next_address', Integer()), + Column('next_address', BigInteger()), Column('created_at', DateTime()), Column('updated_at', DateTime())) mac_addresses = Table('mac_addresses', meta, Column('id', String(36), primary_key=True, nullable=False), - Column('address', Integer(), nullable=False), + Column('address', BigInteger(), nullable=False), Column('mac_address_range_id', String(36), ForeignKey('mac_address_ranges.id')), Column('interface_id', String(36), ForeignKey('interfaces.id')), diff --git a/melange/ipam/models.py b/melange/ipam/models.py index 7da54ae9..eae5511d 100644 --- a/melange/ipam/models.py +++ b/melange/ipam/models.py @@ -603,7 +603,8 @@ class IpAddress(ModelBase): def deallocate(self): self.update(marked_for_deallocation=True, - deallocated_at=utils.utcnow()) + deallocated_at=utils.utcnow(), + interface_id=None) def restore(self): self.update(marked_for_deallocation=False, deallocated_at=None) @@ -683,7 +684,7 @@ class MacAddressRange(ModelBase): @classmethod def allocate_next_free_mac(cls, **kwargs): - ranges = cls.find_all() + ranges = sorted(cls.find_all(), key=lambda model: model.created_at) for range in ranges: try: return range.allocate_mac(**kwargs) diff --git a/melange/tests/__init__.py b/melange/tests/__init__.py index 845386ef..8156c25b 100644 --- a/melange/tests/__init__.py +++ b/melange/tests/__init__.py @@ -25,6 +25,7 @@ from melange.db import db_api class BaseTest(unittest.TestCase): def setUp(self): + self.maxDiff = None self.mock = mox.Mox() db_api.clean_db() super(BaseTest, self).setUp() diff --git a/melange/tests/unit/test_ipam_models.py b/melange/tests/unit/test_ipam_models.py index 7bc52bc8..bb5c59ee 100644 --- a/melange/tests/unit/test_ipam_models.py +++ b/melange/tests/unit/test_ipam_models.py @@ -179,7 +179,7 @@ class TestIpBlock(tests.BaseTest): factory_models.PrivateIpBlockFactory(cidr="10.0.0.0/8", network_id="18888", tenant_id='xxxx') - saved_block = models.IpBlock.find_by(network_id=18888) + saved_block = models.IpBlock.find_by(network_id="18888") self.assertEqual(saved_block.cidr, "10.0.0.0/8") self.assertEqual(saved_block.network_id, '18888') self.assertEqual(saved_block.type, "private") @@ -196,13 +196,13 @@ class TestIpBlock(tests.BaseTest): def test_valid_cidr(self): factory = factory_models.PrivateIpBlockFactory - block = factory.build(cidr="10.1.1.1////", network_id=111) + block = factory.build(cidr="10.1.1.1////", network_id="111") self.assertFalse(block.is_valid()) self.assertEqual(block.errors, {'cidr': ["cidr is invalid"]}) self.assertRaises(models.InvalidModelError, block.save) self.assertRaises(models.InvalidModelError, models.IpBlock.create, - cidr="10.1.0.0/33", network_id=111) + cidr="10.1.0.0/33", network_id="111") block.cidr = "10.1.1.1/8" self.assertTrue(block.is_valid()) @@ -217,9 +217,9 @@ class TestIpBlock(tests.BaseTest): def test_validates_overlapping_cidr_for_public_ip_blocks(self): factory = factory_models.PublicIpBlockFactory - factory(cidr="10.0.0.0/8", network_id=145) + factory(cidr="10.0.0.0/8", network_id="145") - overlapping_block = factory.build(cidr="10.0.0.0/30", network_id=11) + overlapping_block = factory.build(cidr="10.0.0.0/30", network_id="11") self.assertFalse(overlapping_block.is_valid()) self.assertEqual(overlapping_block.errors, @@ -236,9 +236,9 @@ class TestIpBlock(tests.BaseTest): def test_different_types_of_blocks_cannot_be_created_within_network(self): factory = factory_models.IpBlockFactory - factory(network_id=1, type='private') + factory(network_id="1", type='private') - block_of_different_type = factory.build(network_id=1, type='public') + block_of_different_type = factory.build(network_id="1", type='public') self.assertFalse(block_of_different_type.is_valid()) self.assertEqual(block_of_different_type.errors, @@ -469,7 +469,9 @@ class TestIpBlock(tests.BaseTest): self.assertEqual(found_block.cidr, block1.cidr) def test_find_ip_block_for_nonexistent_block(self): - self.assertRaises(models.ModelNotFoundError, models.IpBlock.find, 123) + self.assertRaises(models.ModelNotFoundError, + models.IpBlock.find, + "123") def test_find_ip(self): block = factory_models.PrivateIpBlockFactory(cidr="10.0.0.1/8") @@ -1061,8 +1063,7 @@ class TestIpAddress(tests.BaseTest): ip_block_id=block1.id) interface = factory_models.InterfaceFactory() - expected_error = ("Failed to save IpAddress because: " - "columns address, ip_block_id are not unique") + expected_error = ("Failed to save IpAddress") self.assertRaisesExcMessage(exception.DBConstraintError, expected_error, models.IpAddress.create, @@ -1106,7 +1107,7 @@ class TestIpAddress(tests.BaseTest): def test_find_ip_address_for_nonexistent_address(self): self.assertRaises(models.ModelNotFoundError, models.IpAddress.find, - 123) + "123") def test_find_all_allocated_ips(self): block1 = factory_models.IpBlockFactory(tenant_id="1") @@ -1476,7 +1477,9 @@ class TestMacAddressRange(tests.BaseTest): self.assertRaises(models.NoMoreMacAddressesError, rng.allocate_mac) def test_allocate_next_free_mac_from_first_free_range(self): - factory_models.MacAddressRangeFactory(cidr="BC:76:4E:20:00:00/47") + two_days_before = datetime.datetime.now() - datetime.timedelta(days=2) + with unit.StubTime(time=two_days_before): + factory_models.MacAddressRangeFactory(cidr="BC:76:4E:20:00:00/47") factory_models.MacAddressRangeFactory(cidr="BC:76:4E:30:00:00/40") mac1 = models.MacAddressRange.allocate_next_free_mac() @@ -1757,7 +1760,7 @@ class TestPolicy(tests.BaseTest): self.assertRaises(models.ModelNotFoundError, policy.find_ip_range, - ip_range_id=122222) + ip_range_id="122222") def test_create_unusable_ip_range(self): policy = factory_models.PolicyFactory(name="BLAH") @@ -1910,25 +1913,25 @@ class TestIpOctet(tests.BaseTest): class TestNetwork(tests.BaseTest): def test_find_when_ip_blocks_for_given_network_exist(self): - ip_block1 = factory_models.PublicIpBlockFactory(network_id=1, - tenant_id=123) - noise_ip_block1 = factory_models.PublicIpBlockFactory(network_id=1, - tenant_id=321) + ip_block1 = factory_models.PublicIpBlockFactory(network_id="1", + tenant_id="123") + noise_ip_block1 = factory_models.PublicIpBlockFactory(network_id="1", + tenant_id="321") - network = models.Network.find_by(id=1, tenant_id=123) + network = models.Network.find_by(id="1", tenant_id="123") - self.assertEqual(network.id, 1) + self.assertEqual(network.id, "1") self.assertEqual(network.ip_blocks, [ip_block1]) def test_find_when_no_ip_blocks_for_given_network_exist(self): - noise_ip_block = factory_models.PublicIpBlockFactory(network_id=9999) + noise_ip_block = factory_models.PublicIpBlockFactory(network_id="9999") self.assertRaises(models.ModelNotFoundError, models.Network.find_by, - id=1) + id="1") def test_find_or_create_when_no_ip_blocks_for_given_network_exist(self): - noise_ip_block = factory_models.PublicIpBlockFactory(network_id=9999) + noise_ip_block = factory_models.PublicIpBlockFactory(network_id="9999") with unit.StubConfig(default_cidr="10.10.10.0/24"): network = models.Network.find_or_create_by(id='1', tenant_id='123') @@ -1941,13 +1944,13 @@ class TestNetwork(tests.BaseTest): self.assertEqual(network.ip_blocks[0].type, 'private') def test_allocate_ip_to_allocate_both_ipv4_and_ipv6_addresses(self): - ipv4_block = factory_models.PublicIpBlockFactory(network_id=1, + ipv4_block = factory_models.PublicIpBlockFactory(network_id="1", cidr="10.0.0.0/24", tenant_id="111") - ipv6_block = factory_models.PublicIpBlockFactory(network_id=1, + ipv6_block = factory_models.PublicIpBlockFactory(network_id="1", cidr="ff::00/120", tenant_id="111") - network = models.Network.find_by(id=1, tenant_id="111") + network = models.Network.find_by(id="1", tenant_id="111") interface = factory_models.InterfaceFactory() models.MacAddress.create(interface_id=interface.id, address="aa:bb:cc:dd:ee:ff") @@ -1961,9 +1964,9 @@ class TestNetwork(tests.BaseTest): def test_allocate_ip_from_first_free_ip_block(self): interface = factory_models.InterfaceFactory() - full_ip_block = factory_models.PublicIpBlockFactory(network_id=1, + full_ip_block = factory_models.PublicIpBlockFactory(network_id="1", is_full=True) - free_ip_block = factory_models.PublicIpBlockFactory(network_id=1, + free_ip_block = factory_models.PublicIpBlockFactory(network_id="1", is_full=False) network = models.Network(ip_blocks=[full_ip_block, free_ip_block]) [allocated_ipv4] = network.allocate_ips(interface=interface) @@ -1973,21 +1976,21 @@ class TestNetwork(tests.BaseTest): def test_allocate_ip_raises_error_when_all_ip_blocks_are_full(self): interface = factory_models.InterfaceFactory() - full_ip_block = factory_models.PublicIpBlockFactory(network_id=1, + full_ip_block = factory_models.PublicIpBlockFactory(network_id="1", is_full=True, tenant_id="111") - network = models.Network.find_by(id=1, tenant_id="111") + network = models.Network.find_by(id="1", tenant_id="111") self.assertRaises(exception.NoMoreAddressesError, network.allocate_ips, interface=interface) def test_allocate_ip_assigns_given_interface_and_addresses(self): - factory_models.PublicIpBlockFactory(network_id=1, cidr="10.0.0.0/24") - block = factory_models.PublicIpBlockFactory(network_id=1, + factory_models.PublicIpBlockFactory(network_id="1", cidr="10.0.0.0/24") + block = factory_models.PublicIpBlockFactory(network_id="1", cidr="169.0.0.0/24") addresses = ["10.0.0.7", "169.0.0.2", "10.0.0.3"] - network = models.Network.find_by(id=1, tenant_id=block.tenant_id) + network = models.Network.find_by(id="1", tenant_id=block.tenant_id) interface = factory_models.InterfaceFactory() allocated_ips = network.allocate_ips(interface=interface, @@ -1999,9 +2002,9 @@ class TestNetwork(tests.BaseTest): def test_allocate_ip_assigns_given_address_from_its_block(self): interface = factory_models.InterfaceFactory() - ip_block1 = factory_models.PublicIpBlockFactory(network_id=1, + ip_block1 = factory_models.PublicIpBlockFactory(network_id="1", cidr="10.0.0.0/24") - ip_block2 = factory_models.PublicIpBlockFactory(network_id=1, + ip_block2 = factory_models.PublicIpBlockFactory(network_id="1", cidr="20.0.0.0/24") network = models.Network(ip_blocks=[ip_block1, ip_block2]) @@ -2013,9 +2016,9 @@ class TestNetwork(tests.BaseTest): def test_allocate_ip_ignores_already_allocated_addresses(self): interface = factory_models.InterfaceFactory() - ip_block1 = factory_models.PublicIpBlockFactory(network_id=1, + ip_block1 = factory_models.PublicIpBlockFactory(network_id="1", cidr="10.0.0.0/24") - ip_block2 = factory_models.PublicIpBlockFactory(network_id=1, + ip_block2 = factory_models.PublicIpBlockFactory(network_id="1", cidr="20.0.0.0/24") factory_models.IpAddressFactory(ip_block_id=ip_block1.id, address="10.0.0.0") @@ -2027,12 +2030,12 @@ class TestNetwork(tests.BaseTest): self.assertEqual(allocted_ips[0].address, "20.0.0.0") def test_deallocate_ips(self): - ip_block1 = factory_models.IpBlockFactory(network_id=1, + ip_block1 = factory_models.IpBlockFactory(network_id="1", cidr="10.0.0.0/24") - ip_block2 = factory_models.IpBlockFactory(network_id=1, + ip_block2 = factory_models.IpBlockFactory(network_id="1", cidr="fe80::/64") - network = models.Network(id=1, ip_blocks=[ip_block1, ip_block2]) + network = models.Network(id="1", ip_blocks=[ip_block1, ip_block2]) interface = factory_models.InterfaceFactory() ip1 = factory_models.IpAddressFactory(ip_block_id=ip_block1.id, interface_id=interface.id) @@ -2056,29 +2059,29 @@ class TestNetwork(tests.BaseTest): ip3 = _allocate_ip(ip_block2, interface=interface2) ip4 = _allocate_ip(ip_block2, interface=interface1) - network = models.Network.find_by(id=1) + network = models.Network.find_by(id="1") allocated_ips = network.allocated_ips(interface_id=interface1.id) self.assertModelsEqual(allocated_ips, [ip1, ip2, ip4]) def test_find_allocated_ip(self): - ip_block1 = factory_models.IpBlockFactory(network_id=1, + ip_block1 = factory_models.IpBlockFactory(network_id="1", cidr="10.0.0.0/24") - ip_block2 = factory_models.IpBlockFactory(network_id=1, + ip_block2 = factory_models.IpBlockFactory(network_id="1", cidr="20.0.0.0/24") - noise_block = factory_models.IpBlockFactory(network_id=1, + noise_block = factory_models.IpBlockFactory(network_id="1", cidr="30.0.0.0/24") ip1 = _allocate_ip(ip_block1) ip2 = _allocate_ip(ip_block2) - network = models.Network.find_by(id=1) + network = models.Network.find_by(id="1") self.assertEqual(network.find_allocated_ip(address=ip1.address), ip1) self.assertEqual(network.find_allocated_ip(address=ip2.address), ip2) def test_find_allocated_ip_raises_model_not_found_for_invalid_ip(self): - ip_block = factory_models.IpBlockFactory(network_id=1, + ip_block = factory_models.IpBlockFactory(network_id="1", cidr="10.0.0.0/24") - network = models.Network.find_by(id=1) + network = models.Network.find_by(id="1") self.assertRaisesExcMessage(models.ModelNotFoundError, ("IpAddress with {'address': '10.1.1.1'} " "for network %s not found" % network.id), @@ -2216,6 +2219,8 @@ class TestInterface(tests.BaseTest): self.assertIsNone(models.Interface.get(interface.id)) self.assertTrue(models.IpAddress.get(ip1.id).locked()) self.assertTrue(models.IpAddress.get(ip2.id).locked()) + self.assertIsNone(models.IpAddress.get(ip1.id).interface_id) + self.assertIsNone(models.IpAddress.get(ip2.id).interface_id) def test_data(self): interface = factory_models.InterfaceFactory() diff --git a/melange/tests/unit/test_ipam_service.py b/melange/tests/unit/test_ipam_service.py index 0f35c78d..f675cfc2 100644 --- a/melange/tests/unit/test_ipam_service.py +++ b/melange/tests/unit/test_ipam_service.py @@ -567,7 +567,6 @@ class TestIpAddressController(ControllerTestBase): "interface_id": "iface", "used_by_device": "instance_id"} }) - ip = models.IpAddress.find(response.json['ip_address']['id']) self.assertEqual(ip.mac_address.eui_format, str(netaddr.EUI("BC:AD:CE:0:0:0"))) @@ -1954,7 +1953,7 @@ class TestInterfaceIpAllocationsController(ControllerTestBase): def test_create(self): ip_block = factory_models.PrivateIpBlockFactory(tenant_id="tnt_id", - network_id=1) + network_id="1") response = self.app.post("/ipam/tenants/tnt_id/networks/1/" "interfaces/123/ip_allocations") @@ -1977,7 +1976,7 @@ class TestInterfaceIpAllocationsController(ControllerTestBase): def test_create_with_given_address(self): ip_block = factory_models.PrivateIpBlockFactory(tenant_id="tnt_id", - network_id=1, + network_id="1", cidr="10.0.0.0/24") response = self.app.post_json("/ipam/tenants/tnt_id/networks/1/" @@ -1992,7 +1991,7 @@ class TestInterfaceIpAllocationsController(ControllerTestBase): def test_create_with_optional_params(self): ip_block = factory_models.PrivateIpBlockFactory(tenant_id="tnt_id", - network_id=1, + network_id="1", cidr="10.0.0.0/24") body = { @@ -2014,7 +2013,7 @@ class TestInterfaceIpAllocationsController(ControllerTestBase): def test_create_allocates_a_mac_as_well_when_mac_ranges_exist(self): factory_models.MacAddressRangeFactory(cidr="AD:BC:CE:0:0:0/24") ip_block = factory_models.PrivateIpBlockFactory(tenant_id="tnt_id", - network_id=1, + network_id="1", cidr="10.0.0.0/24") self.app.post_json("/ipam/tenants/tnt_id/networks/1/" @@ -2028,7 +2027,7 @@ class TestInterfaceIpAllocationsController(ControllerTestBase): mac_address = "11-22-33-44-55-66" ipv6_generator = mock_generator.MockIpV6Generator("fe::/96") ipv6_block = factory_models.PrivateIpBlockFactory(tenant_id="tnt_id", - network_id=1, + network_id="1", cidr="fe::/96") self.mock.StubOutWithMock(ipv6, "address_generator_factory") ipv6.address_generator_factory("fe::/96", @@ -2063,7 +2062,7 @@ class TestInterfaceIpAllocationsController(ControllerTestBase): def test_bulk_delete(self): ip_block = factory_models.PrivateIpBlockFactory(tenant_id="tnt_id", - network_id=1) + network_id="1") interface = factory_models.InterfaceFactory(virtual_interface_id="123") ip = ip_block.allocate_ip(interface=interface) diff --git a/melange/tests/unit/test_ipam_views.py b/melange/tests/unit/test_ipam_views.py index a889deb8..2e0a43a1 100644 --- a/melange/tests/unit/test_ipam_views.py +++ b/melange/tests/unit/test_ipam_views.py @@ -23,26 +23,35 @@ from melange.tests.factories import models as factory_models class TestIpConfigurationView(tests.BaseTest): - def test_data_returns_block_ip_and_route_info(self): + def test_data_returns_block_ip_info(self): block1 = factory_models.IpBlockFactory() + block2 = factory_models.IpBlockFactory() interface = factory_models.InterfaceFactory(virtual_interface_id="123") ip1 = factory_models.IpAddressFactory(ip_block_id=block1.id, interface_id=interface.id) - route1 = factory_models.IpRouteFactory(source_block_id=block1.id) - route2 = factory_models.IpRouteFactory(source_block_id=block1.id) - block2 = factory_models.IpBlockFactory() ip2 = factory_models.IpAddressFactory(ip_block_id=block2.id, interface_id=interface.id) + expected_ip1_config = _ip_data(ip1, block1) + expected_ip2_config = _ip_data(ip2, block2) ip_configuration_view = views.IpConfigurationView(ip1, ip2) - expected_ip1_config = _ip_data(ip1, block1) - expected_ip1_config['ip_block']['ip_routes'] = [_route_data(route1), - _route_data(route2)] - expected_ip2_config = _ip_data(ip2, block2) + self.assertEqual(expected_ip1_config, ip_configuration_view.data()[0]) + self.assertEqual(expected_ip2_config, ip_configuration_view.data()[1]) - self.assertEqual(ip_configuration_view.data()[0], expected_ip1_config) - self.assertEqual(ip_configuration_view.data()[1], expected_ip2_config) + def test_data_returns_route_info(self): + block = factory_models.IpBlockFactory() + interface = factory_models.InterfaceFactory(virtual_interface_id="123") + route1 = factory_models.IpRouteFactory(source_block_id=block.id) + route2 = factory_models.IpRouteFactory(source_block_id=block.id) + ip = factory_models.IpAddressFactory(ip_block_id=block.id, + interface_id=interface.id) + expected_ip_config_routes = [_route_data(route1), _route_data(route2)] + + ip_configuration_view = views.IpConfigurationView(ip).data()[0] + ip1_config_routes = ip_configuration_view['ip_block'].pop('ip_routes') + + self.assertItemsEqual(expected_ip_config_routes, ip1_config_routes) def _ip_data(ip, block): @@ -94,5 +103,5 @@ class TestInterfaceConfigurationView(tests.BaseTest): data = views.InterfaceConfigurationView(interface).data() self.assertEqual(len(data['ip_addresses']), 2) - self.assertEqual(data['ip_addresses'], - views.IpConfigurationView(ip1, ip2).data()) + self.assertItemsEqual(data['ip_addresses'], + views.IpConfigurationView(ip1, ip2).data())