Implement router gateway removing

When creating external network in top pod, az hint is passed to
specify which pod bottom external network is located. So plugin
can get bottom router ID with top router ID and bottom pod ID
from resource routing table.

Plugin fist updates router in top pod to remove gateway, then
sends "remove_gateway" request to target bottom pod to update
bottom router.

Change-Id: I69e411188e758016ea789a91298ccd243bdc31cd

tricircle/common/client.py

@@ -514,6 +514,7 @@ class Client(object):
                volume -> set_bootable -> volume, flag -> none
                router -> add_interface -> router, body -> none
                router -> add_gateway -> router, body -> none
+               router -> remove_gateway -> router -> none
                server_volume -> create_server_volume
                              -> server_id, volume_id, device=None
                              -> none

tricircle/network/plugin.py

@@ -898,52 +898,68 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
             query = context.session.query(l3_db.RouterPort)
             query.filter_by(port_id=port_id, router_id=router_id).delete()
 
-    def _update_bottom_router_gateway(self, context, router_id, router_data):
+    def _add_router_gateway(self, context, router_id, router_data):
+        # get top external network information
         ext_net_id = router_data[l3.EXTERNAL_GW_INFO].get('network_id')
-        if ext_net_id:
+        t_ctx = t_context.get_context_from_neutron_context(context)
-            # add router gateway
+        network = self.get_network(context, ext_net_id)
-            t_ctx = t_context.get_context_from_neutron_context(context)
-            network = self.get_network(context, ext_net_id)
-            pod_name = network[az_ext.AZ_HINTS][0]
-            pod = db_api.get_pod_by_name(t_ctx, pod_name)
-            b_net_id = db_api.get_bottom_id_by_top_id_pod_name(
-                t_ctx, ext_net_id, pod_name, t_constants.RT_NETWORK)
-            t_router = self._get_router(context, router_id)
-            body = {'router': {'name': router_id,
-                               'distributed': False}}
-            _, b_router_id = self._prepare_bottom_element(
-                t_ctx, t_router['tenant_id'], pod, t_router,
-                t_constants.RT_ROUTER, body)
-            b_client = self._get_client(pod_name)
-            t_info = router_data[l3.EXTERNAL_GW_INFO]
-            b_info = {'network_id': b_net_id}
-            if 'enable_snat' in t_info:
-                b_info['enable_snat'] = t_info['enable_snat']
-            if 'external_fixed_ips' in t_info:
-                fixed_ips = []
-                for ip in t_info['external_fixed_ips']:
-                    t_subnet_id = ip['subnet_id']
-                    b_subnet_id = db_api.get_bottom_id_by_top_id_pod_name(
-                        t_ctx, t_subnet_id, pod_name,
-                        t_constants.RT_SUBNET)
-                    fixed_ips.append({'subnet_id': b_subnet_id,
-                                      'ip_address': ip['ip_address']})
-                b_info['external_fixed_ips'] = fixed_ips
-            b_client.action_routers(t_ctx, 'add_gateway', b_router_id, b_info)
 
-            # create bridge network and attach to router
+        # when creating external network in top pod, pod name is passed via
-            t_pod = db_api.get_top_pod(t_ctx)
+        # az hint parameter, so tricircle plugin knows where to create the
-            project_id = t_router['tenant_id']
+        # corresponding bottom external network. here we get bottom external
-            admin_project_id = 'admin_project_id'
+        # network ID from resource routing table.
-            pool_id = self._get_bridge_subnet_pool_id(
+        pod_name = network[az_ext.AZ_HINTS][0]
-                t_ctx, context, admin_project_id, t_pod, False)
+        pod = db_api.get_pod_by_name(t_ctx, pod_name)
-            t_bridge_net, t_bridge_subnet = self._get_bridge_network_subnet(
+        b_net_id = db_api.get_bottom_id_by_top_id_pod_name(
-                t_ctx, context, project_id, t_pod, pool_id, False)
+            t_ctx, ext_net_id, pod_name, t_constants.RT_NETWORK)
-            (_, _, b_bridge_subnet_id,
+
-             b_bridge_net_id) = self._get_bottom_bridge_elements(
+        # create corresponding bottom router in the pod where external network
-                context, project_id, pod, t_bridge_net, False, t_bridge_subnet,
+        # is located.
-                None)
+        t_router = self._get_router(context, router_id)
-            is_attach = False
+        body = {'router': {'name': router_id,
+                           'distributed': False}}
+        _, b_router_id = self._prepare_bottom_element(
+            t_ctx, t_router['tenant_id'], pod, t_router,
+            t_constants.RT_ROUTER, body)
+
+        # both router and external network in bottom pod are ready, attach
+        # external network to router in bottom pod.
+        b_client = self._get_client(pod_name)
+        t_info = router_data[l3.EXTERNAL_GW_INFO]
+        b_info = {'network_id': b_net_id}
+        if 'enable_snat' in t_info:
+            b_info['enable_snat'] = t_info['enable_snat']
+        if 'external_fixed_ips' in t_info:
+            fixed_ips = []
+            for ip in t_info['external_fixed_ips']:
+                t_subnet_id = ip['subnet_id']
+                b_subnet_id = db_api.get_bottom_id_by_top_id_pod_name(
+                    t_ctx, t_subnet_id, pod_name,
+                    t_constants.RT_SUBNET)
+                fixed_ips.append({'subnet_id': b_subnet_id,
+                                  'ip_address': ip['ip_address']})
+            b_info['external_fixed_ips'] = fixed_ips
+        b_client.action_routers(t_ctx, 'add_gateway', b_router_id, b_info)
+
+        # when internal network(providing fixed ip) and external network
+        # (providing floating ip) are in different bottom pods, we utilize a
+        # bridge network to connect these two networks. here we create the
+        # bridge network.
+        t_pod = db_api.get_top_pod(t_ctx)
+        project_id = t_router['tenant_id']
+        pool_id = self._get_bridge_subnet_pool_id(
+            t_ctx, context, None, t_pod, False)
+        t_bridge_net, t_bridge_subnet = self._get_bridge_network_subnet(
+            t_ctx, context, project_id, t_pod, pool_id, False)
+        (_, _, b_bridge_subnet_id,
+         b_bridge_net_id) = self._get_bottom_bridge_elements(
+            context, project_id, pod, t_bridge_net, False, t_bridge_subnet,
+            None)
+
+        # here we attach the bridge network to the router in bottom pod. to
+        # make this method reentrant, we check if the interface is already
+        # attached before attaching the interface.
+        def _is_bridge_network_attached():
             interfaces = b_client.list_ports(t_ctx,
                                              filters=[{'key': 'device_id',
                                                        'comparator': 'eq',
@@ -951,13 +967,39 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
             for interface in interfaces:
                 for fixed_ip in interface['fixed_ips']:
                     if fixed_ip['subnet_id'] == b_bridge_subnet_id:
-                        is_attach = True
+                        return True
-                        break
+            return False
-                if is_attach:
+
-                    break
+        is_attach = _is_bridge_network_attached()
-            if not is_attach:
+        if not is_attach:
-                b_client.action_routers(t_ctx, 'add_interface', b_router_id,
+            b_client.action_routers(t_ctx, 'add_interface', b_router_id,
-                                        {'subnet_id': b_bridge_subnet_id})
+                                    {'subnet_id': b_bridge_subnet_id})
+
+    def _remove_router_gateway(self, context, router_id):
+        t_ctx = t_context.get_context_from_neutron_context(context)
+        t_router = self._get_router(context, router_id)
+        gw_port = t_router.gw_port
+        if not gw_port:
+            return
+        ext_net_id = gw_port['network_id']
+        t_network = self.get_network(context, ext_net_id)
+        if az_ext.AZ_HINTS not in t_network:
+            raise t_exceptions.ExternalNetPodNotSpecify()
+        if not t_network[az_ext.AZ_HINTS]:
+            raise t_exceptions.ExternalNetPodNotSpecify()
+
+        pod_name = t_network[az_ext.AZ_HINTS][0]
+        b_router_id = db_api.get_bottom_id_by_top_id_pod_name(
+            t_ctx, router_id, pod_name, t_constants.RT_ROUTER)
+        b_client = self._get_client(pod_name)
+        b_client.action_routers(t_ctx, 'remove_gateway', b_router_id)
+
+    def _update_bottom_router_gateway(self, context, router_id, router_data):
+        ext_net_id = router_data[l3.EXTERNAL_GW_INFO].get('network_id')
+        if ext_net_id:
+            self._add_router_gateway(context, router_id, router_data)
+        else:
+            self._remove_router_gateway(context, router_id)
 
     def update_router(self, context, router_id, router):
         router_data = router['router']
@@ -982,7 +1024,6 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
 
         router = self._get_router(context, router_id)
         project_id = router['tenant_id']
-        admin_project_id = 'admin_project_id'
         add_by_port, _ = self._validate_interface_info(interface_info)
         # make sure network not crosses pods
         # TODO(zhiyuan) support cross-pod tenant network
@@ -999,7 +1040,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
 
         # bridge network for E-W networking
         pool_id = self._get_bridge_subnet_pool_id(
-            t_ctx, context, admin_project_id, t_pod, True)
+            t_ctx, context, None, t_pod, True)
         t_bridge_net, t_bridge_subnet = self._get_bridge_network_subnet(
             t_ctx, context, project_id, t_pod, pool_id, True)
         t_bridge_port = self._get_bridge_interface(
@@ -1022,7 +1063,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
                 need_ns_bridge = True
         if need_ns_bridge:
             pool_id = self._get_bridge_subnet_pool_id(
-                t_ctx, context, admin_project_id, t_pod, False)
+                t_ctx, context, None, t_pod, False)
             t_bridge_net, t_bridge_subnet = self._get_bridge_network_subnet(
                 t_ctx, context, project_id, t_pod, pool_id, False)
             (_, _, b_bridge_subnet_id,

tricircle/tests/unit/network/test_plugin.py

@@ -470,6 +470,9 @@ class FakeQuery(object):
     def all(self):
         return self.records
 
+    def count(self):
+        return len(self.records)
+
     def __iter__(self):
         return self
 
@@ -562,6 +565,9 @@ class FakeSession(object):
     def flush(self):
         pass
 
+    def expire(self, obj):
+        pass
+
 
 class FakeRPCAPI(object):
     def configure_extra_routes(self, context, router_id):
@@ -1549,6 +1555,83 @@ class PluginTest(unittest.TestCase,
                            {'subnet_id': b_ns_bridge_subnet_id})]
         mock_action.assert_has_calls(calls)
 
+    @patch.object(ipam_non_pluggable_backend.IpamNonPluggableBackend,
+                  '_allocate_specific_ip', new=_allocate_specific_ip)
+    @patch.object(ipam_non_pluggable_backend.IpamNonPluggableBackend,
+                  '_generate_ip', new=fake_generate_ip)
+    @patch.object(l3_db.L3_NAT_dbonly_mixin, '_make_router_dict',
+                  new=fake_make_router_dict)
+    @patch.object(db_base_plugin_common.DbBasePluginCommon,
+                  '_make_subnet_dict', new=fake_make_subnet_dict)
+    @patch.object(subnet_alloc.SubnetAllocator, '_lock_subnetpool',
+                  new=mock.Mock)
+    @patch.object(FakeClient, 'action_routers')
+    @patch.object(context, 'get_context_from_neutron_context')
+    def test_unset_gateway(self, mock_context, mock_action):
+        plugin_path = 'tricircle.tests.unit.network.test_plugin.FakePlugin'
+        cfg.CONF.set_override('core_plugin', plugin_path)
+
+        self._basic_pod_route_setup()
+
+        fake_plugin = FakePlugin()
+        q_ctx = FakeNeutronContext()
+        t_ctx = context.get_db_context()
+        mock_context.return_value = t_ctx
+
+        tenant_id = 'test_tenant_id'
+        t_net_body = {
+            'name': 'ext_net',
+            'availability_zone_hints': ['pod_1'],
+            'tenant_id': tenant_id,
+            'router:external': True,
+            'admin_state_up': True,
+            'shared': False,
+        }
+        fake_plugin.create_network(q_ctx, {'network': t_net_body})
+        t_net_id = TOP_NETS[0]['id']
+
+        t_subnet_body = {
+            'network_id': t_net_id,  # only one network created
+            'name': 'ext_subnet',
+            'ip_version': 4,
+            'cidr': '100.64.0.0/24',
+            'allocation_pools': [],
+            'enable_dhcp': False,
+            'gateway_ip': '100.64.0.1',
+            'dns_nameservers': '',
+            'host_routes': '',
+            'tenant_id': tenant_id
+        }
+        fake_plugin.create_subnet(q_ctx, {'subnet': t_subnet_body})
+        t_subnet_id = TOP_SUBNETS[0]['id']
+
+        t_router_id = uuidutils.generate_uuid()
+        t_router = {
+            'id': t_router_id,
+            'name': 'router',
+            'distributed': False,
+            'tenant_id': tenant_id,
+            'attached_ports': []
+        }
+
+        TOP_ROUTERS.append(DotDict(t_router))
+        # first add router gateway
+        fake_plugin.update_router(
+            q_ctx, t_router_id,
+            {'router': {'external_gateway_info': {
+                'network_id': t_net_id,
+                'enable_snat': False,
+                'external_fixed_ips': [{'subnet_id': t_subnet_id,
+                                        'ip_address': '100.64.0.5'}]}}})
+        _, b_router_id = db_api.get_bottom_mappings_by_top_id(
+            t_ctx, t_router_id, constants.RT_ROUTER)[0]
+
+        # then remove router gateway
+        fake_plugin.update_router(
+            q_ctx, t_router_id,
+            {'router': {'external_gateway_info': {}}})
+        mock_action.assert_called_with(t_ctx, 'remove_gateway', b_router_id)
+
     def _prepare_associate_floatingip_test(self, t_ctx, q_ctx, fake_plugin):
         tenant_id = 'test_tenant_id'
         self._basic_pod_route_setup()