diff --git a/vmware_nsx/plugins/nsx_v/drivers/exclusive_router_driver.py b/vmware_nsx/plugins/nsx_v/drivers/exclusive_router_driver.py
index c645c78fbf..826573345d 100644
--- a/vmware_nsx/plugins/nsx_v/drivers/exclusive_router_driver.py
+++ b/vmware_nsx/plugins/nsx_v/drivers/exclusive_router_driver.py
@@ -243,7 +243,7 @@ class RouterExclusiveDriver(router_driver.RouterBaseDriver):
                     'fixed_ips', [])]
             subnet = port_subnets[0]
 
-        if subnet and self._check_lb_on_subnet(context, subnet):
+        if subnet and self._check_lb_on_subnet(context, subnet, router_id):
             error = _('Cannot delete router %(rtr)s interface while '
                       'loadbalancers are provisioned on attached '
                       'subnet %(subnet)s') % {'rtr': router_id,
@@ -277,7 +277,7 @@ class RouterExclusiveDriver(router_driver.RouterBaseDriver):
                                                      address_groups)
         return info
 
-    def _check_lb_on_subnet(self, context, subnet_id):
+    def _check_lb_on_subnet(self, context, subnet_id, router_id):
         # Check lbaas
         dev_owner_v1 = 'neutron:' + plugin_const.LOADBALANCER
         dev_owner_v2 = 'neutron:' + plugin_const.LOADBALANCERV2
@@ -285,7 +285,11 @@ class RouterExclusiveDriver(router_driver.RouterBaseDriver):
                    'fixed_ips': {'subnet_id': [subnet_id]}}
         ports = super(nsx_v.NsxVPluginV2, self.plugin).get_ports(
             context, filters=filters)
-        return (len(ports) >= 1)
+
+        edge_id = self._get_router_edge_id(context, router_id)
+        lb_binding = nsxv_db.get_nsxv_lbaas_loadbalancer_binding_by_edge(
+            context.session, edge_id)
+        return (len(ports) >= 1) and lb_binding
 
     def _update_edge_router(self, context, router_id):
         router = self.plugin._get_router(context.elevated(), router_id)
diff --git a/vmware_nsx/tests/unit/nsx_v/test_plugin.py b/vmware_nsx/tests/unit/nsx_v/test_plugin.py
index 7df13916ef..43e91c6d95 100644
--- a/vmware_nsx/tests/unit/nsx_v/test_plugin.py
+++ b/vmware_nsx/tests/unit/nsx_v/test_plugin.py
@@ -3248,19 +3248,6 @@ class TestExclusiveRouterTestCase(L3NatTest, L3NatTestCaseBase,
                 body = self._show('routers', router_id)
                 self.assertEqual('shared', body['router']['router_type'])
 
-    def test_router_remove_interface_with_load_balancer(self):
-        with self.router() as router, self.subnet() as subnet:
-            fixed_ips = [{'subnet_id': subnet['subnet']['id']}]
-            with self.port(subnet,
-                           device_owner='neutron:LOADBALANCER',
-                           fixed_ips=fixed_ips):
-                expected_code = webob.exc.HTTPInternalServerError.code
-                self._router_interface_action('remove',
-                                              router['router']['id'],
-                                              subnet['subnet']['id'],
-                                              None,
-                                              expected_code=expected_code)
-
     @mock.patch.object(edge_utils.EdgeManager,
                        'update_interface_addr')
     def test_router_update_gateway_with_different_external_subnet(self, mock):