NSX|V3: Simplify LBaaS implementation

Until know, for scale issues, the creation of some NSX backend resources
for loadbalancing was postpone until the first member creation.
This complicates the code unnecessarily, since the scale issues were
already resolved.

The new code will create the matching backend objects for each
LBaaS/Octavia object upon creation.
In case external vip loadbalancer - the service will be created without an attachement,
which will be added upon member creation.

In addition a DB migration is added to mark as ERROR old incomplete load
balancers.

Depends-on: Ic4e604883a7b1437af995110d2d684c0bd396a52
Change-Id: Ib478c336840c2e441bbaeffe94700a5e267c6bef

releasenotes/notes/nsxv3-lbaasv2-error-no-member-635ffc6308289aca.yaml

@@ -0,0 +1,7 @@
+---
+prelude: >
+    NSXv3 plugin will mark unused loadbalancers in ERROR state.
+features:
+  - |
+    Upon upgrade to Stein, unused LBaaS-v2 loadbalancers, which have no members
+    will be marked in ERROR state, and cannot be used.

vmware_nsx/db/db.py

@@ -562,6 +562,14 @@ def get_nsx_lbaas_loadbalancer_binding(session, loadbalancer_id):
         return
 
 
+def update_nsx_lbaas_loadbalancer_binding(session, loadbalancer_id,
+                                          lb_router_id):
+    with session.begin(subtransactions=True):
+        binding = (session.query(nsx_models.NsxLbaasLoadbalancer).
+                   filter_by(loadbalancer_id=loadbalancer_id).one())
+        binding.lb_router_id = lb_router_id
+
+
 def get_nsx_lbaas_loadbalancer_bindings(session):
     return session.query(nsx_models.NsxLbaasLoadbalancer).all()
 

vmware_nsx/db/migration/alembic_migrations/versions/EXPAND_HEAD

@@ -1 +1 @@
-fc6308289aca
+99bfcb6003c6

vmware_nsx/db/migration/alembic_migrations/versions/stein/expand/99bfcb6003c6_lbaas_error_no_member.py

@@ -0,0 +1,38 @@
+# Copyright 2019 VMware, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""lbaas_error_no_member
+
+Revision ID: 99bfcb6003c6
+Revises: fc6308289aca
+Create Date: 2019-03-07 11:27:00.000000
+"""
+
+from alembic import op
+
+from neutron.db import migration
+
+# revision identifiers, used by Alembic.
+revision = '99bfcb6003c6'
+down_revision = 'fc6308289aca'
+
+
+def upgrade():
+    if (migration.schema_has_table('nsxv3_lbaas_loadbalancers') and
+        migration.schema_has_table('lbaas_loadbalancers')):
+        # Mark as ERROR loadbalancers without nsx mapping
+        op.execute("UPDATE lbaas_loadbalancers "
+                   "SET provisioning_status='ERROR' "
+                   "WHERE id not in (SELECT loadbalancer_id FROM "
+                   "nsxv3_lbaas_loadbalancers)")

vmware_nsx/services/lbaas/nsx_v3/implementation/lb_utils.py

@@ -24,6 +24,7 @@ from vmware_nsx.services.lbaas import lb_const
 from vmware_nsxlib.v3 import nsx_constants
 
 ADV_RULE_NAME = 'LB external VIP advertisement'
+NO_ROUTER_ID = 'NO ROUTER'
 
 
 @log_helpers.log_method_call

vmware_nsx/services/lbaas/nsx_v3/implementation/listener_mgr.py

@@ -149,16 +149,20 @@ class EdgeListenerManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
         # server to the lb service
         binding = nsx_db.get_nsx_lbaas_loadbalancer_binding(
             context.session, lb_id)
-        if binding:
+        if not binding:
-            lb_service_id = binding['lb_service_id']
+            completor(success=False)
-            try:
+            msg = _('Failed to get loadbalancer %s binding') % lb_id
-                service_client.add_virtual_server(lb_service_id,
+            raise n_exc.BadRequest(resource='lbaas-listener', msg=msg)
-                                                  virtual_server['id'])
+
-            except nsxlib_exc.ManagerError:
+        lb_service_id = binding['lb_service_id']
-                completor(success=False)
+        try:
-                msg = _('Failed to add virtual server to lb service '
+            service_client.add_virtual_server(lb_service_id,
-                        'at NSX backend')
+                                              virtual_server['id'])
-                raise n_exc.BadRequest(resource='lbaas-listener', msg=msg)
+        except nsxlib_exc.ManagerError:
+            completor(success=False)
+            msg = _('Failed to add virtual server to lb service '
+                    'at NSX backend')
+            raise n_exc.BadRequest(resource='lbaas-listener', msg=msg)
 
         nsx_db.add_nsx_lbaas_listener_binding(
             context.session, lb_id, listener['id'], app_profile_id,
@@ -219,19 +223,24 @@ class EdgeListenerManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
             app_profile_id = binding['app_profile_id']
             lb_binding = nsx_db.get_nsx_lbaas_loadbalancer_binding(
                 context.session, lb_id)
-            if lb_binding:
+            if not lb_binding:
-                try:
+                completor(success=False)
-                    lbs_id = lb_binding.get('lb_service_id')
+                msg = (_('Failed to delete virtual server: %(listener)s: '
-                    lb_service = service_client.get(lbs_id)
+                         'loadbalancer %(lb)s mapping was not found') %
-                    vs_list = lb_service.get('virtual_server_ids')
+                       {'listener': listener['id'], 'lb': lb_id})
-                    if vs_list and vs_id in vs_list:
+                raise n_exc.BadRequest(resource='lbaas-listener', msg=msg)
-                        service_client.remove_virtual_server(lbs_id, vs_id)
+            try:
-                except nsxlib_exc.ManagerError:
+                lbs_id = lb_binding.get('lb_service_id')
-                    completor(success=False)
+                lb_service = service_client.get(lbs_id)
-                    msg = (_('Failed to remove virtual server: %(listener)s '
+                vs_list = lb_service.get('virtual_server_ids')
-                             'from lb service %(lbs)s') %
+                if vs_list and vs_id in vs_list:
-                           {'listener': listener['id'], 'lbs': lbs_id})
+                    service_client.remove_virtual_server(lbs_id, vs_id)
-                    raise n_exc.BadRequest(resource='lbaas-listener', msg=msg)
+            except nsxlib_exc.ManagerError:
+                completor(success=False)
+                msg = (_('Failed to remove virtual server: %(listener)s '
+                         'from lb service %(lbs)s') %
+                       {'listener': listener['id'], 'lbs': lbs_id})
+                raise n_exc.BadRequest(resource='lbaas-listener', msg=msg)
             try:
                 if listener.get('default_pool_id'):
                     vs_client.update(vs_id, pool_id='')

vmware_nsx/services/lbaas/nsx_v3/implementation/loadbalancer_mgr.py

@@ -20,6 +20,7 @@ from oslo_log import log as logging
 from oslo_utils import excutils
 
 from vmware_nsx._i18n import _
+from vmware_nsx.common import exceptions as nsx_exc
 from vmware_nsx.db import db as nsx_db
 from vmware_nsx.services.lbaas import base_mgr
 from vmware_nsx.services.lbaas import lb_const
@@ -34,10 +35,8 @@ class EdgeLoadBalancerManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
 
     @log_helpers.log_method_call
     def create(self, context, lb, completor):
-        if lb_utils.validate_lb_subnet(context, self.core_plugin,
+        if not lb_utils.validate_lb_subnet(context, self.core_plugin,
-                                       lb['vip_subnet_id']):
+                                           lb['vip_subnet_id']):
-            completor(success=True)
-        else:
             completor(success=False)
             msg = (_('Cannot create lb on subnet %(sub)s for '
                      'loadbalancer %(lb)s. The subnet needs to connect a '
@@ -45,6 +44,98 @@ class EdgeLoadBalancerManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
                    {'sub': lb['vip_subnet_id'], 'lb': lb['id']})
             raise n_exc.BadRequest(resource='lbaas-subnet', msg=msg)
 
+        service_client = self.core_plugin.nsxlib.load_balancer.service
+        nsx_router_id = None
+        lb_service = None
+        nsx_router_id = lb_utils.NO_ROUTER_ID
+        router_id = lb_utils.get_router_from_network(
+            context, self.core_plugin, lb['vip_subnet_id'])
+        if router_id:
+            nsx_router_id = nsx_db.get_nsx_router_id(context.session,
+                                                     router_id)
+            lb_service = service_client.get_router_lb_service(nsx_router_id)
+        if not lb_service:
+            lb_size = lb_utils.get_lb_flavor_size(
+                self.flavor_plugin, context, lb.get('flavor_id'))
+            if router_id:
+                # Make sure the NSX service router exists
+                if not self.core_plugin.service_router_has_services(
+                        context, router_id):
+                    self.core_plugin.create_service_router(context, router_id)
+                lb_service = self._create_lb_service(
+                    context, service_client, lb['tenant_id'],
+                    router_id, nsx_router_id, lb['id'], lb_size)
+            else:
+                lb_service = self._create_lb_service_without_router(
+                    context, service_client, lb['tenant_id'],
+                    lb, lb_size)
+            if not lb_service:
+                completor(success=False)
+                msg = (_('Failed to create lb service for loadbalancer '
+                         '%s') % lb['id'])
+                raise nsx_exc.NsxPluginException(err_msg=msg)
+
+        nsx_db.add_nsx_lbaas_loadbalancer_binding(
+            context.session, lb['id'], lb_service['id'],
+            nsx_router_id, lb['vip_address'])
+
+        completor(success=True)
+
+    @log_helpers.log_method_call
+    def _create_lb_service(self, context, service_client, tenant_id,
+                           router_id, nsx_router_id, lb_id, lb_size):
+        """Create NSX LB service for a specific neutron router"""
+        router = self.core_plugin.get_router(context, router_id)
+        if not router.get('external_gateway_info'):
+            msg = (_('Tenant router %(router)s does not connect to '
+                     'external gateway') % {'router': router['id']})
+            raise n_exc.BadRequest(resource='lbaas-lbservice-create',
+                                   msg=msg)
+        lb_name = utils.get_name_and_uuid(router['name'] or 'router',
+                                          router_id)
+        tags = lb_utils.get_tags(self.core_plugin, router_id,
+                                 lb_const.LR_ROUTER_TYPE,
+                                 tenant_id, context.project_name)
+        attachment = {'target_id': nsx_router_id,
+                      'target_type': 'LogicalRouter'}
+        try:
+            lb_service = service_client.create(display_name=lb_name,
+                                               tags=tags,
+                                               attachment=attachment,
+                                               size=lb_size)
+        except nsxlib_exc.ManagerError as e:
+            LOG.error("Failed to create LB service: %s", e)
+            return
+
+        # Add rule to advertise external vips
+        lb_utils.update_router_lb_vip_advertisement(
+            context, self.core_plugin, router, nsx_router_id)
+
+        return lb_service
+
+    @log_helpers.log_method_call
+    def _create_lb_service_without_router(self, context, service_client,
+                                          tenant_id, lb, lb_size):
+        """Create NSX LB service for an external VIP
+        This service will not be attached to a router yet, and it will be
+        updated once the first member is created.
+        """
+        lb_id = lb['id']
+        lb_name = utils.get_name_and_uuid(lb['name'] or 'loadbalancer',
+                                          lb_id)
+        tags = lb_utils.get_tags(self.core_plugin, '',
+                                 lb_const.LR_ROUTER_TYPE,
+                                 tenant_id, context.project_name)
+        try:
+            lb_service = service_client.create(display_name=lb_name,
+                                               tags=tags,
+                                               size=lb_size)
+        except nsxlib_exc.ManagerError as e:
+            LOG.error("Failed to create LB service: %s", e)
+            return
+
+        return lb_service
+
     @log_helpers.log_method_call
     def update(self, context, old_lb, new_lb, completor):
         vs_client = self.core_plugin.nsxlib.load_balancer.virtual_server
@@ -82,6 +173,7 @@ class EdgeLoadBalancerManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
     @log_helpers.log_method_call
     def delete(self, context, lb, completor):
         service_client = self.core_plugin.nsxlib.load_balancer.service
+        router_client = self.core_plugin.nsxlib.logical_router
         lb_binding = nsx_db.get_nsx_lbaas_loadbalancer_binding(
             context.session, lb['id'])
         if lb_binding:
@@ -99,10 +191,10 @@ class EdgeLoadBalancerManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
                         service_client.delete(lb_service_id)
                         # If there is no lb service attached to the router,
                         # delete the router advertise_lb_vip rule.
-                        router_client = self.core_plugin.nsxlib.logical_router
+                        if nsx_router_id != lb_utils.NO_ROUTER_ID:
-                        router_client.update_advertisement_rules(
+                            router_client.update_advertisement_rules(
-                            nsx_router_id, [],
+                                nsx_router_id, [],
-                            name_prefix=lb_utils.ADV_RULE_NAME)
+                                name_prefix=lb_utils.ADV_RULE_NAME)
                     except nsxlib_exc.ManagerError:
                         completor(success=False)
                         msg = (_('Failed to delete lb service %(lbs)s from nsx'
@@ -110,15 +202,16 @@ class EdgeLoadBalancerManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
                         raise n_exc.BadRequest(resource='lbaas-lb', msg=msg)
             nsx_db.delete_nsx_lbaas_loadbalancer_binding(
                 context.session, lb['id'])
-            router_id = nsx_db.get_neutron_from_nsx_router_id(
+            if nsx_router_id != lb_utils.NO_ROUTER_ID:
-                context.session, nsx_router_id)
+                router_id = nsx_db.get_neutron_from_nsx_router_id(
-            # Service router is needed only when the LB exist, and
+                    context.session, nsx_router_id)
-            # no other services are using it.
+                # Service router is needed only when the LB exist, and
-            if not self.core_plugin.service_router_has_services(
+                # no other services are using it.
-                    context,
+                if not self.core_plugin.service_router_has_services(
-                    router_id):
+                        context,
-                self.core_plugin.delete_service_router(context,
+                        router_id):
-                                                       router_id)
+                    self.core_plugin.delete_service_router(context,
+                                                           router_id)
         completor(success=True)
 
     @log_helpers.log_method_call
@@ -189,7 +282,8 @@ class EdgeLoadBalancerManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
         lb_binding = nsx_db.get_nsx_lbaas_loadbalancer_binding(
             context.session, id)
         if not lb_binding:
-            # No service yet
+            LOG.warning("Failed to get loadbalancer %s operating status. "
+                        "Mapping was not found", id)
             return {}
 
         lb_service_id = lb_binding['lb_service_id']

vmware_nsx/services/lbaas/nsx_v3/implementation/member_mgr.py

@@ -26,7 +26,6 @@ from vmware_nsx.services.lbaas import base_mgr
 from vmware_nsx.services.lbaas import lb_const
 from vmware_nsx.services.lbaas.nsx_v3.implementation import lb_utils
 from vmware_nsxlib.v3 import exceptions as nsxlib_exc
-from vmware_nsxlib.v3 import utils
 
 LOG = logging.getLogger(__name__)
 
@@ -45,37 +44,6 @@ class EdgeMemberManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
                      'be a floating IP') % {'fip': fip})
             raise n_exc.BadRequest(resource='lbaas-vip', msg=msg)
 
-    @log_helpers.log_method_call
-    def _create_lb_service(self, context, service_client, tenant_id,
-                           router_id, nsx_router_id, lb_id, lb_size):
-        router = self.core_plugin.get_router(context, router_id)
-        if not router.get('external_gateway_info'):
-            msg = (_('Tenant router %(router)s does not connect to '
-                     'external gateway') % {'router': router['id']})
-            raise n_exc.BadRequest(resource='lbaas-lbservice-create',
-                                   msg=msg)
-        lb_name = utils.get_name_and_uuid(router['name'] or 'router',
-                                          router_id)
-        tags = lb_utils.get_tags(self.core_plugin, router_id,
-                                 lb_const.LR_ROUTER_TYPE,
-                                 tenant_id, context.project_name)
-        attachment = {'target_id': nsx_router_id,
-                      'target_type': 'LogicalRouter'}
-        try:
-            lb_service = service_client.create(display_name=lb_name,
-                                               tags=tags,
-                                               attachment=attachment,
-                                               size=lb_size)
-        except nsxlib_exc.ManagerError as e:
-            LOG.error("Failed to create LB service: %s", e)
-            return
-
-        # Add rule to advertise external vips
-        lb_utils.update_router_lb_vip_advertisement(
-            context, self.core_plugin, router, nsx_router_id)
-
-        return lb_service
-
     @log_helpers.log_method_call
     def _get_updated_pool_members(self, context, lb_pool, member):
         network = lb_utils.get_network_from_subnet(
@@ -91,21 +59,6 @@ class EdgeMemberManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
                 m['weight'] = member['weight']
         return lb_pool['members']
 
-    @log_helpers.log_method_call
-    def _add_loadbalancer_binding(self, context, lb_id, lbs_id,
-                                  nsx_router_id, vip_address):
-        # First check if there is already binding for the lb.
-        # If there is no binding for the lb, add the db binding.
-        binding = nsx_db.get_nsx_lbaas_loadbalancer_binding(
-            context.session, lb_id)
-        if not binding:
-            nsx_db.add_nsx_lbaas_loadbalancer_binding(
-                context.session, lb_id, lbs_id,
-                nsx_router_id, vip_address)
-        else:
-            LOG.debug("LB binding has already been added, and no need "
-                      "to add here.")
-
     @log_helpers.log_method_call
     def create(self, context, member, completor):
         with locking.LockManager.get_lock(
@@ -148,43 +101,44 @@ class EdgeMemberManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
         binding = nsx_db.get_nsx_lbaas_pool_binding(context.session,
                                                     lb_id, pool_id)
         if binding:
-            vs_id = binding.get('lb_vs_id')
             lb_pool_id = binding.get('lb_pool_id')
             lb_binding = nsx_db.get_nsx_lbaas_loadbalancer_binding(
                 context.session, lb_id)
-            lb_service = None
             if not lb_binding:
+                completor(success=False)
+                msg = (_('Failed to get LB binding for member %s') %
+                       member['id'])
+                raise nsx_exc.NsxPluginException(err_msg=msg)
+            if lb_binding.lb_router_id == lb_utils.NO_ROUTER_ID:
+                # Need to attach the LB service to the router now
+                # This will happen here in case of external vip
                 nsx_router_id = nsx_db.get_nsx_router_id(context.session,
                                                          router_id)
-                lb_service = service_client.get_router_lb_service(
+                try:
-                    nsx_router_id)
+                    tags = lb_utils.get_tags(self.core_plugin, router_id,
-                virtual_server_ids = (
+                                             lb_const.LR_ROUTER_TYPE,
-                    lb_service and
+                                             member['tenant_id'],
-                    lb_service.get('virtual_server_ids', []) or [])
+                                             context.project_name)
-                if not lb_service:
+                    service_client.update_service_with_attachment(
-                    lb_size = lb_utils.get_lb_flavor_size(
+                        lb_binding.lb_service_id, nsx_router_id, tags=tags)
-                        self.flavor_plugin, context,
+                    # TODO(asarfaty): Also update the tags
-                        loadbalancer.get('flavor_id'))
+                except nsxlib_exc.ManagerError as e:
-                    if not self.core_plugin.service_router_has_services(
+                    # This will happen if there is another service already
-                            context,
+                    # attached to this router.
-                            router_id):
+                    # This is currently a limitation.
-                        self.core_plugin.create_service_router(context,
+                    completor(success=False)
-                                                               router_id)
+                    msg = (_('Failed to attach router %(rtr)s to LB service '
-                    lb_service = self._create_lb_service(
+                             '%(srv)s: %(e)s') %
-                        context, service_client, member['tenant_id'],
+                           {'rtr': router_id, 'srv': lb_binding.lb_service_id,
-                        router_id, nsx_router_id, loadbalancer['id'], lb_size)
+                            'e': e})
-                    if not lb_service:
+                    raise nsx_exc.NsxPluginException(err_msg=msg)
-                        completor(success=False)
+                # Update the nsx router in the DB binding
-                        msg = (_('Failed to create lb service to attach '
+                nsx_db.update_nsx_lbaas_loadbalancer_binding(
-                                 'virtual server %(vs)s for member '
+                    context.session, lb_id, nsx_router_id)
-                                 '%(member)s') %
+                # Add rule to advertise external vips
-                               {'vs': vs_id, 'member': member['id']})
+                router = self.core_plugin.get_router(context, router_id)
-                        raise nsx_exc.NsxPluginException(err_msg=msg)
+                lb_utils.update_router_lb_vip_advertisement(
-
+                    context, self.core_plugin, router, nsx_router_id)
-                lb_service_id = lb_service['id']
-                self._add_loadbalancer_binding(
-                    context, loadbalancer['id'], lb_service_id,
-                    nsx_router_id, loadbalancer['vip_address'])
 
             with locking.LockManager.get_lock('pool-member-%s' % lb_pool_id):
                 lb_pool = pool_client.get(lb_pool_id)
@@ -197,30 +151,6 @@ class EdgeMemberManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
                 members = (old_m + new_m) if old_m else new_m
                 pool_client.update_pool_with_members(lb_pool_id, members)
 
-            # Check whether the virtual server should be added to the load
-            # balancing server. It is safe to perform this operation after the
-            # member has been added to the pool. This allows us to skip this
-            # check if there is already a member in the pool
-            if vs_id and not old_m:
-                # load the LB service if not already loaded
-                if not lb_service:
-                    nsx_router_id = nsx_db.get_nsx_router_id(context.session,
-                                                             router_id)
-                    lb_service = service_client.get_router_lb_service(
-                        nsx_router_id)
-                    lb_service_id = lb_service['id']
-                    virtual_server_ids = lb_service.get('virtual_server_ids',
-                                                        [])
-                if vs_id not in virtual_server_ids:
-                    try:
-                        service_client.add_virtual_server(lb_service_id, vs_id)
-                    except nsxlib_exc.ManagerError:
-                        completor(success=False)
-                        msg = (_('Failed to attach virtual server %(vs)s '
-                                'to lb service %(service)s') %
-                                {'vs': vs_id, 'service': lb_service_id})
-                        raise n_exc.BadRequest(resource='lbaas-member',
-                                               msg=msg)
         else:
             completor(success=False)
             msg = (_('Failed to get pool binding to add member %s') %

vmware_nsx/tests/unit/services/lbaas/test_nsxv3_driver.py

@@ -20,6 +20,7 @@ from neutron_lib import context
 from neutron_lib import exceptions as n_exc
 
 from vmware_nsx.db import db as nsx_db
+from vmware_nsx.db import nsx_models
 from vmware_nsx.services.lbaas import base_mgr
 from vmware_nsx.services.lbaas.nsx_v3.implementation import lb_utils
 from vmware_nsx.services.lbaas.nsx_v3.v2 import lb_driver_v2
@@ -31,10 +32,16 @@ ROUTER_ID = 'neutron-router-x'
 LB_ID = 'xxx-xxx'
 LB_TENANT_ID = 'yyy-yyy'
 LB_SERVICE_ID = 'service-1'
-LB_BINDING = {'loadbalancer_id': LB_ID,
+LB_BINDING = nsx_models.NsxLbaasLoadbalancer(
-              'lb_service_id': LB_SERVICE_ID,
+    loadbalancer_id=LB_ID,
-              'lb_router_id': LB_ROUTER_ID,
+    lb_service_id=LB_SERVICE_ID,
-              'vip_address': LB_VIP}
+    lb_router_id=LB_ROUTER_ID,
+    vip_address=LB_VIP)
+LB_BINDING_NO_RTR = nsx_models.NsxLbaasLoadbalancer(
+    loadbalancer_id=LB_ID,
+    lb_service_id=LB_SERVICE_ID,
+    lb_router_id=lb_utils.NO_ROUTER_ID,
+    vip_address=LB_VIP)
 LB_NETWORK = {'router:external': False,
               'id': 'xxxxx',
               'name': 'network-1'}
@@ -52,10 +59,10 @@ LB_APP_PROFILE = {
     "idle_timeout": 1800,
     "x_forwarded_for": "INSERT",
 }
-LISTENER_BINDING = {'loadbalancer_id': LB_ID,
+LISTENER_BINDING = nsx_models.NsxLbaasListener(loadbalancer_id=LB_ID,
-                    'listener_id': LISTENER_ID,
+                                               listener_id=LISTENER_ID,
-                    'app_profile_id': APP_PROFILE_ID,
+                                               app_profile_id=APP_PROFILE_ID,
-                    'lb_vs_id': LB_VS_ID}
+                                               lb_vs_id=LB_VS_ID)
 POOL_ID = 'ppp-qqq'
 LB_POOL_ID = 'pool-xx'
 LB_POOL = {
@@ -64,10 +71,10 @@ LB_POOL = {
     "id": LB_POOL_ID,
     "algorithm": "ROUND_ROBIN",
 }
-POOL_BINDING = {'loadbalancer_id': LB_ID,
+POOL_BINDING = nsx_models.NsxLbaasPool(loadbalancer_id=LB_ID,
-                'pool_id': POOL_ID,
+                                       pool_id=POOL_ID,
-                'lb_pool_id': LB_POOL_ID,
+                                       lb_pool_id=LB_POOL_ID,
-                'lb_vs_id': LB_VS_ID}
+                                       lb_vs_id=LB_VS_ID)
 MEMBER_ID = 'mmm-mmm'
 MEMBER_ADDRESS = '10.0.0.200'
 LB_MEMBER = {'display_name': 'member1_' + MEMBER_ID,
@@ -90,17 +97,17 @@ LB_POOL_WITH_MEMBER = {
 HM_ID = 'hhh-mmm'
 LB_MONITOR_ID = 'mmm-ddd'
 
-HM_BINDING = {'loadbalancer_id': LB_ID,
+HM_BINDING = nsx_models.NsxLbaasMonitor(loadbalancer_id=LB_ID,
-              'pool_id': POOL_ID,
+                                        pool_id=POOL_ID,
-              'hm_id': HM_ID,
+                                        hm_id=HM_ID,
-              'lb_monitor_id': LB_MONITOR_ID,
+                                        lb_monitor_id=LB_MONITOR_ID,
-              'lb_pool_id': LB_POOL_ID}
+                                        lb_pool_id=LB_POOL_ID)
 L7POLICY_ID = 'l7policy-xxx'
 LB_RULE_ID = 'lb-rule-xx'
 L7RULE_ID = 'l7rule-111'
-L7POLICY_BINDING = {'l7policy_id': L7POLICY_ID,
+L7POLICY_BINDING = nsx_models.NsxLbaasL7Policy(l7policy_id=L7POLICY_ID,
-                    'lb_vs_id': LB_VS_ID,
+                                               lb_vs_id=LB_VS_ID,
-                    'lb_rule_id': LB_RULE_ID}
+                                               lb_rule_id=LB_RULE_ID)
 LB_PP_ID = "ppp-ppp"
 
 FAKE_CERT = {'id': 'cert-xyz'}
@@ -247,8 +254,26 @@ class TestEdgeLbaasV2Loadbalancer(BaseTestEdgeLbaasV2):
         return 'load_balancer'
 
     def test_create(self):
+        neutron_router = {'id': ROUTER_ID, 'name': 'dummy',
+                          'external_gateway_info': {'external_fixed_ips': []}}
         with mock.patch.object(lb_utils, 'validate_lb_subnet'
-                               ) as mock_validate_lb_subnet:
+                               ) as mock_validate_lb_subnet,\
+            mock.patch.object(lb_utils, 'get_router_from_network',
+                              return_value=ROUTER_ID),\
+            mock.patch.object(self.core_plugin, 'get_router',
+                              return_value=neutron_router), \
+            mock.patch.object(self.core_plugin, '_find_router_gw_subnets',
+                              return_value=[]),\
+            mock.patch.object(nsx_db, 'get_nsx_lbaas_loadbalancer_binding'),\
+            mock.patch.object(nsx_db, 'get_nsx_router_id',
+                              return_value=LB_ROUTER_ID),\
+            mock.patch.object(self.service_client, 'get_router_lb_service',
+                              return_value=None),\
+            mock.patch.object(self.service_client, 'create',
+                              return_value={'id': LB_SERVICE_ID}
+                              ) as create_service,\
+            mock.patch.object(nsx_db, 'add_nsx_lbaas_loadbalancer_binding'
+                              ) as add_binding:
             mock_validate_lb_subnet.return_value = True
 
             self.edge_driver.loadbalancer.create(self.context, self.lb)
@@ -258,6 +283,61 @@ class TestEdgeLbaasV2Loadbalancer(BaseTestEdgeLbaasV2):
             mock_successful_completion.assert_called_with(self.context,
                                                           self.lb,
                                                           delete=False)
+            add_binding.assert_called_once_with(mock.ANY, LB_ID, LB_SERVICE_ID,
+                                                LB_ROUTER_ID, LB_VIP)
+            create_service.assert_called_once()
+
+    def test_create_service_exists(self):
+        with mock.patch.object(lb_utils, 'validate_lb_subnet'
+                               ) as mock_validate_lb_subnet,\
+            mock.patch.object(lb_utils, 'get_router_from_network'),\
+            mock.patch.object(nsx_db, 'get_nsx_lbaas_loadbalancer_binding'),\
+            mock.patch.object(nsx_db, 'get_nsx_router_id',
+                              return_value=LB_ROUTER_ID),\
+            mock.patch.object(self.service_client, 'get_router_lb_service',
+                              return_value={'id': LB_SERVICE_ID}),\
+            mock.patch.object(self.service_client, 'create') as create_service,\
+            mock.patch.object(nsx_db, 'add_nsx_lbaas_loadbalancer_binding'
+                              ) as add_binding:
+            mock_validate_lb_subnet.return_value = True
+
+            self.edge_driver.loadbalancer.create(self.context, self.lb)
+
+            mock_successful_completion = (
+                self.lbv2_driver.load_balancer.successful_completion)
+            mock_successful_completion.assert_called_with(self.context,
+                                                          self.lb,
+                                                          delete=False)
+            add_binding.assert_called_once_with(mock.ANY, LB_ID, LB_SERVICE_ID,
+                                                LB_ROUTER_ID, LB_VIP)
+            create_service.assert_not_called()
+
+    def test_create_external_vip(self):
+        with mock.patch.object(lb_utils, 'validate_lb_subnet'
+                               ) as mock_validate_lb_subnet,\
+            mock.patch.object(lb_utils, 'get_router_from_network',
+                              return_value=None),\
+            mock.patch.object(nsx_db, 'get_nsx_lbaas_loadbalancer_binding'),\
+            mock.patch.object(nsx_db, 'get_nsx_router_id'),\
+            mock.patch.object(self.service_client, 'get_router_lb_service',
+                              return_value=None),\
+            mock.patch.object(self.service_client, 'create',
+                              return_value={'id': LB_SERVICE_ID}
+                              ) as create_service,\
+            mock.patch.object(nsx_db, 'add_nsx_lbaas_loadbalancer_binding'
+                              ) as add_binding:
+            mock_validate_lb_subnet.return_value = True
+
+            self.edge_driver.loadbalancer.create(self.context, self.lb)
+
+            mock_successful_completion = (
+                self.lbv2_driver.load_balancer.successful_completion)
+            mock_successful_completion.assert_called_with(self.context,
+                                                          self.lb,
+                                                          delete=False)
+            add_binding.assert_called_once_with(mock.ANY, LB_ID, LB_SERVICE_ID,
+                                                lb_utils.NO_ROUTER_ID, LB_VIP)
+            create_service.assert_called_once()
 
     def test_update(self):
         new_lb = lb_models.LoadBalancer(LB_ID, 'yyy-yyy', 'lb1-new',
@@ -371,7 +451,6 @@ class TestEdgeLbaasV2Listener(BaseTestEdgeLbaasV2):
             mock_add_listener_binding.assert_called_with(
                 self.context.session, LB_ID, LISTENER_ID, APP_PROFILE_ID,
                 LB_VS_ID)
-
             mock_successful_completion = (
                 self.lbv2_driver.listener.successful_completion)
             mock_successful_completion.assert_called_with(self.context,
@@ -889,7 +968,7 @@ class TestEdgeLbaasV2Member(BaseTestEdgeLbaasV2):
     def _tested_entity(self):
         return 'member'
 
-    def _test_create(self, lb_binding, pool_binding):
+    def test_create(self):
         with mock.patch.object(lb_utils, 'validate_lb_member_subnet'
                                ) as mock_validate_lb_subnet, \
             mock.patch.object(self.lbv2_driver.plugin, 'get_pool_members'
@@ -906,11 +985,6 @@ class TestEdgeLbaasV2Member(BaseTestEdgeLbaasV2):
                               ) as mock_get_nsx_router_id, \
             mock.patch.object(self.service_client, 'get_router_lb_service'
                               ) as mock_get_lb_service, \
-            mock.patch.object(nsx_db, 'add_nsx_lbaas_loadbalancer_binding'
-                              ) as mock_add_loadbalancer_binding, \
-            mock.patch.object(self.service_client,
-                              'add_virtual_server'
-                              ) as mock_add_vs_to_service, \
             mock.patch.object(self.pool_client, 'get'
                               ) as mock_get_pool, \
             mock.patch.object(self.pool_client, 'update_pool_with_members'
@@ -919,20 +993,13 @@ class TestEdgeLbaasV2Member(BaseTestEdgeLbaasV2):
             mock_get_pool_members.return_value = [self.member]
             mock_get_network.return_value = LB_NETWORK
             mock_get_router.return_value = LB_ROUTER_ID
-            mock_get_pool_binding.return_value = pool_binding
+            mock_get_pool_binding.return_value = POOL_BINDING
-            mock_get_lb_binding.return_value = lb_binding
+            mock_get_lb_binding.return_value = LB_BINDING
             mock_get_nsx_router_id.return_value = LB_ROUTER_ID
             mock_get_lb_service.return_value = {'id': LB_SERVICE_ID}
             mock_get_pool.return_value = LB_POOL
 
             self.edge_driver.member.create(self.context, self.member)
-            if not lb_binding:
-                mock_add_loadbalancer_binding.assert_called_with(
-                    self.context.session, LB_ID, LB_SERVICE_ID, LB_ROUTER_ID,
-                    LB_VIP)
-            else:
-                mock_add_loadbalancer_binding.assert_not_called()
-            mock_add_vs_to_service.assert_called_with(LB_SERVICE_ID, LB_VS_ID)
             mock_update_pool_with_members.assert_called_with(LB_POOL_ID,
                                                              [LB_MEMBER])
             mock_successful_completion = (
@@ -941,14 +1008,7 @@ class TestEdgeLbaasV2Member(BaseTestEdgeLbaasV2):
                                                           self.member,
                                                           delete=False)
 
-    def test_create(self):
+    def test_create_external_vip(self):
-        self._test_create(None, POOL_BINDING)
-
-    def test_create_existing_binding(self):
-        self._test_create(LB_BINDING, POOL_BINDING)
-
-    def test_create_with_service(self):
-        ext_cidr = '1.1.1.0/24'
         with mock.patch.object(lb_utils, 'validate_lb_member_subnet'
                                ) as mock_validate_lb_subnet, \
             mock.patch.object(self.lbv2_driver.plugin, 'get_pool_members'
@@ -959,100 +1019,40 @@ class TestEdgeLbaasV2Member(BaseTestEdgeLbaasV2):
                               ) as mock_get_router, \
             mock.patch.object(nsx_db, 'get_nsx_lbaas_pool_binding'
                               ) as mock_get_pool_binding, \
-            mock.patch.object(nsx_db, 'get_nsx_lbaas_loadbalancer_binding'
+            mock.patch.object(nsx_db, 'get_nsx_lbaas_loadbalancer_binding',
                               ) as mock_get_lb_binding, \
+            mock.patch.object(nsx_db, 'update_nsx_lbaas_loadbalancer_binding',
+                              ) as mock_update_lb_binding, \
             mock.patch.object(nsx_db, 'get_nsx_router_id'
                               ) as mock_get_nsx_router_id, \
             mock.patch.object(self.service_client, 'get_router_lb_service'
                               ) as mock_get_lb_service, \
-            mock.patch.object(self.service_client, 'create'
-                              ) as mock_create_lb_service, \
-            mock.patch.object(nsx_db, 'add_nsx_lbaas_loadbalancer_binding'
-                              ) as mock_add_loadbalancer_bidning, \
-            mock.patch.object(self.service_client,
-                              'add_virtual_server'
-                              ) as mock_add_vs_to_service, \
             mock.patch.object(self.pool_client, 'get'
                               ) as mock_get_pool, \
+            mock.patch.object(self.core_plugin, '_find_router_gw_subnets',
+                              return_value=[]),\
             mock.patch.object(self.pool_client, 'update_pool_with_members'
-                              ) as mock_update_pool_with_members,\
+                              ) as mock_update_pool_with_members:
-            mock.patch.object(self.core_plugin.nsxlib.logical_router,
-                              'update_advertisement_rules') as update_adv,\
-            mock.patch.object(self.core_plugin, '_find_router_gw_subnets'
-                              ) as mock_get_subnets,\
-            mock.patch.object(self.core_plugin, 'get_router'
-                              ) as mock_core_get_router:
             mock_validate_lb_subnet.return_value = True
             mock_get_pool_members.return_value = [self.member]
             mock_get_network.return_value = LB_NETWORK
             mock_get_router.return_value = LB_ROUTER_ID
             mock_get_pool_binding.return_value = POOL_BINDING
-            mock_get_lb_binding.return_value = None
+            mock_get_lb_binding.return_value = LB_BINDING_NO_RTR
             mock_get_nsx_router_id.return_value = LB_ROUTER_ID
-            mock_get_lb_service.return_value = {}
+            mock_get_lb_service.return_value = {'id': LB_SERVICE_ID}
-            mock_create_lb_service.return_value = {'id': LB_SERVICE_ID}
             mock_get_pool.return_value = LB_POOL
-            mock_core_get_router.return_value = {
-                'id': LB_ROUTER_ID,
-                'name': 'router1',
-                'external_gateway_info': 'dummy'}
-            mock_get_subnets.return_value = [{'cidr': ext_cidr}]
 
             self.edge_driver.member.create(self.context, self.member)
-
-            mock_add_loadbalancer_bidning.assert_called_with(
-                self.context.session, LB_ID, LB_SERVICE_ID, LB_ROUTER_ID,
-                LB_VIP)
-            mock_add_vs_to_service.assert_called_with(LB_SERVICE_ID, LB_VS_ID)
             mock_update_pool_with_members.assert_called_with(LB_POOL_ID,
                                                              [LB_MEMBER])
-            update_adv.assert_called_with(
-                LB_ROUTER_ID,
-                [{'networks': [ext_cidr],
-                  'display_name': lb_utils.ADV_RULE_NAME,
-                  'rule_filter': {'match_route_types': ['T1_LB_VIP'],
-                  'prefix_operator': 'GE'},
-                  'action': 'ALLOW'}],
-                name_prefix=lb_utils.ADV_RULE_NAME)
             mock_successful_completion = (
                 self.lbv2_driver.member.successful_completion)
             mock_successful_completion.assert_called_with(self.context,
                                                           self.member,
                                                           delete=False)
-
+            mock_update_lb_binding.assert_called_once_with(
-    def test_create_lbs_no_router_gateway(self):
+                mock.ANY, LB_ID, LB_ROUTER_ID)
-        with mock.patch.object(lb_utils, 'validate_lb_member_subnet'
-                               ) as mock_validate_lb_subnet, \
-            mock.patch.object(self.lbv2_driver.plugin, 'get_pool_members'
-                              ) as mock_get_pool_members, \
-            mock.patch.object(lb_utils, 'get_network_from_subnet'
-                              ) as mock_get_network, \
-            mock.patch.object(lb_utils, 'get_router_from_network'
-                              ) as mock_get_router_from_network, \
-            mock.patch.object(nsx_db, 'get_nsx_lbaas_pool_binding'
-                              ) as mock_get_pool_binding, \
-            mock.patch.object(nsx_db, 'get_nsx_lbaas_loadbalancer_binding'
-                              ) as mock_get_lb_binding, \
-            mock.patch.object(nsx_db, 'get_nsx_router_id'
-                              ) as mock_get_nsx_router_id, \
-            mock.patch.object(self.service_client, 'get_router_lb_service'
-                              ) as mock_get_lb_service, \
-            mock.patch.object(self.core_plugin, 'get_router'
-                              ) as mock_get_router:
-            mock_validate_lb_subnet.return_value = True
-            mock_get_pool_members.return_value = [self.member]
-            mock_get_network.return_value = LB_NETWORK
-            mock_get_router_from_network.return_value = LB_ROUTER_ID
-            mock_get_pool_binding.return_value = POOL_BINDING
-            mock_get_lb_binding.return_value = None
-            mock_get_nsx_router_id.return_value = LB_ROUTER_ID
-            mock_get_lb_service.return_value = None
-            mock_get_router.return_value = {'id': 'router1-xxx'}
-
-            self.assertRaises(n_exc.BadRequest,
-                              self.edge_driver.member.create,
-                              self.context,
-                              self.member)
 
     def test_create_member_different_router(self):
         with mock.patch.object(self.lbv2_driver.plugin, 'get_pool_members'