NSX|V3: LBaaS operating status support

The LBaaS V2 plugin expects the driver to update the LB objects operating
status from a separate process/thread.
When the user requests the LB status (or just the LB object itself with GET),
the operating status is retrived from the LBaaS DB, without calling the driver.

To avoid adding a process to actively query and update all objects statuses,
this patch creates a new LBaaSV2 plugin, to be used instead of the default one.
This plugin (vmware_nsx_lbaasv2) will issue a get-statuses call to the driver,
update the current statuses in the DB, and call the original plugin.

Depends-on: I71a56b87144aad743795ad1295ec636b17429035
Change-Id: I3c4e75d92a1bacdb14292a8db727deb4923a85d9

doc/source/devstack.rst

@@ -16,12 +16,17 @@ Add lbaas repo as an external repository and configure following flags in ``loca
     [[local]|[localrc]]
     enable_plugin neutron-lbaas https://git.openstack.org/openstack/neutron-lbaas
     enable_service q-lbaasv2
+    Q_SERVICE_PLUGIN_CLASSES=vmware_nsx_lbaasv2
 
 Configure the service provider::
     [[post-config|$NEUTRON_LBAAS_CONF]]
     [service_providers]
     service_provider = LOADBALANCERV2:VMWareEdge:neutron_lbaas.drivers.vmware.edge_driver_v2.EdgeLoadBalancerDriverV2:default
 
+    [[post-config|$NEUTRON_CONF]]
+    [DEFAULT]
+    api_extensions_path = $DEST/neutron-lbaas/neutron_lbaas/extensions
+
 QoS Driver
 ~~~~~~~~~~
 
@@ -208,12 +213,17 @@ Add lbaas repo as an external repository and configure following flags in ``loca
     [[local]|[localrc]]
     enable_plugin neutron-lbaas https://git.openstack.org/openstack/neutron-lbaas
     enable_service q-lbaasv2
+    Q_SERVICE_PLUGIN_CLASSES=vmware_nsx_lbaasv2
 
 Configure the service provider::
     [[post-config|$NEUTRON_LBAAS_CONF]]
     [service_providers]
     service_provider = LOADBALANCERV2:VMWareEdge:neutron_lbaas.drivers.vmware.edge_driver_v2.EdgeLoadBalancerDriverV2:default
 
+    [[post-config|$NEUTRON_CONF]]
+    [DEFAULT]
+    api_extensions_path = $DEST/neutron-lbaas/neutron_lbaas/extensions
+
 Neutron VPNaaS
 ~~~~~~~~~~~~~~
 

setup.cfg

@@ -43,6 +43,7 @@ firewall_drivers =
     vmware_nsxtvd_edge_v2 = vmware_nsx.services.fwaas.nsx_tv.edge_fwaas_driver_v2:EdgeFwaasTVDriverV2
 neutron.service_plugins =
     vmware_nsxv_qos = vmware_nsx.services.qos.nsx_v.plugin:NsxVQosPlugin
+    vmware_nsx_lbaasv2 = vmware_nsx.services.lbaas.nsx_plugin:LoadBalancerNSXPluginV2
     vmware_nsxtvd_lbaasv2 = vmware_nsx.services.lbaas.nsx.plugin:LoadBalancerTVPluginV2
     vmware_nsxtvd_fwaasv1 = vmware_nsx.services.fwaas.nsx_tv.plugin_v1:FwaasTVPluginV1
     vmware_nsxtvd_fwaasv2 = vmware_nsx.services.fwaas.nsx_tv.plugin_v2:FwaasTVPluginV2

vmware_nsx/db/db.py

@@ -591,6 +591,16 @@ def get_nsx_lbaas_listener_binding(session, loadbalancer_id, listener_id):
         return
 
 
+def get_nsx_lbaas_listener_binding_by_vs(session, loadbalancer_id, lb_vs_id):
+    try:
+        return session.query(
+            nsx_models.NsxLbaasListener).filter_by(
+                loadbalancer_id=loadbalancer_id,
+                lb_vs_id=lb_vs_id).one()
+    except exc.NoResultFound:
+        return
+
+
 def delete_nsx_lbaas_listener_binding(session, loadbalancer_id, listener_id):
     return (session.query(nsx_models.NsxLbaasListener).
             filter_by(loadbalancer_id=loadbalancer_id,
@@ -616,6 +626,15 @@ def get_nsx_lbaas_pool_binding(session, loadbalancer_id, pool_id):
         return
 
 
+def get_nsx_lbaas_pool_binding_by_lb_pool(session, loadbalancer_id,
+                                          lb_pool_id):
+    try:
+        return session.query(nsx_models.NsxLbaasPool).filter_by(
+            loadbalancer_id=loadbalancer_id, lb_pool_id=lb_pool_id).one()
+    except exc.NoResultFound:
+        return
+
+
 def update_nsx_lbaas_pool_binding(session, loadbalancer_id, pool_id,
                                   lb_vs_id):
     try:

vmware_nsx/services/lbaas/lb_const.py

@@ -115,3 +115,8 @@ LB_HTTP_REJECT_STATUS = '403'
 LB_RULE_HTTP_REQUEST_REWRITE = 'HTTP_REQUEST_REWRITE'
 LB_RULE_HTTP_FORWARDING = 'HTTP_FORWARDING'
 LB_RULE_HTTP_RESPONSE_REWRITE = 'HTTP_RESPONSE_REWRITE'
+
+LOADBALANCERS = 'loadbalancers'
+LISTENERS = 'listeners'
+POOLS = 'pools'
+MEMBERS = 'members'

vmware_nsx/services/lbaas/lb_helper.py

@@ -107,3 +107,12 @@ class LBaaSNSXObjectManagerWrapper(object):
             raise n_exc.BadRequest(resource='edge', msg=msg)
         obj_dict = self.translator(obj)
         return self.implementor.stats(context, obj_dict)
+
+    @log_helpers.log_method_call
+    def get_operating_status(self, context, id, **args):
+        # verify that this api exists (supported only for loadbalancer)
+        if not hasattr(self.implementor, 'get_operating_status'):
+            msg = (_("LBaaS object %s does not support get_operating_status "
+                     "api") % self.object_type)
+            raise n_exc.BadRequest(resource='edge', msg=msg)
+        return self.implementor.get_operating_status(context, id, **args)

vmware_nsx/services/lbaas/nsx/plugin.py

@@ -13,13 +13,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from neutron_lbaas.services.loadbalancer import plugin
+from vmware_nsx.services.lbaas import nsx_plugin
 
 from vmware_nsx.plugins.nsx import utils as tvd_utils
 
 
 @tvd_utils.filter_plugins
-class LoadBalancerTVPluginV2(plugin.LoadBalancerPluginv2):
+class LoadBalancerTVPluginV2(nsx_plugin.LoadBalancerNSXPluginV2):
     """NSX-TV plugin for LBaaS V2.
 
     This plugin adds separation between T/V instances

vmware_nsx/services/lbaas/nsx_plugin.py

@@ -0,0 +1,85 @@
+# Copyright 2018 VMware, Inc.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_log import log as logging
+
+from neutron_lbaas.db.loadbalancer import models
+from neutron_lbaas.services.loadbalancer import plugin
+
+from vmware_nsx.services.lbaas import lb_const
+
+LOG = logging.getLogger(__name__)
+
+
+class LoadBalancerNSXPluginV2(plugin.LoadBalancerPluginv2):
+    """NSX Plugin for LBaaS V2.
+
+    This plugin overrides the statuses call to issue the DB update before
+    displaying the results.
+    """
+
+    def nsx_update_operational_statuses(self, context, loadbalancer_id,
+                                        with_members=False):
+        """Update LB objects operating status
+
+        Call the driver to get the current statuses, and update those in the DB
+        """
+        # get the driver
+        driver = self._get_driver_for_loadbalancer(
+            context, loadbalancer_id)
+        driver_obj = driver.load_balancer.lbv2_driver
+
+        # Get the current statuses from the driver
+        lb_statuses = driver_obj.loadbalancer.implementor.get_operating_status(
+            context, loadbalancer_id, with_members=with_members)
+        if not lb_statuses:
+            return
+
+        # update the new statuses in the LBaaS DB
+        if lb_const.LOADBALANCERS in lb_statuses:
+            for lb in lb_statuses[lb_const.LOADBALANCERS]:
+                self.db.update_status(context, models.LoadBalancer, lb['id'],
+                                      operating_status=lb['status'])
+        if lb_const.LISTENERS in lb_statuses:
+            for listener in lb_statuses[lb_const.LISTENERS]:
+                self.db.update_status(context, models.Listener, listener['id'],
+                                      operating_status=listener['status'])
+        if lb_const.POOLS in lb_statuses:
+            for pool in lb_statuses[lb_const.POOLS]:
+                self.db.update_status(context, models.PoolV2, pool['id'],
+                                      operating_status=pool['status'])
+        if lb_const.MEMBERS in lb_statuses:
+            for member in lb_statuses[lb_const.MEMBERS]:
+                self.db.update_status(context, models.MemberV2, member['id'],
+                                      operating_status=member['status'])
+
+    def statuses(self, context, loadbalancer_id):
+        # Update the LB statuses before letting the plugin display them
+        self.nsx_update_operational_statuses(context, loadbalancer_id,
+                                             with_members=True)
+
+        # use super code to get the updated statuses
+        return super(LoadBalancerNSXPluginV2, self).statuses(
+            context, loadbalancer_id)
+
+    def get_loadbalancer(self, context, loadbalancer_id, fields=None):
+        # Update the LB status before letting the plugin display it in the
+        # loadbalancer display
+        self.nsx_update_operational_statuses(context, loadbalancer_id)
+
+        return super(LoadBalancerNSXPluginV2, self).get_loadbalancer(
+            context, loadbalancer_id, fields=fields)
+
+    # TODO(asarfaty) : do the implementation for V objects as well

vmware_nsx/services/lbaas/nsx_v/implementation/loadbalancer_mgr.py

@@ -189,6 +189,11 @@ class EdgeLoadBalancerManagerFromDict(base_mgr.EdgeLoadbalancerBaseManager):
 
         return stats
 
+    def get_operating_status(self, context, id):
+        """Return a map of the operating status of all connected LB objects """
+        #TODO(asarfaty) implement
+        return {}
+
     def _handle_subnet_gw_change(self, *args, **kwargs):
         # As the Edge appliance doesn't use DHCP, we should change the
         # default gateway here when the subnet GW changes.

vmware_nsx/services/lbaas/nsx_v3/implementation/loadbalancer_mgr.py

@@ -18,6 +18,8 @@ from neutron_lib import exceptions as n_exc
 from oslo_log import log as logging
 from oslo_utils import excutils
 
+from neutron_lbaas.services.loadbalancer import constants
+
 from vmware_nsx._i18n import _
 from vmware_nsx.db import db as nsx_db
 from vmware_nsx.services.lbaas import base_mgr
@@ -107,9 +109,114 @@ class EdgeLoadBalancerManagerFromDict(base_mgr.Nsxv3LoadbalancerBaseManager):
         completor(success=True)
 
     def refresh(self, context, lb):
-        # TODO(tongl): implememnt
+        # TODO(tongl): implement
         pass
 
+    def _nsx_status_to_lb_status(self, nsx_status):
+        if not nsx_status:
+            # default fallback
+            return constants.ONLINE
+
+        # Statuses that are considered ONLINE:
+        if nsx_status.upper() in ['UP', 'UNKNOWN', 'PARTIALLY_UP',
+                                  'NO_STANDBY']:
+            return constants.ONLINE
+        # Statuses that are considered OFFLINE:
+        if nsx_status.upper() in ['PRIMARY_DOWN', 'DETACHED', 'DOWN', 'ERROR']:
+            return constants.OFFLINE
+        if nsx_status.upper() == 'DISABLED':
+            return constants.DISABLED
+
+        # default fallback
+        LOG.debug("NSX LB status %s - interpreted as ONLINE", nsx_status)
+        return constants.ONLINE
+
+    def get_lb_pool_members_statuses(self, nsx_pool_id, members_statuses):
+        # Combine the NSX pool members data and the NSX statuses to provide
+        # member statuses list
+        # Get the member id from the suffix of the member in the NSX pool list
+        # and find the matching ip+port member in the statuses list
+        # get the members list from the NSX
+        nsx_pool = self.core_plugin.nsxlib.load_balancer.pool.get(nsx_pool_id)
+        if not nsx_pool or not nsx_pool.get('members'):
+            return []
+        # create a map of existing members: ip+port -> lbaas ID (which is the
+        # suffix of the member name)
+        members_map = {}
+        for member in nsx_pool['members']:
+            ip = member['ip_address']
+            port = member['port']
+            if ip not in members_map:
+                members_map[ip] = {}
+            members_map[ip][port] = member['display_name'][-36:]
+        # go over the statuses map, and match the member ip_port, to the ID
+        # in the map
+        statuses = []
+        for member in members_statuses:
+            ip = member['ip_address']
+            port = member['port']
+            if ip in members_map and port in members_map[ip]:
+                member_id = members_map[ip][port]
+                member_status = self._nsx_status_to_lb_status(member['status'])
+                statuses.append({'id': member_id, 'status': member_status})
+        return statuses
+
+    def get_operating_status(self, context, id, with_members=False):
+        """Return a map of the operating status of all connected LB objects """
+        service_client = self.core_plugin.nsxlib.load_balancer.service
+        lb_binding = nsx_db.get_nsx_lbaas_loadbalancer_binding(
+            context.session, id)
+        if not lb_binding:
+            # No service yet
+            return {}
+
+        lb_service_id = lb_binding['lb_service_id']
+        try:
+            service_status = service_client.get_status(lb_service_id)
+            vs_statuses = service_client.get_virtual_servers_status(
+                lb_service_id)
+        except nsxlib_exc.ManagerError:
+            LOG.warning("LB service %(lbs)s is not found",
+                        {'lbs': lb_service_id})
+            return {}
+
+        # get the loadbalancer status from the LB service
+        lb_status = self._nsx_status_to_lb_status(
+            service_status.get('service_status'))
+        statuses = {lb_const.LOADBALANCERS: [{'id': id, 'status': lb_status}],
+                    lb_const.LISTENERS: [],
+                    lb_const.POOLS: [],
+                    lb_const.MEMBERS: []}
+
+        # Add the listeners statuses from the virtual servers statuses
+        for vs in vs_statuses.get('results', []):
+            vs_status = self._nsx_status_to_lb_status(vs.get('status'))
+            vs_id = vs.get('virtual_server_id')
+            listener_binding = nsx_db.get_nsx_lbaas_listener_binding_by_vs(
+                context.session, id, vs_id)
+            if listener_binding:
+                listener_id = listener_binding['listener_id']
+                statuses[lb_const.LISTENERS].append(
+                    {'id': listener_id, 'status': vs_status})
+
+        # Add the pools statuses from the LB service status
+        for pool in service_status.get('pools', []):
+            nsx_pool_id = pool.get('pool_id')
+            pool_status = self._nsx_status_to_lb_status(pool.get('status'))
+            pool_binding = nsx_db.get_nsx_lbaas_pool_binding_by_lb_pool(
+                context.session, id, nsx_pool_id)
+            if pool_binding:
+                pool_id = pool_binding['pool_id']
+                statuses[lb_const.POOLS].append(
+                    {'id': pool_id, 'status': pool_status})
+                # Add the pools members
+                if with_members and pool.get('members'):
+                    statuses[lb_const.MEMBERS].extend(
+                        self.get_lb_pool_members_statuses(
+                            nsx_pool_id, pool['members']))
+
+        return statuses
+
     def stats(self, context, lb):
         # Since multiple LBaaS loadbalancer can share the same LB service,
         # get the corresponding virtual servers' stats instead of LB service.

vmware_nsx/tests/unit/services/lbaas/test_nsxv3_driver.py

@@ -103,6 +103,31 @@ L7POLICY_BINDING = {'l7policy_id': L7POLICY_ID,
 
 FAKE_CERT = {'id': 'cert-xyz'}
 
+SERVICE_STATUSES = {
+    "virtual_servers": [{
+        "virtual_server_id": LB_VS_ID,
+        "status": "UP"
+    }],
+    "service_id": LB_SERVICE_ID,
+    "service_status": "UP",
+    "pools": [{
+        "members": [{
+            "port": "80",
+            "ip_address": MEMBER_ADDRESS,
+            "status": "DOWN"
+        }],
+        "pool_id": LB_POOL_ID,
+        "status": "DOWN"
+    }]
+}
+
+VS_STATUSES = {
+    "results": [{
+        "virtual_server_id": LB_VS_ID,
+        "status": "UP"
+    }]
+}
+
 
 class BaseTestEdgeLbaasV2(base.BaseTestCase):
     def _tested_entity(self):
@@ -257,6 +282,36 @@ class TestEdgeLbaasV2Loadbalancer(BaseTestEdgeLbaasV2):
     def test_refresh(self):
         pass
 
+    def test_status_update(self):
+        with mock.patch.object(nsx_db, 'get_nsx_lbaas_loadbalancer_binding'
+                               ) as mock_get_lb_binding, \
+            mock.patch.object(self.service_client, 'get_status'
+                              ) as mock_get_lb_service_status, \
+            mock.patch.object(self.service_client, 'get_virtual_servers_status'
+                              ) as mock_get_vs_status, \
+            mock.patch.object(nsx_db, 'get_nsx_lbaas_pool_binding_by_lb_pool'
+                              ) as mock_get_pool_binding, \
+            mock.patch.object(self.pool_client, 'get'
+                              ) as mock_get_pool, \
+            mock.patch.object(nsx_db, 'get_nsx_lbaas_listener_binding_by_vs'
+                              ) as mock_get_listener_binding:
+            mock_get_lb_binding.return_value = LB_BINDING
+            mock_get_pool_binding.return_value = POOL_BINDING
+            mock_get_listener_binding.return_value = LISTENER_BINDING
+            mock_get_lb_service_status.return_value = SERVICE_STATUSES
+            mock_get_vs_status.return_value = VS_STATUSES
+            mock_get_pool.return_value = LB_POOL_WITH_MEMBER
+            statuses = self.edge_driver.loadbalancer.get_operating_status(
+                self.context, self.lb.id, with_members=True)
+            self.assertEqual(1, len(statuses['loadbalancers']))
+            self.assertEqual('ONLINE', statuses['loadbalancers'][0]['status'])
+            self.assertEqual(1, len(statuses['pools']))
+            self.assertEqual('OFFLINE', statuses['pools'][0]['status'])
+            self.assertEqual(1, len(statuses['listeners']))
+            self.assertEqual('ONLINE', statuses['listeners'][0]['status'])
+            self.assertEqual(1, len(statuses['members']))
+            self.assertEqual('OFFLINE', statuses['members'][0]['status'])
+
 
 class TestEdgeLbaasV2Listener(BaseTestEdgeLbaasV2):
     def setUp(self):