Octavia LB status admin util

When Octavia loadbalancer hangs in PENDING status, it becomes immutable
and cannot be recovered or deleted in any way.
The following admin util commmand updates the loadbalancer status to
ERROR so it can be deleted successfully, but sending an RPC message to
Octavia driver agent service.

Change-Id: I7ee1ba445ab4526ef8b2d271574319114a31d631

doc/source/admin_util.rst

@@ -351,6 +351,13 @@ Config
 
     nsxadmin -r config -o validate
 
+Loadbalancers
+~~~~~~~~~~~~~
+
+- Set an Octavia loadbalancer id to ERROR status. Useful while the loadbalancer hangs in PENDING status::
+
+    nsxadmin -r loadbalancers -o set-status-error --property loadbalancer-id=<loadbalancer id>
+
 NSX-T Plugin
 ------------
 
@@ -628,6 +635,13 @@ T2P migration
 
     nsxadmin -r nsx-migrate-t2p -o clean-all
 
+Loadbalancers
+~~~~~~~~~~~~~
+
+- Set an Octavia loadbalancer id to ERROR status. Useful while the loadbalancer hangs in PENDING status::
+
+    nsxadmin -r loadbalancers -o set-status-error --property loadbalancer-id=<loadbalancer id>
+
 NSXtvd Plugin
 -------------
 
@@ -694,6 +708,10 @@ NSX Policy Plugin
 
     nsxadmin -r nsx-migrate-v2t -o validate --property ext-net=<path> --property ext-cidr=<path>
 
+- Set an Octavia loadbalancer id to ERROR status. Useful while the loadbalancer hangs in PENDING status::
+
+    nsxadmin -r loadbalancers -o set-status-error --property loadbalancer-id=<loadbalancer id>
+
 Client Certificate
 ~~~~~~~~~~~~~~~~~~
 

vmware_nsx/services/lbaas/octavia/octavia_driver.py

@@ -44,7 +44,7 @@ RPC_SERVER = None
 
 
 def _log_before_retry(retry_state):
-    if retry_state.attempt_number > 0:
+    if retry_state.attempt_number > 1:
         LOG.warning("Retrying the call to _update_loadbalancer_status due to "
                     "timeout")
 
@@ -704,7 +704,7 @@ class NSXOctaviaDriverEndpoint(driver_lib.DriverLibrary):
         retry=tenacity.retry_if_exception_type(
             exceptions.UpdateStatusError))
     def _update_loadbalancer_status(self, status):
-        super(NSXOctaviaDriverEndpoint,
+        return super(NSXOctaviaDriverEndpoint,
               self).update_loadbalancer_status(status)
 
     @log_helpers.log_method_call

vmware_nsx/shell/admin/plugins/common/constants.py

@@ -37,6 +37,7 @@ CONFIG = 'config'
 ORPHANED_NETWORKS = 'orphaned-networks'
 ORPHANED_ROUTERS = 'orphaned-routers'
 SYSTEM = 'system'
+LOADBALANCERS = 'loadbalancers'
 
 # NSXV3 only Resource Constants
 PORTS = 'ports'

vmware_nsx/shell/admin/plugins/common/loadbalancers.py

@@ -0,0 +1,52 @@
+# Copyright 2021 VMware, Inc.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+import oslo_messaging as messaging
+
+from vmware_nsx.services.lbaas.octavia import constants as octavia_const
+from vmware_nsx.shell.admin.plugins.common import utils as admin_utils
+
+LOG = logging.getLogger(__name__)
+
+
+@admin_utils.output_header
+def set_loadbalancer_status_error(resource, event, trigger, **kwargs):
+    usage_msg = ("Loadbalancer id should be specified with "
+                 "--property loadbalancer-id=<id>")
+    if not kwargs.get('property'):
+        LOG.error(usage_msg)
+        return
+    properties = admin_utils.parse_multi_keyval_opt(kwargs['property'])
+    lb_id = properties.get('loadbalancer-id')
+    if not lb_id:
+        LOG.error("Need to specify loadbalancer-id. "
+                  "Add --property loadbalancer-id=<id>")
+        return
+
+    status_dict = {
+        octavia_const.LOADBALANCERS: [{
+            'id': lb_id,
+            octavia_const.PROVISIONING_STATUS: octavia_const.ERROR}]}
+    kw = {'status': status_dict}
+
+    topic = octavia_const.DRIVER_TO_OCTAVIA_TOPIC
+    transport = messaging.get_rpc_transport(cfg.CONF)
+    target = messaging.Target(topic=topic, exchange="common",
+                              namespace='control', fanout=False,
+                              version='1.0')
+    client = messaging.RPCClient(transport, target)
+    client.cast({}, 'update_loadbalancer_status', **kw)

vmware_nsx/shell/admin/plugins/nsxp/resources/loadbalancers.py

@@ -0,0 +1,25 @@
+# Copyright 2021 VMware, Inc.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron_lib.callbacks import registry
+
+from vmware_nsx.shell.admin.plugins.common import constants
+from vmware_nsx.shell.admin.plugins.common import loadbalancers
+from vmware_nsx.shell import resources as shell
+
+
+registry.subscribe(loadbalancers.set_loadbalancer_status_error,
+                   constants.LOADBALANCERS,
+                   shell.Operations.SET_STATUS_ERROR.value)

vmware_nsx/shell/admin/plugins/nsxv/resources/loadbalancers.py

@@ -0,0 +1,25 @@
+# Copyright 2021 VMware, Inc.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron_lib.callbacks import registry
+
+from vmware_nsx.shell.admin.plugins.common import constants
+from vmware_nsx.shell.admin.plugins.common import loadbalancers
+from vmware_nsx.shell import resources as shell
+
+
+registry.subscribe(loadbalancers.set_loadbalancer_status_error,
+                   constants.LOADBALANCERS,
+                   shell.Operations.SET_STATUS_ERROR.value)

vmware_nsx/shell/admin/plugins/nsxv3/resources/loadbalancers.py

@@ -0,0 +1,25 @@
+# Copyright 2021 VMware, Inc.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron_lib.callbacks import registry
+
+from vmware_nsx.shell.admin.plugins.common import constants
+from vmware_nsx.shell.admin.plugins.common import loadbalancers
+from vmware_nsx.shell import resources as shell
+
+
+registry.subscribe(loadbalancers.set_loadbalancer_status_error,
+                   constants.LOADBALANCERS,
+                   shell.Operations.SET_STATUS_ERROR.value)

vmware_nsx/shell/resources.py

@@ -77,6 +77,7 @@ class Operations(enum.Enum):
     REUSE = 'reuse'
     UPDATE_TIER0 = 'update-tier0'
     UPDATE_FIREWALL_MATCH = 'update-nat-firewall-match'
+    SET_STATUS_ERROR = 'set-status-error'
 
 
 ops = [op.value for op in Operations]
@@ -162,6 +163,8 @@ nsxv3_resources = {
                                         [Operations.IMPORT.value,
                                          Operations.CLEAN_ALL.value,
                                          Operations.VALIDATE.value]),
+    constants.LOADBALANCERS: Resource(constants.LOADBALANCERS,
+                                      [Operations.SET_STATUS_ERROR.value]),
 }
 
 # Add supported NSX-V resources in this dictionary
@@ -258,6 +261,8 @@ nsxv_resources = {
                                         [Operations.VALIDATE.value]),
     constants.PORTS: Resource(constants.PORTS,
                               [Operations.LIST.value]),
+    constants.LOADBALANCERS: Resource(constants.LOADBALANCERS,
+                                      [Operations.SET_STATUS_ERROR.value]),
 }
 
 
@@ -296,6 +301,8 @@ nsxp_resources = {
                                         [Operations.CLEAN_ALL.value,
                                          Operations.VALIDATE.value,
                                          Operations.NSX_REDISTRIBUTE.value]),
+    constants.LOADBALANCERS: Resource(constants.LOADBALANCERS,
+                                      [Operations.SET_STATUS_ERROR.value]),
 }
 
 nsxv3_resources_names = list(nsxv3_resources.keys())