NSX v3 multi-manager

This patch adds multi-manager support to the
NSX v3 plugin thereby enabling the ability to specify multiple
NSX managers for the v3 plugin.

This implementation supports the same basic features
as the MH multi-manager support does including;
timeouts, retries, keep-alive, etc..

The approach in a nutshell is to introduce a "proxy"
class which looks like a requests or requests.Session
object and can be used in place of requests in the
NSX REST API client. Under the covers this class handles
management of endpoint selection and connectivity.

Also note that with this patch your devstack local rc / conf
no longer needs to specify NSX_CONTROLLERS when using
the v3 plugin. Instead a comma list of managers is supported
on the NSX_MANAGERS devstack var.

Closes-Bug: #1524046

Change-Id: I433a4b9ea73de0680d64d86e2f826c092adfba87

devstack/lib/vmware_nsx_v3

@@ -98,17 +98,18 @@ function neutron_plugin_configure_service {
         Q_L3_ENABLED=True
         Q_L3_ROUTER_PER_TENANT=True
     fi
-    # NSX_CONTROLLERS must be a comma separated string
+    # NSX_MANAGER must be a comma separated string
-    if [[ "$NSX_CONTROLLERS" != "" ]]; then
+    if [[ "$NSX_MANAGERS" != "" ]]; then
-        iniset /$Q_PLUGIN_CONF_FILE DEFAULT nsx_controllers $NSX_CONTROLLERS
+        _nsxv3_ini_set nsx_managers $NSX_MANAGERS
+    elif [[ "$NSX_MANAGER" != "" ]]; then
+        _nsxv3_ini_set nsx_managers $NSX_MANAGER
     else
-        die $LINENO "The VMware NSX plugin needs at least an NSX controller."
+        die $LINENO "The VMware NSX plugin needs at least one NSX manager."
     fi
     if [[ "$NSX_L2GW_DRIVER" != "" ]]; then
         iniset /$Q_PLUGIN_CONF_FILE DEFAULT nsx_l2gw_driver $NSX_L2GW_DRIVER
     fi
     _nsxv3_ini_set default_tier0_router_uuid $DEFAULT_TIER0_ROUTER_UUID
-    _nsxv3_ini_set nsx_manager $NSX_MANAGER "The VMWare NSX plugin needs a NSX manager."
     _nsxv3_ini_set nsx_user $NSX_USER
     _nsxv3_ini_set nsx_password $NSX_PASSWORD
     _nsxv3_ini_set retries $NSX_RETRIES

etc/nsx.ini

@@ -315,8 +315,12 @@
 # metadata_shared_secret =
 
 [nsx_v3]
-# IP address of NSX manager
+# IP address of one or more NSX managers separated by commas.
-# nsx_manager = 1.2.3.4
+# The IP address should be of the form:
+#   [<scheme>://]<ip_adress>[:<port>]
+# If scheme is not provided https is used. If port is not provided
+# port 80 is used for http and port 443 for https.
+# nsx_managers = 1.2.3.4
 
 # User name of NSX Manager
 # nsx_user = admin
@@ -340,12 +344,25 @@
 
 # Specify a CA bundle file to use in verifying the NSX Manager
 # server certificate. This option is ignored if "insecure" is set to True.
+# If "insecure" is set to False and ca_file is unset, the system root CAs
+# will be used to verify the server certificate.
 # ca_file =
 
-# If true, the NSX Manager server certificate is not verified. If false,
+# If true, the NSX Manager server certificate is not verified. If false
-# then the default CA truststore is used for verification.
+# the CA bundle specified via "ca_file" will be used or if unsest the
+# default system root CAs will be used.
 # insecure = True
 
+# The time in seconds before aborting an HTTP request to a NSX manager.
+# http_timeout = 75
+
+# Maxiumum number of connection connections to each NSX manager.
+# concurrent_connections = 10
+
+# The amount of time in seconds to wait before ensuring connectivity to
+# the NSX manager if no manager connection has been used.
+# conn_idle_timeout = 60
+
 # UUID of the default tier0 router that will be used for connecting to
 # tier1 logical routers and configuring external network
 # default_tier0_router_uuid = 412983fd-9016-45e5-93f2-48ba2a931225

vmware_nsx/common/config.py

@@ -176,8 +176,12 @@ nsx_v3_opts = [
                default='default',
                secret=True,
                help=_('Password for the NSX manager')),
-    cfg.StrOpt('nsx_manager',
+    cfg.ListOpt('nsx_managers',
-               help=_('IP address of the NSX manager')),
+                deprecated_name='nsx_manager',
+                help=_('IP address of one or more NSX managers separated '
+                       'by commas. The IP address can optionally specify a '
+                       'scheme (e.g. http or https) and port using the format '
+                       '<scheme>://<ip_address>:<port>')),
     cfg.StrOpt('default_overlay_tz_uuid',
                deprecated_name='default_tz_uuid',
                help=_("This is the UUID of the default NSX overlay transport "
@@ -200,13 +204,30 @@ nsx_v3_opts = [
                help=_('Maximum number of times to retry API request')),
     cfg.StrOpt('ca_file',
                help=_('Specify a CA bundle file to use in verifying the NSX '
-                      'Manager server certificate.')),
+                      'Manager server certificate. This option is ignored if '
+                      '"insecure" is set to True. If "insecure" is set to '
+                      'False and ca_file is unset, the system root CAs will '
+                      'be used to verify the server certificate.')),
     cfg.BoolOpt('insecure',
                 default=True,
                 help=_('If true, the NSX Manager server certificate is not '
-                       'verified. If false, then the default CA truststore is '
+                       'verified. If false the CA bundle specified via '
-                       'used for verification. This option is ignored if '
+                       '"ca_file" will be used or if unsest the default '
-                       '"ca_file" is set.')),
+                       'system root CAs will be used.')),
+    cfg.IntOpt('http_timeout',
+               default=75,
+               help=_('Time before aborting a HTTP request to a '
+                      'NSX manager.')),
+    cfg.IntOpt('concurrent_connections', default=10,
+               help=_("Maximum concurrent connections to each NSX "
+                      "manager.")),
+    cfg.IntOpt('conn_idle_timeout',
+               default=60,
+               help=_('Ensure connectivity to the NSX manager if a connection '
+                      'is not used within timeout seconds.')),
+    cfg.IntOpt('redirects',
+               default=2,
+               help=_('Number of times a HTTP redirect should be followed.')),
     cfg.StrOpt('default_tier0_router_uuid',
                help=_("Default tier0 router identifier")),
 ]

vmware_nsx/nsxlib/v3/__init__.py

@@ -56,7 +56,8 @@ def delete_resource_by_values(resource, skip_not_found=True, **kwargs):
             err_msg = (_("No resource in %(res)s matched for values: "
                          "%(values)s") % {'res': resource,
                                           'values': kwargs})
-            raise nsx_exc.ResourceNotFound(manager=client._get_manager_ip(),
+            raise nsx_exc.ResourceNotFound(
+                manager=client._get_nsx_managers_from_conf(),
                 operation=err_msg)
     elif matched_num > 1:
         LOG.warning(_LW("%(num)s resources in %(res)s matched for values: "

vmware_nsx/nsxlib/v3/client.py

@@ -14,11 +14,11 @@
 #    under the License.
 #
 import requests
+import urlparse
 
 from oslo_config import cfg
 from oslo_log import log
 from oslo_serialization import jsonutils
-
 from vmware_nsx._i18n import _, _LW
 from vmware_nsx.common import exceptions as nsx_exc
 
@@ -37,37 +37,19 @@ class RESTClient(object):
         'delete': [requests.codes.ok]
     }
 
-    def __init__(self, host_ip=None, user_name=None,
+    def __init__(self, connection, url_prefix=None,
-                 password=None, insecure=None,
+                 default_headers=None):
-                 url_prefix=None, default_headers=None,
+        self._conn = connection
-                 cert_file=None):
-        self._host_ip = host_ip
-        self._user_name = user_name
-        self._password = password
-        self._insecure = insecure if insecure is not None else False
         self._url_prefix = url_prefix or ""
         self._default_headers = default_headers or {}
-        self._cert_file = cert_file
-
-        self._session = requests.Session()
-        self._session.auth = (self._user_name, self._password)
-        if not insecure and self._cert_file:
-            self._session.cert = self._cert_file
 
     def new_client_for(self, *uri_segments):
-        uri = "%s/%s" % (self._url_prefix, '/'.join(uri_segments))
+        uri = self._build_url('/'.join(uri_segments))
-        uri = uri.replace('//', '/')
 
         return self.__class__(
-            host_ip=self._host_ip, user_name=self._user_name,
+            self._conn,
-            password=self._password, insecure=self._insecure,
             url_prefix=uri,
-            default_headers=self._default_headers,
+            default_headers=self._default_headers)
-            cert_file=self._cert_file)
-
-    @property
-    def validate_certificate(self):
-        return not self._insecure
 
     def list(self, headers=None):
         return self.url_list('')
@@ -115,7 +97,7 @@ class RESTClient(object):
             if type(result_msg) is dict:
                 result_msg = result_msg.get('error_message', result_msg)
             raise manager_error(
-                manager=self._host_ip,
+                manager=_get_nsx_managers_from_conf(),
                 operation=operation,
                 details=result_msg)
 
@@ -128,25 +110,28 @@ class RESTClient(object):
         return merged
 
     def _build_url(self, uri):
-        uri = ("/%s/%s" % (self._url_prefix, uri)).replace('//', '/')
+        prefix = urlparse.urlparse(self._url_prefix)
-        return ("https://%s%s" % (self._host_ip, uri)).strip('/')
+        uri = ("/%s/%s" % (prefix.path, uri)).replace('//', '/').strip('/')
+        if prefix.netloc:
+            uri = "%s/%s" % (prefix.netloc, uri)
+        if prefix.scheme:
+            uri = "%s://%s" % (prefix.scheme, uri)
+        return uri
 
     def _rest_call(self, url, method='GET', body=None, headers=None):
         request_headers = headers.copy() if headers else {}
         request_headers.update(self._default_headers)
         request_url = self._build_url(url)
 
-        do_request = getattr(self._session, method.lower())
+        do_request = getattr(self._conn, method.lower())
 
         LOG.debug("REST call: %s %s\nHeaders: %s\nBody: %s",
                   method, request_url, request_headers, body)
 
         result = do_request(
             request_url,
-            verify=self.validate_certificate,
             data=body,
-            headers=request_headers,
+            headers=request_headers)
-            cert=self._cert_file)
 
         self._validate_result(
             result, RESTClient._VERB_RESP_CODES[method.lower()],
@@ -161,18 +146,14 @@ class JSONRESTClient(RESTClient):
         'Content-Type': 'application/json'
     }
 
-    def __init__(self, host_ip=None, user_name=None,
+    def __init__(self, connection, url_prefix=None,
-                 password=None, insecure=None,
+                 default_headers=None):
-                 url_prefix=None, default_headers=None,
-                 cert_file=None):
 
         super(JSONRESTClient, self).__init__(
-            host_ip=host_ip, user_name=user_name,
+            connection,
-            password=password, insecure=insecure,
             url_prefix=url_prefix,
             default_headers=RESTClient.merge_headers(
-                JSONRESTClient._DEFAULT_HEADERS, default_headers),
+                JSONRESTClient._DEFAULT_HEADERS, default_headers))
-            cert_file=cert_file)
 
     def _rest_call(self, *args, **kwargs):
         if kwargs.get('body') is not None:
@@ -183,60 +164,64 @@ class JSONRESTClient(RESTClient):
 
 class NSX3Client(JSONRESTClient):
 
-    _NSX_V1_API_PREFIX = '/api/v1/'
+    _NSX_V1_API_PREFIX = 'api/v1/'
 
-    def __init__(self, host_ip=None, user_name=None,
+    def __init__(self, connection, url_prefix=None,
-                 password=None, insecure=None,
+                 default_headers=None):
-                 url_prefix=None, default_headers=None,
-                 cert_file=None):
 
         url_prefix = url_prefix or NSX3Client._NSX_V1_API_PREFIX
-        if (url_prefix and not url_prefix.startswith(
+        if url_prefix and NSX3Client._NSX_V1_API_PREFIX not in url_prefix:
-                NSX3Client._NSX_V1_API_PREFIX)):
+            if url_prefix.startswith('http'):
+                url_prefix += '/' + NSX3Client._NSX_V1_API_PREFIX
+            else:
                 url_prefix = "%s/%s" % (NSX3Client._NSX_V1_API_PREFIX,
                                         url_prefix or '')
-        host_ip = host_ip or cfg.CONF.nsx_v3.nsx_manager
-        user_name = user_name or cfg.CONF.nsx_v3.nsx_user
-        password = password or cfg.CONF.nsx_v3.nsx_password
-        cert_file = cert_file or cfg.CONF.nsx_v3.ca_file
-        insecure = (insecure if insecure is not None
-                    else cfg.CONF.nsx_v3.insecure)
 
         super(NSX3Client, self).__init__(
-            host_ip=host_ip, user_name=user_name,
+            connection, url_prefix=url_prefix,
-            password=password, insecure=insecure,
+            default_headers=default_headers)
-            url_prefix=url_prefix,
-            default_headers=default_headers,
-            cert_file=cert_file)
 
 
-# NOTE(boden): tmp until all refs use client class
+# TODO(boden): remove mod level fns and vars below
-def _get_client(client, *args, **kwargs):
+_DEFAULT_API_CLUSTER = None
-    return client or NSX3Client(*args, **kwargs)
+
+
+def _get_default_api_cluster():
+    global _DEFAULT_API_CLUSTER
+    if _DEFAULT_API_CLUSTER is None:
+        # removes circular ref between client / cluster
+        import vmware_nsx.nsxlib.v3.cluster as nsx_cluster
+        _DEFAULT_API_CLUSTER = nsx_cluster.NSXClusteredAPI()
+    return _DEFAULT_API_CLUSTER
+
+
+def _set_default_api_cluster(cluster):
+    global _DEFAULT_API_CLUSTER
+    old = _DEFAULT_API_CLUSTER
+    _DEFAULT_API_CLUSTER = cluster
+    return old
+
+
+def _get_client(client):
+    return client or NSX3Client(_get_default_api_cluster())
 
 
 # NOTE(shihli): tmp until all refs use client class
-def _get_manager_ip(client=None):
+def _get_nsx_managers_from_conf():
-    # NOTE: In future this may return the IP address from a pool
+    return cfg.CONF.nsx_v3.nsx_managers
-    return (client._host_ip if client is not None
-            else cfg.CONF.nsx_v3.nsx_manager)
 
 
-# NOTE(boden): tmp until all refs use client class
 def get_resource(resource, client=None):
     return _get_client(client).get(resource)
 
 
-# NOTE(boden): tmp until all refs use client class
 def create_resource(resource, data, client=None):
     return _get_client(client).url_post(resource, body=data)
 
 
-# NOTE(boden): tmp until all refs use client class
 def update_resource(resource, data, client=None):
     return _get_client(client).update(resource, body=data)
 
 
-# NOTE(boden): tmp until all refs use client class
 def delete_resource(resource, client=None):
     return _get_client(client).delete(resource)

vmware_nsx/nsxlib/v3/cluster.py

@@ -0,0 +1,479 @@
+# Copyright 2015 VMware, Inc.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import abc
+import contextlib
+import copy
+import datetime
+import eventlet
+import logging
+import random
+import requests
+import six
+import urlparse
+
+from eventlet import greenpool
+from eventlet import pools
+from oslo_config import cfg
+from oslo_log import log
+from oslo_service import loopingcall
+from requests import adapters
+from vmware_nsx._i18n import _, _LI, _LW
+from vmware_nsx.common import exceptions as nsx_err
+from vmware_nsx.common import exceptions as nsx_exc
+from vmware_nsx.nsxlib.v3 import client as nsx_client
+
+LOG = log.getLogger(__name__)
+
+# disable warning message for each HTTP retry
+logging.getLogger(
+    "requests.packages.urllib3.connectionpool").setLevel(logging.ERROR)
+
+
+@six.add_metaclass(abc.ABCMeta)
+class AbstractHTTPProvider(object):
+    """Interface for providers of HTTP connections which
+    are responsible for creating and validating connections
+    for their underlying HTTP support.
+    """
+
+    @property
+    def default_scheme(self):
+        return 'https'
+
+    @abc.abstractproperty
+    def provider_id(self):
+        """A unique string name for this provider."""
+        pass
+
+    @abc.abstractmethod
+    def validate_connection(self, cluster_api, endpoint, conn):
+        """Validate the said connection for the given endpoint and cluster.
+        """
+        pass
+
+    @abc.abstractmethod
+    def new_connection(self, cluster_api, provider):
+        """Create a new http connection for the said cluster and
+        cluster provider. The actual connection should duck type
+        requests.Session http methods (get(), put(), etc.).
+        """
+        pass
+
+
+class TimeoutSession(requests.Session):
+    """Extends requests.Session to support timeout
+    at the session level.
+    """
+
+    def __init__(self, timeout=cfg.CONF.nsx_v3.http_timeout):
+        self.timeout = timeout
+        super(TimeoutSession, self).__init__()
+
+    # wrapper timeouts at the session level
+    # see: https://goo.gl/xNk7aM
+    def request(self, *args, **kwargs):
+        if 'timeout' not in kwargs:
+            kwargs['timeout'] = self.timeout
+        return super(TimeoutSession, self).request(*args, **kwargs)
+
+
+class NSXRequestsHTTPProvider(AbstractHTTPProvider):
+    """Concrete implementation of AbstractHTTPProvider
+    using requests.Session() as the underlying connection.
+    """
+
+    @property
+    def provider_id(self):
+        return "%s-%s" % (requests.__title__, requests.__version__)
+
+    def validate_connection(self, cluster_api, endpoint, conn):
+        client = nsx_client.NSX3Client(conn, url_prefix=endpoint.provider.url)
+        zones = client.get('transport-zones')
+        if not zones or zones['result_count'] <= 0:
+            msg = _("No transport zones found "
+                    "for '%s'") % endpoint.provider.url
+            LOG.warning(msg)
+            raise nsx_exc.ResourceNotFound(
+                manager=endpoint.provider.url, operation=msg)
+
+    def new_connection(self, cluster_api, provider):
+        session = TimeoutSession(cluster_api.http_timeout)
+        session.auth = (cluster_api.username, cluster_api.password)
+        # NSX v3 doesn't use redirects
+        session.max_redirects = 0
+
+        session.verify = not cluster_api.insecure
+        if session.verify and cluster_api.ca_file:
+            # verify using the said ca bundle path
+            session.verify = cluster_api.ca_file
+
+        # we are pooling with eventlet in the cluster class
+        adapter = adapters.HTTPAdapter(
+            pool_connections=1, pool_maxsize=1,
+            max_retries=cluster_api.retries,
+            pool_block=False)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+
+        return session
+
+
+class ClusterHealth(object):
+    """Indicator of overall cluster health with respect
+    to the connectivity of the clusters managed endpoints.
+    """
+    # all endpoints are UP
+    GREEN = 'GREEN'
+    # at least 1 endpoint is UP, but 1 or more are DOWN
+    ORANGE = 'ORANGE'
+    # all endpoints are DOWN
+    RED = 'RED'
+
+
+class EndpointState(object):
+    """Tracks the connectivity state for a said endpoint.
+    """
+    # no UP or DOWN state recorded yet
+    INITIALIZED = 'INITIALIZED'
+    # endpoint has been validate and is good
+    UP = 'UP'
+    # endpoint can't be reached or validated
+    DOWN = 'DOWN'
+
+
+class Provider(object):
+    """Data holder for a provider which has a unique id
+    and a connection URL.
+    """
+
+    def __init__(self, provider_id, provider_url):
+        self.id = provider_id
+        self.url = provider_url
+
+    def __str__(self):
+        return str(self.url)
+
+
+class Endpoint(object):
+    """A single NSX manager endpoint (host) which includes
+    related information such as the endpoint's provider,
+    state, etc.. A pool is used to hold connections to the
+    endpoint which are doled out when proxying HTTP methods
+    to the underlying connections.
+    """
+
+    def __init__(self, provider, pool):
+        self.provider = provider
+        self.pool = pool
+        self._state = EndpointState.INITIALIZED
+        self._last_updated = datetime.datetime.now()
+
+    @property
+    def last_updated(self):
+        return self._last_updated
+
+    @property
+    def state(self):
+        return self._state
+
+    def set_state(self, state):
+        if self.state != state:
+            LOG.info(_LI("Endpoint '%(ep)s' changing from state"
+                         " '%(old)s' to '%(new)s'"),
+                     {'ep': self.provider,
+                      'old': self.state,
+                      'new': state})
+        old_state = self._state
+        self._state = state
+
+        self._last_updated = datetime.datetime.now()
+
+        return old_state
+
+    def __str__(self):
+        return "[%s] %s" % (self.state, self.provider)
+
+
+class EndpointConnection(object):
+    """Simple data holder which contains an endpoint and
+    a connection for that endpoint.
+    """
+
+    def __init__(self, endpoint, connection):
+        self.endpoint = endpoint
+        self.connection = connection
+
+
+class ClusteredAPI(object):
+    """Duck types the major HTTP based methods of a
+    requests.Session such as get(), put(), post(), etc.
+    and transparently proxies those calls to one of
+    its managed NSX manager endpoints.
+    """
+    _HTTP_VERBS = ['get', 'delete', 'head', 'put', 'post', 'patch', 'create']
+
+    def __init__(self, providers,
+                 http_provider,
+                 min_conns_per_pool=1,
+                 max_conns_per_pool=500,
+                 keepalive_interval=33):
+
+        self._http_provider = http_provider
+        self._keepalive_interval = keepalive_interval
+
+        def _create_conn(p):
+            def _conn():
+                # called when a pool needs to create a new connection
+                return self._http_provider.new_connection(self, p)
+            return _conn
+
+        self._endpoints = {}
+        for provider in providers:
+            pool = pools.Pool(
+                min_size=min_conns_per_pool,
+                max_size=max_conns_per_pool,
+                order_as_stack=True,
+                create=_create_conn(provider))
+
+            endpoint = Endpoint(provider, pool)
+            self._endpoints[provider.id] = endpoint
+
+        # duck type to proxy http invocations
+        for method in ClusteredAPI._HTTP_VERBS:
+            setattr(self, method, self._proxy_stub(method))
+
+        LOG.debug("Initializing API endpoints")
+        conns = greenpool.GreenPool()
+        for endpoint in self._endpoints.values():
+            conns.spawn(self._validate, endpoint)
+        eventlet.sleep(0)
+        while conns.running():
+            if (self.health == ClusterHealth.GREEN
+                    or self.health == ClusterHealth.ORANGE):
+                # only wait for 1 or more endpoints to reduce init time
+                break
+            eventlet.sleep(0.5)
+
+        for endpoint in self._endpoints.values():
+            # dynamic loop for each endpoint to ensure connectivity
+            loop = loopingcall.DynamicLoopingCall(
+                self._endpoint_keepalive, endpoint)
+            loop.start(initial_delay=self._keepalive_interval,
+                       periodic_interval_max=self._keepalive_interval,
+                       stop_on_exception=False)
+
+        LOG.debug("Done initializing API endpoint(s). "
+                  "API cluster health: %s", self.health)
+
+    def _endpoint_keepalive(self, endpoint):
+        delta = datetime.datetime.now() - endpoint.last_updated
+        if delta.seconds >= self._keepalive_interval:
+            # TODO(boden): backoff on validation failure
+            self._validate(endpoint)
+            return self._keepalive_interval
+        return self._keepalive_interval - delta.seconds
+
+    @property
+    def providers(self):
+        return [ep.provider for ep in self._endpoints.values()]
+
+    @property
+    def endpoints(self):
+        return copy.copy(self._endpoints)
+
+    @property
+    def http_provider(self):
+        return self._http_provider
+
+    @property
+    def health(self):
+        down = 0
+        up = 0
+        for endpoint in self._endpoints.values():
+            if endpoint.state != EndpointState.UP:
+                down += 1
+            else:
+                up += 1
+
+        if down == len(self._endpoints):
+            return ClusterHealth.RED
+        return (ClusterHealth.GREEN
+                if up == len(self._endpoints)
+                else ClusterHealth.ORANGE)
+
+    def revalidate_endpoints(self):
+        # validate each endpoint in serial
+        for endpoint in self._endpoints.values():
+            self._validate(endpoint)
+
+    def _validate(self, endpoint):
+        try:
+            with endpoint.pool.item() as conn:
+                self._http_provider.validate_connection(self, endpoint, conn)
+                endpoint.set_state(EndpointState.UP)
+                LOG.debug("Validated API cluster endpoint: %s", endpoint)
+        except Exception as e:
+            endpoint.set_state(EndpointState.DOWN)
+            LOG.warning(_LW("Failed to validate API cluster endpoint "
+                            "'%(ep)s' due to: %(err)s"),
+                        {'ep': endpoint, 'err': e})
+
+    def _select_endpoint(self, revalidate=False):
+        connected = {}
+        for provider_id, endpoint in self._endpoints.items():
+            if endpoint.state == EndpointState.UP:
+                connected[provider_id] = endpoint
+                if endpoint.pool.free():
+                    # connection can be used now
+                    return endpoint
+
+        if not connected and revalidate:
+            LOG.debug("All endpoints DOWN; revalidating.")
+            # endpoints may have become available, try to revalidate
+            self.revalidate_endpoints()
+            return self._select_endpoint(revalidate=False)
+
+        # no free connections; randomly select a connected endpoint
+        # which will likely wait on pool.item() until a connection frees up
+        return (connected[random.choice(connected.keys())]
+                if connected else None)
+
+    def endpoint_for_connection(self, conn):
+        # check all endpoint pools
+        for endpoint in self._endpoints.values():
+            if (conn in endpoint.pool.channel.queue or
+                    conn in endpoint.pool.free_items):
+                return endpoint
+
+    @property
+    def cluster_id(self):
+        return ','.join([str(ep.provider.url)
+                         for ep in self._endpoints.values()])
+
+    @contextlib.contextmanager
+    def connection(self):
+        with self.endpoint_connection() as conn_data:
+            yield conn_data.connection
+
+    @contextlib.contextmanager
+    def endpoint_connection(self):
+        endpoint = self._select_endpoint(revalidate=True)
+        if not endpoint:
+            raise nsx_err.ServiceClusterUnavailable(
+                cluster_id=self.cluster_id)
+
+        if endpoint.pool.free() == 0:
+            LOG.info(_LI("API endpoint %(ep)s at connection "
+                         "capacity %(max)s and has %(waiting)s waiting"),
+                     {'ep': endpoint,
+                      'max': endpoint.pool.max_size,
+                      'waiting': endpoint.pool.waiting()})
+        # pool.item() will wait if pool has 0 free
+        with endpoint.pool.item() as conn:
+            yield EndpointConnection(endpoint, conn)
+
+    def _proxy_stub(self, proxy_for):
+        def _call_proxy(url, *args, **kwargs):
+            return self._proxy(proxy_for, url, *args, **kwargs)
+        return _call_proxy
+
+    def _proxy(self, proxy_for, uri, *args, **kwargs):
+        # proxy http request call to an avail endpoint
+        with self.endpoint_connection() as conn_data:
+            conn = conn_data.connection
+            endpoint = conn_data.endpoint
+
+            # http conn must support requests style interface
+            do_request = getattr(conn, proxy_for)
+
+            if not uri.startswith('/'):
+                uri = "/%s" % uri
+            url = "%s%s" % (endpoint.provider.url, uri)
+            try:
+                LOG.debug("API cluster proxy %s %s to %s",
+                          proxy_for.upper(), uri, url)
+                # call the actual connection method to do the
+                # http request/response over the wire
+                response = do_request(url, *args, **kwargs)
+                endpoint.set_state(EndpointState.UP)
+
+                return response
+            except Exception as e:
+                LOG.warning(_LW("Request failed due to: %s"), e)
+                endpoint.set_state(EndpointState.DOWN)
+                # retry until exhausting endpoints
+                return self._proxy(proxy_for, uri, *args, **kwargs)
+
+
+class NSXClusteredAPI(ClusteredAPI):
+    """Extends ClusteredAPI to get conf values and setup the
+    NSX v3 cluster.
+    """
+
+    def __init__(self,
+                 username=None,
+                 password=None,
+                 retries=None,
+                 insecure=None,
+                 ca_file=None,
+                 concurrent_connections=None,
+                 http_timeout=None,
+                 conn_idle_timeout=None,
+                 http_provider=None):
+        self.username = username or cfg.CONF.nsx_v3.nsx_user
+        self.password = password or cfg.CONF.nsx_v3.nsx_password
+        self.retries = retries or cfg.CONF.nsx_v3.retries
+        self.insecure = insecure or cfg.CONF.nsx_v3.insecure
+        self.ca_file = ca_file or cfg.CONF.nsx_v3.ca_file
+        self.conns_per_pool = (concurrent_connections or
+                               cfg.CONF.nsx_v3.concurrent_connections)
+        self.http_timeout = http_timeout or cfg.CONF.nsx_v3.http_timeout
+        self.conn_idle_timeout = (conn_idle_timeout or
+                                  cfg.CONF.nsx_v3.conn_idle_timeout)
+
+        self._http_provider = http_provider or NSXRequestsHTTPProvider()
+
+        super(NSXClusteredAPI, self).__init__(
+            self._build_conf_providers(),
+            self._http_provider,
+            max_conns_per_pool=self.conns_per_pool,
+            keepalive_interval=self.conn_idle_timeout)
+
+        LOG.debug("Created NSX clustered API with '%s' "
+                  "provider", self._http_provider.provider_id)
+
+    def _build_conf_providers(self):
+
+        def _schemed_url(uri):
+            uri = uri.strip('/')
+            return urlparse.urlparse(
+                uri if uri.startswith('http') else
+                "%s://%s" % (self._http_provider.default_scheme, uri))
+
+        conf_urls = cfg.CONF.nsx_v3.nsx_managers[:]
+        urls = []
+        providers = []
+
+        for conf_url in conf_urls:
+            conf_url = _schemed_url(conf_url)
+            if conf_url in urls:
+                LOG.warning(_LW("'%s' already defined in configuration file. "
+                                "Skipping."), urlparse.urlunparse(conf_url))
+                continue
+            urls.append(conf_url)
+            providers.append(Provider(
+                conf_url.netloc, urlparse.urlunparse(conf_url)))
+        return providers

vmware_nsx/nsxlib/v3/resources.py

@@ -368,7 +368,8 @@ class LogicalRouterPort(AbstractRESTResource):
         else:
             err_msg = (_("Logical router link port not found on logical "
                          "switch %s") % logical_switch_id)
-            raise nsx_exc.ResourceNotFound(manager=client._get_manager_ip(),
+            raise nsx_exc.ResourceNotFound(
+                manager=client._get_nsx_managers_from_conf(),
                 operation=err_msg)
 
     def update_by_lswitch_id(self, logical_router_id, ls_id, **payload):
@@ -390,5 +391,5 @@ class LogicalRouterPort(AbstractRESTResource):
             if port['resource_type'] == nsx_constants.LROUTERPORT_LINKONTIER1:
                 return port
         raise nsx_exc.ResourceNotFound(
-            manager=client._get_manager_ip(),
+            manager=client._get_nsx_managers_from_conf(),
             operation="get router link port")

vmware_nsx/plugins/nsx_v3/plugin.py

@@ -66,6 +66,7 @@ from vmware_nsx.common import utils
 from vmware_nsx.db import db as nsx_db
 from vmware_nsx.nsxlib import v3 as nsxlib
 from vmware_nsx.nsxlib.v3 import client as nsx_client
+from vmware_nsx.nsxlib.v3 import cluster as nsx_cluster
 from vmware_nsx.nsxlib.v3 import dfw_api as firewall
 from vmware_nsx.nsxlib.v3 import resources as nsx_resources
 from vmware_nsx.nsxlib.v3 import router
@@ -110,6 +111,10 @@ class NsxV3Plugin(addr_pair_db.AllowedAddressPairsMixin,
         super(NsxV3Plugin, self).__init__()
         LOG.info(_LI("Starting NsxV3Plugin"))
 
+        self._api_cluster = nsx_cluster.NSXClusteredAPI()
+        self._nsx_client = nsx_client.NSX3Client(self._api_cluster)
+        nsx_client._set_default_api_cluster(self._api_cluster)
+
         self.base_binding_dict = {
             pbin.VIF_TYPE: pbin.VIF_TYPE_OVS,
             pbin.VIF_DETAILS: {
@@ -119,7 +124,7 @@ class NsxV3Plugin(addr_pair_db.AllowedAddressPairsMixin,
         self.tier0_groups_dict = {}
         self._setup_dhcp()
         self._start_rpc_notifiers()
-        self._nsx_client = nsx_client.NSX3Client()
+
         self._port_client = nsx_resources.LogicalPort(self._nsx_client)
         self.nsgroup_container, self.default_section = (
             security.init_nsgroup_container_and_default_section_rules())

vmware_nsx/tests/unit/nsx_v3/mocks.py

@@ -192,6 +192,9 @@ class MockRequestSessionApi(object):
         else:
             response_content = self._create(url, body, **kwargs)
 
+        if isinstance(response_content, MockRequestsResponse):
+            return response_content
+
         return self._build_response(
             url, content=response_content,
             status=requests.codes.created, **kwargs)

vmware_nsx/tests/unit/nsx_v3/test_plugin.py

@@ -12,6 +12,7 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import mock
 import six
 
 from neutron.api.v2 import attributes
@@ -39,6 +40,7 @@ from oslo_utils import uuidutils
 
 from vmware_nsx.common import utils
 from vmware_nsx.nsxlib.v3 import client as nsx_client
+from vmware_nsx.nsxlib.v3 import cluster as nsx_cluster
 from vmware_nsx.plugins.nsx_v3 import plugin as nsx_plugin
 from vmware_nsx.tests import unit as vmware
 from vmware_nsx.tests.unit.nsx_v3 import mocks as nsx_v3_mocks
@@ -54,26 +56,65 @@ class NsxV3PluginTestCaseMixin(test_plugin.NeutronDbPluginV2TestCase,
     def setUp(self, plugin=PLUGIN_NAME,
               ext_mgr=None,
               service_plugins=None):
+
         self._patchers = []
+
         self.mock_api = nsx_v3_mocks.MockRequestSessionApi()
-        self.client = nsx_client.NSX3Client()
+        nsxlib_testcase.NsxClientTestCase.setup_conf_overrides()
+        self.cluster = nsx_cluster.NSXClusteredAPI(
+            http_provider=nsxlib_testcase.MemoryMockAPIProvider(self.mock_api))
 
-        def mock_client_module(mod):
+        self.cluster.revalidate_endpoints()
-            mocked = nsxlib_testcase.NsxClientTestCase.mocked_session_module(
-                mod, self.client,
-                mock_session=self.mock_api)
-            mocked.start()
-            self._patchers.append(mocked)
 
-        mock_client_module(nsx_plugin.security.firewall)
+        def _patch_object(*args, **kwargs):
-        mock_client_module(nsx_plugin.router.nsxlib)
+            patcher = mock.patch.object(*args, **kwargs)
-        mock_client_module(nsx_plugin)
+            patcher.start()
+            self._patchers.append(patcher)
+
+        def _new_cluster(*args, **kwargs):
+            return self.cluster
+
+        self.mocked_rest_fns(
+            nsx_plugin.security.firewall, 'nsxclient',
+            mock_cluster=self.cluster)
+        self.mocked_rest_fns(
+            nsx_plugin.router.nsxlib, 'client', mock_cluster=self.cluster)
+
+        mock_client_module = mock.Mock()
+        mock_cluster_module = mock.Mock()
+        mocked_client = self.new_mocked_client(
+            nsx_client.NSX3Client, mock_cluster=self.cluster)
+        mock_cluster_module.NSXClusteredAPI.return_value = self.cluster
+        mock_client_module.NSX3Client.return_value = mocked_client
+        _patch_object(nsx_plugin, 'nsx_client', new=mock_client_module)
+        _patch_object(nsx_plugin, 'nsx_cluster', new=mock_cluster_module)
 
         super(NsxV3PluginTestCaseMixin, self).setUp(plugin=plugin,
                                                     ext_mgr=ext_mgr)
 
         self.maxDiff = None
 
+        # populate pre-existing mock resources
+        cluster_id = uuidutils.generate_uuid()
+        self.mock_api.post(
+            'api/v1/logical-routers',
+            data=jsonutils.dumps({
+                'display_name': nsx_v3_mocks.DEFAULT_TIER0_ROUTER_UUID,
+                'router_type': "TIER0",
+                'id': nsx_v3_mocks.DEFAULT_TIER0_ROUTER_UUID,
+                'edge_cluster_id': cluster_id}),
+            headers=nsx_client.JSONRESTClient._DEFAULT_HEADERS)
+
+        self.mock_api.post(
+            'api/v1/edge-clusters',
+            data=jsonutils.dumps({
+                'id': cluster_id,
+                'members': [
+                    {'member_index': 0},
+                    {'member_index': 1}
+                ]}),
+            headers=nsx_client.JSONRESTClient._DEFAULT_HEADERS)
+
     def tearDown(self):
         for patcher in self._patchers:
             patcher.stop()
@@ -215,27 +256,6 @@ class TestL3NatTestCase(L3NatTest,
               service_plugins=None):
         super(TestL3NatTestCase, self).setUp(plugin=plugin, ext_mgr=ext_mgr)
 
-        cluster_id = uuidutils.generate_uuid()
-
-        self.mock_api.post(
-            'api/v1/logical-routers',
-            data=jsonutils.dumps({
-                'display_name': nsx_v3_mocks.DEFAULT_TIER0_ROUTER_UUID,
-                'router_type': "TIER0",
-                'id': nsx_v3_mocks.DEFAULT_TIER0_ROUTER_UUID,
-                'edge_cluster_id': cluster_id}),
-            headers=nsx_client.JSONRESTClient._DEFAULT_HEADERS)
-
-        self.mock_api.post(
-            'api/v1/edge-clusters',
-            data=jsonutils.dumps({
-                'id': cluster_id,
-                'members': [
-                    {'member_index': 0},
-                    {'member_index': 1}
-                ]}),
-            headers=nsx_client.JSONRESTClient._DEFAULT_HEADERS)
-
     def _test_create_l3_ext_network(
             self, physical_network=nsx_v3_mocks.DEFAULT_TIER0_ROUTER_UUID):
         name = 'l3_ext_net'

vmware_nsx/tests/unit/nsxlib/v3/nsxlib_testcase.py

@@ -13,20 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import contextlib
+import copy
 import mock
-import types
 import unittest
 
 from oslo_config import cfg
 from oslo_utils import uuidutils
 from vmware_nsx.nsxlib.v3 import client as nsx_client
-from vmware_nsx.tests.unit.nsx_v3 import mocks
+from vmware_nsx.nsxlib.v3 import cluster as nsx_cluster
 
 NSX_USER = 'admin'
 NSX_PASSWORD = 'default'
 NSX_MANAGER = '1.2.3.4'
-NSX_INSECURE = True
+NSX_INSECURE = False
 NSX_CERT = '/opt/stack/certs/nsx.pem'
 
 V3_CLIENT_PKG = 'vmware_nsx.nsxlib.v3.client'
@@ -35,26 +34,45 @@ BRIDGE_FNS = ['create_resource', 'delete_resource',
 
 
 class NsxLibTestCase(unittest.TestCase):
-    def setUp(self, *args, **kwargs):
-        super(NsxLibTestCase, self).setUp()
-        cfg.CONF.set_override('nsx_user', NSX_USER)
-        cfg.CONF.set_override('nsx_password', NSX_PASSWORD)
-        cfg.CONF.set_override('default_tz_uuid',
-                              uuidutils.generate_uuid())
-        cfg.CONF.set_override('nsx_controllers', ['11.9.8.7', '11.9.8.77'])
 
+    @classmethod
+    def setup_conf_overrides(cls):
+        cfg.CONF.set_override('default_overlay_tz_uuid',
+                              uuidutils.generate_uuid(), 'nsx_v3')
         cfg.CONF.set_override('nsx_user', NSX_USER, 'nsx_v3')
         cfg.CONF.set_override('nsx_password', NSX_PASSWORD, 'nsx_v3')
-        cfg.CONF.set_override('nsx_manager', NSX_MANAGER, 'nsx_v3')
+        cfg.CONF.set_override('nsx_managers', [NSX_MANAGER], 'nsx_v3')
         cfg.CONF.set_override('insecure', NSX_INSECURE, 'nsx_v3')
         cfg.CONF.set_override('ca_file', NSX_CERT, 'nsx_v3')
 
+    def setUp(self, *args, **kwargs):
+        super(NsxLibTestCase, self).setUp()
+        NsxClientTestCase.setup_conf_overrides()
+
         # print diffs when assert comparisons fail
         self.maxDiff = None
 
 
-# NOTE(boden): a lot of the hackery and magic below can be removed
+class MemoryMockAPIProvider(nsx_cluster.AbstractHTTPProvider):
-# once we move all v3 rest function calls to OO based on rest resource
+    """Acts as a HTTP provider for mocking which is backed
+    by a MockRequestSessionApi.
+    """
+
+    def __init__(self, mock_session_api):
+        self._store = mock_session_api
+
+    @property
+    def provider_id(self):
+        return "Memory mock API"
+
+    def validate_connection(self, cluster_api, endpoint, conn):
+        return
+
+    def new_connection(self, cluster_api, provider):
+        # all callers use the same backing
+        return self._store
+
+
 class NsxClientTestCase(NsxLibTestCase):
 
     class MockBridge(object):
@@ -77,114 +95,145 @@ class NsxClientTestCase(NsxLibTestCase):
             return nsx_client.create_resource(
                 resource, data, client=self._client)
 
-    def new_client(
+    class MockNSXClusteredAPI(nsx_cluster.NSXClusteredAPI):
-            self, clazz, host_ip=NSX_MANAGER,
-            user_name=NSX_USER,
-            password=NSX_PASSWORD,
-            insecure=NSX_INSECURE,
-            url_prefix=None,
-            default_headers=None,
-            cert_file=NSX_CERT):
 
-        return clazz(host_ip=host_ip, user_name=user_name,
+        def __init__(self, session_response=None):
-                     password=password, insecure=insecure,
+            super(NsxClientTestCase.MockNSXClusteredAPI, self).__init__(
-                     url_prefix=url_prefix, default_headers=default_headers,
+                http_provider=NsxClientTestCase.MockHTTPProvider(
-                     cert_file=cert_file)
+                    session_response=session_response))
+            self._record = mock.Mock()
 
-    @contextlib.contextmanager
+        def record_call(self, request, **kwargs):
-    def mocked_client(self, client, mock_validate=True):
+            verb = request.method.lower()
-        session = client._session
+
-        with mock.patch.object(session, 'get') as _get:
+            # filter out requests specific attributes
-            with mock.patch.object(session, 'post') as _post:
+            checked_kwargs = copy.copy(kwargs)
-                with mock.patch.object(session, 'delete') as _delete:
+            del checked_kwargs['proxies']
-                    with mock.patch.object(session, 'put') as _put:
+            del checked_kwargs['stream']
-                        rep = {
+            del checked_kwargs['timeout']
-                            'get': _get,
+            if 'allow_redirects' in checked_kwargs:
-                            'put': _put,
+                del checked_kwargs['allow_redirects']
-                            'delete': _delete,
+
-                            'post': _post
+            for attr in ['url', 'body']:
-                        }
+                checked_kwargs[attr] = getattr(request, attr, None)
+
+            # remove headers we don't need to verify
+            checked_kwargs['headers'] = copy.copy(request.headers)
+            for header in ['Accept-Encoding', 'User-Agent',
+                           'Connection', 'Authorization',
+                           'Content-Length']:
+                if header in checked_kwargs['headers']:
+                    del checked_kwargs['headers'][header]
+
+            checked_kwargs['headers'] = request.headers
+
+            # record the call in the mock object
+            method = getattr(self._record, verb)
+            method(**checked_kwargs)
+
+        def assert_called_once(self, verb, **kwargs):
+            mock_call = getattr(self._record, verb.lower())
+            mock_call.assert_called_once_with(**kwargs)
+
+        @property
+        def recorded_calls(self):
+            return self._record
+
+    class MockHTTPProvider(nsx_cluster.NSXRequestsHTTPProvider):
+
+        def __init__(self, session_response=None):
+            super(NsxClientTestCase.MockHTTPProvider, self).__init__()
+            self._session_response = session_response
+
+        def new_connection(self, cluster_api, provider):
+            # wrapper the session so we can intercept and record calls
+            session = super(NsxClientTestCase.MockHTTPProvider,
+                            self).new_connection(cluster_api, provider)
+
+            mock_adapter = mock.Mock()
+            session_send = session.send
+
+            def _adapter_send(request, **kwargs):
+                # record calls at the requests HTTP adapter level
+                mock_response = mock.Mock()
+                mock_response.history = None
+                # needed to bypass requests internal checks for mock
+                mock_response.raw._original_response = {}
+
+                # record the request for later verification
+                cluster_api.record_call(request, **kwargs)
+                return mock_response
+
+            def _session_send(request, **kwargs):
+                # calls at the Session level
+                if self._session_response:
+                    # consumer has setup a response for the session
+                    cluster_api.record_call(request, **kwargs)
+                    return self._session_response
+
+                # bypass requests redirect handling for mock
+                kwargs['allow_redirects'] = False
+
+                # session send will end up calling adapter send
+                return session_send(request, **kwargs)
+
+            mock_adapter.send = _adapter_send
+            session.send = _session_send
+
+            def _mock_adapter(*args, **kwargs):
+                # use our mock adapter rather than requests adapter
+                return mock_adapter
+
+            session.get_adapter = _mock_adapter
+            return session
+
+        def validate_connection(self, cluster_api, endpoint, conn):
+            assert conn is not None
+
+    def mock_nsx_clustered_api(self, session_response=None):
+        return NsxClientTestCase.MockNSXClusteredAPI(
+            session_response=session_response)
+
+    def mocked_resource(self, resource_class, mock_validate=True,
+                        session_response=None):
+        mocked = resource_class(nsx_client.NSX3Client(
+            self.mock_nsx_clustered_api(session_response=session_response)))
         if mock_validate:
-                            with mock.patch.object(client, '_validate_result'):
+            mock.patch.object(mocked._client, '_validate_result').start()
-                                yield rep
-                        else:
-                            yield rep
 
-    @contextlib.contextmanager
+        return mocked
-    def mocked_resource(self, resource, mock_validate=True):
-        with self.mocked_client(resource._client,
-                                mock_validate=mock_validate) as _client:
-            yield _client
 
-    @contextlib.contextmanager
+    def new_mocked_client(self, client_class, mock_validate=True,
-    def mocked_client_bridge(self, client, module, attr, mock_validate=True):
+                          session_response=None, mock_cluster=None,
-        mocked_bridge = NsxClientTestCase.MockBridge(client)
+                          **kwargs):
-        mocked_bridge.JSONRESTClient = nsx_client.JSONRESTClient
+        client = client_class(mock_cluster or self.mock_nsx_clustered_api(
-        with self.mocked_client(client, mock_validate=mock_validate) as mocked:
+            session_response=session_response), **kwargs)
-            with mock.patch.object(module, attr, new=mocked_bridge):
-                yield mocked
 
-    @classmethod
+        if mock_validate:
-    def patch_client_module(cls, in_module, fn_map):
+            mock.patch.object(client, '_validate_result').start()
-        mock_client = mock.Mock()
-        for name, clazz in in_module.__dict__.items():
-            if (isinstance(clazz, types.ModuleType) and
-                    clazz.__name__ == V3_CLIENT_PKG):
-                for fn_name in BRIDGE_FNS:
-                    mock_call = fn_map.get(fn_name, getattr(mocks, fn_name))
-                    setattr(mock_client, fn_name, mock_call)
-                for fn_name, fn_call in fn_map.items():
-                    if fn_name not in BRIDGE_FNS:
-                        setattr(mock_client, fn_name, fn_call)
-                return mock.patch.object(in_module, name, new=mock_client)
-        return None
 
-    @classmethod
+        new_client_for = client.new_client_for
-    def mocked_session_module(cls, in_module, with_client,
-                              mock_session=None):
-        mock_session = mock_session or mocks.MockRequestSessionApi()
-        with_client._session = mock_session
 
-        def _call_client(fn_name):
+        def _new_client_for(*args, **kwargs):
-            def _client(*args, **kwargs):
+            sub_client = new_client_for(*args, **kwargs)
-                client_fn = getattr(nsx_client, fn_name)
+            if mock_validate:
-                kwargs['client'] = with_client
+                mock.patch.object(sub_client, '_validate_result').start()
-                return client_fn(*args, **kwargs)
+            return sub_client
-            return _client
 
-        def _proxy_new_client_for(client):
+        client.new_client_for = _new_client_for
 
-            new_client_fn = client.new_client_for
-
-            def _new_client_for(*uri_segs):
-                new_client = new_client_fn(*uri_segs)
-                new_client._session = mock_session
-                new_client.new_client_for = _proxy_new_client_for(new_client)
-                return new_client
-            return _new_client_for
-
-        def _proxy_init(class_name):
-            client_init = getattr(nsx_client, class_name)
-
-            def _init_client(*args, **kwargs):
-                if (not args and not kwargs and
-                        with_client.__class__.__name__ == class_name):
-                    return with_client
-
-                client = client_init(*args, **kwargs)
-                client._session = mock_session
         return client
 
-            return _init_client
+    def mocked_rest_fns(self, module, attr, mock_validate=True,
+                        mock_cluster=None):
+        client = nsx_client.NSX3Client(
+            mock_cluster or self.mock_nsx_clustered_api())
+        mocked_fns = NsxClientTestCase.MockBridge(client)
+        mocked_fns.JSONRESTClient = nsx_client.JSONRESTClient
 
-        fn_map = {}
+        if mock_validate:
-        for fn in BRIDGE_FNS:
+            mock.patch.object(client, '_validate_result').start()
-            fn_map[fn] = _call_client(fn)
 
-        fn_map['NSX3Client'] = _proxy_init('NSX3Client')
+        mock.patch.object(module, attr, new=mocked_fns).start()
-        fn_map['JSONRESTClient'] = _proxy_init('JSONRESTClient')
-        fn_map['RESTClient'] = _proxy_init('RESTClient')
 
-        with_client.new_client_for = _proxy_new_client_for(with_client)
+        return mocked_fns
-
-        return cls.patch_client_module(in_module, fn_map)

vmware_nsx/tests/unit/nsxlib/v3/test_client.py

@@ -13,6 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
+import copy
+
 from oslo_log import log
 from oslo_serialization import jsonutils
 
@@ -26,213 +28,209 @@ LOG = log.getLogger(__name__)
 
 CLIENT_PKG = 'vmware_nsx.nsxlib.v3.client'
 
+DFT_ACCEPT_HEADERS = {
+    'Accept': '*/*'
+}
 
-def assert_session_call(mock_call, url, verify, data, headers, cert):
+
-    mock_call.assert_called_once_with(
+def _headers(**kwargs):
-        url, verify=verify, data=data, headers=headers, cert=cert)
+    headers = copy.copy(DFT_ACCEPT_HEADERS)
+    headers.update(kwargs)
+    return headers
+
+
+def assert_call(verb, client_or_resource,
+                url, verify=nsxlib_testcase.NSX_CERT,
+                data=None, headers=DFT_ACCEPT_HEADERS):
+    nsx_client = client_or_resource
+    if getattr(nsx_client, '_client', None) is not None:
+        nsx_client = nsx_client._client
+    cluster = nsx_client._conn
+    cluster.assert_called_once(
+        verb,
+        **{'url': url, 'verify': verify, 'body': data,
+           'headers': headers, 'cert': None})
+
+
+def assert_json_call(verb, client_or_resource, url,
+                     verify=nsxlib_testcase.NSX_CERT,
+                     data=None,
+                     headers=client.JSONRESTClient._DEFAULT_HEADERS):
+    return assert_call(verb, client_or_resource, url,
+                       verify=verify, data=data,
+                       headers=headers)
 
 
 class NsxV3RESTClientTestCase(nsxlib_testcase.NsxClientTestCase):
 
-    def test_client_conf_init(self):
-        api = self.new_client(client.RESTClient)
-        self.assertEqual((
-            nsxlib_testcase.NSX_USER, nsxlib_testcase.NSX_PASSWORD),
-            api._session.auth)
-        self.assertEqual(nsxlib_testcase.NSX_MANAGER, api._host_ip)
-        self.assertEqual(nsxlib_testcase.NSX_CERT, api._cert_file)
-
-    def test_client_params_init(self):
-        api = self.new_client(
-            client.RESTClient, host_ip='11.12.13.14', password='mypass')
-        self.assertEqual((
-            nsxlib_testcase.NSX_USER, 'mypass'),
-            api._session.auth)
-        self.assertEqual('11.12.13.14', api._host_ip)
-        self.assertEqual(nsxlib_testcase.NSX_CERT, api._cert_file)
-
     def test_client_url_prefix(self):
-        api = self.new_client(client.RESTClient, url_prefix='/cloud/api')
+        api = self.new_mocked_client(client.RESTClient,
-        with self.mocked_client(api) as mocked:
+                                     url_prefix='/cloud/api')
-            mock_get = mocked.get('get')
+
-            mock_get.return_value = {}
         api.list()
 
-            assert_session_call(
+        assert_call(
-                mocked.get('get'),
+            'get', api,
-                'https://1.2.3.4/cloud/api',
+            'https://1.2.3.4/cloud/api')
-                False, None, {}, nsxlib_testcase.NSX_CERT)
 
-            mock_get.reset_mock()
+        api = self.new_mocked_client(client.RESTClient,
+                                     url_prefix='/cloud/api')
 
         api.url_list('v1/ports')
-            assert_session_call(
+
-                mock_get,
+        assert_call(
-                'https://1.2.3.4/cloud/api/v1/ports', False, None, {},
+            'get', api,
-                nsxlib_testcase.NSX_CERT)
+            'https://1.2.3.4/cloud/api/v1/ports')
 
     def test_client_headers(self):
         default_headers = {'Content-Type': 'application/golang'}
-        api = self.new_client(
+        api = self.new_mocked_client(
             client.RESTClient, default_headers=default_headers,
             url_prefix='/v1/api')
 
-        with self.mocked_client(api) as mocked:
-            mock_get = mocked.get('get')
-
-            mock_get.return_value = {}
-
         api.list()
 
-            assert_session_call(
+        assert_call(
-                mock_get,
+            'get', api,
             'https://1.2.3.4/v1/api',
-                False, None, default_headers, nsxlib_testcase.NSX_CERT)
+            headers=_headers(**default_headers))
 
-            mock_get.reset_mock()
+        api = self.new_mocked_client(
+            client.RESTClient,
+            default_headers=default_headers,
+            url_prefix='/v1/api')
 
         method_headers = {'X-API-Key': 'strong-crypt'}
         api.url_list('ports/33', headers=method_headers)
         method_headers.update(default_headers)
-            assert_session_call(
+        assert_call(
-                mock_get,
+            'get', api,
-                'https://1.2.3.4/v1/api/ports/33', False, None,
+            'https://1.2.3.4/v1/api/ports/33',
-                method_headers,
+            headers=_headers(**method_headers))
-                nsxlib_testcase.NSX_CERT)
 
     def test_client_for(self):
-        api = self.new_client(client.RESTClient, url_prefix='api/v1/')
+        api = self.new_mocked_client(client.RESTClient, url_prefix='api/v1/')
         sub_api = api.new_client_for('switch/ports')
 
-        with self.mocked_client(sub_api) as mocked:
         sub_api.get('11a2b')
 
-            assert_session_call(
+        assert_call(
-                mocked.get('get'),
+            'get', sub_api,
-                'https://1.2.3.4/api/v1/switch/ports/11a2b',
+            'https://1.2.3.4/api/v1/switch/ports/11a2b')
-                False, None, {}, nsxlib_testcase.NSX_CERT)
 
     def test_client_list(self):
-        api = self.new_client(client.RESTClient, url_prefix='api/v1/ports')
+        api = self.new_mocked_client(client.RESTClient,
-        with self.mocked_client(api) as mocked:
+                                     url_prefix='api/v1/ports')
         api.list()
 
-            assert_session_call(
+        assert_call(
-                mocked.get('get'),
+            'get', api,
-                'https://1.2.3.4/api/v1/ports',
+            'https://1.2.3.4/api/v1/ports')
-                False, None, {}, nsxlib_testcase.NSX_CERT)
 
     def test_client_get(self):
-        api = self.new_client(client.RESTClient, url_prefix='api/v1/ports')
+        api = self.new_mocked_client(client.RESTClient,
-        with self.mocked_client(api) as mocked:
+                                     url_prefix='api/v1/ports')
         api.get('unique-id')
 
-            assert_session_call(
+        assert_call(
-                mocked.get('get'),
+            'get', api,
-                'https://1.2.3.4/api/v1/ports/unique-id',
+            'https://1.2.3.4/api/v1/ports/unique-id')
-                False, None, {}, nsxlib_testcase.NSX_CERT)
 
     def test_client_delete(self):
-        api = self.new_client(client.RESTClient, url_prefix='api/v1/ports')
+        api = self.new_mocked_client(client.RESTClient,
-        with self.mocked_client(api) as mocked:
+                                     url_prefix='api/v1/ports')
         api.delete('unique-id')
 
-            assert_session_call(
+        assert_call(
-                mocked.get('delete'),
+            'delete', api,
-                'https://1.2.3.4/api/v1/ports/unique-id',
+            'https://1.2.3.4/api/v1/ports/unique-id')
-                False, None, {}, nsxlib_testcase.NSX_CERT)
 
     def test_client_update(self):
-        api = self.new_client(client.RESTClient, url_prefix='api/v1/ports')
+        api = self.new_mocked_client(client.RESTClient,
-        with self.mocked_client(api) as mocked:
+                                     url_prefix='api/v1/ports')
-            api.update('unique-id', {'name': 'a-new-name'})
+        api.update('unique-id', jsonutils.dumps({'name': 'a-new-name'}))
 
-            assert_session_call(
+        assert_call(
-                mocked.get('put'),
+            'put', api,
             'https://1.2.3.4/api/v1/ports/unique-id',
-                False, {'name': 'a-new-name'},
+            data=jsonutils.dumps({'name': 'a-new-name'}))
-                {}, nsxlib_testcase.NSX_CERT)
 
     def test_client_create(self):
-        api = self.new_client(client.RESTClient, url_prefix='api/v1/ports')
+        api = self.new_mocked_client(client.RESTClient,
-        with self.mocked_client(api) as mocked:
+                                     url_prefix='api/v1/ports')
-            api.create({'resource-name': 'port1'})
+        api.create(jsonutils.dumps({'resource-name': 'port1'}))
 
-            assert_session_call(
+        assert_call(
-                mocked.get('post'),
+            'post', api,
             'https://1.2.3.4/api/v1/ports',
-                False, {'resource-name': 'port1'},
+            data=jsonutils.dumps({'resource-name': 'port1'}))
-                {}, nsxlib_testcase.NSX_CERT)
 
     def test_client_url_list(self):
-        api = self.new_client(client.RESTClient, url_prefix='api/v1/ports')
+        api = self.new_mocked_client(client.RESTClient,
-        with self.mocked_client(api) as mocked:
+                                     url_prefix='api/v1/ports')
-            api.url_list('/connections', {'Content-Type': 'application/json'})
 
-            assert_session_call(
+        json_headers = {'Content-Type': 'application/json'}
-                mocked.get('get'),
+
+        api.url_list('/connections', json_headers)
+
+        assert_call(
+            'get', api,
             'https://1.2.3.4/api/v1/ports/connections',
-                False, None,
+            headers=_headers(**json_headers))
-                {'Content-Type': 'application/json'},
-                nsxlib_testcase.NSX_CERT)
 
     def test_client_url_get(self):
-        api = self.new_client(client.RESTClient, url_prefix='api/v1/ports')
+        api = self.new_mocked_client(client.RESTClient,
-        with self.mocked_client(api) as mocked:
+                                     url_prefix='api/v1/ports')
         api.url_get('connections/1')
 
-            assert_session_call(
+        assert_call(
-                mocked.get('get'),
+            'get', api,
-                'https://1.2.3.4/api/v1/ports/connections/1',
+            'https://1.2.3.4/api/v1/ports/connections/1')
-                False, None, {}, nsxlib_testcase.NSX_CERT)
 
     def test_client_url_delete(self):
-        api = self.new_client(client.RESTClient, url_prefix='api/v1/ports')
+        api = self.new_mocked_client(client.RESTClient,
-        with self.mocked_client(api) as mocked:
+                                     url_prefix='api/v1/ports')
         api.url_delete('1')
 
-            assert_session_call(
+        assert_call(
-                mocked.get('delete'),
+            'delete', api,
-                'https://1.2.3.4/api/v1/ports/1',
+            'https://1.2.3.4/api/v1/ports/1')
-                False, None, {}, nsxlib_testcase.NSX_CERT)
 
     def test_client_url_put(self):
-        api = self.new_client(client.RESTClient, url_prefix='api/v1/ports')
+        api = self.new_mocked_client(client.RESTClient,
-        with self.mocked_client(api) as mocked:
+                                     url_prefix='api/v1/ports')
-            api.url_put('connections/1', {'name': 'conn1'})
+        api.url_put('connections/1', jsonutils.dumps({'name': 'conn1'}))
 
-            assert_session_call(
+        assert_call(
-                mocked.get('put'),
+            'put', api,
             'https://1.2.3.4/api/v1/ports/connections/1',
-                False, {'name': 'conn1'},
+            data=jsonutils.dumps({'name': 'conn1'}))
-                {}, nsxlib_testcase.NSX_CERT)
 
     def test_client_url_post(self):
-        api = self.new_client(client.RESTClient, url_prefix='api/v1/ports')
+        api = self.new_mocked_client(client.RESTClient,
-        with self.mocked_client(api) as mocked:
+                                     url_prefix='api/v1/ports')
-            api.url_post('1/connections', {'name': 'conn1'})
+        api.url_post('1/connections', jsonutils.dumps({'name': 'conn1'}))
 
-            assert_session_call(
+        assert_call(
-                mocked.get('post'),
+            'post', api,
             'https://1.2.3.4/api/v1/ports/1/connections',
-                False, {'name': 'conn1'},
+            data=jsonutils.dumps({'name': 'conn1'}))
-                {}, nsxlib_testcase.NSX_CERT)
 
     def test_client_validate_result(self):
 
-        api = self.new_client(client.RESTClient)
-        with self.mocked_client(api, mock_validate=False) as mocked:
         def _verb_response_code(http_verb, status_code):
             response = mocks.MockRequestsResponse(
                 status_code, None)
-                for _verb in ['get', 'post', 'put', 'delete']:
+
-                    mocked.get(_verb).return_value = response
+            client_api = self.new_mocked_client(
-                client_call = getattr(api, "url_%s" % http_verb)
+                client.RESTClient, mock_validate=False,
+                session_response=response)
+
+            client_call = getattr(client_api, "url_%s" % http_verb)
             client_call('', None)
 
         for verb in ['get', 'post', 'put', 'delete']:
-                for code in client.RESTClient._VERB_RESP_CODES.get(
+            for code in client.RESTClient._VERB_RESP_CODES.get(verb):
-                        verb):
                 _verb_response_code(verb, code)
             self.assertRaises(
                 exep.ManagerError,
@@ -242,20 +240,19 @@ class NsxV3RESTClientTestCase(nsxlib_testcase.NsxClientTestCase):
 class NsxV3JSONClientTestCase(nsxlib_testcase.NsxClientTestCase):
 
     def test_json_request(self):
-        api = self.new_client(client.JSONRESTClient, url_prefix='api/v2/nat')
+        resp = mocks.MockRequestsResponse(
-        with self.mocked_client(api) as mocked:
-            mock_post = mocked.get('post')
-            mock_post.return_value = mocks.MockRequestsResponse(
             200, jsonutils.dumps({'result': {'ok': 200}}))
 
+        api = self.new_mocked_client(client.JSONRESTClient,
+                                     session_response=resp,
+                                     url_prefix='api/v2/nat')
+
         resp = api.create(body={'name': 'mgmt-egress'})
 
-            assert_session_call(
+        assert_json_call(
-                mock_post,
+            'post', api,
             'https://1.2.3.4/api/v2/nat',
-                False, jsonutils.dumps({'name': 'mgmt-egress'}),
+            data=jsonutils.dumps({'name': 'mgmt-egress'}))
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
         self.assertEqual(resp, {'result': {'ok': 200}})
 
@@ -263,67 +260,50 @@ class NsxV3JSONClientTestCase(nsxlib_testcase.NsxClientTestCase):
 class NsxV3APIClientTestCase(nsxlib_testcase.NsxClientTestCase):
 
     def test_api_call(self):
-        api = self.new_client(client.NSX3Client)
+        api = self.new_mocked_client(client.NSX3Client)
-        with self.mocked_client(api) as mocked:
         api.get('ports')
 
-            assert_session_call(
+        assert_json_call(
-                mocked.get('get'),
+            'get', api,
-                'https://1.2.3.4/api/v1/ports',
+            'https://1.2.3.4/api/v1/ports')
-                False, None,
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
 
 # NOTE(boden): remove this when tmp brigding removed
 class NsxV3APIClientBridgeTestCase(nsxlib_testcase.NsxClientTestCase):
 
     def test_get_resource(self):
-        api = self.new_client(client.NSX3Client)
+        api = self.new_mocked_client(client.NSX3Client)
-        with self.mocked_client(api) as mocked:
         client.get_resource('ports', client=api)
 
-            assert_session_call(
+        assert_json_call(
-                mocked.get('get'),
+            'get', api,
-                'https://1.2.3.4/api/v1/ports',
+            'https://1.2.3.4/api/v1/ports')
-                False, None,
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_create_resource(self):
-        api = self.new_client(client.NSX3Client)
+        api = self.new_mocked_client(client.NSX3Client)
-        with self.mocked_client(api) as mocked:
         client.create_resource(
             'ports', {'resource-name': 'port1'},
             client=api)
 
-            assert_session_call(
+        assert_json_call(
-                mocked.get('post'),
+            'post', api,
             'https://1.2.3.4/api/v1/ports',
-                False, jsonutils.dumps({'resource-name': 'port1'}),
+            data=jsonutils.dumps({'resource-name': 'port1'}))
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_update_resource(self):
-        api = self.new_client(client.NSX3Client)
+        api = self.new_mocked_client(client.NSX3Client)
-        with self.mocked_client(api) as mocked:
         client.update_resource(
             'ports/1', {'name': 'a-new-name'}, client=api)
 
-            assert_session_call(
+        assert_json_call(
-                mocked.get('put'),
+            'put', api,
             'https://1.2.3.4/api/v1/ports/1',
-                False, jsonutils.dumps({'name': 'a-new-name'}),
+            data=jsonutils.dumps({'name': 'a-new-name'}))
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_delete_resource(self):
-        api = self.new_client(client.NSX3Client)
+        api = self.new_mocked_client(client.NSX3Client)
-        with self.mocked_client(api) as mocked:
         client.delete_resource('ports/11', client=api)
 
-            assert_session_call(
+        assert_json_call(
-                mocked.get('delete'),
+            'delete', api,
-                'https://1.2.3.4/api/v1/ports/11',
+            'https://1.2.3.4/api/v1/ports/11')
-                False, None, client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)

vmware_nsx/tests/unit/nsxlib/v3/test_cluster.py

@@ -0,0 +1,172 @@
+# Copyright 2015 VMware, Inc.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import mock
+import urlparse
+
+from oslo_config import cfg
+from oslo_serialization import jsonutils
+from vmware_nsx.common import exceptions as nsx_exc
+from vmware_nsx.nsxlib.v3 import client
+from vmware_nsx.nsxlib.v3 import cluster
+from vmware_nsx.tests.unit.nsx_v3 import mocks
+from vmware_nsx.tests.unit.nsxlib.v3 import nsxlib_testcase
+
+
+def _validate_conn_up(*args, **kwargs):
+    return
+
+
+def _validate_conn_down(*args, **kwargs):
+    raise Exception()
+
+
+class RequestsHTTPProviderTestCase(nsxlib_testcase.NsxClientTestCase):
+
+    def test_new_connection(self):
+        mock_api = mock.Mock()
+        mock_api.username = 'nsxuser'
+        mock_api.password = 'nsxpassword'
+        mock_api.retries = 100
+        mock_api.insecure = True
+        mock_api.ca_file = None
+        mock_api.http_timeout = 99
+        mock_api.conn_idle_timeout = 39
+        provider = cluster.NSXRequestsHTTPProvider()
+        session = provider.new_connection(
+            mock_api, cluster.Provider('9.8.7.6', 'https://9.8.7.6'))
+
+        self.assertEqual(session.auth, ('nsxuser', 'nsxpassword'))
+        self.assertEqual(session.verify, False)
+        self.assertEqual(session.cert, None)
+        self.assertEqual(session.adapters['https://'].max_retries.total, 100)
+        self.assertEqual(session.timeout, 99)
+
+    def test_validate_connection(self):
+        mock_conn = mocks.MockRequestSessionApi()
+        mock_ep = mock.Mock()
+        mock_ep.provider.url = 'https://1.2.3.4'
+        provider = cluster.NSXRequestsHTTPProvider()
+        self.assertRaises(nsx_exc.ResourceNotFound,
+                          provider.validate_connection,
+                          mock.Mock(), mock_ep, mock_conn)
+
+        mock_conn.post('api/v1/transport-zones',
+                       data=jsonutils.dumps({'id': 'dummy-tz'}),
+                       headers=client.JSONRESTClient._DEFAULT_HEADERS)
+        provider.validate_connection(mock.Mock(), mock_ep, mock_conn)
+
+
+class NsxV3ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase):
+
+    def _assert_providers(self, cluster_api, provider_tuples):
+        self.assertEqual(len(cluster_api.providers), len(provider_tuples))
+
+        def _assert_provider(pid, purl):
+            for provider in cluster_api.providers:
+                if provider.id == pid and provider.url == purl:
+                    return
+            self.fail("Provider: %s not found" % pid)
+
+        for provider_tuple in provider_tuples:
+            _assert_provider(provider_tuple[0], provider_tuple[1])
+
+    def test_conf_providers_no_scheme(self):
+        conf_managers = ['8.9.10.11', '9.10.11.12:4433']
+        cfg.CONF.set_override(
+            'nsx_managers', conf_managers, 'nsx_v3')
+
+        mock_provider = mock.Mock()
+        mock_provider.default_scheme = 'https'
+        mock_provider.validate_connection = _validate_conn_up
+
+        api = cluster.NSXClusteredAPI(http_provider=mock_provider)
+
+        self._assert_providers(
+            api, [(p, "https://%s" % p) for p in conf_managers])
+
+    def test_conf_providers_with_scheme(self):
+        conf_managers = ['http://8.9.10.11:8080', 'https://9.10.11.12:4433']
+        cfg.CONF.set_override(
+            'nsx_managers', conf_managers, 'nsx_v3')
+
+        mock_provider = mock.Mock()
+        mock_provider.default_scheme = 'https'
+        mock_provider.validate_connection = _validate_conn_up
+
+        api = cluster.NSXClusteredAPI(http_provider=mock_provider)
+
+        self._assert_providers(
+            api, [(urlparse.urlparse(p).netloc, p) for p in conf_managers])
+
+    def test_conns_per_pool(self):
+        cfg.CONF.set_override(
+            'concurrent_connections', 11, 'nsx_v3')
+        conf_managers = ['8.9.10.11', '9.10.11.12:4433']
+        cfg.CONF.set_override(
+            'nsx_managers', conf_managers, 'nsx_v3')
+
+        mock_provider = mock.Mock()
+        mock_provider.default_scheme = 'https'
+        mock_provider.validate_connection = _validate_conn_up
+
+        api = cluster.NSXClusteredAPI(http_provider=mock_provider)
+
+        for ep_id, ep in api.endpoints.items():
+            self.assertEqual(ep.pool.max_size, 11)
+
+
+class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase):
+
+    def _test_health(self, validate_fn, expected_health):
+        conf_managers = ['8.9.10.11', '9.10.11.12']
+        cfg.CONF.set_override(
+            'nsx_managers', conf_managers, 'nsx_v3')
+
+        mock_provider = mock.Mock()
+        mock_provider.default_scheme = 'https'
+
+        mock_provider.validate_connection = validate_fn
+        api = cluster.NSXClusteredAPI(http_provider=mock_provider)
+        self.assertEqual(api.health, expected_health)
+
+    def test_orange_health(self):
+
+        def _validate(cluster_api, endpoint, conn):
+            if endpoint.provider.id == '8.9.10.11':
+                raise Exception()
+
+        self._test_health(_validate, cluster.ClusterHealth.ORANGE)
+
+    def test_green_health(self):
+        self._test_health(_validate_conn_up, cluster.ClusterHealth.GREEN)
+
+    def test_red_health(self):
+        self._test_health(_validate_conn_down, cluster.ClusterHealth.RED)
+
+    def test_cluster_unavailable(self):
+        conf_managers = ['8.9.10.11', '9.10.11.12', '10.11.12.13']
+        cfg.CONF.set_override(
+            'nsx_managers', conf_managers, 'nsx_v3')
+
+        mock_provider = mock.Mock()
+        mock_provider.default_scheme = 'https'
+
+        mock_provider.validate_connection = _validate_conn_down
+        api = cluster.NSXClusteredAPI(http_provider=mock_provider)
+
+        self.assertEqual(len(api.endpoints), 3)
+        self.assertRaises(nsx_exc.ServiceClusterUnavailable,
+                          api.get, 'api/v1/transport-zones')

vmware_nsx/tests/unit/nsxlib/v3/test_qos_switching_profile.py

@@ -17,13 +17,11 @@ from oslo_log import log
 from oslo_serialization import jsonutils
 
 from vmware_nsx.nsxlib import v3 as nsxlib
-from vmware_nsx.nsxlib.v3 import client
 from vmware_nsx.tests.unit.nsx_v3 import test_constants as test_constants_v3
 from vmware_nsx.tests.unit.nsxlib.v3 import nsxlib_testcase
 from vmware_nsx.tests.unit.nsxlib.v3 import test_client
 
 LOG = log.getLogger(__name__)
-_JSON_HEADERS = client.JSONRESTClient._DEFAULT_HEADERS
 
 
 class NsxLibQosTestCase(nsxlib_testcase.NsxClientTestCase):
@@ -47,54 +45,46 @@ class NsxLibQosTestCase(nsxlib_testcase.NsxClientTestCase):
         """
         Test creating a qos-switching profile returns the correct response
         """
-        api = self.new_client(client.NSX3Client)
+        api = self.mocked_rest_fns(nsxlib, 'client')
-        with self.mocked_client_bridge(api, nsxlib, 'client') as mocked:
+
         nsxlib.create_qos_switching_profile(
             qos_marking="untrusted", dscp=25, tags=[],
             name=test_constants_v3.FAKE_NAME,
             description=test_constants_v3.FAKE_NAME)
 
-            test_client.assert_session_call(
+        test_client.assert_json_call(
-                mocked.get('post'),
+            'post', api,
             'https://1.2.3.4/api/v1/switching-profiles',
-                False,
+            data=jsonutils.dumps(self._body(qos_marking='UNTRUSTED', dscp=25),
-                jsonutils.dumps(self._body(qos_marking='UNTRUSTED', dscp=25),
+                                 sort_keys=True))
-                                sort_keys=True),
-                _JSON_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_create_qos_switching_profile_trusted(self):
         """
         Test creating a qos-switching profile returns the correct response
         """
-        api = self.new_client(client.NSX3Client)
+        api = self.mocked_rest_fns(nsxlib, 'client')
-        with self.mocked_client_bridge(api, nsxlib, 'client') as mocked:
+
         nsxlib.create_qos_switching_profile(
             qos_marking="trusted", dscp=0, tags=[],
             name=test_constants_v3.FAKE_NAME,
             description=test_constants_v3.FAKE_NAME)
 
-            test_client.assert_session_call(
+        test_client.assert_json_call(
-                mocked.get('post'),
+            'post', api,
             'https://1.2.3.4/api/v1/switching-profiles',
-                False,
+            data=jsonutils.dumps(self._body(qos_marking='trusted', dscp=0),
-                jsonutils.dumps(self._body(qos_marking='trusted', dscp=0),
+                                 sort_keys=True))
-                                sort_keys=True),
-                _JSON_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_delete_qos_switching_profile(self):
         """
         Test deleting qos-switching-profile
         """
-        api = self.new_client(client.NSX3Client)
+        api = self.mocked_rest_fns(nsxlib, 'client')
-        with self.mocked_client_bridge(api, nsxlib, 'client') as mocked:
+
         nsxlib.delete_qos_switching_profile(
             test_constants_v3.FAKE_QOS_PROFILE['id'])
-            test_client.assert_session_call(
+
-                mocked.get('delete'),
+        test_client.assert_json_call(
+            'delete', api,
             'https://1.2.3.4/api/v1/switching-profiles/%s'
-                % test_constants_v3.FAKE_QOS_PROFILE['id'],
+            % test_constants_v3.FAKE_QOS_PROFILE['id'])
-                False, None,
-                _JSON_HEADERS,
-                nsxlib_testcase.NSX_CERT)

vmware_nsx/tests/unit/nsxlib/v3/test_resources.py

@@ -17,7 +17,6 @@ import mock
 
 from oslo_serialization import jsonutils
 
-from vmware_nsx.nsxlib.v3 import client
 from vmware_nsx.nsxlib.v3 import resources
 from vmware_nsx.tests.unit.nsx_v3 import mocks
 from vmware_nsx.tests.unit.nsx_v3 import test_constants as test_constants_v3
@@ -31,22 +30,24 @@ profile_types = resources.SwitchingProfileTypes
 
 class TestSwitchingProfileTestCase(nsxlib_testcase.NsxClientTestCase):
 
+    def _mocked_switching_profile(self, session_response=None):
+        return self.mocked_resource(
+            resources.SwitchingProfile, session_response=session_response)
+
     def test_switching_profile_create(self):
-        api = resources.SwitchingProfile(client.NSX3Client())
+        mocked_resource = self._mocked_switching_profile()
-        with self.mocked_resource(api) as mocked:
+
-            api.create(profile_types.PORT_MIRRORING,
+        mocked_resource.create(profile_types.PORT_MIRRORING,
                                'pm-profile', 'port mirror prof')
 
-            test_client.assert_session_call(
+        test_client.assert_json_call(
-                mocked.get('post'),
+            'post', mocked_resource,
             'https://1.2.3.4/api/v1/switching-profiles',
-                False, jsonutils.dumps({
+            data=jsonutils.dumps({
                 'resource_type': profile_types.PORT_MIRRORING,
                 'display_name': 'pm-profile',
                 'description': 'port mirror prof'
-                }, sort_keys=True),
+            }, sort_keys=True))
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_switching_profile_update(self):
 
@@ -61,19 +62,18 @@ class TestSwitchingProfileTestCase(nsxlib_testcase.NsxClientTestCase):
             }
         ]
 
-        api = resources.SwitchingProfile(client.NSX3Client())
+        mocked_resource = self._mocked_switching_profile()
-        with self.mocked_resource(api) as mocked:
-            api.update('a12bc1', profile_types.PORT_MIRRORING, tags=tags)
 
-            test_client.assert_session_call(
+        mocked_resource.update(
-                mocked.get('put'),
+            'a12bc1', profile_types.PORT_MIRRORING, tags=tags)
+
+        test_client.assert_json_call(
+            'put', mocked_resource,
             'https://1.2.3.4/api/v1/switching-profiles/a12bc1',
-                False, jsonutils.dumps({
+            data=jsonutils.dumps({
                 'resource_type': profile_types.PORT_MIRRORING,
                 'tags': tags
-                }, sort_keys=True),
+            }, sort_keys=True))
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_spoofgaurd_profile_create(self):
 
@@ -88,25 +88,22 @@ class TestSwitchingProfileTestCase(nsxlib_testcase.NsxClientTestCase):
             }
         ]
 
-        api = resources.SwitchingProfile(client.NSX3Client())
+        mocked_resource = self._mocked_switching_profile()
-        with self.mocked_resource(api) as mocked:
+
-            api.create_spoofguard_profile(
+        mocked_resource.create_spoofguard_profile(
             'neutron-spoof', 'spoofguard-for-neutron',
             whitelist_ports=True, tags=tags)
 
-            test_client.assert_session_call(
+        test_client.assert_json_call(
-                mocked.get('post'),
+            'post', mocked_resource,
             'https://1.2.3.4/api/v1/switching-profiles',
-                False,
+            data=jsonutils.dumps({
-                jsonutils.dumps({
                 'resource_type': profile_types.SPOOF_GUARD,
                 'display_name': 'neutron-spoof',
                 'description': 'spoofguard-for-neutron',
                 'white_list_providers': ['LPORT_BINDINGS'],
                 'tags': tags
-                }, sort_keys=True),
+            }, sort_keys=True))
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_create_dhcp_profile(self):
 
@@ -121,17 +118,16 @@ class TestSwitchingProfileTestCase(nsxlib_testcase.NsxClientTestCase):
             }
         ]
 
-        api = resources.SwitchingProfile(client.NSX3Client())
+        mocked_resource = self._mocked_switching_profile()
-        with self.mocked_resource(api) as mocked:
+
-            api.create_dhcp_profile(
+        mocked_resource.create_dhcp_profile(
             'neutron-dhcp', 'dhcp-for-neutron',
             tags=tags)
 
-            test_client.assert_session_call(
+        test_client.assert_json_call(
-                mocked.get('post'),
+            'post', mocked_resource,
             'https://1.2.3.4/api/v1/switching-profiles',
-                False,
+            data=jsonutils.dumps({
-                jsonutils.dumps({
                 'bpdu_filter': {
                     'enabled': False,
                     'white_list': []
@@ -151,9 +147,7 @@ class TestSwitchingProfileTestCase(nsxlib_testcase.NsxClientTestCase):
                     'rx_multicast': 0,
                     'tx_multicast': 0
                 }
-                }, sort_keys=True),
+            }, sort_keys=True))
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_find_by_display_name(self):
         resp_resources = {
@@ -163,19 +157,17 @@ class TestSwitchingProfileTestCase(nsxlib_testcase.NsxClientTestCase):
                 {'display_name': 'resource-3'}
             ]
         }
-        api = resources.SwitchingProfile(client.NSX3Client())
+        session_response = mocks.MockRequestsResponse(
-        with self.mocked_resource(api) as mocked:
-            mock_get = mocked.get('get')
-            mock_get.return_value = mocks.MockRequestsResponse(
             200, jsonutils.dumps(resp_resources))
-            self.assertEqual([{'display_name': 'resource-1'}],
+        mocked_resource = self._mocked_switching_profile(
-                             api.find_by_display_name('resource-1'))
+            session_response=session_response)
-            self.assertEqual([{'display_name': 'resource-2'}],
-                             api.find_by_display_name('resource-2'))
-            self.assertEqual([{'display_name': 'resource-3'}],
-                             api.find_by_display_name('resource-3'))
 
-            mock_get.reset_mock()
+        self.assertEqual([{'display_name': 'resource-1'}],
+                         mocked_resource.find_by_display_name('resource-1'))
+        self.assertEqual([{'display_name': 'resource-2'}],
+                         mocked_resource.find_by_display_name('resource-2'))
+        self.assertEqual([{'display_name': 'resource-3'}],
+                         mocked_resource.find_by_display_name('resource-3'))
 
         resp_resources = {
             'results': [
@@ -184,14 +176,20 @@ class TestSwitchingProfileTestCase(nsxlib_testcase.NsxClientTestCase):
                 {'display_name': 'resource-1'}
             ]
         }
-            mock_get.return_value = mocks.MockRequestsResponse(
+        session_response = mocks.MockRequestsResponse(
             200, jsonutils.dumps(resp_resources))
+        mocked_resource = self._mocked_switching_profile(
+            session_response=session_response)
         self.assertEqual(resp_resources['results'],
-                             api.find_by_display_name('resource-1'))
+                         mocked_resource.find_by_display_name('resource-1'))
 
 
 class LogicalPortTestCase(nsxlib_testcase.NsxClientTestCase):
 
+    def _mocked_lport(self, session_response=None):
+        return self.mocked_resource(
+            resources.LogicalPort, session_response=session_response)
+
     def test_create_logical_port(self):
         """
         Test creating a port returns the correct response and 200 status
@@ -217,14 +215,10 @@ class LogicalPortTestCase(nsxlib_testcase.NsxClientTestCase):
 
         fake_port['address_bindings'] = binding_repr
 
-        api = resources.LogicalPort(client.NSX3Client())
+        mocked_resource = self._mocked_lport()
-        with self.mocked_resource(api) as mocked:
-
-            mocked.get('post').return_value = mocks.MockRequestsResponse(
-                200, jsonutils.dumps(fake_port))
 
         switch_profile = resources.SwitchingProfile
-            result = api.create(
+        mocked_resource.create(
             fake_port['logical_switch_id'],
             fake_port['attachment']['id'],
             address_bindings=pkt_classifiers,
@@ -242,14 +236,10 @@ class LogicalPortTestCase(nsxlib_testcase.NsxClientTestCase):
             'address_bindings': fake_port['address_bindings']
         }
 
-            self.assertEqual(fake_port, result)
+        test_client.assert_json_call(
-            test_client.assert_session_call(
+            'post', mocked_resource,
-                mocked.get('post'),
             'https://1.2.3.4/api/v1/logical-ports',
-                False,
+            data=jsonutils.dumps(resp_body, sort_keys=True))
-                jsonutils.dumps(resp_body, sort_keys=True),
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_create_logical_port_admin_down(self):
         """
@@ -257,12 +247,12 @@ class LogicalPortTestCase(nsxlib_testcase.NsxClientTestCase):
         """
         fake_port = test_constants_v3.FAKE_PORT
         fake_port['admin_state'] = "DOWN"
-        api = resources.LogicalPort(client.NSX3Client())
-        with self.mocked_resource(api) as mocked:
-            mocked.get('post').return_value = mocks.MockRequestsResponse(
-                200, jsonutils.dumps(fake_port))
 
-            result = api.create(
+        mocked_resource = self._mocked_lport(
+            session_response=mocks.MockRequestsResponse(
+                200, jsonutils.dumps(fake_port)))
+
+        result = mocked_resource.create(
             test_constants_v3.FAKE_PORT['logical_switch_id'],
             test_constants_v3.FAKE_PORT['attachment']['id'],
             tags={}, admin_state=False)
@@ -273,39 +263,31 @@ class LogicalPortTestCase(nsxlib_testcase.NsxClientTestCase):
         """
         Test deleting port
         """
-        api = resources.LogicalPort(client.NSX3Client())
+        mocked_resource = self._mocked_lport()
-        with self.mocked_resource(api) as mocked:
-            mocked.get('delete').return_value = mocks.MockRequestsResponse(
-                200, None)
 
         uuid = test_constants_v3.FAKE_PORT['id']
-            result = api.delete(uuid)
+        mocked_resource.delete(uuid)
-            self.assertIsNone(result.content)
+        test_client.assert_json_call(
-            test_client.assert_session_call(
+            'delete', mocked_resource,
-                mocked.get('delete'),
+            'https://1.2.3.4/api/v1/logical-ports/%s?detach=true' % uuid)
-                'https://1.2.3.4/api/v1/logical-ports/%s?detach=true' % uuid,
-                False,
-                None,
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
 
 class LogicalRouterTestCase(nsxlib_testcase.NsxClientTestCase):
 
+    def _mocked_lrouter(self, session_response=None):
+        return self.mocked_resource(
+            resources.LogicalRouter, session_response=session_response)
+
     def test_create_logical_router(self):
         """
         Test creating a router returns the correct response and 201 status
         """
         fake_router = test_constants_v3.FAKE_ROUTER.copy()
 
-        api = resources.LogicalRouter(client.NSX3Client())
+        router = self._mocked_lrouter()
-        with self.mocked_resource(api) as mocked:
-            mocked.get('post').return_value = mocks.MockRequestsResponse(
-                201, jsonutils.dumps(fake_router))
 
         tier0_router = True
-            result = api.create(fake_router['display_name'], None, None,
+        router.create(fake_router['display_name'], None, None, tier0_router)
-                                tier0_router)
 
         data = {
             'display_name': fake_router['display_name'],
@@ -313,50 +295,38 @@ class LogicalRouterTestCase(nsxlib_testcase.NsxClientTestCase):
             'tags': None
         }
 
-            self.assertEqual(fake_router, result)
+        test_client.assert_json_call(
-            test_client.assert_session_call(
+            'post', router,
-                mocked.get('post'),
             'https://1.2.3.4/api/v1/logical-routers',
-                False,
+            data=jsonutils.dumps(data, sort_keys=True))
-                jsonutils.dumps(data, sort_keys=True),
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_delete_logical_router(self):
         """
         Test deleting router
         """
-        api = resources.LogicalRouter(client.NSX3Client())
+        router = self._mocked_lrouter()
-        with self.mocked_resource(api) as mocked:
-            mocked.get('delete').return_value = mocks.MockRequestsResponse(
-                200, None)
-
         uuid = test_constants_v3.FAKE_ROUTER['id']
-            result = api.delete(uuid)
+        router.delete(uuid)
-            self.assertIsNone(result.content)
+        test_client.assert_json_call(
-            test_client.assert_session_call(
+            'delete', router,
-                mocked.get('delete'),
+            'https://1.2.3.4/api/v1/logical-routers/%s' % uuid)
-                'https://1.2.3.4/api/v1/logical-routers/%s' % uuid,
-                False,
-                None,
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
 
 class LogicalRouterPortTestCase(nsxlib_testcase.NsxClientTestCase):
 
+    def _mocked_lrport(self, session_response=None):
+        return self.mocked_resource(
+            resources.LogicalRouterPort, session_response=session_response)
+
     def test_create_logical_router_port(self):
         """
         Test creating a router port returns the correct response and 201 status
         """
         fake_router_port = test_constants_v3.FAKE_ROUTER_PORT.copy()
 
-        api = resources.LogicalRouterPort(client.NSX3Client())
+        lrport = self._mocked_lrport()
-        with self.mocked_resource(api) as mocked:
-            mocked.get('post').return_value = mocks.MockRequestsResponse(
-                201, jsonutils.dumps(fake_router_port))
 
-            result = api.create(fake_router_port['logical_router_id'],
+        lrport.create(fake_router_port['logical_router_id'],
                       fake_router_port['display_name'],
                       fake_router_port['resource_type'],
                       None, None, None)
@@ -367,34 +337,22 @@ class LogicalRouterPortTestCase(nsxlib_testcase.NsxClientTestCase):
             'resource_type': fake_router_port['resource_type']
         }
 
-            self.assertEqual(fake_router_port, result)
+        test_client.assert_json_call(
-            test_client.assert_session_call(
+            'post', lrport,
-                mocked.get('post'),
             'https://1.2.3.4/api/v1/logical-router-ports',
-                False,
+            data=jsonutils.dumps(data, sort_keys=True))
-                jsonutils.dumps(data, sort_keys=True),
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_delete_logical_router_port(self):
         """
         Test deleting router port
         """
-        api = resources.LogicalRouterPort(client.NSX3Client())
+        lrport = self._mocked_lrport()
-        with self.mocked_resource(api) as mocked:
-            mocked.get('delete').return_value = mocks.MockRequestsResponse(
-                200, None)
 
         uuid = test_constants_v3.FAKE_ROUTER_PORT['id']
-            result = api.delete(uuid)
+        lrport.delete(uuid)
-            self.assertIsNone(result.content)
+        test_client.assert_json_call(
-            test_client.assert_session_call(
+            'delete', lrport,
-                mocked.get('delete'),
+            'https://1.2.3.4/api/v1/logical-router-ports/%s' % uuid)
-                'https://1.2.3.4/api/v1/logical-router-ports/%s' % uuid,
-                False,
-                None,
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_get_logical_router_port_by_router_id(self):
         """
@@ -403,22 +361,17 @@ class LogicalRouterPortTestCase(nsxlib_testcase.NsxClientTestCase):
         fake_router_port = test_constants_v3.FAKE_ROUTER_PORT.copy()
         resp_resources = {'results': [fake_router_port]}
 
-        api = resources.LogicalRouterPort(client.NSX3Client())
+        lrport = self._mocked_lrport(
-        with self.mocked_resource(api) as mocked:
+            session_response=mocks.MockRequestsResponse(
-            mocked.get('get').return_value = mocks.MockRequestsResponse(
+                200, jsonutils.dumps(resp_resources)))
-                200, jsonutils.dumps(resp_resources))
 
         router_id = fake_router_port['logical_router_id']
-            result = api.get_by_router_id(router_id)
+        result = lrport.get_by_router_id(router_id)
         self.assertEqual(fake_router_port, result[0])
-            test_client.assert_session_call(
+        test_client.assert_json_call(
-                mocked.get('get'),
+            'get', lrport,
             'https://1.2.3.4/api/v1/logical-router-ports/?'
-                'logical_router_id=%s' % router_id,
+            'logical_router_id=%s' % router_id)
-                False,
-                None,
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_get_logical_router_port_by_switch_id(self):
         """
@@ -430,19 +383,13 @@ class LogicalRouterPortTestCase(nsxlib_testcase.NsxClientTestCase):
             'results': [fake_router_port]
         }
 
-        api = resources.LogicalRouterPort(client.NSX3Client())
+        lrport = self._mocked_lrport(
-        with self.mocked_resource(api) as mocked:
+            session_response=mocks.MockRequestsResponse(
-            mocked.get('get').return_value = mocks.MockRequestsResponse(
+                200, jsonutils.dumps(resp_resources)))
-                200, jsonutils.dumps(resp_resources))
 
         switch_id = test_constants_v3.FAKE_SWITCH_UUID
-            result = api.get_by_lswitch_id(switch_id)
+        lrport.get_by_lswitch_id(switch_id)
-            self.assertEqual(fake_router_port, result)
+        test_client.assert_json_call(
-            test_client.assert_session_call(
+            'get', lrport,
-                mocked.get('get'),
             'https://1.2.3.4/api/v1/logical-router-ports/?'
-                'logical_switch_id=%s' % switch_id,
+            'logical_switch_id=%s' % switch_id)
-                False,
-                None,
-                client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)

vmware_nsx/tests/unit/nsxlib/v3/test_switch.py

@@ -46,65 +46,59 @@ class NsxLibSwitchTestCase(nsxlib_testcase.NsxClientTestCase):
         """
         Test creating a switch returns the correct response and 200 status
         """
-        api = self.new_client(nsxlib.client.NSX3Client)
+        api = self.mocked_rest_fns(nsxlib, 'client')
-        with self.mocked_client_bridge(api, nsxlib, 'client') as mocked:
+
         nsxlib.create_logical_switch(
             nsx_v3_mocks.FAKE_NAME, NsxLibSwitchTestCase._tz_id, [])
-            test_client.assert_session_call(
+
-                mocked.get('post'),
+        test_client.assert_json_call(
+            'post', api,
             'https://1.2.3.4/api/v1/logical-switches',
-                False, jsonutils.dumps(self._create_body(), sort_keys=True),
+            data=jsonutils.dumps(self._create_body(), sort_keys=True))
-                nsxlib.client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_create_logical_switch_admin_down(self):
         """
         Test creating switch with admin_state down
         """
-        api = self.new_client(nsxlib.client.NSX3Client)
+        api = self.mocked_rest_fns(nsxlib, 'client')
-        with self.mocked_client_bridge(api, nsxlib, 'client') as mocked:
+
         nsxlib.create_logical_switch(
             nsx_v3_mocks.FAKE_NAME, NsxLibSwitchTestCase._tz_id,
             [], admin_state=False)
-            test_client.assert_session_call(
+
-                mocked.get('post'),
+        test_client.assert_json_call(
+            'post', api,
             'https://1.2.3.4/api/v1/logical-switches',
-                False,
+            data=jsonutils.dumps(self._create_body(
-                jsonutils.dumps(self._create_body(
                 admin_state=nsx_constants.ADMIN_STATE_DOWN),
-                    sort_keys=True),
+                sort_keys=True))
-                nsxlib.client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_create_logical_switch_vlan(self):
         """
         Test creating switch with provider:network_type VLAN
         """
-        api = self.new_client(nsxlib.client.NSX3Client)
+        api = self.mocked_rest_fns(nsxlib, 'client')
-        with self.mocked_client_bridge(api, nsxlib, 'client') as mocked:
+
         nsxlib.create_logical_switch(
             nsx_v3_mocks.FAKE_NAME, NsxLibSwitchTestCase._tz_id,
             [], vlan_id='123')
-            test_client.assert_session_call(
+
-                mocked.get('post'),
+        test_client.assert_json_call(
+            'post', api,
             'https://1.2.3.4/api/v1/logical-switches',
-                False, jsonutils.dumps(self._create_body(vlan_id='123'),
+            data=jsonutils.dumps(self._create_body(vlan_id='123'),
-                                       sort_keys=True),
+                                 sort_keys=True))
-                nsxlib.client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)
 
     def test_delete_logical_switch(self):
         """
         Test deleting switch
         """
-        api = self.new_client(nsxlib.client.NSX3Client)
+        api = self.mocked_rest_fns(nsxlib, 'client')
-        with self.mocked_client_bridge(api, nsxlib, 'client') as mocked:
+
         fake_switch = nsx_v3_mocks.make_fake_switch()
         nsxlib.delete_logical_switch(fake_switch['id'])
-            test_client.assert_session_call(
+
-                mocked.get('delete'),
+        test_client.assert_json_call(
+            'delete', api,
             'https://1.2.3.4/api/v1/logical-switches/%s'
-                '?detach=true&cascade=true' % fake_switch['id'],
+            '?detach=true&cascade=true' % fake_switch['id'])
-                False, None,
-                nsxlib.client.JSONRESTClient._DEFAULT_HEADERS,
-                nsxlib_testcase.NSX_CERT)

vmware_nsx/tests/unit/services/l2gateway/test_nsxv3_driver.py

@@ -47,9 +47,6 @@ class TestNsxV3L2GatewayDriver(test_l2gw_db.L2GWTestCase,
 
         self.core_plugin = importutils.import_object(NSX_V3_PLUGIN_CLASS)
 
-        self.core_plugin._nsx_client = self.client
-        self.core_plugin._port_client._client._session = self.mock_api
-
         self.driver = nsx_v3_driver.NsxV3Driver()
         self.l2gw_plugin = l2gw_plugin.NsxL2GatewayPlugin()
         self.context = context.get_admin_context()