Merge "VPNaaS Device Driver for Cisco CSR"

etc/neutron/plugins/cisco/cisco_vpn_agent.ini

@@ -0,0 +1,22 @@
+[cisco_csr_ipsec]
+# Status check interval in seconds, for VPNaaS IPSec connections used on CSR
+# status_check_interval = 60
+
+# Cisco CSR management port information for REST access used by VPNaaS
+# TODO(pcm): Remove once CSR is integrated in as a Neutron router.
+#
+# Format is:
+# [cisco_csr_rest:<public IP>]
+# rest_mgmt = <mgmt port IP>
+# tunnel_ip = <tunnel IP>
+# username = <user>
+# password = <password>
+# timeout = <timeout>
+#
+# where:
+#   public IP ----- Public IP address of router used with a VPN service (1:1 with CSR)
+#   tunnel IP ----- Public IP address of the CSR used for the IPSec tunnel
+#   mgmt port IP -- IP address of CSR for REST API access (not console port)
+#   user ---------- Username for REST management port access to Cisco CSR
+#   password ------ Password for REST management port access to Cisco CSR
+#   timeout ------- REST request timeout to Cisco CSR (optional)

neutron/services/vpn/device_drivers/cisco_csr_rest_client.py

@@ -0,0 +1,254 @@
+# Copyright 2014 Cisco Systems, Inc.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Paul Michali, Cisco Systems, Inc.
+
+import time
+
+import netaddr
+import requests
+from requests import exceptions as r_exc
+
+from neutron.openstack.common import jsonutils
+from neutron.openstack.common import log as logging
+
+
+TIMEOUT = 20.0
+
+LOG = logging.getLogger(__name__)
+HEADER_CONTENT_TYPE_JSON = {'content-type': 'application/json'}
+URL_BASE = 'https://%(host)s/api/v1/%(resource)s'
+
+
+def make_route_id(cidr, interface):
+    """Build ID that will be used to identify route for later deletion."""
+    net = netaddr.IPNetwork(cidr)
+    return '%(network)s_%(prefix)s_%(interface)s' % {
+        'network': net.network,
+        'prefix': net.prefixlen,
+        'interface': interface}
+
+
+class CsrRestClient(object):
+
+    """REST CsrRestClient for accessing the Cisco Cloud Services Router."""
+
+    def __init__(self, host, tunnel_ip, username, password, timeout=None):
+        self.host = host
+        self.tunnel_ip = tunnel_ip
+        self.auth = (username, password)
+        self.token = None
+        self.status = requests.codes.OK
+        self.timeout = timeout
+        self.max_tries = 5
+        self.session = requests.Session()
+
+    def _response_info_for(self, response, method):
+        """Return contents or location from response.
+
+        For a POST or GET with a 200 response, the response content
+        is returned.
+
+        For a POST with a 201 response, return the header's location,
+        which contains the identifier for the created resource.
+
+        If there is an error, return the response content, so that
+        it can be used in error processing ('error-code', 'error-message',
+        and 'detail' fields).
+        """
+        if method in ('POST', 'GET') and self.status == requests.codes.OK:
+            LOG.debug(_('RESPONSE: %s'), response.json())
+            return response.json()
+        if method == 'POST' and self.status == requests.codes.CREATED:
+            return response.headers.get('location', '')
+        if self.status >= requests.codes.BAD_REQUEST and response.content:
+            if 'error-code' in response.content:
+                content = jsonutils.loads(response.content)
+                LOG.debug("Error response content %s", content)
+                return content
+
+    def _request(self, method, url, **kwargs):
+        """Perform REST request and save response info."""
+        try:
+            LOG.debug(_("%(method)s: Request for %(resource)s payload: "
+                        "%(payload)s"),
+                      {'method': method.upper(), 'resource': url,
+                       'payload': kwargs.get('data')})
+            start_time = time.time()
+            response = self.session.request(method, url, verify=False,
+                                            timeout=self.timeout, **kwargs)
+            LOG.debug(_("%(method)s Took %(time).2f seconds to process"),
+                      {'method': method.upper(),
+                       'time': time.time() - start_time})
+        except (r_exc.Timeout, r_exc.SSLError) as te:
+            # Should never see SSLError, unless requests package is old (<2.0)
+            timeout_val = 0.0 if self.timeout is None else self.timeout
+            LOG.warning(_("%(method)s: Request timeout%(ssl)s "
+                          "(%(timeout).3f sec) for CSR(%(host)s)"),
+                        {'method': method,
+                         'timeout': timeout_val,
+                         'ssl': '(SSLError)'
+                         if isinstance(te, r_exc.SSLError) else '',
+                         'host': self.host})
+            self.status = requests.codes.REQUEST_TIMEOUT
+        except r_exc.ConnectionError as ce:
+            LOG.error(_("%(method)s: Unable to connect to CSR(%(host)s): "
+                        "%(error)s"),
+                      {'method': method, 'host': self.host, 'error': ce})
+            self.status = requests.codes.NOT_FOUND
+        except Exception as e:
+            LOG.error(_("%(method)s: Unexpected error for CSR (%(host)s): "
+                        "%(error)s"),
+                      {'method': method, 'host': self.host, 'error': e})
+            self.status = requests.codes.INTERNAL_SERVER_ERROR
+        else:
+            self.status = response.status_code
+            LOG.debug(_("%(method)s: Completed [%(status)s]"),
+                      {'method': method, 'status': self.status})
+            return self._response_info_for(response, method)
+
+    def authenticate(self):
+        """Obtain a token to use for subsequent CSR REST requests.
+
+        This is called when there is no token yet, or if the token has expired
+        and attempts to use it resulted in an UNAUTHORIZED REST response.
+        """
+
+        url = URL_BASE % {'host': self.host, 'resource': 'auth/token-services'}
+        headers = {'Content-Length': '0',
+                   'Accept': 'application/json'}
+        headers.update(HEADER_CONTENT_TYPE_JSON)
+        LOG.debug(_("%(auth)s with CSR %(host)s"),
+                  {'auth': 'Authenticating' if self.token is None
+                   else 'Reauthenticating', 'host': self.host})
+        self.token = None
+        response = self._request("POST", url, headers=headers, auth=self.auth)
+        if response:
+            self.token = response['token-id']
+            LOG.debug(_("Successfully authenticated with CSR %s"), self.host)
+            return True
+        LOG.error(_("Failed authentication with CSR %(host)s [%(status)s]"),
+                  {'host': self.host, 'status': self.status})
+
+    def _do_request(self, method, resource, payload=None, more_headers=None,
+                    full_url=False):
+        """Perform a REST request to a CSR resource.
+
+        If this is the first time interacting with the CSR, a token will
+        be obtained. If the request fails, due to an expired token, the
+        token will be obtained and the request will be retried once more.
+        """
+
+        if self.token is None:
+            if not self.authenticate():
+                return
+
+        if full_url:
+            url = resource
+        else:
+            url = ('https://%(host)s/api/v1/%(resource)s' %
+                   {'host': self.host, 'resource': resource})
+        headers = {'Accept': 'application/json', 'X-auth-token': self.token}
+        if more_headers:
+            headers.update(more_headers)
+        if payload:
+            payload = jsonutils.dumps(payload)
+        response = self._request(method, url, data=payload, headers=headers)
+        if self.status == requests.codes.UNAUTHORIZED:
+            if not self.authenticate():
+                return
+            headers['X-auth-token'] = self.token
+            response = self._request(method, url, data=payload,
+                                     headers=headers)
+        if self.status != requests.codes.REQUEST_TIMEOUT:
+            return response
+        LOG.error(_("%(method)s: Request timeout for CSR(%(host)s)"),
+                  {'method': method, 'host': self.host})
+
+    def get_request(self, resource, full_url=False):
+        """Perform a REST GET requests for a CSR resource."""
+        return self._do_request('GET', resource, full_url=full_url)
+
+    def post_request(self, resource, payload=None):
+        """Perform a POST request to a CSR resource."""
+        return self._do_request('POST', resource, payload=payload,
+                                more_headers=HEADER_CONTENT_TYPE_JSON)
+
+    def put_request(self, resource, payload=None):
+        """Perform a PUT request to a CSR resource."""
+        return self._do_request('PUT', resource, payload=payload,
+                                more_headers=HEADER_CONTENT_TYPE_JSON)
+
+    def delete_request(self, resource):
+        """Perform a DELETE request on a CSR resource."""
+        return self._do_request('DELETE', resource,
+                                more_headers=HEADER_CONTENT_TYPE_JSON)
+
+    def create_ike_policy(self, policy_info):
+        base_ike_policy_info = {u'version': u'v1',
+                                u'local-auth-method': u'pre-share'}
+        base_ike_policy_info.update(policy_info)
+        return self.post_request('vpn-svc/ike/policies',
+                                 payload=base_ike_policy_info)
+
+    def create_ipsec_policy(self, policy_info):
+        base_ipsec_policy_info = {u'mode': u'tunnel'}
+        base_ipsec_policy_info.update(policy_info)
+        return self.post_request('vpn-svc/ipsec/policies',
+                                 payload=base_ipsec_policy_info)
+
+    def create_pre_shared_key(self, psk_info):
+        return self.post_request('vpn-svc/ike/keyrings', payload=psk_info)
+
+    def create_ipsec_connection(self, connection_info):
+        base_conn_info = {u'vpn-type': u'site-to-site',
+                          u'ip-version': u'ipv4'}
+        connection_info.update(base_conn_info)
+        # TODO(pcm) pass in value, when CSR is embedded as Neutron router.
+        # Currently, get this from .INI file.
+        connection_info[u'local-device'][u'tunnel-ip-address'] = self.tunnel_ip
+        return self.post_request('vpn-svc/site-to-site',
+                                 payload=connection_info)
+
+    def configure_ike_keepalive(self, keepalive_info):
+        base_keepalive_info = {u'periodic': True}
+        keepalive_info.update(base_keepalive_info)
+        return self.put_request('vpn-svc/ike/keepalive', keepalive_info)
+
+    def create_static_route(self, route_info):
+        return self.post_request('routing-svc/static-routes',
+                                 payload=route_info)
+
+    def delete_static_route(self, route_id):
+        return self.delete_request('routing-svc/static-routes/%s' % route_id)
+
+    def delete_ipsec_connection(self, conn_id):
+        return self.delete_request('vpn-svc/site-to-site/%s' % conn_id)
+
+    def delete_ipsec_policy(self, policy_id):
+        return self.delete_request('vpn-svc/ipsec/policies/%s' % policy_id)
+
+    def delete_ike_policy(self, policy_id):
+        return self.delete_request('vpn-svc/ike/policies/%s' % policy_id)
+
+    def delete_pre_shared_key(self, key_id):
+        return self.delete_request('vpn-svc/ike/keyrings/%s' % key_id)
+
+    def read_tunnel_statuses(self):
+        results = self.get_request('vpn-svc/site-to-site/active/sessions')
+        if self.status != requests.codes.OK or not results:
+            return []
+        tunnels = [(t[u'vpn-interface-name'], t[u'status'])
+                   for t in results['items']]
+        return tunnels

neutron/services/vpn/device_drivers/cisco_ipsec.py

@@ -0,0 +1,795 @@
+# Copyright 2014 Cisco Systems, Inc.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Paul Michali, Cisco Systems, Inc.
+
+import abc
+from collections import namedtuple
+import httplib
+
+import netaddr
+from oslo.config import cfg
+
+from neutron.common import exceptions
+from neutron.common import rpc as n_rpc
+from neutron import context as ctx
+from neutron.openstack.common import lockutils
+from neutron.openstack.common import log as logging
+from neutron.openstack.common import loopingcall
+from neutron.openstack.common import rpc
+from neutron.openstack.common.rpc import proxy
+from neutron.plugins.common import constants
+from neutron.plugins.common import utils as plugin_utils
+from neutron.services.vpn.common import topics
+from neutron.services.vpn import device_drivers
+from neutron.services.vpn.device_drivers import (
+    cisco_csr_rest_client as csr_client)
+
+
+ipsec_opts = [
+    cfg.IntOpt('status_check_interval',
+               default=60,
+               help=_("Status check interval for Cisco CSR IPSec connections"))
+]
+cfg.CONF.register_opts(ipsec_opts, 'cisco_csr_ipsec')
+
+LOG = logging.getLogger(__name__)
+
+RollbackStep = namedtuple('RollbackStep', ['action', 'resource_id', 'title'])
+
+
+class CsrResourceCreateFailure(exceptions.NeutronException):
+    message = _("Cisco CSR failed to create %(resource)s (%(which)s)")
+
+
+class CsrDriverMismatchError(exceptions.NeutronException):
+    message = _("Required %(resource)s attribute %(attr)s mapping for Cisco "
+                "CSR is missing in device driver")
+
+
+class CsrUnknownMappingError(exceptions.NeutronException):
+    message = _("Device driver does not have a mapping of '%(value)s for "
+                "attribute %(attr)s of %(resource)s")
+
+
+def find_available_csrs_from_config(config_files):
+    """Read INI for available Cisco CSRs that driver can use.
+
+    Loads management port, tunnel IP, user, and password information for
+    available CSRs from configuration file. Driver will use this info to
+    configure VPN connections. The CSR is associated 1:1 with a Neutron
+    router. To identify which CSR to use for a VPN service, the public
+    (GW) IP of the Neutron router will be used as an index into the CSR
+    config info.
+    """
+    multi_parser = cfg.MultiConfigParser()
+    LOG.info(_("Scanning config files %s for Cisco CSR configurations"),
+             config_files)
+    try:
+        read_ok = multi_parser.read(config_files)
+    except cfg.ParseError as pe:
+        LOG.error(_("Config file parse error: %s"), pe)
+        return {}
+
+    if len(read_ok) != len(config_files):
+        raise cfg.Error(_("Unable to parse config files %s for Cisco CSR "
+                          "info") % config_files)
+    csrs_found = {}
+    for parsed_file in multi_parser.parsed:
+        for parsed_item in parsed_file.keys():
+            device_type, sep, for_router = parsed_item.partition(':')
+            if device_type.lower() == 'cisco_csr_rest':
+                try:
+                    netaddr.IPNetwork(for_router)
+                except netaddr.core.AddrFormatError:
+                    LOG.error(_("Ignoring Cisco CSR configuration entry - "
+                                "router IP %s is not valid"), for_router)
+                    continue
+                entry = parsed_file[parsed_item]
+                # Check for missing fields
+                try:
+                    rest_mgmt_ip = entry['rest_mgmt'][0]
+                    tunnel_ip = entry['tunnel_ip'][0]
+                    username = entry['username'][0]
+                    password = entry['password'][0]
+                except KeyError as ke:
+                    LOG.error(_("Ignoring Cisco CSR for router %(router)s "
+                                "- missing %(field)s setting"),
+                              {'router': for_router, 'field': str(ke)})
+                    continue
+                # Validate fields
+                try:
+                    timeout = float(entry['timeout'][0])
+                except ValueError:
+                    LOG.error(_("Ignoring Cisco CSR for router %s - "
+                                "timeout is not a floating point number"),
+                              for_router)
+                    continue
+                except KeyError:
+                    timeout = csr_client.TIMEOUT
+                try:
+                    netaddr.IPAddress(rest_mgmt_ip)
+                except netaddr.core.AddrFormatError:
+                    LOG.error(_("Ignoring Cisco CSR for subnet %s - "
+                                "REST management is not an IP address"),
+                              for_router)
+                    continue
+                try:
+                    netaddr.IPAddress(tunnel_ip)
+                except netaddr.core.AddrFormatError:
+                    LOG.error(_("Ignoring Cisco CSR for router %s - "
+                                "local tunnel is not an IP address"),
+                              for_router)
+                    continue
+                csrs_found[for_router] = {'rest_mgmt': rest_mgmt_ip,
+                                          'tunnel_ip': tunnel_ip,
+                                          'username': username,
+                                          'password': password,
+                                          'timeout': timeout}
+
+                LOG.debug(_("Found CSR for router %(router)s: %(info)s"),
+                          {'router': for_router,
+                           'info': csrs_found[for_router]})
+    return csrs_found
+
+
+class CiscoCsrIPsecVpnDriverApi(proxy.RpcProxy):
+    """RPC API for agent to plugin messaging."""
+
+    def get_vpn_services_on_host(self, context, host):
+        """Get list of vpnservices on this host.
+
+        The vpnservices including related ipsec_site_connection,
+        ikepolicy, ipsecpolicy, and Cisco info on this host.
+        """
+        return self.call(context,
+                         self.make_msg('get_vpn_services_on_host',
+                                       host=host),
+                         topic=self.topic)
+
+    def update_status(self, context, status):
+        """Update status for all VPN services and connections."""
+        return self.cast(context,
+                         self.make_msg('update_status',
+                                       status=status),
+                         topic=self.topic)
+
+
+class CiscoCsrIPsecDriver(device_drivers.DeviceDriver):
+    """Cisco CSR VPN Device Driver for IPSec.
+
+    This class is designed for use with L3-agent now.
+    However this driver will be used with another agent in future.
+    so the use of "Router" is kept minimul now.
+    Insted of router_id,  we are using process_id in this code.
+    """
+
+    # history
+    #   1.0 Initial version
+
+    RPC_API_VERSION = '1.0'
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, agent, host):
+        self.host = host
+        self.conn = rpc.create_connection(new=True)
+        context = ctx.get_admin_context_without_session()
+        node_topic = '%s.%s' % (topics.CISCO_IPSEC_AGENT_TOPIC, self.host)
+
+        self.service_state = {}
+
+        self.conn.create_consumer(
+            node_topic,
+            self.create_rpc_dispatcher(),
+            fanout=False)
+        self.conn.consume_in_thread()
+        self.agent_rpc = (
+            CiscoCsrIPsecVpnDriverApi(topics.CISCO_IPSEC_DRIVER_TOPIC, '1.0'))
+        self.periodic_report = loopingcall.FixedIntervalLoopingCall(
+            self.report_status, context)
+        self.periodic_report.start(
+            interval=agent.conf.cisco_csr_ipsec.status_check_interval)
+
+        csrs_found = find_available_csrs_from_config(cfg.CONF.config_file)
+        if csrs_found:
+            LOG.info(_("Loaded %(num)d Cisco CSR configuration%(plural)s"),
+                     {'num': len(csrs_found),
+                      'plural': 's'[len(csrs_found) == 1:]})
+        else:
+            raise SystemExit(_('No Cisco CSR configurations found in: %s') %
+                             cfg.CONF.config_file)
+        self.csrs = dict([(k, csr_client.CsrRestClient(v['rest_mgmt'],
+                                                       v['tunnel_ip'],
+                                                       v['username'],
+                                                       v['password'],
+                                                       v['timeout']))
+                          for k, v in csrs_found.items()])
+
+    def create_rpc_dispatcher(self):
+        return n_rpc.PluginRpcDispatcher([self])
+
+    def vpnservice_updated(self, context, **kwargs):
+        """Handle VPNaaS service driver change notifications."""
+        LOG.debug(_("Handling VPN service update notification"))
+        self.sync(context, [])
+
+    def create_vpn_service(self, service_data):
+        """Create new entry to track VPN service and its connections."""
+        vpn_service_id = service_data['id']
+        vpn_service_router = service_data['external_ip']
+        self.service_state[vpn_service_id] = CiscoCsrVpnService(
+            service_data, self.csrs.get(vpn_service_router))
+        return self.service_state[vpn_service_id]
+
+    def update_connection(self, context, vpn_service_id, conn_data):
+        """Handle notification for a single IPSec connection."""
+        vpn_service = self.service_state[vpn_service_id]
+        conn_id = conn_data['id']
+        is_admin_up = conn_data[u'admin_state_up']
+        if conn_id in vpn_service.conn_state:
+            ipsec_conn = vpn_service.conn_state[conn_id]
+            ipsec_conn.last_status = conn_data['status']
+            if is_admin_up:
+                ipsec_conn.is_dirty = False
+                LOG.debug(_("Update: IPSec connection %s unchanged - "
+                            "marking clean"), conn_id)
+                # TODO(pcm) FUTURE - Handle update requests (delete/create?)
+                # will need to detect what has changed. For now assume no
+                # change (it is blocked in service driver).
+            else:
+                LOG.debug(_("Update: IPSec connection %s is admin down - "
+                            "will be removed in sweep phase"), conn_id)
+        else:
+            if not is_admin_up:
+                LOG.debug(_("Update: Unknown IPSec connection %s is admin "
+                            "down - ignoring"), conn_id)
+                return
+            LOG.debug(_("Update: New IPSec connection %s - marking clean"),
+                      conn_id)
+            ipsec_conn = vpn_service.create_connection(conn_data)
+            ipsec_conn.create_ipsec_site_connection(context, conn_data)
+        return ipsec_conn
+
+    def update_service(self, context, service_data):
+        """Handle notification for a single VPN Service and its connections."""
+        vpn_service_id = service_data['id']
+        csr_id = service_data['external_ip']
+        if csr_id not in self.csrs:
+            LOG.error(_("Update: Skipping VPN service %(service)s as it's "
+                        "router (%(csr_id)s is not associated with a Cisco "
+                        "CSR"), {'service': vpn_service_id, 'csr_id': csr_id})
+            return
+        is_admin_up = service_data[u'admin_state_up']
+        if vpn_service_id in self.service_state:
+            vpn_service = self.service_state[vpn_service_id]
+            vpn_service.last_status = service_data['status']
+            if is_admin_up:
+                vpn_service.is_dirty = False
+            else:
+                LOG.debug(_("Update: VPN service %s is admin down - will "
+                            "be removed in sweep phase"), vpn_service_id)
+                return vpn_service
+        else:
+            if not is_admin_up:
+                LOG.debug(_("Update: Unknown VPN service %s is admin down - "
+                            "ignoring"), vpn_service_id)
+                return
+            vpn_service = self.create_vpn_service(service_data)
+        # Handle all the IPSec connection notifications in the data
+        LOG.debug(_("Update: Processing IPSec connections for VPN service %s"),
+                  vpn_service_id)
+        for conn_data in service_data['ipsec_conns']:
+            self.update_connection(context, service_data['id'], conn_data)
+        LOG.debug(_("Update: Completed update processing"))
+        return vpn_service
+
+    def update_all_services_and_connections(self, context):
+        """Update services and connections based on plugin info.
+
+        Perform any create and update operations and then update status.
+        Mark every visited connection as no longer "dirty" so they will
+        not be deleted at end of sync processing.
+        """
+        services_data = self.agent_rpc.get_vpn_services_on_host(context,
+                                                                self.host)
+        LOG.debug("Sync updating for %d VPN services", len(services_data))
+        vpn_services = []
+        for service_data in services_data:
+            vpn_service = self.update_service(context, service_data)
+            if vpn_service:
+                vpn_services.append(vpn_service)
+        return vpn_services
+
+    def mark_existing_connections_as_dirty(self):
+        """Mark all existing connections as "dirty" for sync."""
+        service_count = 0
+        connection_count = 0
+        for service_state in self.service_state.values():
+            service_state.is_dirty = True
+            service_count += 1
+            for conn_id in service_state.conn_state:
+                service_state.conn_state[conn_id].is_dirty = True
+                connection_count += 1
+        LOG.debug(_("Mark: %(service)d VPN services and %(conn)d IPSec "
+                    "connections marked dirty"), {'service': service_count,
+                                                  'conn': connection_count})
+
+    def remove_unknown_connections(self, context):
+        """Remove connections that are not known by service driver."""
+        service_count = 0
+        connection_count = 0
+        for vpn_service_id, vpn_service in self.service_state.items():
+            dirty = [c_id for c_id, c in vpn_service.conn_state.items()
+                     if c.is_dirty]
+            for conn_id in dirty:
+                conn_state = vpn_service.conn_state[conn_id]
+                conn_state.delete_ipsec_site_connection(context, conn_id)
+                connection_count += 1
+                del vpn_service.conn_state[conn_id]
+            if vpn_service.is_dirty:
+                service_count += 1
+                del self.service_state[vpn_service_id]
+        LOG.debug(_("Sweep: Removed %(service)d dirty VPN service%(splural)s "
+                    "and %(conn)d dirty IPSec connection%(cplural)s"),
+                  {'service': service_count, 'conn': connection_count,
+                   'splural': 's'[service_count == 1:],
+                   'cplural': 's'[connection_count == 1:]})
+
+    def build_report_for_connections_on(self, vpn_service):
+        """Create the report fragment for IPSec connections on a service.
+
+        Collect the current status from the Cisco CSR and use that to update
+        the status and generate report fragment for each connection on the
+        service. If there is no status information, or no change, then no
+        report info will be created for the connection. The combined report
+        data is returned.
+        """
+        LOG.debug(_("Report: Collecting status for IPSec connections on VPN "
+                    "service %s"), vpn_service.service_id)
+        tunnels = vpn_service.get_ipsec_connections_status()
+        report = {}
+        for connection in vpn_service.conn_state.values():
+            current_status = connection.find_current_status_in(tunnels)
+            frag = connection.build_report_based_on_status(current_status)
+            if frag:
+                LOG.debug(_("Report: Adding info for IPSec connection %s"),
+                          connection.conn_id)
+                report.update(frag)
+        return report
+
+    def build_report_for_service(self, vpn_service):
+        """Create the report info for a VPN service and its IPSec connections.
+
+        Get the report info for the connections on the service, and include
+        it into the report info for the VPN service. If there is no report
+        info for the connection, then no change has occurred and no report
+        will be generated. If there is only one connection for the service,
+        we'll set the service state to match the connection (with ERROR seen
+        as DOWN).
+        """
+        conn_report = self.build_report_for_connections_on(vpn_service)
+        if conn_report:
+            pending_handled = plugin_utils.in_pending_status(
+                vpn_service.last_status)
+            if (len(conn_report) == 1 and
+                conn_report.values()[0]['status'] != constants.ACTIVE):
+                vpn_service.last_status = constants.DOWN
+            else:
+                vpn_service.last_status = constants.ACTIVE
+            LOG.debug(_("Report: Adding info for VPN service %s"),
+                      vpn_service.service_id)
+            return {u'id': vpn_service.service_id,
+                    u'status': vpn_service.last_status,
+                    u'updated_pending_status': pending_handled,
+                    u'ipsec_site_connections': conn_report}
+        else:
+            return {}
+
+    def report_status(self, context):
+        """Report status of all VPN services and IPSec connections to plugin.
+
+        This is called periodically by the agent, to push up status changes,
+        and at the end of any sync operation to reflect the changes due to a
+        sync or change notification.
+        """
+        service_report = []
+        LOG.debug(_("Report: Starting status report"))
+        for vpn_service_id, vpn_service in self.service_state.items():
+            LOG.debug(_("Report: Collecting status for VPN service %s"),
+                      vpn_service_id)
+            report = self.build_report_for_service(vpn_service)
+            if report:
+                service_report.append(report)
+        if service_report:
+            LOG.info(_("Sending status report update to plugin"))
+            self.agent_rpc.update_status(context, service_report)
+        LOG.debug(_("Report: Completed status report processing"))
+        return service_report
+
+    @lockutils.synchronized('vpn-agent', 'neutron-')
+    def sync(self, context, routers):
+        """Synchronize with plugin and report current status.
+
+        Mark all "known" services/connections as dirty, update them based on
+        information from the plugin, remove (sweep) any connections that are
+        not updated (dirty), and report updates, if any, back to plugin.
+        Called when update/delete a service or create/update/delete a
+        connection (vpnservice_updated message), or router change
+        (_process_routers).
+        """
+        self.mark_existing_connections_as_dirty()
+        self.update_all_services_and_connections(context)
+        self.remove_unknown_connections(context)
+        self.report_status(context)
+
+    def create_router(self, process_id):
+        """Actions taken when router created."""
+        # Note: Since Cisco CSR is running out-of-band, nothing to do here
+        pass
+
+    def destroy_router(self, process_id):
+        """Actions taken when router deleted."""
+        # Note: Since Cisco CSR is running out-of-band, nothing to do here
+        pass
+
+
+class CiscoCsrVpnService(object):
+
+    """Maintains state/status information for a service and its connections."""
+
+    def __init__(self, service_data, csr):
+        self.service_id = service_data['id']
+        self.last_status = service_data['status']
+        self.conn_state = {}
+        self.is_dirty = False
+        self.csr = csr
+        # TODO(pcm) FUTURE - handle sharing of policies
+
+    def create_connection(self, conn_data):
+        conn_id = conn_data['id']
+        self.conn_state[conn_id] = CiscoCsrIPSecConnection(conn_data, self.csr)
+        return self.conn_state[conn_id]
+
+    def get_connection(self, conn_id):
+        return self.conn_state.get(conn_id)
+
+    def conn_status(self, conn_id):
+        conn_state = self.get_connection(conn_id)
+        if conn_state:
+            return conn_state.last_status
+
+    def snapshot_conn_state(self, ipsec_conn):
+        """Create/obtain connection state and save current status."""
+        conn_state = self.conn_state.setdefault(
+            ipsec_conn['id'], CiscoCsrIPSecConnection(ipsec_conn, self.csr))
+        conn_state.last_status = ipsec_conn['status']
+        conn_state.is_dirty = False
+        return conn_state
+
+    STATUS_MAP = {'ERROR': constants.ERROR,
+                  'UP-ACTIVE': constants.ACTIVE,
+                  'UP-IDLE': constants.ACTIVE,
+                  'UP-NO-IKE': constants.ACTIVE,
+                  'DOWN': constants.DOWN,
+                  'DOWN-NEGOTIATING': constants.DOWN}
+
+    def get_ipsec_connections_status(self):
+        """Obtain current status of all tunnels on a Cisco CSR.
+
+        Convert them to OpenStack status values.
+        """
+        tunnels = self.csr.read_tunnel_statuses()
+        for tunnel in tunnels:
+            LOG.debug("CSR Reports %(tunnel)s status '%(status)s'",
+                      {'tunnel': tunnel[0], 'status': tunnel[1]})
+        return dict(map(lambda x: (x[0], self.STATUS_MAP[x[1]]), tunnels))
+
+    def find_matching_connection(self, tunnel_id):
+        """Find IPSec connection using Cisco CSR tunnel specified, if any."""
+        for connection in self.conn_state.values():
+            if connection.tunnel == tunnel_id:
+                return connection.conn_id
+
+
+class CiscoCsrIPSecConnection(object):
+
+    """State and actions for IPSec site-to-site connections."""
+
+    def __init__(self, conn_info, csr):
+        self.conn_id = conn_info['id']
+        self.csr = csr
+        self.steps = []
+        self.is_dirty = False
+        self.last_status = conn_info['status']
+        self.tunnel = conn_info['cisco']['site_conn_id']
+
+    def find_current_status_in(self, statuses):
+        if self.tunnel in statuses:
+            return statuses[self.tunnel]
+        else:
+            return constants.ERROR
+
+    def build_report_based_on_status(self, current_status):
+        if current_status != self.last_status:
+            pending_handled = plugin_utils.in_pending_status(self.last_status)
+            self.last_status = current_status
+            return {self.conn_id: {'status': current_status,
+                                   'updated_pending_status': pending_handled}}
+        else:
+            return {}
+
+    DIALECT_MAP = {'ike_policy': {'name': 'IKE Policy',
+                                  'v1': u'v1',
+                                  # auth_algorithm -> hash
+                                  'sha1': u'sha',
+                                  # encryption_algorithm -> encryption
+                                  '3des': u'3des',
+                                  'aes-128': u'aes',
+                                  # TODO(pcm) update these 2 once CSR updated
+                                  'aes-192': u'aes',
+                                  'aes-256': u'aes',
+                                  # pfs -> dhGroup
+                                  'group2': 2,
+                                  'group5': 5,
+                                  'group14': 14},
+                   'ipsec_policy': {'name': 'IPSec Policy',
+                                    # auth_algorithm -> esp-authentication
+                                    'sha1': u'esp-sha-hmac',
+                                    # transform_protocol -> ah
+                                    'esp': None,
+                                    'ah': u'ah-sha-hmac',
+                                    'ah-esp': u'ah-sha-hmac',
+                                    # encryption_algorithm -> esp-encryption
+                                    '3des': u'esp-3des',
+                                    'aes-128': u'esp-aes',
+                                    # TODO(pcm) update these 2 once CSR updated
+                                    'aes-192': u'esp-aes',
+                                    'aes-256': u'esp-aes',
+                                    # pfs -> pfs
+                                    'group2': u'group2',
+                                    'group5': u'group5',
+                                    'group14': u'group14'}}
+
+    def translate_dialect(self, resource, attribute, info):
+        """Map VPNaaS attributes values to CSR values for a resource."""
+        name = self.DIALECT_MAP[resource]['name']
+        if attribute not in info:
+            raise CsrDriverMismatchError(resource=name, attr=attribute)
+        value = info[attribute].lower()
+        if value in self.DIALECT_MAP[resource]:
+            return self.DIALECT_MAP[resource][value]
+        raise CsrUnknownMappingError(resource=name, attr=attribute,
+                                     value=value)
+
+    def create_psk_info(self, psk_id, conn_info):
+        """Collect/create attributes needed for pre-shared key."""
+        return {u'keyring-name': psk_id,
+                u'pre-shared-key-list': [
+                    {u'key': conn_info['psk'],
+                     u'encrypted': False,
+                     u'peer-address': conn_info['peer_address']}]}
+
+    def create_ike_policy_info(self, ike_policy_id, conn_info):
+        """Collect/create/map attributes needed for IKE policy."""
+        for_ike = 'ike_policy'
+        policy_info = conn_info[for_ike]
+        version = self.translate_dialect(for_ike,
+                                         'ike_version',
+                                         policy_info)
+        encrypt_algorithm = self.translate_dialect(for_ike,
+                                                   'encryption_algorithm',
+                                                   policy_info)
+        auth_algorithm = self.translate_dialect(for_ike,
+                                                'auth_algorithm',
+                                                policy_info)
+        group = self.translate_dialect(for_ike,
+                                       'pfs',
+                                       policy_info)
+        lifetime = policy_info['lifetime_value']
+        return {u'version': version,
+                u'priority-id': ike_policy_id,
+                u'encryption': encrypt_algorithm,
+                u'hash': auth_algorithm,
+                u'dhGroup': group,
+                u'lifetime': lifetime}
+
+    def create_ipsec_policy_info(self, ipsec_policy_id, info):
+        """Collect/create attributes needed for IPSec policy.
+
+        Note: OpenStack will provide a default encryption algorithm, if one is
+        not provided, so a authentication only configuration of (ah, sha1),
+        which maps to ah-sha-hmac transform protocol, cannot be selected.
+        As a result, we'll always configure the encryption algorithm, and
+        will select ah-sha-hmac for transform protocol.
+        """
+
+        for_ipsec = 'ipsec_policy'
+        policy_info = info[for_ipsec]
+        transform_protocol = self.translate_dialect(for_ipsec,
+                                                    'transform_protocol',
+                                                    policy_info)
+        auth_algorithm = self.translate_dialect(for_ipsec,
+                                                'auth_algorithm',
+                                                policy_info)
+        encrypt_algorithm = self.translate_dialect(for_ipsec,
+                                                   'encryption_algorithm',
+                                                   policy_info)
+        group = self.translate_dialect(for_ipsec, 'pfs', policy_info)
+        lifetime = policy_info['lifetime_value']
+        settings = {u'policy-id': ipsec_policy_id,
+                    u'protection-suite': {
+                        u'esp-encryption': encrypt_algorithm,
+                        u'esp-authentication': auth_algorithm},
+                    u'lifetime-sec': lifetime,
+                    u'pfs': group,
+                    # TODO(pcm): Remove when CSR fixes 'Disable'
+                    u'anti-replay-window-size': u'64'}
+        if transform_protocol:
+            settings[u'protection-suite'][u'ah'] = transform_protocol
+        return settings
+
+    def create_site_connection_info(self, site_conn_id, ipsec_policy_id,
+                                    conn_info):
+        """Collect/create attributes needed for the IPSec connection."""
+        # TODO(pcm) Enable, once CSR is embedded as a Neutron router
+        # gw_ip = vpnservice['external_ip'] (need to pass in)
+        mtu = conn_info['mtu']
+        return {
+            u'vpn-interface-name': site_conn_id,
+            u'ipsec-policy-id': ipsec_policy_id,
+            u'local-device': {
+                # TODO(pcm): FUTURE - Get CSR port of interface with
+                # local subnet
+                u'ip-address': u'GigabitEthernet3',
+                # TODO(pcm): FUTURE - Get IP address of router's public
+                # I/F, once CSR is used as embedded router.
+                u'tunnel-ip-address': u'172.24.4.23'
+                # u'tunnel-ip-address': u'%s' % gw_ip
+            },
+            u'remote-device': {
+                u'tunnel-ip-address': conn_info['peer_address']
+            },
+            u'mtu': mtu
+        }
+
+    def create_routes_info(self, site_conn_id, conn_info):
+        """Collect/create attributes for static routes."""
+        routes_info = []
+        for peer_cidr in conn_info.get('peer_cidrs', []):
+            route = {u'destination-network': peer_cidr,
+                     u'outgoing-interface': site_conn_id}
+            route_id = csr_client.make_route_id(peer_cidr, site_conn_id)
+            routes_info.append((route_id, route))
+        return routes_info
+
+    def _check_create(self, resource, which):
+        """Determine if REST create request was successful."""
+        if self.csr.status == httplib.CREATED:
+            LOG.debug("%(resource)s %(which)s is configured",
+                      {'resource': resource, 'which': which})
+            return
+        LOG.error(_("Unable to create %(resource)s %(which)s: "
+                    "%(status)d"),
+                  {'resource': resource, 'which': which,
+                   'status': self.csr.status})
+        # ToDO(pcm): Set state to error
+        raise CsrResourceCreateFailure(resource=resource, which=which)
+
+    def do_create_action(self, action_suffix, info, resource_id, title):
+        """Perform a single REST step for IPSec site connection create."""
+        create_action = 'create_%s' % action_suffix
+        try:
+            getattr(self.csr, create_action)(info)
+        except AttributeError:
+            LOG.exception(_("Internal error - '%s' is not defined"),
+                          create_action)
+            raise CsrResourceCreateFailure(resource=title,
+                                           which=resource_id)
+        self._check_create(title, resource_id)
+        self.steps.append(RollbackStep(action_suffix, resource_id, title))
+
+    def _verify_deleted(self, status, resource, which):
+        """Determine if REST delete request was successful."""
+        if status in (httplib.NO_CONTENT, httplib.NOT_FOUND):
+            LOG.debug("%(resource)s configuration %(which)s was removed",
+                      {'resource': resource, 'which': which})
+        else:
+            LOG.warning(_("Unable to delete %(resource)s %(which)s: "
+                          "%(status)d"), {'resource': resource,
+                                          'which': which,
+                                          'status': status})
+
+    def do_rollback(self):
+        """Undo create steps that were completed successfully."""
+        for step in reversed(self.steps):
+            delete_action = 'delete_%s' % step.action
+            LOG.debug(_("Performing rollback action %(action)s for "
+                        "resource %(resource)s"), {'action': delete_action,
+                                                   'resource': step.title})
+            try:
+                getattr(self.csr, delete_action)(step.resource_id)
+            except AttributeError:
+                LOG.exception(_("Internal error - '%s' is not defined"),
+                              delete_action)
+                raise CsrResourceCreateFailure(resource=step.title,
+                                               which=step.resource_id)
+            self._verify_deleted(self.csr.status, step.title, step.resource_id)
+        self.steps = []
+
+    def create_ipsec_site_connection(self, context, conn_info):
+        """Creates an IPSec site-to-site connection on CSR.
+
+        Create the PSK, IKE policy, IPSec policy, connection, static route,
+        and (future) DPD.
+        """
+        # Get all the IDs
+        conn_id = conn_info['id']
+        psk_id = conn_id
+        site_conn_id = conn_info['cisco']['site_conn_id']
+        ike_policy_id = conn_info['cisco']['ike_policy_id']
+        ipsec_policy_id = conn_info['cisco']['ipsec_policy_id']
+
+        LOG.debug(_('Creating IPSec connection %s'), conn_id)
+        # Get all the attributes needed to create
+        try:
+            psk_info = self.create_psk_info(psk_id, conn_info)
+            ike_policy_info = self.create_ike_policy_info(ike_policy_id,
+                                                          conn_info)
+            ipsec_policy_info = self.create_ipsec_policy_info(ipsec_policy_id,
+                                                              conn_info)
+            connection_info = self.create_site_connection_info(site_conn_id,
+                                                               ipsec_policy_id,
+                                                               conn_info)
+            routes_info = self.create_routes_info(site_conn_id, conn_info)
+        except (CsrUnknownMappingError, CsrDriverMismatchError) as e:
+            LOG.exception(e)
+            return
+
+        try:
+            self.do_create_action('pre_shared_key', psk_info,
+                                  conn_id, 'Pre-Shared Key')
+            self.do_create_action('ike_policy', ike_policy_info,
+                                  ike_policy_id, 'IKE Policy')
+            self.do_create_action('ipsec_policy', ipsec_policy_info,
+                                  ipsec_policy_id, 'IPSec Policy')
+            self.do_create_action('ipsec_connection', connection_info,
+                                  site_conn_id, 'IPSec Connection')
+
+            # TODO(pcm): FUTURE - Do DPD for v1 and handle if >1 connection
+            # and different DPD settings
+            for route_id, route_info in routes_info:
+                self.do_create_action('static_route', route_info,
+                                      route_id, 'Static Route')
+        except CsrResourceCreateFailure:
+            self.do_rollback()
+            LOG.info(_("FAILED: Create of IPSec site-to-site connection %s"),
+                     conn_id)
+        else:
+            LOG.info(_("SUCCESS: Created IPSec site-to-site connection %s"),
+                     conn_id)
+
+    def delete_ipsec_site_connection(self, context, conn_id):
+        """Delete the site-to-site IPSec connection.
+
+        This will be best effort and will continue, if there are any
+        failures.
+        """
+        LOG.debug(_('Deleting IPSec connection %s'), conn_id)
+        if not self.steps:
+            LOG.warning(_('Unable to find connection %s'), conn_id)
+        else:
+            self.do_rollback()
+
+        LOG.info(_("SUCCESS: Deleted IPSec site-to-site connection %s"),
+                 conn_id)

neutron/services/vpn/service_drivers/cisco_csr_db.py

@@ -11,6 +11,8 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+#
+# @author: Paul Michali, Cisco Systems, Inc.
 
 import sqlalchemy as sa
 from sqlalchemy.orm import exc as sql_exc
@@ -105,8 +107,9 @@ def get_next_available_ipsec_policy_id(session):
 def find_conn_with_policy(policy_field, policy_id, conn_id, session):
     """Return ID of another conneciton (if any) that uses same policy ID."""
     qry = session.query(vpn_db.IPsecSiteConnection.id)
-    match = qry.filter(policy_field == policy_id,
-                       vpn_db.IPsecSiteConnection.id != conn_id).first()
+    match = qry.filter_request(
+        policy_field == policy_id,
+        vpn_db.IPsecSiteConnection.id != conn_id).first()
     if match:
         return match[0]
 
@@ -215,6 +218,7 @@ def create_tunnel_mapping(context, conn_info):
                                   csr_ipsec_policy_id=csr_ipsec_id)
         try:
             context.session.add(map_entry)
+            # Force committing to database
             context.session.flush()
         except db_exc.DBDuplicateEntry:
             msg = _("Attempt to create duplicate entry in Cisco CSR "

neutron/tests/unit/services/vpn/device_drivers/cisco_csr_mock.py

@@ -0,0 +1,538 @@
+# Copyright 2014 Cisco Systems, Inc.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Paul Michali, Cisco Systems, Inc.
+
+"""Mock REST requests to Cisco Cloud Services Router."""
+
+import re
+
+from functools import wraps
+# import httmock
+import requests
+from requests import exceptions as r_exc
+
+from neutron.openstack.common import log as logging
+# TODO(pcm) Remove once httmock package is added to test-requirements. For
+# now, uncomment and include httmock source to UT
+from neutron.tests.unit.services.vpn.device_drivers import httmock
+
+# TODO(pcm) Remove, once verified these have been fixed
+FIXED_CSCum50512 = False
+FIXED_CSCum35484 = False
+FIXED_CSCul82396 = False
+FIXED_CSCum10324 = False
+
+LOG = logging.getLogger(__name__)
+
+
+def repeat(n):
+    """Decorator to limit the number of times a handler is called.
+
+    Will allow the wrapped function (handler) to be called 'n' times.
+    After that, this will return None for any additional calls,
+    allowing other handlers, if any, to be invoked.
+    """
+
+    class static:
+        retries = n
+
+    def decorator(func):
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            if static.retries == 0:
+                return None
+            static.retries -= 1
+            return func(*args, **kwargs)
+        return wrapped
+    return decorator
+
+
+def filter_request(methods, resource):
+    """Decorator to invoke handler once for a specific resource.
+
+    This will call the handler only for a specific resource using
+    a specific method(s). Any other resource request or method will
+    return None, allowing other handlers, if any, to be invoked.
+    """
+
+    class static:
+        target_methods = [m.upper() for m in methods]
+        target_resource = resource
+
+    def decorator(func):
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            if (args[1].method in static.target_methods and
+                static.target_resource in args[0].path):
+                return func(*args, **kwargs)
+            else:
+                return None  # Not for this resource
+        return wrapped
+    return decorator
+
+
+@httmock.urlmatch(netloc=r'localhost')
+def token(url, request):
+    if 'auth/token-services' in url.path:
+        return {'status_code': requests.codes.OK,
+                'content': {'token-id': 'dummy-token'}}
+
+
+@httmock.urlmatch(netloc=r'localhost')
+def token_unauthorized(url, request):
+    if 'auth/token-services' in url.path:
+        return {'status_code': requests.codes.UNAUTHORIZED}
+
+
+@httmock.urlmatch(netloc=r'wrong-host')
+def token_wrong_host(url, request):
+    raise r_exc.ConnectionError()
+
+
+@httmock.all_requests
+def token_timeout(url, request):
+    raise r_exc.Timeout()
+
+
+@filter_request(['get'], 'global/host-name')
+@httmock.all_requests
+def timeout(url, request):
+    """Simulated timeout of a normal request."""
+
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    raise r_exc.Timeout()
+
+
+@httmock.urlmatch(netloc=r'localhost')
+def no_such_resource(url, request):
+    """Indicate not found error, when invalid resource requested."""
+    return {'status_code': requests.codes.NOT_FOUND}
+
+
+@filter_request(['get'], 'global/host-name')
+@repeat(1)
+@httmock.urlmatch(netloc=r'localhost')
+def expired_request(url, request):
+    """Simulate access denied failure on first request for this resource.
+
+    Intent here is to simulate that the token has expired, by failing
+    the first request to the resource. Because of the repeat=1, this
+    will only be called once, and subsequent calls will not be handled
+    by this function, but instead will access the normal handler and
+    will pass. Currently configured for a GET request, but will work
+    with POST and PUT as well. For DELETE, would need to filter_request on a
+    different resource (e.g. 'global/local-users')
+    """
+
+    return {'status_code': requests.codes.UNAUTHORIZED}
+
+
+@httmock.urlmatch(netloc=r'localhost')
+def normal_get(url, request):
+    if request.method != 'GET':
+        return
+    LOG.debug("DEBUG: GET mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    if 'global/host-name' in url.path:
+        content = {u'kind': u'object#host-name',
+                   u'host-name': u'Router'}
+        return httmock.response(requests.codes.OK, content=content)
+    if 'global/local-users' in url.path:
+        content = {u'kind': u'collection#local-user',
+                   u'users': ['peter', 'paul', 'mary']}
+        return httmock.response(requests.codes.OK, content=content)
+    if 'interfaces/GigabitEthernet' in url.path:
+        actual_interface = url.path.split('/')[-1]
+        ip = actual_interface[-1]
+        content = {u'kind': u'object#interface',
+                   u'description': u'Changed description',
+                   u'if-name': actual_interface,
+                   u'proxy-arp': True,
+                   u'subnet-mask': u'255.255.255.0',
+                   u'icmp-unreachable': True,
+                   u'nat-direction': u'',
+                   u'icmp-redirects': True,
+                   u'ip-address': u'192.168.200.%s' % ip,
+                   u'verify-unicast-source': False,
+                   u'type': u'ethernet'}
+        return httmock.response(requests.codes.OK, content=content)
+    if 'vpn-svc/ike/policies/2' in url.path:
+        content = {u'kind': u'object#ike-policy',
+                   u'priority-id': u'2',
+                   u'version': u'v1',
+                   u'local-auth-method': u'pre-share',
+                   u'encryption': u'aes',
+                   u'hash': u'sha',
+                   u'dhGroup': 5,
+                   u'lifetime': 3600}
+        return httmock.response(requests.codes.OK, content=content)
+    if 'vpn-svc/ike/keyrings' in url.path:
+        content = {u'kind': u'object#ike-keyring',
+                   u'keyring-name': u'5',
+                   u'pre-shared-key-list': [
+                       {u'key': u'super-secret',
+                        u'encrypted': False,
+                        u'peer-address': u'10.10.10.20 255.255.255.0'}
+                   ]}
+        return httmock.response(requests.codes.OK, content=content)
+    if 'vpn-svc/ipsec/policies/' in url.path:
+        ipsec_policy_id = url.path.split('/')[-1]
+        content = {u'kind': u'object#ipsec-policy',
+                   u'mode': u'tunnel',
+                   u'policy-id': u'%s' % ipsec_policy_id,
+                   u'protection-suite': {
+                       u'esp-encryption': u'esp-aes',
+                       u'esp-authentication': u'esp-sha-hmac',
+                       u'ah': u'ah-sha-hmac',
+                   },
+                   u'anti-replay-window-size': u'128',
+                   u'lifetime-sec': 120,
+                   u'pfs': u'group5',
+                   u'lifetime-kb': 4608000,
+                   u'idle-time': None}
+        return httmock.response(requests.codes.OK, content=content)
+    if 'vpn-svc/site-to-site/Tunnel' in url.path:
+        tunnel = url.path.split('/')[-1]
+        # Use same number, to allow mock to generate IPSec policy ID
+        ipsec_policy_id = tunnel[6:]
+        content = {u'kind': u'object#vpn-site-to-site',
+                   u'vpn-interface-name': u'%s' % tunnel,
+                   u'ip-version': u'ipv4',
+                   u'vpn-type': u'site-to-site',
+                   u'ipsec-policy-id': u'%s' % ipsec_policy_id,
+                   u'ike-profile-id': None,
+                   u'mtu': 1500,
+                   u'local-device': {
+                       u'ip-address': '10.3.0.1/24',
+                       u'tunnel-ip-address': '10.10.10.10'
+                   },
+                   u'remote-device': {
+                       u'tunnel-ip-address': '10.10.10.20'
+                   }}
+        return httmock.response(requests.codes.OK, content=content)
+    if 'vpn-svc/ike/keepalive' in url.path:
+        content = {u'interval': 60,
+                   u'retry': 4,
+                   u'periodic': True}
+        return httmock.response(requests.codes.OK, content=content)
+    if 'routing-svc/static-routes' in url.path:
+        content = {u'destination-network': u'10.1.0.0/24',
+                   u'kind': u'object#static-route',
+                   u'next-hop-router': None,
+                   u'outgoing-interface': u'GigabitEthernet1',
+                   u'admin-distance': 1}
+        return httmock.response(requests.codes.OK, content=content)
+    if 'vpn-svc/site-to-site/active/sessions':
+        # Only including needed fields for mock
+        content = {u'kind': u'collection#vpn-active-sessions',
+                   u'items': [{u'status': u'DOWN-NEGOTIATING',
+                               u'vpn-interface-name': u'Tunnel123'}, ]}
+        return httmock.response(requests.codes.OK, content=content)
+
+
+@filter_request(['get'], 'vpn-svc/ike/keyrings')
+@httmock.urlmatch(netloc=r'localhost')
+def get_fqdn(url, request):
+    LOG.debug("DEBUG: GET FQDN mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    content = {u'kind': u'object#ike-keyring',
+               u'keyring-name': u'5',
+               u'pre-shared-key-list': [
+                   {u'key': u'super-secret',
+                    u'encrypted': False,
+                    u'peer-address': u'cisco.com'}
+               ]}
+    return httmock.response(requests.codes.OK, content=content)
+
+
+@filter_request(['get'], 'vpn-svc/ipsec/policies/')
+@httmock.urlmatch(netloc=r'localhost')
+def get_no_ah(url, request):
+    LOG.debug("DEBUG: GET No AH mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    ipsec_policy_id = url.path.split('/')[-1]
+    content = {u'kind': u'object#ipsec-policy',
+               u'mode': u'tunnel',
+               u'anti-replay-window-size': u'128',
+               u'policy-id': u'%s' % ipsec_policy_id,
+               u'protection-suite': {
+                   u'esp-encryption': u'esp-aes',
+                   u'esp-authentication': u'esp-sha-hmac',
+               },
+               u'lifetime-sec': 120,
+               u'pfs': u'group5',
+               u'lifetime-kb': 4608000,
+               u'idle-time': None}
+    return httmock.response(requests.codes.OK, content=content)
+
+
+@httmock.urlmatch(netloc=r'localhost')
+def get_defaults(url, request):
+    if request.method != 'GET':
+        return
+    LOG.debug("DEBUG: GET mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    if 'vpn-svc/ike/policies/2' in url.path:
+        content = {u'kind': u'object#ike-policy',
+                   u'priority-id': u'2',
+                   u'version': u'v1',
+                   u'local-auth-method': u'pre-share',
+                   u'encryption': u'des',
+                   u'hash': u'sha',
+                   u'dhGroup': 1,
+                   u'lifetime': 86400}
+        return httmock.response(requests.codes.OK, content=content)
+    if 'vpn-svc/ipsec/policies/' in url.path:
+        ipsec_policy_id = url.path.split('/')[-1]
+        content = {u'kind': u'object#ipsec-policy',
+                   u'mode': u'tunnel',
+                   u'policy-id': u'%s' % ipsec_policy_id,
+                   u'protection-suite': {},
+                   u'lifetime-sec': 3600,
+                   u'pfs': u'Disable',
+                   u'anti-replay-window-size': u'None',
+                   u'lifetime-kb': 4608000,
+                   u'idle-time': None}
+        return httmock.response(requests.codes.OK, content=content)
+
+
+@filter_request(['get'], 'vpn-svc/site-to-site')
+@httmock.urlmatch(netloc=r'localhost')
+def get_unnumbered(url, request):
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    if FIXED_CSCum50512:
+        tunnel = url.path.split('/')[-1]
+        ipsec_policy_id = tunnel[6:]
+        content = {u'kind': u'object#vpn-site-to-site',
+                   u'vpn-interface-name': u'%s' % tunnel,
+                   u'ip-version': u'ipv4',
+                   u'vpn-type': u'site-to-site',
+                   u'ipsec-policy-id': u'%s' % ipsec_policy_id,
+                   u'ike-profile-id': None,
+                   u'mtu': 1500,
+                   u'local-device': {
+                       u'ip-address': u'unnumbered GigabitEthernet3',
+                       u'tunnel-ip-address': u'10.10.10.10'
+                   },
+                   u'remote-device': {
+                       u'tunnel-ip-address': u'10.10.10.20'
+                   }}
+        return httmock.response(requests.codes.OK, content=content)
+    else:
+        return httmock.response(requests.codes.INTERNAL_SERVER_ERROR)
+
+
+@filter_request(['get'], 'vpn-svc/site-to-site')
+@httmock.urlmatch(netloc=r'localhost')
+def get_mtu(url, request):
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    tunnel = url.path.split('/')[-1]
+    ipsec_policy_id = tunnel[6:]
+    content = {u'kind': u'object#vpn-site-to-site',
+               u'vpn-interface-name': u'%s' % tunnel,
+               u'ip-version': u'ipv4',
+               u'vpn-type': u'site-to-site',
+               u'ipsec-policy-id': u'%s' % ipsec_policy_id,
+               u'ike-profile-id': None,
+               u'mtu': 9192,
+               u'local-device': {
+                   u'ip-address': u'10.3.0.1/24',
+                   u'tunnel-ip-address': u'10.10.10.10'
+               },
+               u'remote-device': {
+                   u'tunnel-ip-address': u'10.10.10.20'
+               }}
+    return httmock.response(requests.codes.OK, content=content)
+
+
+@filter_request(['get'], 'vpn-svc/ike/keepalive')
+@httmock.urlmatch(netloc=r'localhost')
+def get_not_configured(url, request):
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    return {'status_code': requests.codes.NOT_FOUND}
+
+
+@filter_request(['get'], 'vpn-svc/site-to-site/active/sessions')
+@httmock.urlmatch(netloc=r'localhost')
+def get_none(url, request):
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    content = {u'kind': u'collection#vpn-active-sessions',
+               u'items': []}
+    return httmock.response(requests.codes.OK, content=content)
+
+
+@httmock.urlmatch(netloc=r'localhost')
+def post(url, request):
+    if request.method != 'POST':
+        return
+    LOG.debug("DEBUG: POST mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    if 'interfaces/GigabitEthernet' in url.path:
+        return {'status_code': requests.codes.NO_CONTENT}
+    if 'global/local-users' in url.path:
+        if 'username' not in request.body:
+            return {'status_code': requests.codes.BAD_REQUEST}
+        if '"privilege": 20' in request.body:
+            return {'status_code': requests.codes.BAD_REQUEST}
+        headers = {'location': '%s/test-user' % url.geturl()}
+        return httmock.response(requests.codes.CREATED, headers=headers)
+    if 'vpn-svc/ike/policies' in url.path:
+        headers = {'location': "%s/2" % url.geturl()}
+        return httmock.response(requests.codes.CREATED, headers=headers)
+    if 'vpn-svc/ipsec/policies' in url.path:
+        m = re.search(r'"policy-id": "(\S+)"', request.body)
+        if m:
+            headers = {'location': "%s/%s" % (url.geturl(), m.group(1))}
+            return httmock.response(requests.codes.CREATED, headers=headers)
+        return {'status_code': requests.codes.BAD_REQUEST}
+    if 'vpn-svc/ike/keyrings' in url.path:
+        headers = {'location': "%s/5" % url.geturl()}
+        return httmock.response(requests.codes.CREATED, headers=headers)
+    if 'vpn-svc/site-to-site' in url.path:
+        m = re.search(r'"vpn-interface-name": "(\S+)"', request.body)
+        if m:
+            headers = {'location': "%s/%s" % (url.geturl(), m.group(1))}
+            return httmock.response(requests.codes.CREATED, headers=headers)
+        return {'status_code': requests.codes.BAD_REQUEST}
+    if 'routing-svc/static-routes' in url.path:
+        headers = {'location':
+                   "%s/10.1.0.0_24_GigabitEthernet1" % url.geturl()}
+        return httmock.response(requests.codes.CREATED, headers=headers)
+
+
+@filter_request(['post'], 'global/local-users')
+@httmock.urlmatch(netloc=r'localhost')
+def post_change_attempt(url, request):
+    LOG.debug("DEBUG: POST change value mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    return {'status_code': requests.codes.NOT_FOUND,
+            'content': {
+                u'error-code': -1,
+                u'error-message': u'user test-user already exists'}}
+
+
+@httmock.urlmatch(netloc=r'localhost')
+def post_duplicate(url, request):
+    LOG.debug("DEBUG: POST duplicate mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    return {'status_code': requests.codes.BAD_REQUEST,
+            'content': {
+                u'error-code': -1,
+                u'error-message': u'policy 2 exist, not allow to '
+                                  u'update policy using POST method'}}
+
+
+@filter_request(['post'], 'vpn-svc/site-to-site')
+@httmock.urlmatch(netloc=r'localhost')
+def post_missing_ipsec_policy(url, request):
+    LOG.debug("DEBUG: POST missing ipsec policy mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    return {'status_code': requests.codes.BAD_REQUEST}
+
+
+@filter_request(['post'], 'vpn-svc/site-to-site')
+@httmock.urlmatch(netloc=r'localhost')
+def post_missing_ike_policy(url, request):
+    LOG.debug("DEBUG: POST missing ike policy mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    return {'status_code': requests.codes.BAD_REQUEST}
+
+
+@filter_request(['post'], 'vpn-svc/site-to-site')
+@httmock.urlmatch(netloc=r'localhost')
+def post_bad_ip(url, request):
+    LOG.debug("DEBUG: POST bad IP mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    return {'status_code': requests.codes.BAD_REQUEST}
+
+
+@filter_request(['post'], 'vpn-svc/site-to-site')
+@httmock.urlmatch(netloc=r'localhost')
+def post_bad_mtu(url, request):
+    LOG.debug("DEBUG: POST bad mtu mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    return {'status_code': requests.codes.BAD_REQUEST}
+
+
+@filter_request(['post'], 'vpn-svc/ipsec/policies')
+@httmock.urlmatch(netloc=r'localhost')
+def post_bad_lifetime(url, request):
+    LOG.debug("DEBUG: POST bad lifetime mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    return {'status_code': requests.codes.BAD_REQUEST}
+
+
+@httmock.urlmatch(netloc=r'localhost')
+def put(url, request):
+    if request.method != 'PUT':
+        return
+    LOG.debug("DEBUG: PUT mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    # Any resource
+    return {'status_code': requests.codes.NO_CONTENT}
+
+
+@httmock.urlmatch(netloc=r'localhost')
+def delete(url, request):
+    if request.method != 'DELETE':
+        return
+    LOG.debug("DEBUG: DELETE mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    # Any resource
+    return {'status_code': requests.codes.NO_CONTENT}
+
+
+@httmock.urlmatch(netloc=r'localhost')
+def delete_unknown(url, request):
+    if request.method != 'DELETE':
+        return
+    LOG.debug("DEBUG: DELETE unknown mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    # Any resource
+    return {'status_code': requests.codes.NOT_FOUND,
+            'content': {
+                u'error-code': -1,
+                u'error-message': 'user unknown not found'}}
+
+
+@httmock.urlmatch(netloc=r'localhost')
+def delete_not_allowed(url, request):
+    if request.method != 'DELETE':
+        return
+    LOG.debug("DEBUG: DELETE not allowed mock for %s", url)
+    if not request.headers.get('X-auth-token', None):
+        return {'status_code': requests.codes.UNAUTHORIZED}
+    # Any resource
+    return {'status_code': requests.codes.METHOD_NOT_ALLOWED}