Merge "Implements sync mechanism between Neutron and Nuage VSD"

etc/neutron/plugins/nuage/nuage_plugin.ini

@@ -1,10 +1,41 @@
 # Please fill in the correct data for all the keys below and uncomment key-value pairs
 [restproxy]
+# (StrOpt) Default Network partition in which VSD will
+# orchestrate network resources using openstack
+#
 #default_net_partition_name = <default-net-partition-name>
+
+# (StrOpt) Nuage provided uri for initial authorization to
+# access VSD
+#
 #auth_resource = /auth
+
+# (StrOpt) IP Address and Port of VSD
+#
 #server = ip:port
+
+# (StrOpt) Organization name in which VSD will orchestrate
+# network resources using openstack
+#
 #organization = org
+
+# (StrOpt) Username and password of VSD for authentication
+#
 #serverauth = uname:pass
+
+# (BoolOpt) Boolean for SSL connection with VSD server
+#
 #serverssl = True
+
+# (StrOpt) Nuage provided base uri to reach out to VSD
+#
 #base_uri = /base
 
+[syncmanager]
+# (BoolOpt) Boolean to enable sync between openstack and VSD
+#
+#enable_sync = False
+
+# (IntOpt) Sync interval in seconds between openstack and VSD
+#
+#sync_interval = 0

neutron/plugins/nuage/common/config.py

@@ -42,6 +42,18 @@ restproxy_opts = [
                help=_("Per Net Partition quota of floating ips")),
 ]
 
+syncmanager_opts = [
+    cfg.BoolOpt('enable_sync', default=False,
+                help=_("Nuage plugin will sync resources between openstack "
+                       "and VSD")),
+    cfg.IntOpt('sync_interval', default=0,
+               help=_("Sync interval in seconds between openstack and VSD. "
+                      "It defines how often the synchronization is done. "
+                      "If not set, value of 0 is assumed and sync will be "
+                      "performed only once, at the Neutron startup time.")),
+]
+
 
 def nuage_register_cfg_opts():
     cfg.CONF.register_opts(restproxy_opts, "RESTPROXY")
+    cfg.CONF.register_opts(syncmanager_opts, "SYNCMANAGER")

neutron/plugins/nuage/nuagedb.py

@@ -15,6 +15,10 @@
 # @author: Ronak Shah, Nuage Networks, Alcatel-Lucent USA Inc.
 
 from neutron.db import common_db_mixin
+from neutron.db import extraroute_db
+from neutron.db import l3_db
+from neutron.db import models_v2
+from neutron.db import securitygroups_db
 from neutron.plugins.nuage import nuage_models
 
 
@@ -33,6 +37,11 @@ def delete_net_partition(session, net_partition):
     session.delete(net_partition)
 
 
+def delete_net_partition_by_id(session, netpart_id):
+    query = session.query(nuage_models.NetPartition)
+    query.filter_by(id=netpart_id).delete()
+
+
 def get_net_partition_by_name(session, name):
     query = session.query(nuage_models.NetPartition)
     return query.filter_by(name=name).first()
@@ -52,6 +61,74 @@ def get_net_partitions(session, filters=None, fields=None):
     return query
 
 
+def get_net_partition_ids(session):
+    query = session.query(nuage_models.NetPartition.id)
+    return [netpart[0] for netpart in query]
+
+
+def get_net_partition_with_lock(session, netpart_id):
+    query = session.query(nuage_models.NetPartition)
+    netpart_db = query.filter_by(id=netpart_id).with_lockmode('update').one()
+    return make_net_partition_dict(netpart_db)
+
+
+def get_subnet_ids(session):
+    query = session.query(models_v2.Subnet.id)
+    return [subn[0] for subn in query]
+
+
+def get_subnet_with_lock(session, sub_id):
+    query = session.query(models_v2.Subnet)
+    subnet_db = query.filter_by(id=sub_id).with_lockmode('update').one()
+    return subnet_db
+
+
+def get_router_ids(session):
+    query = session.query(l3_db.Router.id)
+    return [router[0] for router in query]
+
+
+def get_router_with_lock(session, router_id):
+    query = session.query(l3_db.Router)
+    router_db = query.filter_by(id=router_id).with_lockmode('update').one()
+    return router_db
+
+
+def get_secgrp_ids(session):
+    query = session.query(securitygroups_db.SecurityGroup.id)
+    return [secgrp[0] for secgrp in query]
+
+
+def get_secgrp_with_lock(session, secgrp_id):
+    query = session.query(securitygroups_db.SecurityGroup)
+    secgrp_db = query.filter_by(id=secgrp_id).with_lockmode('update').one()
+    return secgrp_db
+
+
+def get_secgrprule_ids(session):
+    query = session.query(securitygroups_db.SecurityGroupRule.id)
+    return [secgrprule[0] for secgrprule in query]
+
+
+def get_secgrprule_with_lock(session, secgrprule_id):
+    query = session.query(securitygroups_db.SecurityGroupRule)
+    secgrprule_db = (query.filter_by(id=secgrprule_id).with_lockmode(
+        'update').one())
+    return secgrprule_db
+
+
+def get_port_with_lock(session, port_id):
+    query = session.query(models_v2.Port)
+    port_db = query.filter_by(id=port_id).with_lockmode('update').one()
+    return port_db
+
+
+def get_fip_with_lock(session, fip_id):
+    query = session.query(l3_db.FloatingIP)
+    fip_db = query.filter_by(id=fip_id).with_lockmode('update').one()
+    return fip_db
+
+
 def add_entrouter_mapping(session, np_id,
                           router_id,
                           n_l3id):
@@ -81,6 +158,20 @@ def update_subnetl2dom_mapping(subnet_l2dom,
     subnet_l2dom.update(new_dict)
 
 
+def get_update_subnetl2dom_mapping(session, new_dict):
+    subnet_l2dom = get_subnet_l2dom_with_lock(session, new_dict['subnet_id'])
+    subnet_l2dom.update(new_dict)
+
+
+def update_entrtr_mapping(ent_rtr, new_dict):
+    ent_rtr.update(new_dict)
+
+
+def get_update_entrtr_mapping(session, new_dict):
+    ent_rtr = get_ent_rtr_mapping_with_lock(session, new_dict['router_id'])
+    ent_rtr.update(new_dict)
+
+
 def delete_subnetl2dom_mapping(session, subnet_l2dom):
     session.delete(subnet_l2dom)
 
@@ -90,8 +181,13 @@ def get_subnet_l2dom_by_id(session, id):
     return query.filter_by(subnet_id=id).first()
 
 
-def get_ent_rtr_mapping_by_entid(session,
-                                 entid):
+def get_subnet_l2dom_with_lock(session, id):
+    query = session.query(nuage_models.SubnetL2Domain)
+    subl2dom = query.filter_by(subnet_id=id).with_lockmode('update').one()
+    return subl2dom
+
+
+def get_ent_rtr_mapping_by_entid(session, entid):
     query = session.query(nuage_models.NetPartitionRouter)
     return query.filter_by(net_partition_id=entid).all()
 
@@ -115,3 +211,82 @@ def get_network_binding(session, network_id):
     return (session.query(nuage_models.ProviderNetBinding).
             filter_by(network_id=network_id).
             first())
+
+
+def get_ent_rtr_mapping_with_lock(session, rtrid):
+    query = session.query(nuage_models.NetPartitionRouter)
+    entrtr = query.filter_by(router_id=rtrid).with_lockmode('update').one()
+    return entrtr
+
+
+def get_ipalloc_for_fip(session, network_id, ip, lock=False):
+    query = session.query(models_v2.IPAllocation)
+    if lock:
+        # Lock is required when the resource is synced
+        ipalloc_db = (query.filter_by(network_id=network_id).filter_by(
+            ip_address=ip).with_lockmode('update').one())
+    else:
+        ipalloc_db = (query.filter_by(network_id=network_id).filter_by(
+            ip_address=ip).one())
+    return make_ipalloc_dict(ipalloc_db)
+
+
+def get_all_net_partitions(session):
+    net_partitions = get_net_partitions(session)
+    return make_net_partition_list(net_partitions)
+
+
+def get_all_routes(session):
+    routes = session.query(extraroute_db.RouterRoute)
+    return make_route_list(routes)
+
+
+def get_route_with_lock(session, dest, nhop):
+    query = session.query(extraroute_db.RouterRoute)
+    route_db = (query.filter_by(destination=dest).filter_by(nexthop=nhop)
+                .with_lockmode('update').one())
+    return make_route_dict(route_db)
+
+
+def make_ipalloc_dict(subnet_db):
+    return {'port_id': subnet_db['port_id'],
+            'subnet_id': subnet_db['subnet_id'],
+            'network_id': subnet_db['network_id'],
+            'ip_address': subnet_db['ip_address']}
+
+
+def make_net_partition_dict(net_partition):
+    return {'id': net_partition['id'],
+            'name': net_partition['name'],
+            'l3dom_tmplt_id': net_partition['l3dom_tmplt_id'],
+            'l2dom_tmplt_id': net_partition['l2dom_tmplt_id']}
+
+
+def make_net_partition_list(net_partitions):
+    return [make_net_partition_dict(net_partition) for net_partition in
+            net_partitions]
+
+
+def make_route_dict(route):
+    return {'destination': route['destination'],
+            'nexthop': route['nexthop'],
+            'router_id': route['router_id']}
+
+
+def make_route_list(routes):
+    return [make_route_dict(route) for route in routes]
+
+
+def make_subnl2dom_dict(subl2dom):
+    return {'subnet_id': subl2dom['subnet_id'],
+            'net_partition_id': subl2dom['net_partition_id'],
+            'nuage_subnet_id': subl2dom['nuage_subnet_id'],
+            'nuage_l2dom_tmplt_id': subl2dom['nuage_l2dom_tmplt_id'],
+            'nuage_user_id': subl2dom['nuage_user_id'],
+            'nuage_group_id': subl2dom['nuage_group_id']}
+
+
+def make_entrtr_dict(entrtr):
+    return {'net_partition_id': entrtr['net_partition_id'],
+            'router_id': entrtr['router_id'],
+            'nuage_router_id': entrtr['nuage_router_id']}

neutron/plugins/nuage/plugin.py

@@ -41,12 +41,14 @@ from neutron.extensions import securitygroup as ext_sg
 from neutron.openstack.common import excutils
 from neutron.openstack.common import importutils
 from neutron.openstack.common import lockutils
+from neutron.openstack.common import loopingcall
 from neutron.plugins.nuage.common import config
 from neutron.plugins.nuage.common import constants
 from neutron.plugins.nuage.common import exceptions as nuage_exc
 from neutron.plugins.nuage import extensions
 from neutron.plugins.nuage.extensions import netpartition
 from neutron.plugins.nuage import nuagedb
+from neutron.plugins.nuage import syncmanager
 from neutron import policy
 
 
@@ -71,6 +73,9 @@ class NuagePlugin(db_base_plugin_v2.NeutronDbPluginV2,
         self.nuageclient_init()
         net_partition = cfg.CONF.RESTPROXY.default_net_partition_name
         self._create_default_net_partition(net_partition)
+        if cfg.CONF.SYNCMANAGER.enable_sync:
+            self.syncmanager = syncmanager.SyncManager(self.nuageclient)
+            self._synchronization_thread()
 
     def nuageclient_init(self):
         server = cfg.CONF.RESTPROXY.server
@@ -85,6 +90,16 @@ class NuagePlugin(db_base_plugin_v2.NeutronDbPluginV2,
                                                    auth_resource,
                                                    organization)
 
+    def _synchronization_thread(self):
+        sync_interval = cfg.CONF.SYNCMANAGER.sync_interval
+        fip_quota = str(cfg.CONF.RESTPROXY.default_floatingip_quota)
+        if sync_interval > 0:
+            sync_loop = loopingcall.FixedIntervalLoopingCall(
+                self.syncmanager.synchronize, fip_quota)
+            sync_loop.start(interval=sync_interval)
+        else:
+            self.syncmanager.synchronize(fip_quota)
+
     def _resource_finder(self, context, for_resource, resource, user_req):
         match = re.match(attributes.UUID_PATTERN, user_req[resource])
         if match:
@@ -1084,10 +1099,13 @@ class NuagePlugin(db_base_plugin_v2.NeutronDbPluginV2,
                                   neutron_fip, port_id):
         rtr_id = neutron_fip['router_id']
         net_id = neutron_fip['floating_network_id']
+        subn = nuagedb.get_ipalloc_for_fip(context.session,
+                                           net_id,
+                                           neutron_fip['floating_ip_address'])
 
-        fip_pool = self.nuageclient.get_nuage_fip_pool_by_id(net_id)
+        fip_pool = self.nuageclient.get_nuage_fip_pool_by_id(subn['subnet_id'])
         if not fip_pool:
-            msg = _('sharedresource %s not found on VSD') % net_id
+            msg = _('sharedresource %s not found on VSD') % subn['subnet_id']
             raise n_exc.BadRequest(resource='floatingip',
                                    msg=msg)
 

neutron/plugins/nuage/syncmanager.py

@@ -0,0 +1,423 @@
+# Copyright 2014 Alcatel-Lucent USA Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Sayaji Patil, Nuage Networks, Alcatel-Lucent USA Inc.
+
+from oslo.config import cfg
+import sqlalchemy.orm.exc as db_exc
+
+from neutron import context as ncontext
+from neutron.db import db_base_plugin_v2
+from neutron.db import extraroute_db
+from neutron.db import securitygroups_db
+from neutron.openstack.common import importutils
+from neutron.openstack.common import log
+from neutron.openstack.common.gettextutils import _LE, _LI, _LW
+from neutron.plugins.nuage.common import config
+from neutron.plugins.nuage import nuagedb
+
+
+LOG = log.getLogger(__name__)
+NUAGE_CONFIG_FILE = '/etc/neutron/plugins/nuage/nuage_plugin.ini'
+
+
+class SyncManager(db_base_plugin_v2.NeutronDbPluginV2,
+                  extraroute_db.ExtraRoute_db_mixin,
+                  securitygroups_db.SecurityGroupDbMixin):
+    """
+    This class provides functionality to sync data between OpenStack and VSD.
+    """
+
+    def __init__(self, nuageclient):
+        self.context = ncontext.get_admin_context()
+        self.nuageclient = nuageclient
+
+    def synchronize(self, fipquota):
+        LOG.info(_LI("Starting the sync between Neutron and VSD"))
+        try:
+            # Get all data to determine the resources to sync
+            data = self._get_all_data()
+            resources = self.nuageclient.get_resources_to_sync(data)
+
+            # Sync all resources
+            self._sync(resources, fipquota)
+        except Exception as e:
+            LOG.error(_LE("Cannot complete the sync between Neutron and VSD "
+                          "because of error:%s"), str(e))
+            return
+
+        LOG.info(_LI("Sync between Neutron and VSD completed successfully"))
+
+    def _get_all_data(self):
+        # Get all net-partitions
+        net_partition_list = nuagedb.get_all_net_partitions(
+            self.context.session)
+
+        # Get all subnet ids
+        subnet_id_list = nuagedb.get_subnet_ids(self.context.session)
+
+        # Get all router ids
+        router_id_list = nuagedb.get_router_ids(self.context.session)
+
+        # Get all ports
+        port_list = self.get_ports(self.context)
+
+        # Get all routes
+        route_list = nuagedb.get_all_routes(self.context.session)
+
+        # Get all floatingips
+        fip_list = self.get_floatingips(self.context)
+
+        # Get all securitygrp ids
+        secgrp_id_list = nuagedb.get_secgrp_ids(self.context.session)
+
+        # Get all securitygrprules
+        secgrprule_id_list = self.get_security_group_rules(self.context)
+
+        # Get all portbindings
+        portbinding_list = self._get_port_security_group_bindings(self.context)
+
+        data = {
+            'netpartition': net_partition_list,
+            'subnet': subnet_id_list,
+            'router': router_id_list,
+            'port': port_list,
+            'route': route_list,
+            'fip': fip_list,
+            'secgroup': secgrp_id_list,
+            'secgrouprule': secgrprule_id_list,
+            'portbinding': portbinding_list,
+        }
+        return data
+
+    def _sync(self, resources, fip_quota):
+        # Sync net-partitions
+        net_partition_id_dict = self.sync_net_partitions(fip_quota, resources)
+
+        # Sync sharednetworks
+        self.sync_sharednetworks(resources)
+
+        # Sync l2domains
+        self.sync_l2domains(net_partition_id_dict, resources)
+
+        # Sync domains
+        self.sync_domains(net_partition_id_dict, resources)
+
+        # Sync domainsubnets
+        self.sync_domainsubnets(resources)
+
+        # Sync routes
+        self.sync_routes(resources)
+
+        # Sync vms
+        self.sync_vms(resources)
+
+        # Sync secgrps
+        self.sync_secgrps(resources)
+
+        # Sync secgrprules
+        self.sync_secgrp_rules(resources)
+
+        # Sync fips
+        self._sync_fips(resources)
+
+        # Delete the old net-partitions
+        for net_id in net_partition_id_dict:
+            nuagedb.delete_net_partition_by_id(self.context.session,
+                                               net_id)
+
+    def sync_net_partitions(self, fip_quota, resources):
+        net_partition_id_dict = {}
+        for netpart_id in resources['netpartition']['add']:
+            with self.context.session.begin(subtransactions=True):
+                netpart = self._get_netpart_data(netpart_id)
+                if netpart:
+                    result = self.nuageclient.create_netpart(netpart,
+                                                             fip_quota)
+                    netpart = result.get(netpart_id)
+                    if netpart:
+                        net_partition_id_dict[netpart_id] = netpart['id']
+                        nuagedb.add_net_partition(
+                            self.context.session,
+                            netpart['id'],
+                            netpart['l3dom_tmplt_id'],
+                            netpart['l2dom_tmplt_id'],
+                            netpart['name'])
+
+        return net_partition_id_dict
+
+    def sync_sharednetworks(self, resources):
+        for sharednet_id in resources['sharednetwork']['add']:
+            with self.context.session.begin(subtransactions=True):
+                subnet, subl2dom = self._get_subnet_data(
+                    sharednet_id,
+                    get_mapping=False)
+                if subnet:
+                    self.nuageclient.create_sharednetwork(subnet)
+
+    def sync_l2domains(self, net_partition_id_dict, resources):
+        for l2dom_id in resources['l2domain']['add']:
+            with self.context.session.begin(subtransactions=True):
+                subnet, subl2dom = self._get_subnet_data(l2dom_id)
+                if subnet:
+                    # if subnet exists, subl2dom will exist
+                    netpart_id = subl2dom['net_partition_id']
+                    if netpart_id in net_partition_id_dict.keys():
+                        # Use the id of the newly created net_partition
+                        netpart_id = net_partition_id_dict[netpart_id]
+
+                    result = self.nuageclient.create_l2domain(netpart_id,
+                                                              subnet)
+                    if result:
+                        nuagedb.get_update_subnetl2dom_mapping(
+                            self.context.session,
+                            result)
+
+    def sync_domains(self, net_partition_id_dict, resources):
+        for domain_id in resources['domain']['add']:
+            with self.context.session.begin(subtransactions=True):
+                router, entrtr = self._get_router_data(domain_id)
+                if router:
+                    # if router exists, entrtr will exist
+                    netpart_id = entrtr['net_partition_id']
+                    if netpart_id in net_partition_id_dict.keys():
+                        # Use the id of the newly created net_partition
+                        netpart_id = net_partition_id_dict[netpart_id]
+
+                    netpart = nuagedb.get_net_partition_by_id(
+                        self.context.session,
+                        netpart_id)
+                    result = self.nuageclient.create_domain(netpart, router)
+                    if result:
+                        nuagedb.get_update_entrtr_mapping(self.context.session,
+                                                          result)
+
+    def sync_domainsubnets(self, resources):
+        for domsubn_id in resources['domainsubnet']['add']:
+            # This is a dict of subn_id and the router interface port
+            subn_rtr_intf_port_dict = (
+                resources['port']['sub_rtr_intf_port_dict'])
+            port_id = subn_rtr_intf_port_dict[domsubn_id]
+            port = self._get_port_data(port_id)
+            if port:
+                with self.context.session.begin(subtransactions=True):
+                    subnet, subl2dom = self._get_subnet_data(domsubn_id)
+                    if subnet:
+                        result = self.nuageclient.create_domainsubnet(subnet,
+                                                                      port)
+                        if result:
+                            nuagedb.get_update_subnetl2dom_mapping(
+                                self.context.session,
+                                result)
+
+    def sync_routes(self, resources):
+        for rt in resources['route']['add']:
+            with self.context.session.begin(subtransactions=True):
+                route = self._get_route_data(rt)
+                if route:
+                    self.nuageclient.create_route(route)
+
+    def sync_vms(self, resources):
+        for port_id in resources['port']['vm']:
+            port = self._get_port_data(port_id)
+            if port:
+                self.nuageclient.create_vm(port)
+
+    def sync_secgrps(self, resources):
+        secgrp_dict = resources['security']['secgroup']
+        for secgrp_id, ports in secgrp_dict['l2domain']['add'].iteritems():
+            with self.context.session.begin(subtransactions=True):
+                secgrp = self._get_sec_grp_data(secgrp_id)
+                if secgrp:
+                    self.nuageclient.create_security_group(secgrp, ports)
+
+        for secgrp_id, ports in secgrp_dict['domain']['add'].iteritems():
+            with self.context.session.begin(subtransactions=True):
+                secgrp = self._get_sec_grp_data(secgrp_id)
+                if secgrp:
+                    self.nuageclient.create_security_group(secgrp, ports)
+
+    def sync_secgrp_rules(self, resources):
+        secrule_list = resources['security']['secgrouprule']
+        for secrule_id in secrule_list['l2domain']['add']:
+            with self.context.session.begin(subtransactions=True):
+                secgrprule = self._get_sec_grp_rule_data(secrule_id)
+                if secgrprule:
+                    self.nuageclient.create_security_group_rule(secgrprule)
+
+        for secrule_id in secrule_list['domain']['add']:
+            with self.context.session.begin(subtransactions=True):
+                secgrprule = self._get_sec_grp_rule_data(secrule_id)
+                if secgrprule:
+                    self.nuageclient.create_security_group_rule(secgrprule)
+
+    def _sync_fips(self, resources):
+        for fip_id in resources['fip']['add']:
+            with self.context.session.begin(subtransactions=True):
+                fip = self._get_fip_data(fip_id)
+                if fip:
+                    ipalloc = self._get_ipalloc_for_fip(fip)
+                    self.nuageclient.create_fip(fip, ipalloc)
+
+        for fip_id in resources['fip']['disassociate']:
+            with self.context.session.begin(subtransactions=True):
+                fip = self._get_fip_data(fip_id)
+                if fip:
+                    self.nuageclient.disassociate_fip(fip)
+
+        for fip_id in resources['fip']['associate']:
+            with self.context.session.begin(subtransactions=True):
+                fip = self._get_fip_data(fip_id)
+                if fip:
+                    self.nuageclient.associate_fip(fip)
+
+    def _get_subnet_data(self, subnet_id, get_mapping=True):
+        subnet = None
+        subl2dom = None
+        try:
+            if get_mapping:
+                subl2dom_db = nuagedb.get_subnet_l2dom_with_lock(
+                    self.context.session,
+                    subnet_id)
+                subl2dom = nuagedb.make_subnl2dom_dict(subl2dom_db)
+
+            subnet_db = nuagedb.get_subnet_with_lock(self.context.session,
+                                                     subnet_id)
+            subnet = self._make_subnet_dict(subnet_db)
+        except db_exc.NoResultFound:
+            LOG.warning(_LW("Subnet %s not found in neutron for sync"),
+                        subnet_id)
+
+        return subnet, subl2dom
+
+    def _get_router_data(self, router_id):
+        router = None
+        entrtr = None
+        try:
+            entrtr_db = nuagedb.get_ent_rtr_mapping_with_lock(
+                self.context.session,
+                router_id)
+            entrtr = nuagedb.make_entrtr_dict(entrtr_db)
+
+            router_db = nuagedb.get_router_with_lock(self.context.session,
+                                                     router_id)
+            router = self._make_router_dict(router_db)
+        except db_exc.NoResultFound:
+            LOG.warning(_LW("Router %s not found in neutron for sync"),
+                        router_id)
+
+        return router, entrtr
+
+    def _get_route_data(self, rt):
+        route = None
+        try:
+            route = nuagedb.get_route_with_lock(self.context.session,
+                                                rt['destination'],
+                                                rt['nexthop'])
+        except db_exc.NoResultFound:
+            LOG.warning(_LW("Route with destination %(dest)s and nexthop "
+                            "%(hop)s not found in neutron for sync"),
+                        {'dest': rt['destination'],
+                         'hop': rt['nexthop']})
+
+        return route
+
+    def _get_sec_grp_data(self, secgrp_id):
+        secgrp = None
+        try:
+            secgrp_db = nuagedb.get_secgrp_with_lock(self.context.session,
+                                                     secgrp_id)
+            secgrp = self._make_security_group_dict(secgrp_db)
+        except db_exc.NoResultFound:
+            LOG.warning(_LW("Security group %s not found in neutron for sync"),
+                        secgrp_id)
+        return secgrp
+
+    def _get_sec_grp_rule_data(self, secgrprule_id):
+        secgrprule = None
+        try:
+            secrule_db = nuagedb.get_secgrprule_with_lock(self.context.session,
+                                                          secgrprule_id)
+            secgrprule = self._make_security_group_rule_dict(secrule_db)
+        except db_exc.NoResultFound:
+            LOG.warning(_LW("Security group rule %s not found in neutron for "
+                            "sync"), secgrprule_id)
+        return secgrprule
+
+    def _get_fip_data(self, fip_id):
+        fip = None
+        try:
+            fip_db = nuagedb.get_fip_with_lock(self.context.session, fip_id)
+            fip = self._make_floatingip_dict(fip_db)
+        except db_exc.NoResultFound:
+            LOG.warning(_LW("Floating ip %s not found in neutron for sync"),
+                        fip_id)
+        return fip
+
+    def _get_ipalloc_for_fip(self, fip):
+        ipalloc = None
+        try:
+            ipalloc = nuagedb.get_ipalloc_for_fip(self.context.session,
+                                                  fip['floating_network_id'],
+                                                  fip['floating_ip_address'],
+                                                  lock=True)
+        except db_exc.NoResultFound:
+            LOG.warning(_LW("IP allocation for floating ip %s not found in "
+                            "neutron for sync"), fip['id'])
+        return ipalloc
+
+    def _get_netpart_data(self, netpart_id):
+        netpart = None
+        try:
+            netpart = nuagedb.get_net_partition_with_lock(
+                self.context.session,
+                netpart_id)
+        except db_exc.NoResultFound:
+            LOG.warning(_LW("Net-partition %s not found in neutron for sync"),
+                        netpart_id)
+        return netpart
+
+    def _get_port_data(self, port_id):
+        port = None
+        try:
+            port_db = nuagedb.get_port_with_lock(self.context.session, port_id)
+            port = self._make_port_dict(port_db)
+        except db_exc.NoResultFound:
+            LOG.warning(_LW("VM port %s not found in neutron for sync"),
+                        port_id)
+        return port
+
+
+def main():
+    cfg.CONF(default_config_files=(
+        [NUAGE_CONFIG_FILE]))
+    config.nuage_register_cfg_opts()
+    server = cfg.CONF.RESTPROXY.server
+    serverauth = cfg.CONF.RESTPROXY.serverauth
+    serverssl = cfg.CONF.RESTPROXY.serverssl
+    base_uri = cfg.CONF.RESTPROXY.base_uri
+    auth_resource = cfg.CONF.RESTPROXY.auth_resource
+    organization = cfg.CONF.RESTPROXY.organization
+    fipquota = str(cfg.CONF.RESTPROXY.default_floatingip_quota)
+    logging = importutils.import_module('logging')
+    nuageclientinst = importutils.import_module('nuagenetlib.nuageclient')
+    nuageclient = nuageclientinst.NuageClient(server, base_uri,
+                                              serverssl, serverauth,
+                                              auth_resource,
+                                              organization)
+    logging.basicConfig(level=logging.DEBUG)
+    SyncManager(nuageclient).synchronize(fipquota)
+
+if __name__ == '__main__':
+    main()

neutron/tests/unit/nuage/fake_nuageclient.py

@@ -197,3 +197,126 @@ class FakeNuageClient(object):
 
     def remove_router_interface(self, params):
         pass
+
+    def get_resources_to_sync(self, data):
+        netpart_id_list = []
+        for netpart in data['netpartition']:
+            netpart_id_list.append(netpart['id'])
+
+        netpart_dict = {
+            'add': netpart_id_list,
+            'sync': []
+        }
+
+        subn_id_list = []
+        if data['subnet']:
+            subn_id_list.append(data['subnet'][0])
+
+        l2domain_dict = {
+            'add': subn_id_list
+        }
+
+        rtr_id_list = []
+        if data['router']:
+            rtr_id_list.append(data['router'][0])
+
+        domain_dict = {
+            'add': rtr_id_list
+        }
+
+        domain_subn_id = uuidutils.generate_uuid()
+
+        result = {
+            'netpartition': netpart_dict,
+            'l2domain': l2domain_dict,
+            'domain': domain_dict,
+            'domainsubnet': {'add': [domain_subn_id]},
+            'sharednetwork': {'add': [uuidutils.generate_uuid()]},
+            'route': {'add': []},
+            'security': {
+                'secgroup': {
+                    'l2domain': {'add': {
+                        uuidutils.generate_uuid(): [uuidutils.generate_uuid()]
+                    }},
+                    'domain': {'add': {
+                        uuidutils.generate_uuid(): [uuidutils.generate_uuid()]
+                    }}
+                },
+                'secgrouprule': {
+                    'l2domain': {'add': [uuidutils.generate_uuid()]},
+                    'domain': {'add': [uuidutils.generate_uuid()]}
+                },
+            },
+            'port': {
+                'vm': [uuidutils.generate_uuid()],
+                'sub_rtr_intf_port_dict': {
+                    domain_subn_id: uuidutils.generate_uuid()
+                },
+                'secgroup': [uuidutils.generate_uuid()]
+            },
+            'subl2dommapping': [uuidutils.generate_uuid()],
+            'fip': {
+                'add': [uuidutils.generate_uuid()],
+                'associate': [uuidutils.generate_uuid()],
+                'disassociate': [uuidutils.generate_uuid()]
+            }
+        }
+        return result
+
+    def create_netpart(self, netpart, fip_quota):
+        if netpart['name'] == 'sync-new-netpartition':
+            oldid = netpart['id']
+            netpart['id'] = 'a917924f-3139-4bdb-a4c3-ea7c8011582f'
+            netpart = {
+                oldid: netpart
+            }
+            return netpart
+        return {}
+
+    def create_sharednetwork(self, subnet):
+        pass
+
+    def create_l2domain(self, netpart_id, subnet):
+        subl2dom = {
+            'subnet_id': subnet['id'],
+            'nuage_subnet_id': '52daa465-cf33-4efd-91d3-f5bc2aebd',
+            'net_partition_id': netpart_id,
+            'nuage_l2dom_tmplt_id': uuidutils.generate_uuid(),
+            'nuage_user_id': uuidutils.generate_uuid(),
+            'nuage_group_id': uuidutils.generate_uuid(),
+        }
+
+        return subl2dom
+
+    def create_domain(self, netpart, router):
+        entrtr = {
+            'router_id': router['id'],
+            'nuage_router_id': '2d782c02-b88e-44ad-a79b-4bdf11f7df3d',
+            'net_partition_id': netpart['id']
+        }
+
+        return entrtr
+
+    def create_domainsubnet(self, subnet, ports):
+        pass
+
+    def create_route(self, route):
+        pass
+
+    def create_vm(self, port):
+        pass
+
+    def create_security_group(self, secgrp, ports):
+        pass
+
+    def create_security_group_rule(self, secgrprule):
+        pass
+
+    def create_fip(self, fip, ipalloc):
+        pass
+
+    def associate_fip(self, fip):
+        pass
+
+    def disassociate_fip(self, fip):
+        pass

neutron/tests/unit/nuage/test_nuage_plugin.py

@@ -59,6 +59,22 @@ FAKE_ORGANIZATION = 'fake_org'
 _plugin_name = ('%s.NuagePlugin' % NUAGE_PLUGIN_PATH)
 
 
+def getNuageClient():
+    server = FAKE_SERVER
+    serverauth = FAKE_SERVER_AUTH
+    serverssl = FAKE_SERVER_SSL
+    base_uri = FAKE_BASE_URI
+    auth_resource = FAKE_AUTH_RESOURCE
+    organization = FAKE_ORGANIZATION
+    nuageclient = fake_nuageclient.FakeNuageClient(server,
+                                                   base_uri,
+                                                   serverssl,
+                                                   serverauth,
+                                                   auth_resource,
+                                                   organization)
+    return nuageclient
+
+
 class NuagePluginV2TestCase(test_db_plugin.NeutronDbPluginV2TestCase):
     def setUp(self, plugin=_plugin_name,
               ext_mgr=None, service_plugins=None):
@@ -67,19 +83,7 @@ class NuagePluginV2TestCase(test_db_plugin.NeutronDbPluginV2TestCase):
             self.skipTest("Nuage Plugin does not support IPV6.")
 
         def mock_nuageClient_init(self):
-            server = FAKE_SERVER
-            serverauth = FAKE_SERVER_AUTH
-            serverssl = FAKE_SERVER_SSL
-            base_uri = FAKE_BASE_URI
-            auth_resource = FAKE_AUTH_RESOURCE
-            organization = FAKE_ORGANIZATION
-            self.nuageclient = None
-            self.nuageclient = fake_nuageclient.FakeNuageClient(server,
-                                                                base_uri,
-                                                                serverssl,
-                                                                serverauth,
-                                                                auth_resource,
-                                                                organization)
+            self.nuageclient = getNuageClient()
 
         with mock.patch.object(nuage_plugin.NuagePlugin,
                                'nuageclient_init', new=mock_nuageClient_init):
@@ -541,4 +545,4 @@ class TestNuageSecurityGroupTestCase(NuagePluginV2TestCase,
                 # The Nuage plugin reserve the first port
                 port = ports['ports'][1]
                 self.assertEqual(1, len(port[ext_sg.SECURITYGROUPS]))
-                self._delete('ports', port['id'])
+                self._delete('ports', port['id'])

neutron/tests/unit/nuage/test_syncmanager.py

@@ -0,0 +1,343 @@
+# Copyright 2014 Alcatel-Lucent USA Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Sayaji Patil, Nuage Networks, Alcatel-Lucent USA Inc.
+
+import contextlib
+
+from neutron import context
+from neutron.openstack.common import uuidutils
+from neutron.plugins.nuage import nuage_models
+from neutron.plugins.nuage import syncmanager as sync
+from neutron.tests.unit.nuage import test_netpartition
+from neutron.tests.unit.nuage import test_nuage_plugin
+from neutron.tests.unit import test_extension_extraroute as extraroute_test
+from neutron.tests.unit import test_extension_security_group as test_sg
+from neutron.tests.unit import test_l3_plugin
+
+_uuid = uuidutils.generate_uuid
+
+
+class TestL3Sync(test_nuage_plugin.NuagePluginV2TestCase,
+                 test_l3_plugin.L3NatDBIntTestCase):
+
+    def setUp(self):
+        self.session = context.get_admin_context().session
+        self.syncmanager = sync.SyncManager(
+            test_nuage_plugin.getNuageClient())
+        super(TestL3Sync, self).setUp()
+
+    def _make_floatingip_for_tenant_port(self, net_id, port_id, tenant_id):
+        data = {'floatingip': {'floating_network_id': net_id,
+                               'tenant_id': tenant_id,
+                               'port_id': port_id}}
+        floatingip_req = self.new_create_request('floatingips', data, self.fmt)
+        res = floatingip_req.get_response(self.ext_api)
+        return self.deserialize(self.fmt, res)
+
+    def test_router_sync(self):
+        # If the router exists in neutron and not in VSD,
+        # sync will create it in VSD. But the nuage_router_id
+        # will now change and will be updated in neutron
+        # accordingly
+        rtr_res = self._create_router('json', 'foo', 'test-router', True)
+        router = self.deserialize('json', rtr_res)
+
+        self.syncmanager.synchronize('250')
+
+        # Check that the nuage_router_id is updated in entrtrmapping table
+        router_db = self.session.query(
+            nuage_models.NetPartitionRouter).filter_by(
+                router_id=router['router']['id']).first()
+
+        self.assertEqual('2d782c02-b88e-44ad-a79b-4bdf11f7df3d',
+                         router_db['nuage_router_id'])
+
+        self._delete('routers', router['router']['id'])
+
+    def test_router_deleted_get(self):
+        data = self.syncmanager._get_router_data(_uuid())
+        self.assertIsNone(data[0])
+        self.assertIsNone(data[1])
+
+    def test_fip_sync(self):
+        with self.subnet(cidr='200.0.0.0/24') as public_sub:
+            self._set_net_external(public_sub['subnet']['network_id'])
+            with contextlib.nested(self.port(), self.port(), self.port()) as (
+                p1, p2, p3):
+                p1_id = p1['port']['id']
+                p2_id = p2['port']['id']
+                p3_id = p3['port']['id']
+                with contextlib.nested(self.floatingip_with_assoc(
+                    port_id=p1_id), self.floatingip_with_assoc(
+                    port_id=p2_id), self.floatingip_with_assoc(
+                    port_id=p3_id)) as (fip1, fip2, fip3):
+                    fip_dict = {'fip': {
+                        'add': [fip1['floatingip']['id']],
+                        'associate': [fip2['floatingip']['id']],
+                        'disassociate': [fip3['floatingip']['id']]
+                    }}
+                    self.syncmanager._sync_fips(fip_dict)
+
+    def test_deleted_fip_sync(self):
+        fip_dict = {'fip': {
+            'add': [_uuid()],
+            'associate': [_uuid()],
+            'disassociate': [_uuid()]
+        }}
+        self.syncmanager._sync_fips(fip_dict)
+
+    def test_fip_and_ipalloc_get(self):
+        with self.subnet(cidr='200.0.0.0/24') as public_sub:
+            self._set_net_external(public_sub['subnet']['network_id'])
+            with self.port() as port:
+                p_id = port['port']['id']
+                with self.floatingip_with_assoc(port_id=p_id) as fip:
+
+                    data = self.syncmanager._get_fip_data(
+                        fip['floatingip']['id'])
+
+                    self.assertEqual(fip['floatingip']['id'], data['id'])
+
+                    data = self.syncmanager._get_ipalloc_for_fip(
+                        fip['floatingip'])
+                    self.assertEqual(fip['floatingip']['floating_ip_address'],
+                                     data['ip_address'])
+
+    def test_fip_and_ipalloc_deleted_get(self):
+        data = self.syncmanager._get_fip_data(_uuid())
+        self.assertIsNone(data)
+
+        fip = {
+            'id': _uuid(),
+            'floating_network_id': _uuid(),
+            'floating_ip_address': '176.176.10.10'
+        }
+        data = self.syncmanager._get_ipalloc_for_fip(fip)
+        self.assertIsNone(data)
+
+    def test_domainsubnet_sync(self):
+        with self.subnet() as s1:
+            with contextlib.nested(
+                    self.router(),
+                    self.port()) as (r1, p1):
+                self._router_interface_action(
+                    'add', r1['router']['id'],
+                    s1['subnet']['id'], p1['port']['id'])
+                domainsubn_dict = {
+                    'domainsubnet': {'add': [s1['subnet']['id']]},
+                    'port': {'sub_rtr_intf_port_dict': {s1['subnet']['id']:
+                                                        p1['port']['id']}}}
+                self.syncmanager.sync_domainsubnets(domainsubn_dict)
+                self._router_interface_action('remove', r1['router']['id'],
+                                              s1['subnet']['id'], None)
+
+    def test_floatingip_update_different_router(self):
+        self._test_floatingip_update_different_router()
+
+    def test_floatingip_update_different_fixed_ip_same_port(self):
+        self._test_floatingip_update_different_fixed_ip_same_port()
+
+    def test_floatingip_create_different_fixed_ip_same_port(self):
+        self._test_floatingip_create_different_fixed_ip_same_port()
+
+    def test_network_update_external_failure(self):
+        self._test_network_update_external_failure()
+
+
+class TestExtraRouteSync(extraroute_test.ExtraRouteDBIntTestCase):
+
+    def setUp(self):
+        self.session = context.get_admin_context().session
+        self.syncmanager = sync.SyncManager(
+            test_nuage_plugin.getNuageClient())
+        super(TestExtraRouteSync, self).setUp()
+
+    def test_route_sync(self):
+        route = {'destination': '135.207.0.0/16', 'nexthop': '10.0.1.3'}
+        with self.router() as r:
+            with self.subnet(cidr='10.0.1.0/24') as s:
+                net_id = s['subnet']['network_id']
+                res = self._create_port('json', net_id)
+                p = self.deserialize(self.fmt, res)
+                self._routes_update_prepare(r['router']['id'],
+                                            None, p['port']['id'], [route])
+
+                route_dict = {'route': {'add': [route]}}
+                self.syncmanager.sync_routes(route_dict)
+
+                self._routes_update_cleanup(p['port']['id'],
+                                            None, r['router']['id'], [])
+
+    def test_route_get(self):
+        routes = [{'destination': '135.207.0.0/16', 'nexthop': '10.0.1.3'}]
+        with self.router() as r:
+            with self.subnet(cidr='10.0.1.0/24') as s:
+                net_id = s['subnet']['network_id']
+                res = self._create_port('json', net_id)
+                p = self.deserialize(self.fmt, res)
+                self._routes_update_prepare(r['router']['id'],
+                                            None, p['port']['id'], routes)
+
+                data = self.syncmanager._get_route_data(routes[0])
+                self.assertEqual(routes[0]['destination'], data['destination'])
+                self.assertEqual(routes[0]['nexthop'], data['nexthop'])
+                self._routes_update_cleanup(p['port']['id'],
+                                            None, r['router']['id'], [])
+
+    def test_route_deleted_get(self):
+        route = {'destination': '135.207.0.0/16', 'nexthop': '10.0.1.3'}
+        data = self.syncmanager._get_route_data(route)
+        self.assertIsNone(data)
+
+
+class TestNetPartSync(test_netpartition.NetPartitionTestCase):
+
+    def setUp(self):
+        self.session = context.get_admin_context().session
+        self.syncmanager = sync.SyncManager(
+            test_nuage_plugin.getNuageClient())
+        super(TestNetPartSync, self).setUp()
+
+    def test_net_partition_sync(self):
+        # If the net-partition exists in neutron and not in VSD,
+        # sync will create it in VSD. But the net-partition
+        # id will now change and has to be updated in neutron
+        # accordingly
+        netpart = self._make_netpartition('json', 'sync-new-netpartition')
+
+        self.syncmanager.synchronize('250')
+
+        # Check that the net-partition id is updated in db
+        netpart_db = self.session.query(
+            nuage_models.NetPartition).filter_by(name=netpart['net_partition'][
+                'name']).first()
+
+        self.assertEqual('a917924f-3139-4bdb-a4c3-ea7c8011582f',
+                         netpart_db['id'])
+        self._del_netpartition(netpart_db['id'])
+
+    def test_net_partition_deleted_get(self):
+        data = self.syncmanager._get_netpart_data(_uuid())
+        self.assertIsNone(data)
+
+
+class TestL2Sync(test_nuage_plugin.NuagePluginV2TestCase):
+
+    def setUp(self):
+        self.session = context.get_admin_context().session
+        self.syncmanager = sync.SyncManager(
+            test_nuage_plugin.getNuageClient())
+        super(TestL2Sync, self).setUp()
+
+    def test_subnet_sync(self):
+        # If the subnet exists in neutron and not in VSD,
+        # sync will create it in VSD. But the nuage_subnet_id
+        # will now change and will be updated in neutron
+        # accordingly
+        net_res = self._create_network("json", "pub", True)
+        network = self.deserialize('json', net_res)
+
+        sub_res = self._create_subnet("json", network['network']['id'],
+                                      '10.0.0.0/24')
+        subnet = self.deserialize('json', sub_res)
+
+        self.syncmanager.synchronize('250')
+
+        # Check that the nuage_subnet_id is updated in db
+        subl2dom_db = self.session.query(
+            nuage_models.SubnetL2Domain).filter_by(subnet_id=subnet[
+                'subnet']['id']).first()
+        self.assertEqual('52daa465-cf33-4efd-91d3-f5bc2aebd',
+                         subl2dom_db['nuage_subnet_id'])
+
+        self._delete('subnets', subnet['subnet']['id'])
+        self._delete('networks', network['network']['id'])
+
+    def test_subnet_deleted_get(self):
+        data = self.syncmanager._get_subnet_data(_uuid())
+        self.assertIsNone(data[0])
+        self.assertIsNone(data[1])
+
+    def test_sharednetwork_sync(self):
+        with self.subnet(cidr='200.0.0.0/24') as public_sub:
+            sharednet_dict = {'sharednetwork': {'add': [public_sub['subnet'][
+                                                        'id']]}}
+            self.syncmanager.sync_sharednetworks(sharednet_dict)
+
+    def test_vm_sync(self):
+        with self.port() as p:
+            port_dict = {'port': {'vm': [p['port']['id']]}}
+            self.syncmanager.sync_vms(port_dict)
+
+
+class TestSecurityGroupSync(test_sg.TestSecurityGroups):
+
+    def setUp(self):
+        self.session = context.get_admin_context().session
+        self.syncmanager = sync.SyncManager(
+            test_nuage_plugin.getNuageClient())
+        super(TestSecurityGroupSync, self).setUp()
+
+    def test_sg_get(self):
+        with self.security_group() as sg:
+            data = self.syncmanager._get_sec_grp_data(
+                sg['security_group']['id'])
+            self.assertEqual(sg['security_group']['id'], data['id'])
+
+    def test_sg_deleted_get(self):
+        data = self.syncmanager._get_sec_grp_data(_uuid())
+        self.assertIsNone(data)
+
+    def test_sg_rule_get(self):
+        with self.security_group() as sg:
+            sg_rule_id = sg['security_group']['security_group_rules'][0]['id']
+            data = self.syncmanager._get_sec_grp_rule_data(sg_rule_id)
+            self.assertEqual(sg_rule_id, data['id'])
+
+    def test_sg_rule_deleted_get(self):
+        data = self.syncmanager._get_sec_grp_rule_data(_uuid())
+        self.assertIsNone(data)
+
+    def test_sg_grp_sync(self):
+        with contextlib.nested(self.security_group(),
+                               self.security_group()) as (sg1, sg2):
+            sg1_id = sg1['security_group']['id']
+            sg2_id = sg2['security_group']['id']
+            sg_dict = {'security': {'secgroup': {'l2domain': {'add': {sg1_id: [
+                _uuid()]}}, 'domain': {'add': {sg2_id: [_uuid()]}}}}}
+            self.syncmanager.sync_secgrps(sg_dict)
+
+    def test_deleted_sg_grp_sync(self):
+        sg_dict = {'security': {'secgroup': {'l2domain': {'add': {_uuid(): [
+            _uuid()]}}, 'domain': {'add': {_uuid(): [_uuid()]}}}}}
+        self.syncmanager.sync_secgrps(sg_dict)
+
+    def test_sg_rule_sync(self):
+        with contextlib.nested(self.security_group(),
+                               self.security_group()) as (sg1, sg2):
+            sg1_rule_id = (
+                sg1['security_group']['security_group_rules'][0]['id'])
+            sg2_rule_id = (
+                sg2['security_group']['security_group_rules'][0]['id'])
+
+            sg_dict = {'security': {'secgrouprule': {'l2domain': {
+                'add': [sg1_rule_id]}, 'domain': {'add': [sg2_rule_id]}}}}
+            self.syncmanager.sync_secgrp_rules(sg_dict)
+
+    def test_deleted_sg_grp_rule_sync(self):
+        sg_dict = {'security': {'secgrouprule':
+                                {'l2domain': {'add': [_uuid()]},
+                                 'domain': {'add': [_uuid()]}}}}
+        self.syncmanager.sync_secgrp_rules(sg_dict)