Merge "NSXv3Admin: Fix mismatches for security-groups"

vmware_nsx/nsxlib/v3/dfw_api.py

@@ -139,9 +139,12 @@ def _update_nsgroup_with_members(nsgroup_id, members, action):
     return nsxclient.create_resource(members_update, members)
 
 
-def add_nsgroup_member(nsgroup_id, target_type, target_id):
+def add_nsgroup_members(nsgroup_id, target_type, target_ids):
+    members = []
+    for target_id in target_ids:
         member_expr = get_nsgroup_member_expression(target_type, target_id)
-    members = {'members': [member_expr]}
+        members.append(member_expr)
+    members = {'members': members}
     try:
         return _update_nsgroup_with_members(nsgroup_id, members, ADD_MEMBERS)
     except (nsx_exc.StaleRevision, nsx_exc.ResourceNotFound):
@@ -149,10 +152,10 @@ def add_nsgroup_member(nsgroup_id, target_type, target_id):
     except nsx_exc.ManagerError:
         # REVISIT(roeyc): A ManagerError might have been raised for a
         # different reason, e.g - NSGroup does not exists.
-        LOG.warning(_LW("Failed to add %(target_type)s %(target_id)s to "
+        LOG.warning(_LW("Failed to add %(target_type)s resources "
-                        "NSGroup %(nsgroup_id)s"),
+                        "(%(target_ids))s to NSGroup %(nsgroup_id)s"),
                     {'target_type': target_type,
-                     'target_id': target_id,
+                     'target_ids': target_ids,
                      'nsgroup_id': nsgroup_id})
         raise NSGroupIsFull(nsgroup_id=nsgroup_id)
 
@@ -175,7 +178,13 @@ def read_nsgroup(nsgroup_id):
 
 
 def delete_nsgroup(nsgroup_id):
-    return nsxclient.delete_resource('ns-groups/%s?force=true' % nsgroup_id)
+    try:
+        return nsxclient.delete_resource('ns-groups/%s?force=true'
+                                         % nsgroup_id)
+    #FIXME(roeyc): Should only except NotFound error.
+    except Exception:
+        LOG.debug("NSGroup %s does not exists for delete request.",
+                  nsgroup_id)
 
 
 def _build_section(display_name, description, applied_tos, tags):
@@ -231,7 +240,12 @@ def list_sections():
 
 def delete_section(section_id):
     resource = 'firewall/sections/%s?cascade=true' % section_id
+    try:
         return nsxclient.delete_resource(resource)
+    #FIXME(roeyc): Should only except NotFound error.
+    except Exception:
+        LOG.debug("Firewall section %s does not exists for delete request.",
+                  section_id)
 
 
 def get_nsgroup_reference(nsgroup_id):

vmware_nsx/nsxlib/v3/security.py

@@ -254,8 +254,8 @@ def update_lport_with_security_groups(context, lport_id, original, updated):
     for sg_id in added:
         nsgroup_id, s = get_sg_mappings(context.session, sg_id)
         try:
-            firewall.add_nsgroup_member(
+            firewall.add_nsgroup_members(
-                nsgroup_id, firewall.LOGICAL_PORT, lport_id)
+                nsgroup_id, firewall.LOGICAL_PORT, [lport_id])
         except firewall.NSGroupIsFull:
             for sg_id in added:
                 nsgroup_id, s = get_sg_mappings(context.session, sg_id)
@@ -415,9 +415,9 @@ class NSGroupManager(object):
             try:
                 LOG.debug("Adding NSGroup %s to nested group %s",
                           nsgroup_id, group)
-                firewall.add_nsgroup_member(group,
+                firewall.add_nsgroup_members(group,
                                              firewall.NSGROUP,
-                                            nsgroup_id)
+                                             [nsgroup_id])
                 break
             except firewall.NSGroupIsFull:
                 LOG.debug("Nested group %(group_id)s is full, trying the "

vmware_nsx/plugins/nsx_v3/plugin.py

@@ -2661,14 +2661,45 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin,
                 marker=marker, page_reverse=page_reverse,
                 default_sg=default_sg)
 
+    def _create_fw_section_for_secgroup(self, nsgroup, is_provider):
+        # NOTE(arosen): if a security group is provider we want to
+        # insert our rules at the top.
+        operation = (firewall.INSERT_TOP
+                     if is_provider
+                     else firewall.INSERT_BEFORE)
+
+        # security-group rules are located in a dedicated firewall section.
+        firewall_section = (
+            firewall.create_empty_section(
+                nsgroup['display_name'], nsgroup['description'],
+                [nsgroup['id']], nsgroup['tags'],
+                operation=operation,
+                other_section=self.default_section))
+        return firewall_section
+
+    def _create_security_group_backend_resources(self, secgroup):
+        tags = utils.build_v3_tags_payload(
+            secgroup, resource_type='os-neutron-secgr-id',
+            project_name=secgroup['tenant_id'])
+        name = security.get_nsgroup_name(secgroup)
+
+        if utils.is_nsx_version_1_1_0(self._nsx_version):
+                tag_expression = (
+                    firewall.get_nsgroup_port_tag_expression(
+                        security.PORT_SG_SCOPE, secgroup['id']))
+        else:
+            tag_expression = None
+
+        ns_group = firewall.create_nsgroup(
+            name, secgroup['description'], tags, tag_expression)
+        # security-group rules are located in a dedicated firewall section.
+        firewall_section = self._create_fw_section_for_secgroup(
+            ns_group, secgroup.get(provider_sg.PROVIDER))
+        return ns_group, firewall_section
+
     def create_security_group(self, context, security_group, default_sg=False):
         secgroup = security_group['security_group']
         secgroup['id'] = secgroup.get('id') or uuidutils.generate_uuid()
-
-        tags = utils.build_v3_tags_payload(
-            secgroup, resource_type='os-neutron-secgr-id',
-            project_name=context.tenant_name)
-        name = security.get_nsgroup_name(secgroup)
         ns_group = {}
         firewall_section = {}
 
@@ -2676,34 +2707,8 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin,
             tenant_id = secgroup['tenant_id']
             self._ensure_default_security_group(context, tenant_id)
         try:
-            if utils.is_nsx_version_1_1_0(self._nsx_version):
+            ns_group, firewall_section = (
-                tag_expression = (
+                self._create_security_group_backend_resources(secgroup))
-                    firewall.get_nsgroup_port_tag_expression(
-                        security.PORT_SG_SCOPE, secgroup['id']))
-            else:
-                tag_expression = None
-            # NOTE(roeyc): We first create the nsgroup so that once the sg is
-            # saved into db its already backed up by an nsx resource.
-            ns_group = firewall.create_nsgroup(
-                name, secgroup['description'], tags, tag_expression)
-
-            operation = firewall.INSERT_BEFORE
-            action = firewall.ALLOW
-            if secgroup.get(provider_sg.PROVIDER) is True:
-                # NOTE(arosen): if a security group is provider we want to
-                # insert our rules at the top.
-                operation = firewall.INSERT_TOP
-                # We also want to block the traffic as provider rules are
-                # drops.
-                action = firewall.DROP
-
-            # security-group rules are located in a dedicated firewall section.
-            firewall_section = (
-                firewall.create_empty_section(
-                    name, secgroup.get('description', ''), [ns_group['id']],
-                    tags, operation=operation,
-                    other_section=self.default_section))
-
             # REVISIT(roeyc): Ideally, at this point we need not be under an
             # open db transactions, however, unittests fail if omitting
             # subtransactions=True.
@@ -2753,6 +2758,9 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin,
                 # translate and creates firewall rules.
                 logging = (cfg.CONF.nsx_v3.log_security_groups_allowed_traffic
                            or secgroup.get(sg_logging.LOGGING, False))
+                action = (firewall.DROP
+                          if secgroup.get(provider_sg.PROVIDER)
+                          else firewall.ALLOW)
                 rules = security.create_firewall_rules(
                     context, firewall_section['id'], ns_group['id'],
                     logging, action, sg_rules)

vmware_nsx/shell/admin/plugins/common/utils.py

@@ -17,6 +17,8 @@ import six
 import sys
 
 from vmware_nsx._i18n import _LI, _
+from neutron.callbacks import registry
+from vmware_nsx.shell import resources as nsxadmin
 
 LOG = logging.getLogger(__name__)
 
@@ -83,3 +85,27 @@ def query_yes_no(question, default="yes"):
         else:
             sys.stdout.write("Please respond with 'yes' or 'no' "
                              "(or 'y' or 'n').\n")
+
+
+def list_handler(resource):
+    def wrap(func):
+        registry.subscribe(func, resource,
+                           nsxadmin.Operations.LIST.value)
+        return func
+    return wrap
+
+
+def list_mismatches_handler(resource):
+    def wrap(func):
+        registry.subscribe(func, resource,
+                           nsxadmin.Operations.LIST_MISMATCHES.value)
+        return func
+    return wrap
+
+
+def fix_mismatches_handler(resource):
+    def wrap(func):
+        registry.subscribe(func, resource,
+                           nsxadmin.Operations.FIX_MISMATCH.value)
+        return func
+    return wrap

vmware_nsx/shell/admin/plugins/nsxv/resources/securitygroups.py

@@ -16,7 +16,6 @@
 import logging
 import xml.etree.ElementTree as et
 
-from neutron.callbacks import registry
 from neutron import context
 from neutron.db import models_v2
 from neutron.db import securitygroups_db
@@ -28,7 +27,6 @@ from vmware_nsx.shell.admin.plugins.common import constants
 from vmware_nsx.shell.admin.plugins.common import formatters
 from vmware_nsx.shell.admin.plugins.common import utils as admin_utils
 from vmware_nsx.shell.admin.plugins.nsxv.resources import utils
-from vmware_nsx.shell import resources as nsxadmin
 
 
 LOG = logging.getLogger(__name__)
@@ -133,31 +131,7 @@ def _log_info(resource, data, attrs=['name', 'id']):
     LOG.info(formatters.output_formatter(resource, data, attrs))
 
 
-def list_handler(resource):
+@admin_utils.list_handler(constants.SECURITY_GROUPS)
-    def wrap(func):
-        registry.subscribe(func, resource,
-                           nsxadmin.Operations.LIST.value)
-        return func
-    return wrap
-
-
-def list_mismatches_handler(resource):
-    def wrap(func):
-        registry.subscribe(func, resource,
-                           nsxadmin.Operations.LIST_MISMATCHES.value)
-        return func
-    return wrap
-
-
-def fix_mismatches_handler(resource):
-    def wrap(func):
-        registry.subscribe(func, resource,
-                           nsxadmin.Operations.FIX_MISMATCH.value)
-        return func
-    return wrap
-
-
-@list_handler(constants.SECURITY_GROUPS)
 @admin_utils.output_header
 def neutron_list_security_groups_mappings(resource, event, trigger, **kwargs):
     sg_mappings = neutron_sg.get_security_groups_mappings()
@@ -167,7 +141,7 @@ def neutron_list_security_groups_mappings(resource, event, trigger, **kwargs):
     return bool(sg_mappings)
 
 
-@list_handler(constants.FIREWALL_SECTIONS)
+@admin_utils.list_handler(constants.FIREWALL_SECTIONS)
 @admin_utils.output_header
 def nsx_list_dfw_sections(resource, event, trigger, **kwargs):
     fw_sections = nsxv_firewall.list_fw_sections()
@@ -175,7 +149,7 @@ def nsx_list_dfw_sections(resource, event, trigger, **kwargs):
     return bool(fw_sections)
 
 
-@list_handler(constants.FIREWALL_NSX_GROUPS)
+@admin_utils.list_handler(constants.FIREWALL_NSX_GROUPS)
 @admin_utils.output_header
 def nsx_list_security_groups(resource, event, trigger, **kwargs):
     nsx_secgroups = nsxv_firewall.list_security_groups()
@@ -196,7 +170,7 @@ def _find_missing_security_groups():
     return missing_secgroups
 
 
-@list_mismatches_handler(constants.FIREWALL_NSX_GROUPS)
+@admin_utils.list_mismatches_handler(constants.FIREWALL_NSX_GROUPS)
 @admin_utils.output_header
 def list_missing_security_groups(resource, event, trigger, **kwargs):
     sgs_with_missing_nsx_group = _find_missing_security_groups()
@@ -225,7 +199,7 @@ def _find_missing_sections():
     return missing_sections
 
 
-@list_mismatches_handler(constants.FIREWALL_SECTIONS)
+@admin_utils.list_mismatches_handler(constants.FIREWALL_SECTIONS)
 @admin_utils.output_header
 def list_missing_firewall_sections(resource, event, trigger, **kwargs):
     sgs_with_missing_section = _find_missing_sections()
@@ -238,7 +212,7 @@ def list_missing_firewall_sections(resource, event, trigger, **kwargs):
     return bool(missing_sections_info)
 
 
-@fix_mismatches_handler(constants.SECURITY_GROUPS)
+@admin_utils.fix_mismatches_handler(constants.SECURITY_GROUPS)
 @admin_utils.output_header
 def fix_security_groups(resource, event, trigger, **kwargs):
     context_ = context.get_admin_context()

vmware_nsx/shell/admin/plugins/nsxv3/resources/securitygroups.py

@@ -17,17 +17,20 @@ import logging
 from neutron.callbacks import registry
 from neutron import context as neutron_context
 from neutron.db import common_db_mixin as common_db
-from neutron.db import securitygroups_db as sg_db
+from neutron.db import securitygroups_db
 
 from vmware_nsx.common import utils
 from vmware_nsx.db import db as nsx_db
+from vmware_nsx.db import nsx_models
+from vmware_nsx.extensions import providersecuritygroup as provider_sg
+from vmware_nsx.extensions import securitygrouplogging as sg_logging
 from vmware_nsx.shell.admin.plugins.common import constants
 from vmware_nsx.shell.admin.plugins.common import formatters
 from vmware_nsx.shell.admin.plugins.common import utils as admin_utils
 from vmware_nsx.shell.admin.plugins.nsxv3.resources import ports
 from vmware_nsx.shell.admin.plugins.nsxv3.resources import utils as v3_utils
 from vmware_nsx.shell import resources as shell
-from vmware_nsx._i18n import _LE, _LI, _LW
+from vmware_nsx._i18n import _LE, _LW
 from vmware_nsx.nsxlib import v3 as nsxlib
 from vmware_nsx.nsxlib.v3 import dfw_api as firewall
 from vmware_nsx.nsxlib.v3 import security
@@ -35,7 +38,7 @@ from vmware_nsx.nsxlib.v3 import security
 LOG = logging.getLogger(__name__)
 
 
-class NeutronSecurityGroupApi(sg_db.SecurityGroupDbMixin,
+class NeutronSecurityGroupApi(securitygroups_db.SecurityGroupDbMixin,
                               common_db.CommonDbMixin):
     def __init__(self):
         super(NeutronSecurityGroupApi, self)
@@ -58,96 +61,183 @@ class NeutronSecurityGroupApi(sg_db.SecurityGroupDbMixin,
             self.context, {'port_id': [port_id]})
         return [b['security_group_id'] for b in secgroups_bindings]
 
-    def get_security_group_ports(self, security_group_id):
+    def get_ports_in_security_group(self, security_group_id):
         secgroups_bindings = self._get_port_security_group_bindings(
             self.context, {'security_group_id': [security_group_id]})
         return [b['port_id'] for b in secgroups_bindings]
 
+    def delete_security_group_section_mapping(self, sg_id):
+        fw_mapping = self.context.session.query(
+            nsx_models.NeutronNsxFirewallSectionMapping).filter_by(
+                neutron_id=sg_id).one_or_none()
+        if fw_mapping:
+            with self.context.session.begin(subtransactions=True):
+                self.context.session.delete(fw_mapping)
+
+    def delete_security_group_backend_mapping(self, sg_id):
+        sg_mapping = self.context.session.query(
+            nsx_models.NeutronNsxSecurityGroupMapping).filter_by(
+                neutron_id=sg_id).one_or_none()
+        if sg_mapping:
+            with self.context.session.begin(subtransactions=True):
+                self.context.session.delete(sg_mapping)
+
+    def get_security_groups_mappings(self):
+        q = self.context.session.query(
+            securitygroups_db.SecurityGroup.name,
+            securitygroups_db.SecurityGroup.id,
+            nsx_models.NeutronNsxFirewallSectionMapping.nsx_id,
+            nsx_models.NeutronNsxSecurityGroupMapping.nsx_id).join(
+                nsx_models.NeutronNsxFirewallSectionMapping,
+                nsx_models.NeutronNsxSecurityGroupMapping).all()
+        sg_mappings = [{'name': mapp[0],
+                        'id': mapp[1],
+                        'section-id': mapp[2],
+                        'nsx-securitygroup-id': mapp[3]}
+                       for mapp in q]
+        return sg_mappings
+
+    def get_logical_port_id(self, port_id):
+        mapping = self.context.session.query(
+            nsx_models.NeutronNsxPortMapping).filter_by(
+                neutron_id=port_id).one_or_none()
+        if mapping:
+            return mapping.nsx_id
+
 
 neutron_sg = NeutronSecurityGroupApi()
 neutron_db = v3_utils.NeutronDbClient()
 
 
+def _log_info(resource, data, attrs=['display_name', 'id']):
+    LOG.info(formatters.output_formatter(resource, data, attrs))
+
+
+@admin_utils.list_handler(constants.SECURITY_GROUPS)
+@admin_utils.output_header
+def list_security_groups_mappings(resource, event, trigger, **kwargs):
+    sg_mappings = neutron_sg.get_security_groups_mappings()
+    _log_info(constants.SECURITY_GROUPS,
+              sg_mappings,
+              attrs=['name', 'id', 'section-id', 'nsx-securitygroup-id'])
+    return bool(sg_mappings)
+
+
+@admin_utils.list_handler(constants.FIREWALL_SECTIONS)
+@admin_utils.output_header
+def nsx_list_dfw_sections(resource, event, trigger, **kwargs):
+    fw_sections = firewall.list_sections()
+    _log_info(constants.FIREWALL_SECTIONS, fw_sections)
+    return bool(fw_sections)
+
+
+@admin_utils.list_handler(constants.FIREWALL_NSX_GROUPS)
 @admin_utils.output_header
 def nsx_list_security_groups(resource, event, trigger, **kwargs):
-    sections = firewall.list_sections()
+    nsx_secgroups = firewall.list_nsgroups()
-    LOG.info(formatters.output_formatter(constants.FIREWALL_SECTIONS,
+    _log_info(constants.FIREWALL_NSX_GROUPS, nsx_secgroups)
-                                         sections, ['display_name', 'id']))
+    return bool(nsx_secgroups)
-    nsgroups = firewall.list_nsgroups()
-    LOG.info(formatters.output_formatter(constants.FIREWALL_NSX_GROUPS,
-                                         nsgroups, ['display_name', 'id']))
-    return bool(sections) or bool(nsgroups)
 
 
+def _find_missing_security_groups():
+    nsx_secgroups = firewall.list_nsgroups()
+    sg_mappings = neutron_sg.get_security_groups_mappings()
+    missing_secgroups = {}
+    for sg_db in sg_mappings:
+        for nsx_sg in nsx_secgroups:
+            if nsx_sg['id'] == sg_db['nsx-securitygroup-id']:
+                break
+        else:
+            missing_secgroups[sg_db['id']] = sg_db
+    return missing_secgroups
+
+
+@admin_utils.list_mismatches_handler(constants.FIREWALL_NSX_GROUPS)
 @admin_utils.output_header
-def nsx_delete_security_groups(resource, event, trigger, **kwargs):
+def list_missing_security_groups(resource, event, trigger, **kwargs):
-    if 'force' in kwargs and kwargs['force'] is False:
+    sgs_with_missing_nsx_group = _find_missing_security_groups()
-        if nsx_list_security_groups(resource, event, trigger, **kwargs):
+    missing_securitgroups_info = [
-            msg = ('Do you want to delete the following NSX firewall '
+        {'securitygroup-name': sg['name'],
-                   'sections/nsgroups?')
+         'securitygroup-id': sg['id'],
-            user_confirm = admin_utils.query_yes_no(msg, default='no')
+         'nsx-securitygroup-id':
-
+         sg['nsx-securitygroup-id']}
-            if user_confirm is False:
+        for sg in sgs_with_missing_nsx_group.values()]
-                LOG.info(_LI('NSX security groups cleanup aborted by user'))
+    _log_info(constants.FIREWALL_NSX_GROUPS, missing_securitgroups_info,
-                return
+              attrs=['securitygroup-name', 'securitygroup-id',
-
+                     'nsx-securitygroup-id'])
-    sections = firewall.list_sections()
+    return bool(missing_securitgroups_info)
-    # NOTE(roeyc): We use -2 indexing because don't want to delete the
-    # default firewall sections.
-    if sections:
-        NON_DEFAULT_SECURITY_GROUPS = -2
-        for section in sections[:NON_DEFAULT_SECURITY_GROUPS]:
-            LOG.info(_LI("Deleting firewall section %(display_name)s, "
-                         "section id %(id)s"),
-                     {'display_name': section['display_name'],
-                      'id': section['id']})
-            firewall.delete_section(section['id'])
-
-    nsgroups = firewall.list_nsgroups()
-    if nsgroups:
-        for nsgroup in [nsg for nsg in nsgroups
-                        if not utils.is_internal_resource(nsg)]:
-            LOG.info(_LI("Deleting ns-group %(display_name)s, "
-                         "ns-group id %(id)s"),
-                     {'display_name': nsgroup['display_name'],
-                      'id': nsgroup['id']})
-            firewall.delete_nsgroup(nsgroup['id'])
 
 
+def _find_missing_sections():
+    fw_sections = firewall.list_sections()
+    sg_mappings = neutron_sg.get_security_groups_mappings()
+    missing_sections = {}
+    for sg_db in sg_mappings:
+        for fw_section in fw_sections:
+            if fw_section['id'] == sg_db['section-id']:
+                break
+        else:
+            missing_sections[sg_db['id']] = sg_db
+    return missing_sections
+
+
+@admin_utils.list_mismatches_handler(constants.FIREWALL_SECTIONS)
 @admin_utils.output_header
-def neutron_list_security_groups(resource, event, trigger, **kwargs):
+def list_missing_firewall_sections(resource, event, trigger, **kwargs):
-    security_groups = neutron_sg.get_security_groups()
+    sgs_with_missing_section = _find_missing_sections()
-    LOG.info(formatters.output_formatter(constants.SECURITY_GROUPS,
+    missing_sections_info = [{'securitygroup-name': sg['name'],
-                                         security_groups, ['name', 'id']))
+                              'securitygroup-id': sg['id'],
-    return bool(security_groups)
+                              'section-id': sg['section-id']}
+                             for sg in sgs_with_missing_section.values()]
+    _log_info(constants.FIREWALL_SECTIONS, missing_sections_info,
+              attrs=['securitygroup-name', 'securitygroup-id', 'section-id'])
+    return bool(missing_sections_info)
 
 
+@admin_utils.fix_mismatches_handler(constants.SECURITY_GROUPS)
 @admin_utils.output_header
-def neutron_delete_security_groups(resource, event, trigger, **kwargs):
+def fix_security_groups(resource, event, trigger, **kwargs):
-    if 'force' in kwargs and kwargs['force'] is False:
+    context_ = neutron_context.get_admin_context()
-        if neutron_list_security_groups(resource, event, trigger, **kwargs):
+    plugin = v3_utils.NsxV3PluginWrapper()
-            msg = ('Do you want to delete the following neutron '
+    inconsistent_secgroups = _find_missing_sections()
-                   'security groups?')
+    inconsistent_secgroups.update(_find_missing_security_groups())
-            user_confirm = admin_utils.query_yes_no(msg, default='no')
-            if user_confirm is False:
-                LOG.info(_LI('Neutron security groups cleanup aborted by '
-                             'user'))
-                return
 
-    security_groups = neutron_sg.get_security_groups()
+    for sg_id, sg in inconsistent_secgroups.items():
-    if not security_groups:
+        secgroup = plugin.get_security_group(context_, sg_id)
-        return
+        firewall.delete_section(sg['section-id'])
+        firewall.delete_nsgroup(sg['nsx-securitygroup-id'])
+        neutron_sg.delete_security_group_section_mapping(sg_id)
+        neutron_sg.delete_security_group_backend_mapping(sg_id)
+        nsgroup, fw_section = (
+            plugin._create_security_group_backend_resources(secgroup))
+        security.save_sg_mappings(
+            context_.session, sg_id, nsgroup['id'], fw_section['id'])
+        # If version > 1.1 then we use dynamic criteria tags, and the port
+        # should already have them.
+        if not utils.is_nsx_version_1_1_0(plugin._nsx_version):
+            members = []
+            for port_id in neutron_db.get_ports_in_security_group(sg_id):
+                lport_id = neutron_db.get_logical_port_id(port_id)
+                members.append(lport_id)
+            firewall.add_nsgroup_members(
+                nsgroup['id'], firewall.LOGICAL_PORT, members)
 
-    for security_group in security_groups:
+        for rule in secgroup['security_group_rules']:
-        try:
+            rule_mapping = (context_.session.query(
-            LOG.info(_LI('Trying to delete %(sg_id)s'),
+                nsx_models.NeutronNsxRuleMapping).filter_by(
-                     {'sg_id': security_group['id']})
+                    neutron_id=rule['id']).one())
-            neutron_sg.delete_security_group(security_group['id'])
+            with context_.session.begin(subtransactions=True):
-            LOG.info(_LI("Deleted security group name: %(name)s id: %(id)s"),
+                context_.session.delete(rule_mapping)
-                     {'name': security_group['name'],
+        action = (firewall.DROP
-                      'id': security_group['id']})
+                  if secgroup.get(provider_sg.PROVIDER)
-        except Exception as e:
+                  else firewall.ALLOW)
-            LOG.warning(str(e))
+        rules = security.create_firewall_rules(
+            context_, fw_section['id'], nsgroup['id'],
+            secgroup.get(sg_logging.LOGGING, False), action,
+            secgroup['security_group_rules'])
+        security.save_sg_rule_mappings(context_.session, rules['rules'])
+        # Add nsgroup to a nested group
+        plugin.nsgroup_manager.add_nsgroup(nsgroup['id'])
 
 
 def _update_ports_dynamic_criteria_tags():
@@ -197,33 +287,6 @@ def migrate_nsgroups_to_dynamic_criteria(resource, event, trigger, **kwargs):
     _update_security_group_dynamic_criteria()
 
 
-registry.subscribe(nsx_list_security_groups,
-                   constants.SECURITY_GROUPS,
-                   shell.Operations.LIST.value)
-registry.subscribe(nsx_list_security_groups,
-                   constants.SECURITY_GROUPS,
-                   shell.Operations.NSX_LIST.value)
-
-registry.subscribe(neutron_list_security_groups,
-                   constants.SECURITY_GROUPS,
-                   shell.Operations.LIST.value)
-registry.subscribe(neutron_list_security_groups,
-                   constants.SECURITY_GROUPS,
-                   shell.Operations.NEUTRON_LIST.value)
-
-registry.subscribe(nsx_delete_security_groups,
-                   constants.SECURITY_GROUPS,
-                   shell.Operations.CLEAN.value)
-registry.subscribe(nsx_delete_security_groups,
-                   constants.SECURITY_GROUPS,
-                   shell.Operations.NSX_CLEAN.value)
-
-registry.subscribe(neutron_delete_security_groups,
-                   constants.SECURITY_GROUPS,
-                   shell.Operations.CLEAN.value)
-registry.subscribe(neutron_delete_security_groups,
-                   constants.SECURITY_GROUPS,
-                   shell.Operations.NEUTRON_CLEAN.value)
 registry.subscribe(migrate_nsgroups_to_dynamic_criteria,
                    constants.FIREWALL_NSX_GROUPS,
                    shell.Operations.MIGRATE_TO_DYNAMIC_CRITERIA.value)

vmware_nsx/shell/admin/plugins/nsxv3/resources/utils.py

@@ -16,6 +16,7 @@ from neutron import context
 from neutron.db import db_base_plugin_v2
 
 from vmware_nsx.db import db as nsx_db
+from vmware_nsx.plugins.nsx_v3 import plugin
 
 
 class NeutronDbClient(db_base_plugin_v2.NeutronDbPluginV2):
@@ -48,3 +49,22 @@ class NeutronDbClient(db_base_plugin_v2.NeutronDbPluginV2):
     def net_id_to_lswitch_id(self, net_id):
         lswitch_ids = nsx_db.get_nsx_switch_ids(self.context.session, net_id)
         return lswitch_ids[0] if lswitch_ids else None
+
+
+class NsxV3PluginWrapper(plugin.NsxV3Plugin):
+    def _init_dhcp_metadata(self):
+        pass
+
+    def _process_security_group_logging(self):
+        pass
+
+    def _init_port_security_profile(self):
+        return True
+
+    def _init_dhcp_switching_profile(self):
+        pass
+
+    def _extend_get_network_dict_provider(self, context, net):
+        self._extend_network_dict_provider(context, net)
+        # skip getting the Qos policy ID because get_object calls
+        # plugin init again on admin-util environment

vmware_nsx/shell/resources.py

@@ -61,14 +61,15 @@ class Resource(object):
 # Add supported NSX-V3 resources in this dictionary
 nsxv3_resources = {
     constants.SECURITY_GROUPS: Resource(constants.SECURITY_GROUPS,
-                                        [Operations.CLEAN.value,
+                                        [Operations.LIST.value,
-                                         Operations.LIST.value,
+                                         Operations.FIX_MISMATCH.value]),
-                                         Operations.NSX_LIST.value,
+    constants.FIREWALL_SECTIONS: Resource(constants.FIREWALL_SECTIONS,
-                                         Operations.NSX_CLEAN.value,
+                                          [Operations.LIST.value,
-                                         Operations.NEUTRON_LIST.value,
+                                           Operations.LIST_MISMATCHES.value]),
-                                         Operations.NEUTRON_CLEAN.value]),
     constants.FIREWALL_NSX_GROUPS: Resource(
         constants.FIREWALL_NSX_GROUPS, [
+            Operations.LIST.value,
+            Operations.LIST_MISMATCHES.value,
             Operations.MIGRATE_TO_DYNAMIC_CRITERIA.value]),
     constants.NETWORKS: Resource(constants.NETWORKS,
                                  [Operations.LIST_MISMATCHES.value]),

vmware_nsx/tests/unit/extensions/test_securitygroup.py

@@ -38,7 +38,7 @@ def _mock_create_and_list_nsgroups(test_method):
     def _create_nsgroup_mock(name, desc, tags, membership_criteria=None):
         nsgroup = {'id': NSG_IDS[len(nsgroups)],
                    'display_name': name,
-                   'desc': desc,
+                   'description': desc,
                    'tags': tags}
         nsgroups.append(nsgroup)
         return nsgroup
@@ -72,7 +72,7 @@ class TestSecurityGroupsNoDynamicCriteria(test_nsxv3.NsxV3PluginTestCaseMixin,
 
     @_mock_create_and_list_nsgroups
     @mock.patch.object(firewall, 'remove_nsgroup_member')
-    @mock.patch.object(firewall, 'add_nsgroup_member')
+    @mock.patch.object(firewall, 'add_nsgroup_members')
     def test_create_port_with_multiple_security_groups(self,
                                                        add_member_mock,
                                                        remove_member_mock):
@@ -87,7 +87,7 @@ class TestSecurityGroupsNoDynamicCriteria(test_nsxv3.NsxV3PluginTestCaseMixin,
 
     @_mock_create_and_list_nsgroups
     @mock.patch.object(firewall, 'remove_nsgroup_member')
-    @mock.patch.object(firewall, 'add_nsgroup_member')
+    @mock.patch.object(firewall, 'add_nsgroup_members')
     def test_update_port_with_multiple_security_groups(self,
                                                        add_member_mock,
                                                        remove_member_mock):
@@ -104,7 +104,7 @@ class TestSecurityGroupsNoDynamicCriteria(test_nsxv3.NsxV3PluginTestCaseMixin,
 
     @_mock_create_and_list_nsgroups
     @mock.patch.object(firewall, 'remove_nsgroup_member')
-    @mock.patch.object(firewall, 'add_nsgroup_member')
+    @mock.patch.object(firewall, 'add_nsgroup_members')
     def test_update_port_remove_security_group_empty_list(self,
                                                           add_member_mock,
                                                           remove_member_mock):
@@ -117,7 +117,7 @@ class TestSecurityGroupsNoDynamicCriteria(test_nsxv3.NsxV3PluginTestCaseMixin,
             NSG_IDS[1], firewall.LOGICAL_PORT, mock.ANY)
 
     @_mock_create_and_list_nsgroups
-    @mock.patch.object(firewall, 'add_nsgroup_member')
+    @mock.patch.object(firewall, 'add_nsgroup_members')
     def test_create_port_with_full_security_group(self, add_member_mock):
 
         def _add_member_mock(nsgroup, target_type, target_id):
@@ -136,7 +136,7 @@ class TestSecurityGroupsNoDynamicCriteria(test_nsxv3.NsxV3PluginTestCaseMixin,
 
     @_mock_create_and_list_nsgroups
     @mock.patch.object(firewall, 'remove_nsgroup_member')
-    @mock.patch.object(firewall, 'add_nsgroup_member')
+    @mock.patch.object(firewall, 'add_nsgroup_members')
     def test_update_port_with_full_security_group(self,
                                                   add_member_mock,
                                                   remove_member_mock):
@@ -204,7 +204,7 @@ class TestNSGroupManager(nsxlib_testcase.NsxLibTestCase):
 
     @_mock_create_and_list_nsgroups
     @mock.patch.object(firewall, 'remove_nsgroup_member')
-    @mock.patch.object(firewall, 'add_nsgroup_member')
+    @mock.patch.object(firewall, 'add_nsgroup_members')
     def test_add_and_remove_nsgroups(self,
                                      add_member_mock,
                                      remove_member_mock):
@@ -222,13 +222,13 @@ class TestNSGroupManager(nsxlib_testcase.NsxLibTestCase):
         # There are 5 nested groups, the hash function will return 7, therefore
         # we expect that the nsgroup will be placed in the 3rd group.
         add_member_mock.assert_called_once_with(
-            NSG_IDS[2], firewall.NSGROUP, nsgroup_id)
+            NSG_IDS[2], firewall.NSGROUP, [nsgroup_id])
         remove_member_mock.assert_called_once_with(
             NSG_IDS[2], firewall.NSGROUP, nsgroup_id, verify=True)
 
     @_mock_create_and_list_nsgroups
     @mock.patch.object(firewall, 'remove_nsgroup_member')
-    @mock.patch.object(firewall, 'add_nsgroup_member')
+    @mock.patch.object(firewall, 'add_nsgroup_members')
     def test_when_nested_group_is_full(self,
                                        add_member_mock,
                                        remove_member_mock):
@@ -255,8 +255,8 @@ class TestNSGroupManager(nsxlib_testcase.NsxLibTestCase):
         # Trying to add nsgroup to the nested group at index 2 will raise
         # NSGroupIsFull exception, we expect that the nsgroup will be added to
         # the nested group at index 3.
-        calls = [mock.call(NSG_IDS[2], firewall.NSGROUP, nsgroup_id),
+        calls = [mock.call(NSG_IDS[2], firewall.NSGROUP, [nsgroup_id]),
-                 mock.call(NSG_IDS[3], firewall.NSGROUP, nsgroup_id)]
+                 mock.call(NSG_IDS[3], firewall.NSGROUP, [nsgroup_id])]
         add_member_mock.assert_has_calls(calls)
 
         # Since the nsgroup was added to the nested group at index 3, it will
@@ -271,7 +271,7 @@ class TestNSGroupManager(nsxlib_testcase.NsxLibTestCase):
 
     @_mock_create_and_list_nsgroups
     @mock.patch.object(firewall, 'remove_nsgroup_member')
-    @mock.patch.object(firewall, 'add_nsgroup_member')
+    @mock.patch.object(firewall, 'add_nsgroup_members')
     def initialize_with_absent_nested_groups(self,
                                              add_member_mock,
                                              remove_member_mock):

vmware_nsx/tests/unit/shell/test_admin_utils.py

@@ -19,6 +19,7 @@ import mock
 import six
 
 from oslo_config import cfg
+from oslo_utils import uuidutils
 
 from neutron.callbacks import registry
 from neutron.common import config as neutron_config
@@ -146,7 +147,10 @@ class TestNsxv3AdminUtils(AbstractTestAdminUtils,
         self._patch_object(nsx_v3_resources.SwitchingProfile,
                            '__init__', return_value=None)
         self._patch_object(nsx_v3_resources.SwitchingProfile,
-                           'find_by_display_name', return_value=None)
+                           'find_by_display_name',
+                           return_value=[{'id': uuidutils.generate_uuid()}])
+        self._patch_object(nsx_v3_resources.LogicalRouterPort,
+                           '__init__', return_value=None)
 
     def _get_plugin_name(self):
         return 'nsxv3'