NSX|V+V3: Move FW section logging update to admin utility

On the plugin init there is a side process going over all the security
group rules in the NSX DFW checking if their logging flag should be
updated according to the global configuration flag.
Since this is relevant only in case the global config flag
log_security_groups_allowed_traffic was updated by the user, which is very rare,
this patch removed it from the code, and replaced it with an admin utility
that can be used.
This will make the plugin initialization process quicker and prevent unnecessary
load on the NSX.

Change-Id: I233915e589b53ccb4b76a3ef3d24bb56c0459e92

doc/source/admin_util.rst

@@ -269,6 +269,10 @@ Security Groups, Firewall and Spoofguard
 
     nsxadmin -r security-groups -o migrate-to-policy --property policy-id=policy-10 --property security-group-id=733f0741-fa2c-4b32-811c-b78e4dc8ec39
 
+- Update logging flag of the security groups on the NSX DFW
+
+    nsxadmin -r security-groups -o update-logging --property log-allowed-traffic=true
+
 - Spoofguard support::
 
     nsxadmin -r spoofguard-policy -o list-mismatches
@@ -390,6 +394,10 @@ Security Groups & NSX Security Groups
 
     nsxadmin -r nsx-security-groups -o migrate-to-dynamic-criteria
 
+- Update logging flag of the security groups on the NSX DFW
+
+    nsxadmin -r security-groups -o update-logging --property log-allowed-traffic=true
+
 Firewall Sections
 ~~~~~~~~~~~~~~~~~
 

vmware_nsx/plugins/nsx_v/plugin.py

@@ -295,7 +295,6 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
                             "NSX 6.3 onwards")
         self.sg_container_id = self._create_security_group_container()
         self.default_section = self._create_cluster_default_fw_section()
-        self._process_security_groups_rules_logging()
 
         self._router_managers = managers.RouterTypeManager(self)
 
@@ -574,47 +573,6 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
                 section_id = self.nsx_sg_utils.parse_and_get_section_id(c)
         return section_id
 
-    def _process_security_groups_rules_logging(self):
-
-        def process_security_groups_rules_logging(*args, **kwargs):
-            with locking.LockManager.get_lock('nsx-dfw-section',
-                                              lock_file_prefix='dfw-section'):
-                context = n_context.get_admin_context()
-                log_allowed = cfg.CONF.nsxv.log_security_groups_allowed_traffic
-
-                # If the section/sg is already logged, then no action is
-                # required.
-                for sg in [sg for sg in self.get_security_groups(context)
-                           if sg.get(sg_logging.LOGGING) is False]:
-                    if sg.get(sg_policy.POLICY):
-                        # Logging is not relevant with a policy
-                        continue
-
-                    section_uri = self._get_section_uri(context.session,
-                                                        sg['id'])
-                    if section_uri is None:
-                        continue
-
-                    # Section/sg is not logged, update rules logging according
-                    # to the 'log_security_groups_allowed_traffic' config
-                    # option.
-                    try:
-                        h, c = self.nsx_v.vcns.get_section(section_uri)
-                        section = self.nsx_sg_utils.parse_section(c)
-                        section_needs_update = (
-                            self.nsx_sg_utils.set_rules_logged_option(
-                                section, log_allowed))
-                        if section_needs_update:
-                            self.nsx_v.vcns.update_section(
-                                section_uri,
-                                self.nsx_sg_utils.to_xml_string(section), h)
-                    except Exception as exc:
-                        LOG.error('Unable to update security group %(sg)s '
-                                  'section for logging. %(e)s',
-                                  {'e': exc, 'sg': sg['id']})
-
-        c_utils.spawn_n(process_security_groups_rules_logging)
-
     def _create_dhcp_static_binding(self, context, neutron_port_db):
 
         network_id = neutron_port_db['network_id']

vmware_nsx/plugins/nsx_v3/plugin.py

@@ -274,7 +274,6 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin,
         self._init_dhcp_metadata()
 
         self._prepare_default_rules()
-        self._process_security_group_logging()
 
         # init profiles on nsx backend
         self._init_nsx_profiles()
@@ -779,27 +778,6 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin,
                 tags=self.nsxlib.build_v3_api_version_tag())
         return self._get_port_security_profile()
 
-    def _process_security_group_logging(self):
-        def process_security_group_logging(*args, **kwargs):
-            context = q_context.get_admin_context()
-            log_all_rules = cfg.CONF.nsx_v3.log_security_groups_allowed_traffic
-            secgroups = self.get_security_groups(context,
-                                                 fields=['id',
-                                                 sg_logging.LOGGING])
-            for sg in [sg for sg in secgroups
-                       if sg.get(sg_logging.LOGGING) is False]:
-                nsgroup_id, section_id = nsx_db.get_sg_mappings(
-                    context.session, sg['id'])
-                if section_id:
-                    try:
-                        self.nsxlib.firewall_section.set_rule_logging(
-                            section_id, logging=log_all_rules)
-                    except nsx_lib_exc.ManagerError:
-                        LOG.error("Failed to update firewall rule logging "
-                                  "for rule in section %s", section_id)
-
-        utils.spawn_n(process_security_group_logging)
-
     def _init_default_section_rules(self):
         with locking.LockManager.get_lock('nsxv3_default_section'):
             section_description = ("This section is handled by OpenStack to "

vmware_nsx/shell/admin/plugins/nsxv/resources/securitygroups.py

@@ -31,6 +31,7 @@ from vmware_nsx.db import extended_security_group_rule as extend_sg_rule
 from vmware_nsx.db import nsx_models
 from vmware_nsx.db import nsxv_db
 from vmware_nsx.db import nsxv_models
+from vmware_nsx.extensions import securitygrouplogging as sg_logging
 from vmware_nsx.extensions import securitygrouppolicy as sg_policy
 from vmware_nsx.shell.admin.plugins.common import constants
 from vmware_nsx.shell.admin.plugins.common import formatters
@@ -464,6 +465,63 @@ def firewall_update_cluster_default_fw_section(resource, event, trigger,
         LOG.info("Cluster default FW section updated.")
 
 
+@admin_utils.output_header
+def update_security_groups_logging(resource, event, trigger, **kwargs):
+    """Update allowed traffic logging for all neutron security group rules"""
+    errmsg = ("Need to specify log-allowed-traffic property. Add --property "
+              "log-allowed-traffic=true/false")
+    if not kwargs.get('property'):
+        LOG.error("%s", errmsg)
+        return
+    properties = admin_utils.parse_multi_keyval_opt(kwargs['property'])
+    log_allowed_str = properties.get('log-allowed-traffic')
+    if not log_allowed_str or log_allowed_str.lower() not in ['true', 'false']:
+        LOG.error("%s", errmsg)
+        return
+    log_allowed = log_allowed_str.lower() == 'true'
+
+    context = n_context.get_admin_context()
+
+    with utils.NsxVPluginWrapper() as plugin:
+        vcns = plugin.nsx_v.vcns
+        sg_utils = plugin. nsx_sg_utils
+        # If the section/sg is already logged, then no action is
+        # required.
+        security_groups = plugin.get_security_groups(context)
+        LOG.info("Going to update logging of %s sections",
+                 len(security_groups))
+        for sg in [sg for sg in plugin.get_security_groups(context)
+                   if sg.get(sg_logging.LOGGING) is False]:
+            if sg.get(sg_policy.POLICY):
+                # Logging is not relevant with a policy
+                continue
+
+            section_uri = plugin._get_section_uri(context.session,
+                                                  sg['id'])
+            if section_uri is None:
+                continue
+
+            # Section/sg is not logged, update rules logging according
+            # to the 'log_security_groups_allowed_traffic' config
+            # option.
+            try:
+                h, c = vcns.get_section(section_uri)
+                section = sg_utils.parse_section(c)
+                section_needs_update = sg_utils.set_rules_logged_option(
+                    section, log_allowed)
+                if section_needs_update:
+                    vcns.update_section(section_uri,
+                                        sg_utils.to_xml_string(section), h)
+            except Exception as exc:
+                LOG.error('Unable to update security group %(sg)s '
+                          'section for logging. %(e)s',
+                          {'e': exc, 'sg': sg['id']})
+
+
+registry.subscribe(update_security_groups_logging,
+                   constants.SECURITY_GROUPS,
+                   shell.Operations.UPDATE_LOGGING.value)
+
 registry.subscribe(migrate_sg_to_policy,
                    constants.SECURITY_GROUPS,
                    shell.Operations.MIGRATE_TO_POLICY.value)

vmware_nsx/shell/admin/plugins/nsxv/resources/utils.py

@@ -75,9 +75,6 @@ class NsxVPluginWrapper(plugin.NsxVPlugin):
         # skip getting the Qos policy ID because get_object calls
         # plugin init again on admin-util environment
 
-    def _process_security_groups_rules_logging(self):
-        pass
-
     def count_spawn_jobs(self):
         # check if there are any spawn jobs running
         return self.edge_manager._get_worker_pool().running()

vmware_nsx/shell/admin/plugins/nsxv3/resources/securitygroups.py

@@ -29,6 +29,7 @@ from vmware_nsx.shell.admin.plugins.common import formatters
 from vmware_nsx.shell.admin.plugins.common import utils as admin_utils
 from vmware_nsx.shell.admin.plugins.nsxv3.resources import utils as v3_utils
 from vmware_nsx.shell import resources as shell
+from vmware_nsxlib.v3 import exceptions as nsx_lib_exc
 from vmware_nsxlib.v3 import nsx_constants as consts
 from vmware_nsxlib.v3 import security
 
@@ -314,6 +315,46 @@ def clean_orphaned_sections(resource, event, trigger, **kwargs):
             LOG.info("Backend firewall section %s was deleted.", sec['id'])
 
 
+def update_security_groups_logging(resource, event, trigger, **kwargs):
+    """Update allowed traffic logging for all neutron security group rules"""
+    errmsg = ("Need to specify log-allowed-traffic property. Add --property "
+              "log-allowed-traffic=true/false")
+    if not kwargs.get('property'):
+        LOG.error("%s", errmsg)
+        return
+    properties = admin_utils.parse_multi_keyval_opt(kwargs['property'])
+    log_allowed_str = properties.get('log-allowed-traffic')
+    if not log_allowed_str or log_allowed_str.lower() not in ['true', 'false']:
+        LOG.error("%s", errmsg)
+        return
+    log_allowed = log_allowed_str.lower() == 'true'
+
+    context = neutron_context.get_admin_context()
+    nsxlib = v3_utils.get_connected_nsxlib()
+
+    with v3_utils.NsxV3PluginWrapper() as plugin:
+        secgroups = plugin.get_security_groups(context,
+                                             fields=['id',
+                                             sg_logging.LOGGING])
+        LOG.info("Going to update logging of %s sections",
+                 len(secgroups))
+        for sg in [sg for sg in secgroups
+                   if sg.get(sg_logging.LOGGING) is False]:
+            nsgroup_id, section_id = nsx_db.get_sg_mappings(
+                context.session, sg['id'])
+            if section_id:
+                try:
+                    nsxlib.firewall_section.set_rule_logging(
+                        section_id, logging=log_allowed)
+                except nsx_lib_exc.ManagerError:
+                    LOG.error("Failed to update firewall rule logging "
+                              "for rule in section %s", section_id)
+
+
+registry.subscribe(update_security_groups_logging,
+                   constants.SECURITY_GROUPS,
+                   shell.Operations.UPDATE_LOGGING.value)
+
 registry.subscribe(migrate_nsgroups_to_dynamic_criteria,
                    constants.FIREWALL_NSX_GROUPS,
                    shell.Operations.MIGRATE_TO_DYNAMIC_CRITERIA.value)

vmware_nsx/shell/admin/plugins/nsxv3/resources/utils.py

@@ -169,9 +169,6 @@ class NsxV3PluginWrapper(plugin.NsxV3Plugin):
     def _init_dhcp_metadata(self):
         pass
 
-    def _process_security_group_logging(self):
-        pass
-
     def _extend_get_network_dict_provider(self, context, net):
         self._extend_network_dict_provider(context, net)
         # skip getting the Qos policy ID because get_object calls

vmware_nsx/shell/resources.py

@@ -59,6 +59,7 @@ class Operations(enum.Enum):
     MIGRATE_TO_DYNAMIC_CRITERIA = 'migrate-to-dynamic-criteria'
     NSX_MIGRATE_V_V3 = 'nsx-migrate-v-v3'
     MIGRATE_TO_POLICY = 'migrate-to-policy'
+    UPDATE_LOGGING = 'update-logging'
     NSX_MIGRATE_EXCLUDE_PORTS = 'migrate-exclude-ports'
     MIGRATE_VDR_DHCP = 'migrate-vdr-dhcp'
     STATUS = 'status'
@@ -80,7 +81,8 @@ class Resource(object):
 nsxv3_resources = {
     constants.SECURITY_GROUPS: Resource(constants.SECURITY_GROUPS,
                                         [Operations.LIST.value,
-                                         Operations.FIX_MISMATCH.value]),
+                                         Operations.FIX_MISMATCH.value,
+                                         Operations.UPDATE_LOGGING.value]),
     constants.FIREWALL_SECTIONS: Resource(constants.FIREWALL_SECTIONS,
                                           [Operations.LIST.value,
                                            Operations.LIST_MISMATCHES.value]),
@@ -186,7 +188,8 @@ nsxv_resources = {
     constants.SECURITY_GROUPS: Resource(constants.SECURITY_GROUPS,
                                         [Operations.LIST.value,
                                          Operations.FIX_MISMATCH.value,
-                                         Operations.MIGRATE_TO_POLICY.value]),
+                                         Operations.MIGRATE_TO_POLICY.value,
+                                         Operations.UPDATE_LOGGING.value]),
     constants.FIREWALL_NSX_GROUPS: Resource(
         constants.FIREWALL_NSX_GROUPS, [Operations.LIST.value,
                                         Operations.LIST_MISMATCHES.value]),

vmware_nsx/tests/unit/nsx_v/test_plugin.py

@@ -214,10 +214,6 @@ class NsxVPluginV2TestCase(test_plugin.NeutronDbPluginV2TestCase):
         mock_deploy_backup_edges_at_backend = mock.patch("%s.%s" % (
             vmware.EDGE_MANAGE_NAME, '_deploy_backup_edges_at_backend'))
         mock_deploy_backup_edges_at_backend.start()
-        mock_process_security_group_logging = mock.patch(
-            'vmware_nsx.plugin.NsxVPlugin.'
-            '_process_security_groups_rules_logging')
-        mock_process_security_group_logging.start()
 
         self.default_res_pool = 'respool-28'
         cfg.CONF.set_override("resource_pool_id", self.default_res_pool,
@@ -3988,10 +3984,6 @@ class NsxVSecurityGroupsTestCase(ext_sg.SecurityGroupDBTestCase):
         mock_check_backup_edge_pools = mock.patch("%s.%s" % (
             vmware.EDGE_MANAGE_NAME, '_check_backup_edge_pools'))
         mock_check_backup_edge_pools.start()
-        mock_process_security_group_logging = mock.patch(
-            'vmware_nsx.plugin.NsxVPlugin.'
-            '_process_security_groups_rules_logging')
-        mock_process_security_group_logging.start()
 
         c_utils.spawn_n = mock.Mock(side_effect=lambda f: f())
         super(NsxVSecurityGroupsTestCase, self).setUp(plugin=plugin,

vmware_nsx/tests/unit/nsx_v3/test_plugin.py

@@ -197,10 +197,6 @@ class NsxV3PluginTestCaseMixin(test_plugin.NeutronDbPluginV2TestCase,
             'neutron.scheduler.dhcp_agent_scheduler.AZAwareWeightScheduler')
 
     def mock_plugin_methods(self):
-        # mock unnecessary call which causes spawn
-        mock_process_security_group_logging = mock.patch.object(
-            nsx_plugin.NsxV3Plugin, '_process_security_group_logging')
-        mock_process_security_group_logging.start()
         # need to mock the global placeholder. This is due to the fact that
         # the generic security group tests assume that there is just one
         # security group.

vmware_nsx/tests/unit/shell/test_admin_utils.py

@@ -220,7 +220,8 @@ class TestNsxvAdminUtils(AbstractTestAdminUtils,
                 "security-group-id=sg-1",
                 "dvs-id=dvs-1",
                 "moref=virtualwire-1",
-                "teamingpolicy=LACP_ACTIVE"
+                "teamingpolicy=LACP_ACTIVE",
+                "log-allowed-traffic=true"
                 ]
         self._test_resources_with_args(
             resources.nsxv_resources, args)
@@ -281,7 +282,8 @@ class TestNsxv3AdminUtils(AbstractTestAdminUtils,
                 "metadata_proxy_uuid=e5b9b249-0034-4729-8ab6-fe4dacaa3a12",
                 "nsx-id=e5b9b249-0034-4729-8ab6-fe4dacaa3a12",
                 "availability-zone=default",
-                "server-ip=1.1.1.1"
+                "server-ip=1.1.1.1",
+                "log-allowed-traffic=true"
                 ]
         # Create some neutron objects for the utilities to run on
         self._create_router()