AdminUtils: Improve NSXv security admin utils

1. Better explain the security groups / nsx security groups / firewall sections
admiun utilities.
2. Also remove the unrelated firewall sections reorder form the fix-mismatch utility
3. fix some warnings that appeared when runnin g the utilities
4. Add new utilities to list/clean unused NSX sections:
- List NSX firewall sections that does not have a matching neutron security group::

    nsxadmin -r firewall-section -o list-unused

- Delete NSX firewall sections that does not have a matching neutron security group::

    nsxadmin -r firewall-section -o nsx-clean

Change-Id: Ie9868d1fb196964ce479bca2c42d4a6eea7ef427

doc/source/admin_util.rst

@@ -215,21 +215,21 @@ Orphaned Networks
 Security Groups, Firewall and Spoofguard
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Security groups. This adds support to list security-groups mappings and miss-matches between the mappings and backend resources as: firewall-sections and nsx-security-groups::
+- List NSX firewall sections::
 
-    nsxadmin --resource security-groups --operation list
+    nsxadmin -r firewall-section -o list
-    nsxadmin -r nsx-security-groups -o {list, list-missmatches}
-    nsxadmin -r firewall-sections -o {list, list-missmatches, nsx-update}
 
-- Spoofguard support::
+- List neutron security groups that does not have a matching NSX firewall section::
 
-    nsxadmin -r spoofguard-policy -o list-mismatches
+    nsxadmin -r firewall-section -o list-mismatches
-    nsxadmin -r spoofguard-policy -o clean --property policy-id=spoofguardpolicy-10
-    nsxadmin -r spoofguard-policy -o list --property reverse (entries defined on NSXv and not in Neutron)
 
-- Migrate a security group from using rules to using a policy
+- List NSX firewall sections that does not have a matching neutron security group::
 
-    nsxadmin -r security-groups -o migrate-to-policy --property policy-id=policy-10 --property security-group-id=733f0741-fa2c-4b32-811c-b78e4dc8ec39
+    nsxadmin -r firewall-section -o list-unused
+
+- Delete NSX firewall sections that does not have a matching neutron security group::
+
+    nsxadmin -r firewall-section -o nsx-clean
 
 - Reorder the nsx L3 firewall sections to correctly support the policy security groups
 
@@ -239,6 +239,32 @@ Security Groups, Firewall and Spoofguard
 
     nsxadmin -r firewall-sections -o nsx-update
 
+- List NSX security groups::
+
+    nsxadmin -r nsx-security-groups -o list
+
+- List neutron security groups that does not have a matching NSX security group::
+
+    nsxadmin -r nsx-security-groups -o list-mismatches
+
+- List all the neutron security groups together with their NSX security groups and firewall sections::
+
+    nsxadmin -r security-groups -o list
+
+- Recreate missing NSX security groups ans firewall sections
+
+    nsxadmin -r security-groups -o fix-mismatch
+
+- Migrate a security group from using rules to using a policy
+
+    nsxadmin -r security-groups -o migrate-to-policy --property policy-id=policy-10 --property security-group-id=733f0741-fa2c-4b32-811c-b78e4dc8ec39
+
+- Spoofguard support::
+
+    nsxadmin -r spoofguard-policy -o list-mismatches
+    nsxadmin -r spoofguard-policy -o clean --property policy-id=spoofguardpolicy-10
+    nsxadmin -r spoofguard-policy -o list --property reverse (entries defined on NSXv and not in Neutron)
+
 Metadata
 ~~~~~~~~
 

vmware_nsx/shell/admin/plugins/nsxv/resources/securitygroups.py

@@ -12,7 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-
+import re
 import xml.etree.ElementTree as et
 
 from neutron.db import api as db_api
@@ -158,6 +158,11 @@ class NsxFirewallAPI(object):
                              'id': sec_id})
         return sections
 
+    def delete_fw_section(self, section_id):
+        section_uri = ("/api/4.0/firewall/globalroot-0/"
+                       "config/layer3sections/%s" % section_id)
+        self.vcns.delete_section(section_uri)
+
     def reorder_fw_sections(self):
         # read all the sections
         h, firewall_config = self.vcns.get_dfw_config()
@@ -294,7 +299,38 @@ def list_missing_firewall_sections(resource, event, trigger, **kwargs):
     return bool(missing_sections_info)
 
 
-@admin_utils.list_mismatches_handler(constants.FIREWALL_SECTIONS)
+def _get_unused_firewall_sections():
+    fw_sections = nsxv_firewall.list_fw_sections()
+    sg_mappings = neutron_sg.get_security_groups_mappings()
+    unused_sections = []
+    for fw_section in fw_sections:
+        for sg_db in sg_mappings:
+            if fw_section['id'] == sg_db.get('section-uri', '').split('/')[-1]:
+                break
+        else:
+            # skip sections with non neutron like names
+            if re.search("SG Section: .* (.*)", fw_section['name']):
+                unused_sections.append(fw_section)
+    return unused_sections
+
+
+@admin_utils.output_header
+def list_unused_firewall_sections(resource, event, trigger, **kwargs):
+    unused_sections = _get_unused_firewall_sections()
+    _log_info(constants.FIREWALL_SECTIONS, unused_sections,
+              attrs=['name', 'id'])
+    return bool(unused_sections)
+
+
+@admin_utils.output_header
+def clean_unused_firewall_sections(resource, event, trigger, **kwargs):
+    unused_sections = _get_unused_firewall_sections()
+    for fw_section in unused_sections:
+        LOG.info("Deleting firewall section %s", fw_section['id'])
+        nsxv_firewall.delete_fw_section(fw_section['id'])
+    return bool(unused_sections)
+
+
 @admin_utils.output_header
 def reorder_firewall_sections(resource, event, trigger, **kwargs):
     nsxv_firewall.reorder_fw_sections()
@@ -319,6 +355,7 @@ def fix_security_groups(resource, event, trigger, **kwargs):
             plugin._create_fw_section_for_security_group(
                 context_, secgroup,
                 sgs_with_missing_section[sg_id]['nsx-securitygroup-id'])
+            LOG.info("Created NSX section for security group %s", sg_id)
 
         # If nsx security-group is missing then create both nsx security-group
         # and a new fw section (remove old one).
@@ -330,6 +367,8 @@ def fix_security_groups(resource, event, trigger, **kwargs):
             neutron_sg.delete_security_group_backend_mapping(sg_id)
             plugin._process_security_group_create_backend_resources(context_,
                                                                     secgroup)
+            LOG.info("Created NSX section & security group for security group"
+                     " %s", sg_id)
             nsx_id = nsx_db.get_nsx_security_group_id(context_.session, sg_id,
                                                       moref=False)
             for vnic_id in neutron_sg.get_vnics_in_security_group(sg_id):
@@ -440,3 +479,11 @@ registry.subscribe(fix_security_groups,
 registry.subscribe(firewall_update_cluster_default_fw_section,
                    constants.FIREWALL_SECTIONS,
                    shell.Operations.NSX_UPDATE.value)
+
+registry.subscribe(list_unused_firewall_sections,
+                   constants.FIREWALL_SECTIONS,
+                   shell.Operations.LIST_UNUSED.value)
+
+registry.subscribe(clean_unused_firewall_sections,
+                   constants.FIREWALL_SECTIONS,
+                   shell.Operations.NSX_CLEAN.value)

vmware_nsx/shell/admin/plugins/nsxv/resources/utils.py

@@ -75,6 +75,9 @@ class NsxVPluginWrapper(plugin.NsxVPlugin):
         # skip getting the Qos policy ID because get_object calls
         # plugin init again on admin-util environment
 
+    def _process_security_groups_rules_logging(self):
+        pass
+
     def count_spawn_jobs(self):
         # check if there are any spawn jobs running
         return self.edge_manager._get_worker_pool().running()

vmware_nsx/shell/resources.py

@@ -38,6 +38,7 @@ class Operations(enum.Enum):
     DELETE = 'delete'
     LIST_MISMATCHES = 'list-mismatches'
     FIX_MISMATCH = 'fix-mismatch'
+    LIST_UNUSED = 'list-unused'
 
     NEUTRON_LIST = 'neutron-list'
     NEUTRON_CLEAN = 'neutron-clean'
@@ -186,7 +187,9 @@ nsxv_resources = {
                                           [Operations.LIST.value,
                                            Operations.LIST_MISMATCHES.value,
                                            Operations.NSX_UPDATE.value,
-                                           Operations.NSX_REORDER.value]),
+                                           Operations.NSX_REORDER.value,
+                                           Operations.LIST_UNUSED.value,
+                                           Operations.NSX_CLEAN.value]),
     constants.METADATA: Resource(
         constants.METADATA, [Operations.NSX_UPDATE.value,
                              Operations.NSX_UPDATE_SECRET.value,