x/vmware-nsx
Code Issues Proposed changes

NSX cleanup script to clean only related resources

Browse Source 
nsxv_cleanup and nsxv3_cleanup scripts are called by unstack.sh and
removes all backend resources, even resources which may have been
created by other devstack deployments using the same backend.
This patch fix this issue, when calling 'unstack.sh' the script will
only remove backend resources that have db record, if 'clean.sh' is
called, then previous behavior is used and all backend resources created
by openstack are removed.

To run the scripts manually, in such way that only backend resources
with db records are cleaned, one must specify '--db-connection' (e.g -
iniget /etc/neutron/neutron.conf database connection) option so the script can
query the DB.
When '--db-connection' option is not specified then all
backend resources are cleaned.

Change-Id: I2283bdb2758c303a46574296e0067f458a6eefcf
This commit is contained in:
Roey Chen 2017-01-08 06:30:55 -08:00
parent 3756e8bd6f
commit 3d24d19309
3 changed files with 170 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
devstack/plugin.sh
View File

@ -34,8 +34,14 @@ fi
if [[ $Q_PLUGIN == 'vmware_nsx_v' ]]; then
source $dir/lib/vmware_nsx_v
if [[ "$1" == "unstack" ]]; then
python $dir/tools/nsxv_cleanup.py --vsm-ip ${NSXV_MANAGER_URI/https:\/\/} --user $NSXV_USER --password $NSXV_PASSWORD
db_connection=$(iniget $NEUTRON_CONF database connection)
python $dir/tools/nsxv_cleanup.py --vsm-ip ${NSXV_MANAGER_URI/https:\/\/} --user $NSXV_USER --password $NSXV_PASSWORD --db-connection $db_connection
elif [[ "$1" == "clean" ]]; then
if is_service_enabled q-svc || is_service_enabled neutron-api; then
python $dir/tools/nsxv_cleanup.py --vsm-ip ${NSXV_MANAGER_URI/https:\/\/} --user $NSXV_USER --password $NSXV_PASSWORD
fi
fi
elif [[ $Q_PLUGIN == 'vmware_nsx' ]]; then
source $dir/lib/vmware_nsx
if [[ "$1" == "stack" && "$2" == "post-config" ]]; then
@ -50,6 +56,7 @@ elif [[ $Q_PLUGIN == 'vmware_nsx_v3' ]]; then
if [[ "$1" == "stack" && "$2" == "post-config" ]]; then
init_vmware_nsx_v3
elif [[ "$1" == "unstack" ]]; then
db_connection=$(iniget $NEUTRON_CONF database connection)
stop_vmware_nsx_v3
# only clean up when q-svc (legacy support) or neutron-api is enabled
if is_service_enabled q-svc || is_service_enabled neutron-api; then
@ -57,6 +64,10 @@ elif [[ $Q_PLUGIN == 'vmware_nsx_v3' ]]; then
IFS=','
NSX_MANAGER=($NSX_MANAGER)
unset IFS
python $dir/tools/nsxv3_cleanup.py --mgr-ip $NSX_MANAGER --user $NSX_USER --password $NSX_PASSWORD --db-connection $db_connection
fi
elif [[ "$1" == 'clean' ]]; then
if is_service_enabled q-svc || is_service_enabled neutron-api; then
python $dir/tools/nsxv3_cleanup.py --mgr-ip $NSX_MANAGER --user $NSX_USER --password $NSX_PASSWORD
fi
fi

109
devstack/tools/nsxv3_cleanup.py
View File

@ -16,20 +16,56 @@
import base64
import optparse
import requests
import sqlalchemy as sa
from oslo_serialization import jsonutils
import six.moves.urllib.parse as urlparse
from vmware_nsx.db import nsx_models
requests.packages.urllib3.disable_warnings()
class NeutronNsxDB(object):
def __init__(self, db_connection):
super(NeutronNsxDB, self).__init__()
engine = sa.create_engine(db_connection)
self.session = sa.orm.session.sessionmaker()(bind=engine)
def query_all(self, column, model):
return list(set([r[column] for r in self.session.query(model).all()]))
def get_logical_ports(self):
return self.query_all('nsx_port_id',
nsx_models.NeutronNsxPortMapping)
def get_nsgroups(self):
return self.query_all('nsx_id',
nsx_models.NeutronNsxSecurityGroupMapping)
def get_firewall_sections(self):
return self.query_all('nsx_id',
nsx_models.NeutronNsxFirewallSectionMapping)
def get_logical_routers(self):
return self.query_all('nsx_id',
nsx_models.NeutronNsxRouterMapping)
def get_logical_switches(self):
return self.query_all('nsx_id',
nsx_models.NeutronNsxNetworkMapping)
def get_logical_dhcp_servers(self):
return self.query_all('nsx_service_id',
nsx_models.NeutronNsxServiceBinding)
class NSXClient(object):
"""Base NSX REST client"""
API_VERSION = "v1"
NULL_CURSOR_PREFIX = '0000'
def __init__(self, host, username, password, *args, **kwargs):
def __init__(self, host, username, password, db_connection):
self.host = host
self.username = username
self.password = password
@ -43,6 +79,8 @@ class NSXClient(object):
self.url = None
self.headers = None
self.api_version = NSXClient.API_VERSION
self.neutron_db = (NeutronNsxDB(db_connection)
if db_connection else None)
self.__set_headers()
@ -169,8 +207,12 @@ class NSXClient(object):
"""
Retrieve all logical ports created from OpenStack
"""
lports = self.get_logical_ports()
return self.get_os_resources(lports)
lports = self.get_os_resources(
self.get_logical_ports())
if self.neutron_db:
db_lports = self.neutron_db.get_logical_ports()
lports = [lp for lp in lports if lp['id'] in db_lports]
return lports
def update_logical_port_attachment(self, lports):
"""
@ -188,8 +230,7 @@ class NSXClient(object):
"""
Delete all logical ports created by OpenStack
"""
lports = self.get_logical_ports()
os_lports = self.get_os_resources(lports)
os_lports = self.get_os_logical_ports()
print("Number of OS Logical Ports to be deleted: %s" % len(os_lports))
# logical port vif detachment
self.update_logical_port_attachment(os_lports)
@ -221,8 +262,14 @@ class NSXClient(object):
"""
Retrieve all logical switches created from OpenStack
"""
lswitches = self.get_logical_switches()
return self.get_os_resources(lswitches)
lswitches = self.get_os_resources(
self.get_logical_switches())
if self.neutron_db:
db_lswitches = self.neutron_db.get_logical_switches()
lswitches = [ls for ls in lswitches
if ls['id'] in db_lswitches]
return lswitches
def get_lswitch_ports(self, ls_id):
"""
@ -258,8 +305,13 @@ class NSXClient(object):
"""
Retrieve all firewall sections created from OpenStack
"""
fw_sections = self.get_firewall_sections()
return self.get_os_resources(fw_sections)
fw_sections = self.get_os_resources(
self.get_firewall_sections())
if self.neutron_db:
db_sections = self.neutron_db.get_firewall_sections()
fw_sections = [fws for fws in fw_sections
if fws['id'] in db_sections]
return fw_sections
def get_firewall_section_rules(self, fw_section):
"""
@ -306,8 +358,13 @@ class NSXClient(object):
"""
Retrieve all NSGroups on NSX backend
"""
ns_groups = self.get_list_results(endpoint="/ns-groups")
return self.get_os_resources(ns_groups)
ns_groups = self.get_os_resources(
self.get_list_results(endpoint="/ns-groups"))
if self.neutron_db:
db_nsgroups = self.neutron_db.get_nsgroups()
ns_groups = [nsg for nsg in ns_groups
if nsg['id'] in db_nsgroups]
return ns_groups
def cleanup_os_ns_groups(self):
"""
@ -333,8 +390,11 @@ class NSXClient(object):
"""
Retrieve all Switching Profiles created from OpenStack
"""
sw_profiles = self.get_switching_profiles()
return self.get_os_resources(sw_profiles)
sw_profiles = self.get_os_resources(
self.get_switching_profiles())
if self.neutron_db:
sw_profiles = []
return sw_profiles
def cleanup_os_switching_profiles(self):
"""
@ -362,7 +422,13 @@ class NSXClient(object):
endpoint = "/logical-routers?router_type=%s" % tier
else:
endpoint = "/logical-routers"
return self.get_list_results(endpoint=endpoint)
lrouters = self.get_list_results(endpoint=endpoint)
if self.neutron_db:
db_routers = self.neutron_db.get_logical_routers()
lrouters = [lr for lr in lrouters
if lr['id'] in db_routers]
return lrouters
def get_os_logical_routers(self):
"""
@ -438,8 +504,14 @@ class NSXClient(object):
"""
Retrieve all logical DHCP servers created from OpenStack
"""
dhcp_servers = self.get_logical_dhcp_servers()
return self.get_os_resources(dhcp_servers)
dhcp_servers = self.get_os_resources(
self.get_logical_dhcp_servers())
if self.neutron_db:
db_dhcp_servers = self.neutron_db.get_logical_dhcp_servers()
dhcp_servers = [srv for srv in dhcp_servers
if srv['id'] in db_dhcp_servers]
return dhcp_servers
def cleanup_os_logical_dhcp_servers(self):
"""
@ -487,10 +559,13 @@ if __name__ == "__main__":
help="NSX Manager username")
parser.add_option("-p", "--password", default="default", dest="password",
help="NSX Manager password")
parser.add_option("--db-connection", default="", dest="db_connection",
help=("When set, cleaning only backend resources that "
"have db record."))
(options, args) = parser.parse_args()
# Get NSX REST client
nsx_client = NSXClient(options.mgr_ip, options.username,
options.password)
options.password, options.db_connection)
# Clean all objects created by OpenStack
nsx_client.cleanup_all()

75
devstack/tools/nsxv_cleanup.py
View File

@ -52,19 +52,53 @@ Tong Liu <tongl@vmware.com>
import base64
import optparse
import requests
import sqlalchemy as sa
import sys
from oslo_serialization import jsonutils
from vmware_nsx.db import nsx_models
from vmware_nsx.db import nsxv_models
requests.packages.urllib3.disable_warnings()
class NeutronNsxDB(object):
def __init__(self, db_connection):
super(NeutronNsxDB, self).__init__()
engine = sa.create_engine(db_connection)
self.session = sa.orm.session.sessionmaker()(bind=engine)
def query_all(self, column, model):
return list(set([r[column] for r in self.session.query(model).all()]))
def query_all_firewall_sections(self):
return self.query_all('ip_section_id',
nsxv_models.NsxvSecurityGroupSectionMapping)
def query_all_security_groups(self):
return self.query_all('nsx_id',
nsx_models.NeutronNsxSecurityGroupMapping)
def query_all_logical_switches(self):
return self.query_all('nsx_id',
nsx_models.NeutronNsxNetworkMapping)
def query_all_spoofguard_policies(self):
return self.query_all('policy_id',
nsxv_models.NsxvSpoofGuardPolicyNetworkMapping)
def query_all_edges(self):
return self.query_all('edge_id',
nsxv_models.NsxvRouterBinding)
class VSMClient(object):
"""Base VSM REST client """
API_VERSION = "2.0"
def __init__(self, host, username, password, *args, **kwargs):
self.force = True if 'force' in kwargs else False
def __init__(self, host, username, password, db_connection, force):
self.force = force
self.host = host
self.username = username
self.password = password
@ -78,7 +112,8 @@ class VSMClient(object):
self.url = None
self.headers = None
self.api_version = VSMClient.API_VERSION
self.neutron_db = (NeutronNsxDB(db_connection) if db_connection
else None)
self.__set_headers()
def __set_endpoint(self, endpoint):
@ -192,6 +227,11 @@ class VSMClient(object):
temp_lswitches = response.json()['dataPage']['data']
lswitches += temp_lswitches
if self.neutron_db:
db_lswitches = self.neutron_db.query_all_logical_switches()
lswitches = [ls for ls in lswitches
if ls['objectId'] in db_lswitches]
return lswitches
def cleanup_logical_switch(self):
@ -224,6 +264,10 @@ class VSMClient(object):
print("ERROR: wrong response status code! Exiting...")
sys.exit()
if self.neutron_db:
db_sections = self.neutron_db.query_all_firewall_sections()
firewall_sections = [fws for fws in firewall_sections if fws['id']
in db_sections]
return firewall_sections
def cleanup_firewall_section(self):
@ -254,6 +298,11 @@ class VSMClient(object):
# related to any security group created by OpenStack
security_groups = [sg for sg in sg_all if
sg['name'] != "Activity Monitoring Data Collection"]
if self.neutron_db:
db_sgs = self.neutron_db.query_all_security_groups()
security_groups = [sg for sg in security_groups
if sg['objectId'] in db_sgs]
return security_groups
def cleanup_security_group(self):
@ -280,6 +329,10 @@ class VSMClient(object):
sgp_all = response.json()
policies = [sgp for sgp in sgp_all['policies'] if
sgp['name'] != 'Default Policy']
if self.neutron_db:
db_policies = self.neutron_db.query_all_spoofguard_policies()
policies = [p for p in policies if p['policyId'] in db_policies]
return policies
def cleanup_spoofguard_policies(self):
@ -313,6 +366,10 @@ class VSMClient(object):
temp_edges = response.json()['edgePage']['data']
edges += temp_edges
if self.neutron_db:
db_edges = self.neutron_db.query_all_edges()
edges = [e for e in edges if e['id'] in db_edges]
return edges
def cleanup_edge(self):
@ -350,20 +407,20 @@ if __name__ == "__main__":
help="NSX Manager username")
parser.add_option("-p", "--password", default="default", dest="password",
help="NSX Manager password")
parser.add_option("--db-connection", dest="db_connection", default="",
help=("When set, cleaning only backend resources that "
"have db record."))
parser.add_option("-f", "--force", dest="force", action="store_true",
help="Force cleanup option")
(options, args) = parser.parse_args()
print("vsm-ip: %s" % options.vsm_ip)
print("username: %s" % options.username)
print("password: %s" % options.password)
print("db-connection: %s" % options.db_connection)
print("force: %s" % options.force)
# Get VSM REST client
if options.force:
vsm_client = VSMClient(options.vsm_ip, options.username,
options.password, force=options.force)
else:
vsm_client = VSMClient(options.vsm_ip, options.username,
options.password)
vsm_client = VSMClient(options.vsm_ip, options.username, options.password,
options.db_connection, options.force)
# Clean all objects created by OpenStack
vsm_client.cleanup_all()