From fde79f74c1e332f9bb4a18314dfc94e685a5e30f Mon Sep 17 00:00:00 2001 From: venkata anil Date: Fri, 1 Oct 2021 18:04:48 +0530 Subject: [PATCH] improvements in rally cleanup script 1. add security group deletion 2. When we have resources at scale (3k or above), MAX_ATTEMPTS is not sufficient to cleanup the resources. Instead of increasing it to a very high number, we try till we cleanup all the resources 3. Retrive 100 VMs at a time for deletion. Observed deletion is hanging when we fecth 1k VMs and try to delete them. With 100VMs at a time, able to succesfully deleted all VMs. Change-Id: I1973ddfe5f6015149bc5f82c8f7d38f3cddae402 --- rally_cleanup.py | 59 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/rally_cleanup.py b/rally_cleanup.py index 65d039a11..fab7a5afb 100644 --- a/rally_cleanup.py +++ b/rally_cleanup.py @@ -28,6 +28,7 @@ CONF.openstack_client_http_timeout = 180.0 osclient = osclients.Clients.create_from_env() nova_client = osclient.nova(version="2.73") neutron_client = osclient.neutron() +keystone_client = osclient.keystone() # Currently we only check and delete router_interface # and not DVR or HA interface @@ -80,6 +81,22 @@ def delete_network(resource): time.sleep(5) return resource +def delete_security_group(resource): + sg = neutron_client.show_security_group(resource.id)["security_group"] + print("pid {} deleting security_group id {} name {}".format( + os.getpid(), resource.id, sg["name"])) + # delete security_group + neutron_client.delete_security_group(sg["id"]) + # check if security_group got deleted or not + for i in range(0, MAX_CHECK): + try: + neutron_client.show_security_group(resource.id)["security_group"] + print("pid {} security_group {} still exists".format(os.getpid(), resource.id)) + except Exception: + print("pid {} security_group {} succesfully deleted".format(os.getpid(), resource.id)) + break + time.sleep(5) + return resource def delete_floatingip(resource): floatingip = neutron_client.show_floatingip(resource.id)["floatingip"] @@ -179,14 +196,13 @@ def cleanup_with_concurrency(cleanup_fun, resources): def cleanup_nova_vms(): - for i in range(0, MAX_ATTEMPTS): - servers = nova_client.servers.list(detailed=True, search_opts={"all_tenants": 1}) + while True: + servers = nova_client.servers.list(detailed=True, search_opts={"all_tenants": 1}, limit=100) if (len(servers) == 0): break print("Deleting {} servers".format(len(servers))) ids = [Resource(server.id) for server in servers] - if cleanup_with_concurrency(delete_server, ids): - break + cleanup_with_concurrency(delete_server, ids) time.sleep(5) @@ -195,16 +211,42 @@ def cleanup_neutron_networks(): networks = neutron_client.list_networks()["networks"] if (len(networks) == 0): break - print("Deleting {} networks".format(len(networks))) ids = [Resource(network["id"]) for network in networks if network["name"] not in NETWORK_EXCLUDE] + if (len(ids) == 0): + break + print("Deleting {} networks".format(len(ids))) if cleanup_with_concurrency(delete_network, ids): break time.sleep(5) +def get_admin_security_group(): + projects = keystone_client.projects.list() + admin_project = [project.id for project in projects if project.name == "admin"][0] + sgs = neutron_client.list_security_groups(project=admin_project)["security_groups"] + return [sg["id"] for sg in sgs if sg["name"] == "default"][0] + + +def cleanup_neutron_security_groups(): + # we shouldn't cleanup default security group created by admin + default_sg = get_admin_security_group() + while True: + sgs = neutron_client.list_security_groups()["security_groups"] + if (len(sgs) == 0): + break + ids = [Resource(sg["id"]) for sg in sgs + if sg["id"] != default_sg] + if (len(ids) == 0): + break + print("Deleting {} security_groups".format(len(ids))) + if cleanup_with_concurrency(delete_security_group, ids): + break + time.sleep(5) + + def cleanup_neutron_floatingips(): - for i in range(0, MAX_ATTEMPTS): + while True: floatingips = neutron_client.list_floatingips()["floatingips"] if (len(floatingips) == 0): break @@ -216,7 +258,7 @@ def cleanup_neutron_floatingips(): def _cleanup_neutron_router_ports(): - for i in range(0, MAX_ATTEMPTS): + while True: ports = neutron_client.list_ports(device_owner='network:router_interface')["ports"] if (len(ports) == 0): break @@ -229,7 +271,7 @@ def _cleanup_neutron_router_ports(): def cleanup_neutron_routers(): _cleanup_neutron_router_ports() - for i in range(0, MAX_ATTEMPTS): + while True: routers = neutron_client.list_routers()["routers"] if (len(routers) == 0): break @@ -244,6 +286,7 @@ def cleanup_resources(): cleanup_nova_vms() cleanup_neutron_floatingips() cleanup_neutron_routers() + cleanup_neutron_security_groups() cleanup_neutron_networks()