From 8933c56d8bb4e30e181069d8ef96dd6eafcff6ca Mon Sep 17 00:00:00 2001 From: Yichen Wang Date: Thu, 17 Mar 2016 16:53:15 -0700 Subject: [PATCH] Final works for publishing v6 1. Remove old force_cleanup.sh script; 2. Fix running kb_server from git clone; 3. Fix quota calculation issue if disk size is set to 0; 4. Fix the duplicated log issue from Rest API; 5. Change the output level of Pecan to CRITICAL; 6. Enable the logs output of Pecan when running from wrapper script; Change-Id: If5780b740724d5af8d646023f60a5ac635c7c357 --- kb_server/config.py | 2 +- kb_server/kb_server/controllers/__init__.py | 18 +++ kb_server/kb_server/controllers/api_kb.py | 2 +- kloudbuster/force_cleanup.sh | 148 -------------------- kloudbuster/kloudbuster.py | 10 +- kloudbuster/log.py | 4 +- kloudbuster/start_server.py | 6 +- setup.cfg | 1 + 8 files changed, 31 insertions(+), 160 deletions(-) delete mode 100755 kloudbuster/force_cleanup.sh diff --git a/kb_server/config.py b/kb_server/config.py index 389248b..5cf1eb4 100644 --- a/kb_server/config.py +++ b/kb_server/config.py @@ -37,7 +37,7 @@ logging = { 'root': {'level': 'INFO', 'handlers': ['console']}, 'loggers': { 'kb_server': {'level': 'DEBUG', 'handlers': ['console']}, - 'pecan': {'level': 'DEBUG', 'handlers': ['console']}, + 'pecan': {'level': 'CRITICAL', 'handlers': ['console']}, 'py.warnings': {'handlers': ['console']}, '__force_dict__': True }, diff --git a/kb_server/kb_server/controllers/__init__.py b/kb_server/kb_server/controllers/__init__.py index e69de29..b94bcec 100644 --- a/kb_server/kb_server/controllers/__init__.py +++ b/kb_server/kb_server/controllers/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2015 Cisco Systems, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys +kb_main_path = os.path.split(os.path.abspath(__file__))[0] + "/../../.." +sys.path.append(kb_main_path) diff --git a/kb_server/kb_server/controllers/api_kb.py b/kb_server/kb_server/controllers/api_kb.py index abf3cd3..bd48e2d 100644 --- a/kb_server/kb_server/controllers/api_kb.py +++ b/kb_server/kb_server/controllers/api_kb.py @@ -65,7 +65,7 @@ class KBController(object): kloudbuster = kb_session.kloudbuster try: kb_session.sync_cfg(["client_cfg"]) - kloudbuster.run_test(http_test_only=not kb_session.first_run) + kloudbuster.run_test(test_only=not kb_session.first_run) kb_session.first_run = False kb_session.kb_status = 'STAGED' except Exception: diff --git a/kloudbuster/force_cleanup.sh b/kloudbuster/force_cleanup.sh deleted file mode 100755 index c81fa0e..0000000 --- a/kloudbuster/force_cleanup.sh +++ /dev/null @@ -1,148 +0,0 @@ -#! /bin/bash -# Copyright 2015 Cisco Systems, Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -############################################################################### -# # -# This is a helper script which will delete all resources created by # -# KloudBuster. # -# # -# Normally, KloudBuster will clean up automatically when it is done. However, # -# sometimes errors or timeouts happen during the rescource creation stage, # -# which will cause KloudBuster out of sync with the real environment. If that # -# happens, a force cleanup may be needed. # -# # -# It is safe to use the script with the resource list generated by # -# KloudBuster, usage: # -# $ source # -# $ ./force_cleanup --file kb_20150807_183001_svr.log # -# $ ./force_cleanup --file kb_20150807_183001_cnt.log # -# # -# Note: If running under single-tenant or tenant/user reusing mode, you have # -# to cleanup the server resources first, then client resources. # -# # -# When there is no resource list provided, the script will simply grep the # -# resource name with "KB" and delete them. If running on a production # -# network, please double and triple check all resources names are *NOT* # -# containing "KB", otherwise they will be deleted by the script. # -# # -############################################################################### - -# ====================================================== -# WARNING -# ====================================================== -# IMPORTANT FOR RUNNING KLOUDBUSTER ON PRODUCTION CLOUDS -# -# DOUBLE CHECK THE NAMES OF ALL RESOURCES THAT DO NOT -# BELONG TO KLOUDBUSTER ARE *NOT* CONTAINING "KB". -# ====================================================== -function prompt_to_run() { - echo "Warning: You didn't specify a resource list file as the input,"\ - "or the input file is invalid. The script will delete all"\ - "resources of the cloud whose names contain \"KB\". " - read -p "Are you sure? (Y/N) " answer - if [ "$answer" != "Y" ] && [ "$answer" != "y" ]; then - exit 0 - fi -} - -if [ "$1" == "--file" ] && [ -f "$2" ]; then - INSTANCE_LIST=`grep "instances" $2 | cut -d'|' -f3` - SEC_GROUP_LIST=`grep "sec_groups" $2 | cut -d'|' -f3` - FLAVOR_LIST=`grep "flavors" $2 | cut -d'|' -f3` - KEYPAIR_LIST=`grep "keypairs" $2 | cut -d'|' -f2` - ROUTER_LIST=`grep "routers" $2 | cut -d'|' -f3` - NETWORK_LIST=`grep "networks" $2 | cut -d'|' -f3` - TENANT_LIST=`grep "tenants" $2 | cut -d'|' -f3` - USER_LIST=`grep "users" $2 | cut -d'|' -f3` - FLOATINGIP_LIST=`grep "floating_ips" $2 | cut -d'|' -f3` - VOL_LIST=`grep "volumes" $2 | cut -d'|' -f3` -else - prompt_to_run; - INSTANCE_LIST=`nova list --all-tenants | grep KB | cut -d'|' -f2` - SEC_GROUP_LIST=`neutron security-group-list | grep KB | cut -d'|' -f2` - FLAVOR_LIST=`nova flavor-list | grep KB | cut -d'|' -f3` - KEYPAIR_LIST=`nova keypair-list | grep KB | cut -d'|' -f2` - ROUTER_LIST=`neutron router-list | grep KB | cut -d'|' -f2` - NETWORK_LIST=`neutron net-list | grep KB | cut -d'|' -f2` - TENANT_LIST=`keystone tenant-list | grep KB | cut -d'|' -f2` - USER_LIST=`keystone user-list | grep KB | cut -d'|' -f2` - FLOATINGIP_LIST="" - VOL_LIST=`cinder list --all-tenants | grep KB | cut -d'|' -f2` -fi - -vol_list=`cinder list --all-tenants | grep KB` -for line in $VOL_LIST; do - ins_id=`echo $vol_list | grep $line | cut -d'|' -f9 | xargs` - if [ "$ins_id" != "" ]; then - nova volume-detach $ins_id $line - while true; do - vol_st=`cinder list --all-tenants | grep $line | cut -d'|' -f4 | xargs` - if [ $? -ne 0 ]; then break; fi - if [ "$vol_st" == "available" ]; then break; fi - done - fi - cinder force-delete $line & -done; - -for line in $INSTANCE_LIST; do - nova delete $line -done - -for line in $FLAVOR_LIST; do - nova flavor-delete $line & -done; - -for line in $SEC_GROUP_LIST; do - neutron security-group-delete $line & -done; - -for line in $KEYPAIR_LIST; do - nova keypair-delete "$line" & -done; - -if [ "$FLOATINGIP_LIST" == "" ] && [ "$1" == "" ]; then - echo -e "`neutron floatingip-list | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'`" | while read line; do - fid=`echo $line | cut -d'|' -f2 | xargs` - portid=`echo $line | cut -d'|' -f5 | xargs` - if [ "$fid" != "" ] && [ "$portid" = "" ]; then - neutron floatingip-delete $fid & - fi - done; -else - for line in $FLOATINGIP_LIST; do - neutron floatingip-delete $line & - done; -fi - -for line in $ROUTER_LIST; do - neutron router-gateway-clear $line - for line2 in `neutron router-port-list $line | grep subnet_id | cut -d'"' -f4`; do - neutron router-interface-delete $line $line2 - done - neutron router-delete $line -done - -for line in $NETWORK_LIST; do - neutron net-delete $line -done - -for line in $TENANT_LIST; do - keystone tenant-delete $line -done - -for line in $USER_LIST; do - keystone user-delete $line -done diff --git a/kloudbuster/kloudbuster.py b/kloudbuster/kloudbuster.py index 5f4478e..8dccbc1 100755 --- a/kloudbuster/kloudbuster.py +++ b/kloudbuster/kloudbuster.py @@ -642,13 +642,17 @@ class KloudBuster(object): def calc_cinder_quota(self): total_vm = self.get_tenant_vm_count(self.server_cfg) + svr_disk = self.server_cfg['flavor']['disk']\ + if self.server_cfg['flavor']['disk'] != 0 else 20 server_quota = {} - server_quota['gigabytes'] = total_vm * self.server_cfg['flavor']['disk'] + server_quota['gigabytes'] = total_vm * svr_disk server_quota['volumes'] = total_vm - client_quota = {} total_vm = total_vm * self.server_cfg['number_tenants'] - client_quota['gigabytes'] = total_vm * self.client_cfg['flavor']['disk'] + 20 + clt_disk = self.client_cfg['flavor']['disk']\ + if self.client_cfg['flavor']['disk'] != 0 else 20 + client_quota = {} + client_quota['gigabytes'] = total_vm * clt_disk + 20 client_quota['volumes'] = total_vm return [server_quota, client_quota] diff --git a/kloudbuster/log.py b/kloudbuster/log.py index 502774a..b5e222e 100644 --- a/kloudbuster/log.py +++ b/kloudbuster/log.py @@ -55,9 +55,7 @@ def setup(product_name, logfile=None): fmt = logging.Formatter(fmt=CONF.logging_default_format_string) hdlr = logging.FileHandler(logfile) hdlr.setFormatter(fmt) - for name in oslogging._loggers: - if name: - oslogging.getLogger(name).logger.addHandler(hdlr) + oslogging._loggers[product_name].logger.addHandler(hdlr) if CONF.kb_debug: oslogging.getLogger( diff --git a/kloudbuster/start_server.py b/kloudbuster/start_server.py index 3a0b792..248ce46 100755 --- a/kloudbuster/start_server.py +++ b/kloudbuster/start_server.py @@ -18,14 +18,12 @@ import subprocess import sys def exec_command(cmd, cwd=None, show_console=False): - p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if show_console: for line in iter(p.stdout.readline, b""): print line, - (_, stderr) = p.communicate() - if p.returncode: - print stderr + p.communicate() return p.returncode def main(): diff --git a/setup.cfg b/setup.cfg index 8620ce6..a5e8cba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,6 +21,7 @@ classifier = [files] packages = kloudbuster + kb_server package_data = kloudbuster =