x/compass-core
Code Issues Proposed changes

clean installers when refresh

Browse Source 
Change-Id: I86c27b0d9be0af6ef58748d97938a2df54b297c3
This commit is contained in:
xiaodongwang 2014-10-27 15:59:36 -07:00
parent a8b54b2258
commit ce07c3b7dc
15 changed files with 504 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
bin/clean_installers.py Executable file
View File

@ -0,0 +1,163 @@
#!/usr/bin/env python
#
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Scripts to delete cluster and it hosts"""
import logging
import os
import os.path
import sys
current_dir = os.path.dirname(os.path.realpath(__file__))
sys.path.append(current_dir)
import switch_virtualenv
from compass.actions import clean
from compass.db.api import adapter_holder as adapter_api
from compass.db.api import database
from compass.db.api import user as user_api
from compass.tasks.client import celery
from compass.utils import flags
from compass.utils import logsetting
from compass.utils import setting_wrapper as setting
flags.add_bool('async',
help='run in async mode',
default=True)
flags.add('os_installers',
help='comma seperated os installers',
default='')
flags.add('package_installers',
help='comma separated package installers',
default='')
def clean_installers():
os_installers = [
os_installer
for os_installer in flags.OPTIONS.os_installers.split(',')
if os_installer
]
package_installers = [
package_installer
for package_installer in flags.OPTIONS.package_installers.split(',')
if package_installer
]
user = user_api.get_user_object(setting.COMPASS_ADMIN_EMAIL)
adapters = adapter_api.list_adapters(user)
filtered_os_installers = {}
filtered_package_installers = {}
for adapter in adapters:
logging.info(
'got adapter: %s', adapter
)
if 'os_installer' in adapter:
os_installer = adapter['os_installer']
os_installer_name = os_installer['alias']
if not os_installers or os_installer_name in os_installers:
filtered_os_installers[os_installer_name] = os_installer
else:
logging.info(
'ignore os isntaller %s', os_installer_name
)
else:
logging.info(
'cannot find os installer in adapter %s',
adapter['name']
)
if 'package_installer' in adapter:
package_installer = adapter['package_installer']
package_installer_name = package_installer['alias']
if (
not package_installers or
package_installer_name in package_installers
):
filtered_package_installers[package_installer_name] = (
package_installer
)
else:
logging.info(
'ignore package installer %s', package_installer_name
)
else:
logging.info(
'cannot find package installer in adapter %s',
adapter['name']
)
logging.info(
'clean os installers: %s', filtered_os_installers.keys()
)
logging.info(
'clean package installers: %s', filtered_package_installers.keys()
)
if flags.OPTIONS.async:
for os_installer_name, os_installer in filtered_os_installers.items():
celery.send_task(
'compass.tasks.clean_os_installer',
(
os_installer['name'],
os_installer['settings']
)
)
for package_installer_name, package_installer in (
filtered_package_installers.items()
):
celery.send_task(
'compass.tasks.clean_package_installer',
(
package_installer['name'],
package_installer['settings']
)
)
else:
for os_installer_name, os_installer in (
filtered_os_installers.items()
):
try:
clean.clean_os_installer(
os_installer['name'],
os_installer['settings']
)
except Exception as error:
logging.error(
'failed to clean os installer %s', os_installer_name
)
logging.exception(error)
for package_installer_name, package_installer in (
filtered_package_installers.items()
):
try:
clean.clean_package_installer(
package_installer['name'],
package_installer['settings']
)
except Exception as error:
logging.error(
'failed to clean package installer %s',
package_installer_name
)
logging.exception(error)
if __name__ == '__main__':
flags.init()
logsetting.init()
database.init()
clean_installers()

5
bin/delete_clusters.py
View File

@ -56,8 +56,11 @@ def delete_clusters():
if clustername
]
user = user_api.get_user_object(setting.COMPASS_ADMIN_EMAIL)
list_cluster_args = {}
if clusternames:
list_cluster_args['name'] = clusternames
clusters = cluster_api.list_clusters(
user, name=clusternames
user, **list_cluster_args
)
delete_underlying_host = flags.OPTIONS.delete_hosts
for cluster in clusters:

5
bin/refresh.sh
View File

@ -2,10 +2,7 @@
set -e
service mysqld restart
/opt/compass/bin/manage_db.py createdb
echo "You may run '/opt/compass/bin/clean_nodes.sh' to clean nodes on chef server"
echo "You may run '/opt/compass/bin/clean_clients.sh' to clean clients on chef server"
echo "you may run '/opt/compass/bin/clean_environments.sh' to clean environments on chef server"
echo "you may run '/opt/compass/bin/remove_systems.sh' to clean systems on cobbler"
/opt/compass/bin/clean_installers.py
/opt/compass/bin/clean_installation_logs.py
service httpd restart
service rsyslog restart

182
compass/actions/clean.py Normal file
View File

@ -0,0 +1,182 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to clean installers
"""
import chef
import logging
import xmlrpclib
from compass.actions import util
class CobblerInstaller(object):
"""cobbler installer"""
CREDENTIALS = "credentials"
USERNAME = 'username'
PASSWORD = 'password'
INSTALLER_URL = "cobbler_url"
def __init__(self, settings):
username = settings[self.CREDENTIALS][self.USERNAME]
password = settings[self.CREDENTIALS][self.PASSWORD]
cobbler_url = settings[self.INSTALLER_URL]
try:
self.remote = xmlrpclib.Server(cobbler_url)
self.token = self.remote.login(username, password)
logging.info('cobbler %s client created', cobbler_url)
except Exception as error:
logging.error(
'failed to login %s with (%s, %s)',
cobbler_url, username, password
)
logging.exception(error)
def clean(self):
systems = self.remote.get_systems()
for system in systems:
system_name = system['name']
try:
self.remote.remove_system(system_name, self.token)
logging.info('system %s is removed', system_name)
except Exception as error:
logging.error(
'failed to remove system %s', system_name
)
logging.exception(error)
class ChefInstaller(object):
DATABAGS = "databags"
CHEFSERVER_URL = "chef_url"
CHEFSERVER_DNS = "chef_server_dns"
CHEFSERVER_IP = "chef_server_ip"
KEY_DIR = "key_dir"
CLIENT = "client_name"
def __init__(self, settings):
installer_url = settings.get(self.CHEFSERVER_URL, None)
key_dir = settings.get(self.KEY_DIR, None)
client = settings.get(self.CLIENT, None)
try:
if installer_url and key_dir and client:
self.api = chef.ChefAPI(installer_url, key_dir, client)
else:
self.api = chef.autoconfigure()
logging.info(
'chef client created %s(%s, %s)',
installer_url, key_dir, client
)
except Exception as error:
logging.error(
'failed to create chef client %s(%s, %s)',
installer_url, key_dir, client
)
logging.exception(error)
def clean(self):
try:
for node_name in chef.Node.list(api=self.api):
node = chef.Node(node_name, api=self.api)
node.delete()
logging.info('delete node %s', node_name)
except Exception as error:
logging.error('failed to delete some nodes')
logging.exception(error)
try:
for client_name in chef.Client.list(api=self.api):
if client_name in ['chef-webui', 'chef-validator']:
continue
client = chef.Client(client_name, api=self.api)
client.delete()
logging.info('delete client %s', client_name)
except Exception as error:
logging.error('failed to delete some clients')
logging.exception(error)
try:
for env_name in chef.Environment.list(api=self.api):
if env_name == '_default':
continue
env = chef.Environment(env_name, api=self.api)
env.delete()
logging.info('delete env %s', env_name)
except Exception as error:
logging.error('failed to delete some envs')
logging.exception(error)
try:
for databag_name in chef.DataBag.list(api=self.api):
databag = chef.DataBag(databag_name, api=self.api)
for item_name, item in databag.items():
item.delete()
logging.info(
'delete item %s from databag %s',
item_name, databag_name
)
except Exception as error:
logging.error('failed to delete some databag items')
logging.exception(error)
OS_INSTALLERS = {
'cobbler': CobblerInstaller
}
PK_INSTALLERS = {
'chef_installer': ChefInstaller
}
def clean_os_installer(
os_installer_name, os_installer_settings
):
with util.lock('serialized_action', timeout=100) as lock:
if not lock:
raise Exception(
'failed to acquire lock to clean os installer'
)
if os_installer_name not in OS_INSTALLERS:
logging.error(
'%s not found in os_installers',
os_installer_name
)
os_installer = OS_INSTALLERS[os_installer_name](
os_installer_settings
)
os_installer.clean()
def clean_package_installer(
package_installer_name, package_installer_settings
):
with util.lock('serialized_action', timeout=100) as lock:
if not lock:
raise Exception(
'failed to acquire lock to clean package installer'
)
if package_installer_name not in PK_INSTALLERS:
logging.error(
'%s not found in os_installers',
package_installer_name
)
package_installer = PK_INSTALLERS[package_installer_name](
package_installer_settings
)
package_installer.clean()

15
compass/db/api/metadata.py
View File

@ -301,17 +301,28 @@ def _validate_self(
metadata, whole_check,
**kwargs
):
logging.debug('validate config self %s', config_path)
if '_self' not in metadata:
if isinstance(config, dict):
_validate_config(
config_path, config, metadata, whole_check, **kwargs
)
return
field_type = metadata['_self'].get('field_type', 'basestring')
field_type = metadata['_self'].get('field_type', basestring)
if not isinstance(config, field_type):
raise exception.InvalidParameter(
'%s config type is not %s' % (config_path, field_type)
)
is_required = metadata['_self'].get(
'is_required', False
)
required_in_whole_config = metadata['_self'].get(
'required_in_whole_config', False
)
if isinstance(config, basestring):
if config == '' and not is_required and not required_in_whole_config:
# ignore empty config when it is optional
return
required_in_options = metadata['_self'].get(
'required_in_options', False
)
@ -333,6 +344,7 @@ def _validate_self(
'%s config is not in %s' % (config_path, options)
)
validator = metadata['_self'].get('validator', None)
logging.debug('validate by validator %s', validator)
if validator:
if not validator(config_key, config, **kwargs):
raise exception.InvalidParameter(
@ -348,6 +360,7 @@ def _validate_config(
config_path, config, metadata, whole_check,
**kwargs
):
logging.debug('validate config %s', config_path)
generals = {}
specified = {}
for key, value in metadata.items():

96
compass/db/validator.py
View File

@ -13,6 +13,7 @@
# limitations under the License.
"""Validator methods."""
import logging
import netaddr
import re
import socket
@ -23,87 +24,162 @@ from compass.utils import util
def is_valid_ip(name, ip_addr, **kwargs):
"""Valid the format of an IP address."""
if isinstance(ip_addr, list):
return all([
is_valid_ip(name, item, **kwargs) for item in ip_addr
])
try:
netaddr.IPAddress(ip_addr)
except Exception:
logging.debug('%s invalid ip addr %s', name, ip_addr)
return False
return True
def is_valid_network(name, ip_network, **kwargs):
"""Valid the format of an Ip network."""
if isinstance(ip_network, list):
return all([
is_valid_network(name, item, **kwargs) for item in ip_network
])
try:
netaddr.IPNetwork(ip_network)
except Exception:
logging.debug('%s invalid network %s', name, ip_network)
return False
return False
return True
def is_valid_netmask(name, ip_addr, **kwargs):
"""Valid the format of a netmask."""
if isinstance(ip_addr, list):
return all([
is_valid_netmask(name, item, **kwargs) for item in ip_addr
])
if not is_valid_ip(ip_addr):
return False
ip = netaddr.IPAddress(ip_addr)
if ip.is_netmask():
return True
else:
return False
logging.debug('%s invalid netmask %s', name, ip_addr)
return False
def is_valid_gateway(name, ip_addr, **kwargs):
"""Valid the format of gateway."""
if isinstance(ip_addr, list):
return all([
is_valid_gateway(name, item, **kwargs) for item in ip_addr
])
if not is_valid_ip(ip_addr):
return False
ip = netaddr.IPAddress(ip_addr)
if ip.is_private() or ip.is_public():
return True
else:
return False
logging.debug('%s invalid gateway %s', name, ip_addr)
return False
def is_valid_dns(name, dns, **kwargs):
"""Valid the format of DNS."""
if isinstance(dns, list):
return all([is_valid_dns(name, item, **kwargs) for item in dns])
if is_valid_ip(dns):
return True
try:
socket.gethostbyname_ex(dns)
except Exception:
logging.debug('%s invalid dns name %s', name, dns)
return False
return True
def is_valid_url(name, url, **kwargs):
"""Valid the format of url."""
if isinstance(url, list):
return all([
is_valid_url(name, item, **kwargs) for item in url
])
if re.match(
r'^(http|https|ftp)://([0-9A-Za-z_-]+)(\.[0-9a-zA-Z_-]+)*'
r'(:\d+)?(/[0-9a-zA-Z_-]+)*$',
url
):
return True
logging.debug(
'%s invalid url %s', name, url
)
return False
def is_valid_domain(name, domain, **kwargs):
"""Validate the format of domain."""
if isinstance(domain, list):
return all([
is_valid_domain(name, item, **kwargs) for item in domain
])
if re.match(
r'^([0-9a-zA-Z_-]+)(\.[0-9a-zA-Z_-]+)*$',
domain
):
return True
logging.debug(
'%s invalid domain %s', name, domain
)
return False
def is_valid_username(name, username, **kwargs):
"""Valid the format of username."""
return bool(username)
if bool(username):
return True
logging.debug(
'%s username is empty', name
)
def is_valid_password(name, password, **kwargs):
"""Valid the format of password."""
return bool(password)
if bool(password):
return True
logging.debug('%s password is empty', name)
return False
def is_valid_partition(name, partition, **kwargs):
"""Valid the format of partition name."""
if name != 'swap' and not name.startswith('/'):
logging.debug(
'%s is not started with / or swap', name
)
return False
if 'size' not in partition and 'percentage' not in partition:
logging.debug(
'%s partition does not contain sie or percentage',
name
)
return False
return True
def is_valid_percentage(name, percentage, **kwargs):
"""Valid the percentage."""
return 0 <= percentage <= 100
if 0 <= percentage <= 100:
return True
logging.debug('%s invalid percentage %s', name, percentage)
def is_valid_port(name, port, **kwargs):
"""Valid the format of port."""
return 0 < port < 65536
if 0 < port < 65536:
return True
logging.debug('%s invalid port %s', name, port)
def is_valid_size(name, size, **kwargs):
if re.match(r'(\d+)(K|M|G|T)?', size):
if re.match(r'^(\d+)(K|M|G|T)$', size):
return True
logging.debug('%s invalid size %s', name, size)
return False

27
compass/tasks/tasks.py
View File

@ -21,6 +21,7 @@ import logging
from celery.signals import celeryd_init
from celery.signals import setup_logging
from compass.actions import clean
from compass.actions import delete
from compass.actions import deploy
from compass.actions import poll_switch
@ -168,6 +169,32 @@ def delete_host(deleter_email, host_id, cluster_ids):
logging.exception(error)
@celery.task(name='compass.tasks.clean_os_installer')
def clean_os_installer(
os_installer_name, os_installer_settings
):
"""Clean os installer."""
try:
clean.clean_os_installer(
os_installer_name, os_installer_settings
)
except Exception as error:
logging.excception(error)
@celery.task(name='compass.tasks.clean_package_installer')
def clean_package_installer(
package_installer_name, package_installer_settings
):
"""Clean package installer."""
try:
clean.clean_package_installer(
package_installer_name, package_installer_settings
)
except Exception as error:
logging.excception(error)
@celery.task(name='compass.tasks.poweron_host')
def poweron_host(host_id):
"""Deploy the given cluster.

2
conf/os_field/domain.conf Normal file
View File

@ -0,0 +1,2 @@
NAME = 'domain'
VALIDATOR = is_valid_domain

3
conf/os_field/ip_list.conf Normal file
View File

@ -0,0 +1,3 @@
NAME = 'ip_list'
FIELD_TYPE = list
VALIDATOR = is_valid_ip

2
conf/os_field/url.conf Normal file
View File

@ -0,0 +1,2 @@
NAME = 'url'
VALIDATOR = is_valid_url

12
conf/os_metadata/general.conf
View File

@ -29,7 +29,7 @@ METADATA = {
},
'http_proxy': {
'_self': {
'field': 'general',
'field': 'url',
'default_callback': default_proxy,
'options_callback': proxy_options,
'mapping_to': 'http_proxy'
@ -37,7 +37,7 @@ METADATA = {
},
'https_proxy': {
'_self': {
'field': 'general',
'field': 'url',
'default_callback': default_proxy,
'options_callback': proxy_options,
'mapping_to': 'https_proxy'
@ -55,7 +55,7 @@ METADATA = {
'ntp_server': {
'_self': {
'is_required': True,
'field': 'general',
'field': 'ip',
'default_callback': default_ntp_server,
'options_callback': ntp_server_options,
'mapping_to': 'ntp_server'
@ -64,7 +64,7 @@ METADATA = {
'dns_servers': {
'_self': {
'is_required': True,
'field': 'general_list',
'field': 'ip_list',
'default_callback': default_dns_servers,
'options_callback': dns_servers_options,
'mapping_to': 'nameservers'
@ -72,7 +72,7 @@ METADATA = {
},
'domain': {
'_self': {
'field': 'general',
'field': 'domain',
'is_required' : True,
'default_callback': default_domain,
'options_callback': domain_options,
@ -96,7 +96,7 @@ METADATA = {
},
'local_repo': {
'_self': {
'field': 'general',
'field': 'url',
'default_callback': default_localrepo,
'mapping_to': 'local_repo'
}

7
install/install_func.sh
View File

@ -41,21 +41,18 @@ copy2dir()
git clean -x -f
git checkout $git_branch
git reset --hard remotes/origin/$git_branch
git clean -x -f -d -q
cd -
else
echo "create $destdir"
mkdir -p $destdir
git clone $repo $destdir
git clone $repo $destdir -b $git_branch
if [ $? -ne 0 ]; then
echo "failed to git clone $repo $destdir"
exit 1
else
echo "git clone $repo $destdir suceeded"
fi
cd $destdir
git checkout $git_branch
git reset --hard remotes/origin/$git_branch
cd -
fi
cd $destdir
if [[ -z $ZUUL_PROJECT ]]; then

8
install/prepare.sh
View File

@ -125,6 +125,12 @@ else
fi
cd $SCRIPT_DIR
remote_branch=$(git rev-parse --abbrev-ref --symbolic-full-name @{u})
if [[ "$?" != "0" ]]; then
remote_branch="origin/master"
fi
local_branch=$(echo ${remote_branch} | sed -e 's/origin\///g')
if [ -z $WEB_SOURCE ]; then
echo "web source $WEB_SOURCE is not set"
exit 1
@ -135,7 +141,7 @@ if [ -z $ADAPTERS_SOURCE ]; then
echo "adpaters source $ADAPTERS_SOURCE is not set"
exit 1
fi
copy2dir "$ADAPTERS_SOURCE" "$ADAPTERS_HOME" "stackforge/compass-adapters" dev/experimental || exit $?
copy2dir "$ADAPTERS_SOURCE" "$ADAPTERS_HOME" "stackforge/compass-adapters" ${local_branch} || exit $?
if [ "$tempest" == "true" ]; then
echo "download tempest packages"

6
regtest/regtest.sh
View File

@ -79,8 +79,6 @@ for i in `seq $VIRT_NUM`; do
exit 1
fi
echo "make pxe${i} reboot if installation failing."
sed -i "/<boot dev='hd'\/>/ a\ <bios useserial='yes' rebootTimeout='0'\/>" /etc/libvirt/qemu/pxe${i}.xml
echo "check pxe${i} state"
state=$(virsh domstate pxe${i})
if [[ "$state" == "running" ]]; then
@ -92,6 +90,10 @@ for i in `seq $VIRT_NUM`; do
fi
fi
echo "make pxe${i} reboot if installation failing."
sed -i "/<boot dev='hd'\/>/ a\ <bios useserial='yes' rebootTimeout='0'\/>" /etc/libvirt/qemu/pxe${i}.xml
virsh define /etc/libvirt/qemu/pxe${i}.xml
echo "start pxe${i}"
virsh start pxe${i}
if [[ "$?" != "0" ]]; then

2
tox.ini
View File

@ -31,7 +31,7 @@ commands = python setup.py testr --coverage --testr-args='{posargs}'
downloadcache = ~/cache/pip
[flake8]
ignore = H302,H233,H803,F401
ignore = H302,H304,H233,H803,F401
show-source = true
builtins = _
exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build