From e72c9d06b150bb36665f309b096b61f93565c6cf Mon Sep 17 00:00:00 2001 From: xiaodongwang Date: Fri, 10 Oct 2014 21:48:47 -0700 Subject: [PATCH] use local repo in regtest Change-Id: I874e4acb14e2853c903bd4455789b714d9f987dd --- compass/actions/cli.py | 13 +- compass/actions/health_check/base.py | 12 +- compass/actions/health_check/check_apache.py | 6 +- compass/actions/health_check/check_celery.py | 2 +- compass/actions/health_check/check_dhcp.py | 5 +- compass/actions/health_check/check_dns.py | 5 +- compass/actions/health_check/check_hds.py | 11 +- compass/actions/health_check/check_misc.py | 20 ++-- .../health_check/check_os_installer.py | 5 +- .../health_check/check_package_installer.py | 9 +- compass/actions/health_check/check_tftp.py | 5 +- compass/actions/health_check/setting.py | 7 +- compass/db/api/database.py | 5 + compass/log_analyzor/progress_calculator.py | 2 +- install/chef.sh | 31 +++++ install/compass.sh | 68 +---------- install/compass_web.sh | 43 +++++++ install/dependency.sh | 2 +- install/env.conf | 10 -- install/install.conf.template | 10 +- install/install.sh | 7 +- install/install_func.sh | 111 +++++++++++++++--- install/local_repo.sh | 7 -- install/prepare.sh | 5 +- install/setup_env.sh | 1 + regtest/regtest.conf | 6 +- regtest/regtest10.conf | 2 + regtest/regtest7.conf | 2 + regtest/regtest8.conf | 2 + regtest/regtest9.conf | 2 + requirements.txt | 2 + 31 files changed, 259 insertions(+), 159 deletions(-) create mode 100755 install/compass_web.sh delete mode 100755 install/env.conf delete mode 100755 install/local_repo.sh diff --git a/compass/actions/cli.py b/compass/actions/cli.py index 44aba46b..c9058ed0 100644 --- a/compass/actions/cli.py +++ b/compass/actions/cli.py @@ -13,10 +13,16 @@ # limitations under the License. """Compass Command Line Interface""" +import logging import subprocess import sys from compass.actions.health_check import check +from compass.db.api import database + +from compass.utils import flags +from compass.utils import logsetting +from compass.utils import setting_wrapper as setting from compass.utils.util import pretty_print @@ -165,10 +171,9 @@ class BootCLI(object): def main(): """Compass cli entry point.""" + flags.init() + logsetting.init() + database.init() cli = BootCLI() output = cli.run(sys.argv) return sys.exit(output) - - -if __name__ == "__main__": - main() diff --git a/compass/actions/health_check/base.py b/compass/actions/health_check/base.py index 7829139d..141369c0 100644 --- a/compass/actions/health_check/base.py +++ b/compass/actions/health_check/base.py @@ -15,6 +15,7 @@ """Base class for Compass Health Check.""" from compass.actions.health_check import utils as health_check_utils from compass.db.api import database +from compass.db.api import utils from compass.db import models from compass.utils import setting_wrapper as setting @@ -31,24 +32,23 @@ class BaseCheck(object): self.package_installer = self._get_package_installer() def _get_os_installer(self): - database.init() - os_installer = {} with database.session() as session: - installer = session.query( - models.OSInstaller - ).first() + installer = utils.get_db_object( + session, models.OSInstaller + ) + os_installer = {} os_installer['name'] = health_check_utils.strip_name( installer.name) os_installer.update(installer.settings) return os_installer def _get_package_installer(self): - database.init() package_installer = {} with database.session() as session: installer = session.query( models.PackageInstaller ).first() + package_installer = {} package_installer['name'] = health_check_utils.strip_name( installer.name) package_installer.update(installer.settings) diff --git a/compass/actions/health_check/check_apache.py b/compass/actions/health_check/check_apache.py index 524d56a0..294d6f92 100644 --- a/compass/actions/health_check/check_apache.py +++ b/compass/actions/health_check/check_apache.py @@ -79,11 +79,7 @@ class ApacheCheck(base.BaseCheck): % self.NAME) try: html = urllib2.urlopen('http://localhost') - content = html.geturl() - if "http://localhost/ods/ods.html" != content: - self._set_status( - 0, - "[%s]Error: Compass web is not redirected by Apache.") + html.geturl() except Exception: self._set_status( 0, diff --git a/compass/actions/health_check/check_celery.py b/compass/actions/health_check/check_celery.py index 255028a5..2d8d27c1 100644 --- a/compass/actions/health_check/check_celery.py +++ b/compass/actions/health_check/check_celery.py @@ -80,7 +80,7 @@ class CeleryCheck(base.BaseCheck): """Checks if Celery backend is running and configured properly.""" print "Checking Celery Backend......", - if 'celeryd' not in commands.getoutput('ps -ef'): + if 'celery worker' not in commands.getoutput('ps -ef'): self._set_status(0, "[%s]Error: celery is not running" % self.NAME) return True diff --git a/compass/actions/health_check/check_dhcp.py b/compass/actions/health_check/check_dhcp.py index cb3429fe..e3bae1ee 100644 --- a/compass/actions/health_check/check_dhcp.py +++ b/compass/actions/health_check/check_dhcp.py @@ -37,10 +37,11 @@ class DhcpCheck(base.BaseCheck): try: remote = xmlrpclib.Server( - self.os_installer['url'], + self.os_installer['cobbler_url'], allow_none=True) + credentials = self.os_installer['credentials'] remote.login( - *self.os_installer['token']) + credentials['username'], credentials['password']) except Exception: self._set_status( 0, diff --git a/compass/actions/health_check/check_dns.py b/compass/actions/health_check/check_dns.py index 67ee970d..843d7e29 100644 --- a/compass/actions/health_check/check_dns.py +++ b/compass/actions/health_check/check_dns.py @@ -35,10 +35,11 @@ class DnsCheck(base.BaseCheck): """Checks if Cobbler has taken over DNS service.""" try: remote = xmlrpclib.Server( - self.os_installer['url'], + self.os_installer['cobbler_url'], allow_none=True) + credentials = self.os_installer['credentials'] remote.login( - *self.os_installer['token']) + credentials['username'], credentials['password']) except Exception: self._set_status(0, "[%s]Error: Cannot login to Cobbler " diff --git a/compass/actions/health_check/check_hds.py b/compass/actions/health_check/check_hds.py index 2af0be7f..afc7bf5d 100644 --- a/compass/actions/health_check/check_hds.py +++ b/compass/actions/health_check/check_hds.py @@ -13,6 +13,8 @@ # limitations under the License. """Health Check module for Hardware Discovery.""" +import logging + from compass.actions.health_check import base from compass.actions.health_check import utils as health_check_utils @@ -31,10 +33,13 @@ class HdsCheck(base.BaseCheck): try: pkg_module = __import__(pkg_type) except Exception: - self.messages.append("[%s]Error: No module named %s, " - "please install it first." - % (self.NAME, pkg_module)) + self._set_status( + 0, "[%s]Error: No module named %s please install it first." + % (self.NAME, pkg_type) + ) + return (self.code, self.messages) + logging.info('import %s: %s', pkg_type, pkg_module) method_name = 'self.check_' + pkg_type + '_snmp(pkg_module)' eval(method_name) print "[Done]" diff --git a/compass/actions/health_check/check_misc.py b/compass/actions/health_check/check_misc.py index 3cbd1566..b8beb1b5 100644 --- a/compass/actions/health_check/check_misc.py +++ b/compass/actions/health_check/check_misc.py @@ -13,6 +13,8 @@ # limitations under the License. """Miscellaneous Health Check for Compass.""" +import logging + from compass.actions.health_check import base from compass.actions.health_check import utils as health_check_utils @@ -23,16 +25,16 @@ class MiscCheck(base.BaseCheck): MISC_MAPPING = { "yum": "rsyslog ntp iproute openssh-clients python git wget " - "python-setuptools python-netaddr python-flask " - "python-flask-sqlalchemy python-amqplib amqp " - "python-paramiko python-mock mod_wsgi httpd squid " + "python-setuptools " + "amqp mod_wsgi httpd squid " "dhcp bind rsync yum-utils xinetd tftp-server gcc " - "net-snmp-utils net-snmp python-daemon".split(" "), - "pip": "flask-script flask-restful celery six discover " - "unittest2 chef".replace("-", "_").split(" "), + "net-snmp-utils net-snmp".split(" "), + "pip": "netaddr flask flask_script flask_restful amqplib " + "flask_sqlalchemy paramiko mock celery six discover daemon " + "unittest2 chef".split(" "), "disable": "iptables ip6tables".split(" "), "enable": "httpd squid xinetd dhcpd named sshd rsyslog cobblerd " - "ntpd compassd".split(" "), + "ntpd compass-celeryd compass-progress-updated".split(" "), } def run(self): @@ -70,8 +72,10 @@ class MiscCheck(base.BaseCheck): self._set_status( 0, "[%s]Error: No module named %s, " - "please install it first." % (self.NAME, pkg_module)) + "please install it first." % (self.NAME, pkg_type)) + return True + logging.info('import %s: %s', pkg_type, pkg_module) method_name = 'self.check_' + pkg_type + '_dependencies(pkg_module)' eval(method_name) diff --git a/compass/actions/health_check/check_os_installer.py b/compass/actions/health_check/check_os_installer.py index 7e4e25fb..6ef98180 100644 --- a/compass/actions/health_check/check_os_installer.py +++ b/compass/actions/health_check/check_os_installer.py @@ -33,10 +33,11 @@ class OsInstallerCheck(base.BaseCheck): """Runs cobbler check from xmlrpc client.""" try: remote = xmlrpclib.Server( - self.os_installer['url'], + self.os_installer['cobbler_url'], allow_none=True) + credentials = self.os_installer['credentials'] token = remote.login( - *self.os_installer['token']) + credentials['username'], credentials['password']) except Exception: self.code = 0 self.messages.append( diff --git a/compass/actions/health_check/check_package_installer.py b/compass/actions/health_check/check_package_installer.py index 04ec9b1e..c711e891 100644 --- a/compass/actions/health_check/check_package_installer.py +++ b/compass/actions/health_check/check_package_installer.py @@ -13,7 +13,7 @@ # limitations under the License. """Health Check module for Package Installer.""" - +import logging import os import requests @@ -32,10 +32,9 @@ class PackageInstallerCheck(base.BaseCheck): return eval(method_name) def chef_check(self): - """Checks chef setting, cookbooks, databags and roles.""" + """Checks chef setting, cookbooks and roles.""" chef_data_map = { 'CookBook': health_check_setting.COOKBOOKS, - 'DataBag': health_check_setting.DATABAGS, 'Role': health_check_setting.ROLES, } @@ -107,7 +106,7 @@ class PackageInstallerCheck(base.BaseCheck): name for name, item in chef.Role.list(api=api).iteritems() ]) github = set([ - item['name'].replace(".rb", "") + item['name'].replace(".rb", "").replace(".json", "") for item in requests.get(github_url).json() ]) else: @@ -116,6 +115,8 @@ class PackageInstallerCheck(base.BaseCheck): 'chef.' + data_type + '.list(api=api)' ) ]) + logging.info('github %s: %s', data_type, github) + logging.info('local %s: %s', data_type, local) diff = github - local if len(diff) <= 0: diff --git a/compass/actions/health_check/check_tftp.py b/compass/actions/health_check/check_tftp.py index efbcbc8b..7ca6405d 100644 --- a/compass/actions/health_check/check_tftp.py +++ b/compass/actions/health_check/check_tftp.py @@ -39,10 +39,11 @@ class TftpCheck(base.BaseCheck): try: remote = xmlrpclib.Server( - self.os_installer['name'], + self.os_installer['cobbler_url'], allow_none=True) + credentials = self.os_installer['credentials'] remote.login( - *self.os_installer['token']) + credentials['username'], credentials['password']) except Exception: self._set_status( 0, diff --git a/compass/actions/health_check/setting.py b/compass/actions/health_check/setting.py index 16be3ce2..a3e16ce2 100644 --- a/compass/actions/health_check/setting.py +++ b/compass/actions/health_check/setting.py @@ -17,10 +17,7 @@ # Chef data on github COOKBOOKS = ( "https://api.github.com/repos/stackforge" - "/compass-adapters/contents/chef/cookbooks") + "/compass-adapters/contents/chef/cookbooks?ref=dev/experimental") ROLES = ( "https://api.github.com/repos/stackforge" - "/compass-adapters/contents/chef/roles") -DATABAGS = ( - "https://api.github.com/repos/stackforge" - "/compass-adapters/contents/chef/databags") + "/compass-adapters/contents/chef/roles?ref=dev/experimental") diff --git a/compass/db/api/database.py b/compass/db/api/database.py index abfd92a0..67e5289e 100644 --- a/compass/db/api/database.py +++ b/compass/db/api/database.py @@ -20,6 +20,7 @@ import netaddr from contextlib import contextmanager from sqlalchemy import create_engine from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import OperationalError from sqlalchemy.orm import scoped_session from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool @@ -114,6 +115,10 @@ def session(): raise exception.NotAcceptable( 'operation error in database' ) + elif isinstance(error, OperationalError): + raise exception.DatabaseExcedption( + 'operation error in database' + ) elif isinstance(error, exception.DatabaseException): raise error else: diff --git a/compass/log_analyzor/progress_calculator.py b/compass/log_analyzor/progress_calculator.py index 840a6d57..dddb4d38 100644 --- a/compass/log_analyzor/progress_calculator.py +++ b/compass/log_analyzor/progress_calculator.py @@ -71,7 +71,7 @@ OS_INSTALLER_CONFIGURATIONS = { ), match_nextline_next_matcher_name='localechooser' ), - 'localechoose': LineMatcher( + 'localechooser': LineMatcher( pattern=r'Menu.*item.*\'localechooser\'.*selected', progress=.18, message_template='localechooser selected', diff --git a/install/chef.sh b/install/chef.sh index b2647b7d..a9967aa3 100755 --- a/install/chef.sh +++ b/install/chef.sh @@ -56,3 +56,34 @@ $CHEF_PASSWORD EOF sudo sed -i "/node_name/c\node_name \'admin\'" /$USER/.chef/knife.rb sudo sed -i "/client_key/c\client_key \'\/etc\/chef-server\/admin.pem\'" /$USER/.chef/knife.rb + + +sudo rm -rf /var/chef +sudo mkdir -p /var/chef/cookbooks/ +sudo cp -r $ADAPTERS_HOME/chef/cookbooks/* /var/chef/cookbooks/ +if [ $? -ne 0 ]; then + echo "failed to copy cookbooks to /var/chef/cookbooks/" + exit 1 +fi +sudo mkdir -p /var/chef/roles/ +sudo cp -r $ADAPTERS_HOME/chef/roles/* /var/chef/roles/ +if [ $? -ne 0 ]; then + echo "failed to copy roles to /var/chef/roles/" + exit 1 +fi + +knife cookbook upload --all --cookbook-path /var/chef/cookbooks +if [[ "$?" != "0" ]]; then + echo "failed to add cookbooks" + exit 1 +else + echo "cookbooks are added to chef server" +fi + +knife role from file /var/chef/roles/*.json +if [[ "$?" != "0" ]]; then + echo "failed to add roles" + exit 1 +else + echo "roles are added to chef server" +fi diff --git a/install/compass.sh b/install/compass.sh index ee8d13d8..6805ee0e 100755 --- a/install/compass.sh +++ b/install/compass.sh @@ -14,24 +14,11 @@ fi source $DIR/install_func.sh cd $SCRIPT_DIR -if [ -z $WEB_SOURCE ]; then - echo "web source $WEB_SOURCE is not set" - exit 1 -fi -copy2dir "$WEB_SOURCE" "$WEB_HOME" "stackforge/compass-web" || exit $? - -if [ -z $ADAPTERS_SOURCE ]; then - echo "adpaters source $ADAPTERS_SOURCE is not set" - exit 1 -fi -copy2dir "$ADAPTERS_SOURCE" "$ADAPTERS_HOME" "stackforge/compass-adapters" dev/experimental || exit $? mkdir -p /etc/compass rm -rf /etc/compass/* mkdir -p /opt/compass/bin rm -rf /opt/compass/bin/* -mkdir -p /var/www/compass_web -rm -rf /var/www/compass_web/* mkdir -p /var/log/compass rm -rf /var/log/compass/* sudo mkdir -p /var/log/chef @@ -51,29 +38,6 @@ sudo ln -s -f /opt/compass/bin/compass_wsgi.py /var/www/compass/compass.wsgi sudo cp -rf $COMPASSDIR/bin/chef/* /opt/compass/bin/ sudo cp -rf $COMPASSDIR/bin/cobbler/* /opt/compass/bin/ -sudo cp -rf $WEB_HOME/public/* /var/www/compass_web/ -sudo cp -rf $WEB_HOME/v2 /var/www/compass_web/ - -sudo rm -rf /var/chef -sudo mkdir -p /var/chef/cookbooks/ -sudo cp -r $ADAPTERS_HOME/chef/cookbooks/* /var/chef/cookbooks/ -if [ $? -ne 0 ]; then - echo "failed to copy cookbooks to /var/chef/cookbooks/" - exit 1 -fi -sudo mkdir -p /var/chef/databags/ -sudo cp -r $ADAPTERS_HOME/chef/databags/* /var/chef/databags/ -if [ $? -ne 0 ]; then - echo "failed to copy databags to /var/chef/databags/" - exit 1 -fi -sudo mkdir -p /var/chef/roles/ -sudo cp -r $ADAPTERS_HOME/chef/roles/* /var/chef/roles/ -if [ $? -ne 0 ]; then - echo "failed to copy roles to /var/chef/roles/" - exit 1 -fi - # add apache user to the group of virtualenv user sudo usermod -a -G `groups $USER|awk '{print$3}'` apache @@ -89,20 +53,15 @@ sudo echo "export C_FORCE_ROOT=1" > /etc/profile.d/celery_env.sh sudo chmod +x /etc/profile.d/celery_env.sh source `which virtualenvwrapper.sh` if ! lsvirtualenv |grep compass-core>/dev/null; then - mkvirtualenv compass-core + mkvirtualenv --system-site-packages compass-core fi cd $COMPASSDIR workon compass-core -function compass_cleanup { - echo "deactive" - deactivate -} -trap compass_cleanup EXIT - python setup.py install if [[ "$?" != "0" ]]; then echo "failed to install compass package" + deactivate exit 1 else echo "compass package is installed in virtualenv under current dir" @@ -120,27 +79,7 @@ sudo sed -i "s/\$chef_hostname/$HOSTNAME/g" /etc/compass/package_installer/chef- sudo sed -i "s|\$PythonHome|$VIRTUAL_ENV|g" /opt/compass/bin/switch_virtualenv.py sudo ln -s -f $VIRTUAL_ENV/bin/celery /opt/compass/bin/celery -/opt/compass/bin/addcookbooks.py -if [[ "$?" != "0" ]]; then - echo "failed to add cookbooks" - exit 1 -else - echo "cookbooks are added to chef server" -fi -/opt/compass/bin/adddatabags.py -if [[ "$?" != "0" ]]; then - echo "failed to add databags" - exit 1 -else - echo "databags are added to chef server" -fi -/opt/compass/bin/addroles.py -if [[ "$?" != "0" ]]; then - echo "failed to add roles" - exit 1 -else - echo "roles are added to chef server" -fi +deactivate sudo mkdir -p /var/log/redis sudo chown -R redis:root /var/log/redis @@ -168,6 +107,7 @@ if [[ "$?" != "0" ]]; then else echo "compassed service is refreshed" fi + /opt/compass/bin/clean_nodes.sh /opt/compass/bin/clean_clients.sh /opt/compass/bin/clean_environments.sh diff --git a/install/compass_web.sh b/install/compass_web.sh new file mode 100755 index 00000000..9afc85a8 --- /dev/null +++ b/install/compass_web.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Move files to their respective locations + +### BEGIN OF SCRIPT ### +echo "setup compass configuration" +DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +source $DIR/install.conf +if [ -f $DIR/env.conf ]; then + source $DIR/env.conf +else + echo "failed to load environment" + exit 1 +fi +source $DIR/install_func.sh + +mkdir -p /var/www/compass_web +rm -rf /var/www/compass_web/* + +sudo cp -rf $WEB_HOME/public/* /var/www/compass_web/ +sudo cp -rf $WEB_HOME/v2 /var/www/compass_web/ + +if [[ $LOCAL_REPO = "y" ]]; then + echo "setting up local repo" + mkdir -p /tmp/repo + download -f https://s3-us-west-1.amazonaws.com/compass-local-repo/local_repo.tar.gz local_repo.tar.gz unzip /tmp/repo || exit $? + mv -f /tmp/repo/local_repo/* /var/www/compass_web/v2/ + if [[ "$?" != "0" ]]; then + echo "failed to setup local repo" + exit 1 + fi +fi + +sudo service httpd restart +sleep 10 + +echo "Checking if httpd is running" +sudo service httpd status +if [[ "$?" == "0" ]]; then + echo "httpd is running" +else + echo "httpd is not running" + exit 1 +fi diff --git a/install/dependency.sh b/install/dependency.sh index 19a1ae3b..79df9c42 100755 --- a/install/dependency.sh +++ b/install/dependency.sh @@ -10,7 +10,7 @@ if [ "$tempest" == "true" ]; then exit 1 fi fi -sudo yum install -y rsyslog logrotate ntp iproute openssh-clients python python-devel git wget syslinux amqp mod_wsgi httpd squid dhcp bind rsync yum-utils xinetd tftp-server gcc net-snmp-utils net-snmp python-daemon unzip openssl openssl098e ca-certificates redis mysql mysql-server mysql-devel python-virtualenv python-setuptools +sudo yum install -y rsyslog logrotate ntp iproute openssh-clients python python-devel git wget syslinux amqp mod_wsgi httpd squid dhcp bind rsync yum-utils xinetd tftp-server gcc net-snmp-utils net-snmp net-snmp-python unzip openssl openssl098e ca-certificates redis mysql mysql-server mysql-devel python-virtualenv python-setuptools if [[ "$?" != "0" ]]; then echo "failed to install yum dependency" exit 1 diff --git a/install/env.conf b/install/env.conf deleted file mode 100755 index 96326e65..00000000 --- a/install/env.conf +++ /dev/null @@ -1,10 +0,0 @@ -NIC=${NIC:-eth0} -IPADDR=${IPADDR:-10.145.89.100} -NETMASK=${NETMASK:-255.255.254.0} -WEB_SOURCE=${WEB_SOURCE:-http://git.openstack.org/stackforge/compass-web} -ADAPTERS_SOURCE=${ADAPTERS_SOURCE:-http://git.openstack.org/stackforge/compass-adapters} -OPTION_ROUTER=${OPTION_ROUTER:-10.145.88.1} -NAMESERVER_DOMAINS=${NAMESERVER_DOMAINS:-ods.com} -NEXTSERVER=${NEXTSERVER:-10.145.89.100} -IP_START=${IP_START:-10.145.89.100} -IP_END=${IP_END:-10.145.89.250} diff --git a/install/install.conf.template b/install/install.conf.template index e5dc8d30..280332b1 100755 --- a/install/install.conf.template +++ b/install/install.conf.template @@ -14,7 +14,7 @@ export PACKAGE_INSTALLER=chef export NIC=installation # default local repo config is "n" -export LOCAL_REPO=${LOCAL_REPO:-"n"} +export LOCAL_REPO=${LOCAL_REPO:-"y"} # DHCP config export IPADDR=`ifconfig $NIC | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'` @@ -84,8 +84,8 @@ export SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) export COMPASSDIR=${SCRIPT_DIR}/.. # Set test script variables -export NAMESERVERS=$IPADDR -export NTP_SERVER=$IPADDR -export GATEWAY=$IPADDR -export PROXY=http://$IPADDR:3128 +# export NAMESERVERS=$IPADDR +# export NTP_SERVER=$IPADDR +# export GATEWAY=$IPADDR +# export PROXY=http://$IPADDR:3128 export TESTMODE=${TESTMODE:-"True"} diff --git a/install/install.sh b/install/install.sh index fb117d3d..1f82b931 100755 --- a/install/install.sh +++ b/install/install.sh @@ -268,12 +268,11 @@ source ${COMPASSDIR}/install/$OS_INSTALLER.sh || exit $? echo "Install the Package Installer Tool" source ${COMPASSDIR}/install/$PACKAGE_INSTALLER.sh || exit $? +echo "Download and install Compass Web" +source ${COMPASSDIR}/install/compass_web.sh || exit $? + echo "Download and Setup Compass and related services" source ${COMPASSDIR}/install/compass.sh || exit $? -if [[ $LOCAL_REPO = "y" ]]; then - echo "setting up local repo" - source ${COMPASSDIR}/install/local_repo.sh || exit $? -fi figlet -ctf slant Installation Complete! echo -e "It takes\x1b[32m $SECONDS \x1b[0mseconds during the installation." diff --git a/install/install_func.sh b/install/install_func.sh index 9fa0236f..1f4df8e9 100755 --- a/install/install_func.sh +++ b/install/install_func.sh @@ -103,30 +103,84 @@ copy2dir() download() { #download params: [] [] - url=$1 - package=${2:-$(basename $url)} - action=${3:-""} + force=0 + url="" + package="" + options=() + while [ $# -gt 0 ]; do + case "$1" in + -f | --force) + force=1 + shift 1 + ;; + -u | --url): + url=$2 + shift 2 + ;; + -p | --package): + package=$2 + shift 2 + ;; + -*) + echo "Unknown options: $1" + shift 1 + ;; + *) + options+=($1) + shift 1 + ;; + esac + done + set ${options[@]} + if [ -z "$url" ]; then + url=$1 + shift 1 + fi + if [ -z "$package" ]; then + package=${1:-$(basename $url)} + shift 1 + fi + echo "download options: $@" + action=${1:-""} + downloaded=0 echo "download $package from $url and run $action" - if [[ -f /tmp/${package} || -L /tmp/${package} ]]; then - echo "$package already exists" - else - if [[ "$url" =~ (http|https|ftp):// ]]; then - echo "downloading $url to /tmp/${package}" - curl -L -o /tmp/${package}.tmp $url + if [[ "$force" == "0" || "$force" == "false" ]]; then + if [[ -f /tmp/${package} || -L /tmp/${package} ]]; then + echo "$package already exists" + downloaded=1 + fi + fi + if [[ "$url" =~ (http|https|ftp):// ]]; then + if [[ "$downloaded" == "0" ]]; then + echo "downloading $url to /tmp/${package}" + if [[ -f /tmp/${package} || -L /tmp/${package} ]]; then + curl -L -z /tmp/${package} -o /tmp/${package}.tmp $url + else + curl -L -o /tmp/${package}.tmp $url + fi if [[ "$?" != "0" ]]; then echo "failed to download $package" exit 1 else echo "successfully download $package" - mv -f /tmp/${package}.tmp /tmp/${package} + if [[ -f /tmp/${package}.tmp || -L /tmp/${package}.tmp ]]; then + mv -f /tmp/${package}.tmp /tmp/${package} + fi fi - else - cp -rf $url /tmp/${package} - fi - if [[ ! -f /tmp/${package} && ! -L /tmp/${package} ]]; then - echo "/tmp/$package is not created" - exit 1 - fi + fi + else + echo "copy $url to /tmp/${package}" + cp -rf $url /tmp/${package} + fi + if [[ ! -f /tmp/${package} && ! -L /tmp/${package} ]]; then + echo "/tmp/$package is not created" + exit 1 + fi + if [[ -z "$action" ]]; then + echo "download $package is done" + return + else + echo "execute $action after downloading $package" fi if [[ "$action" == "install" ]]; then echo "install /tmp/$package" @@ -138,8 +192,27 @@ download() echo "$package is installed" fi elif [[ "$action" == "copy" ]]; then - echo "copy /tmp/$package to $destdir" - destdir=$4 + destdir=$2 + echo "copy /tmp/$package to $destdir" sudo cp /tmp/$package $destdir + if [[ "$?" != "0" ]]; then + echo "failed to copy $package to $destdir" + exit 1 + else + echo "$package is copied to $destdir" + fi + elif [[ "$action" == "unzip" ]]; then + destdir=$2 + echo "unzip /tmp/$package to $destdir" + sudo tar -C $destdir -xzvf /tmp/$package + if [[ "$?" != "0" ]]; then + echo "failed to unzip $package to $destdir" + exit 1 + else + echo "$package is unziped to $destdir" + fi + else + echo "unknown action $action" + exit 1 fi } diff --git a/install/local_repo.sh b/install/local_repo.sh deleted file mode 100755 index dcb176d6..00000000 --- a/install/local_repo.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -mkdir -p /tmp/repo/ -cd /tmp/repo/ -wget https://s3-us-west-1.amazonaws.com/compass-local-repo/local_repo.tar.gz -tar -xzvf local_repo.tar.gz -mv local_repo/* /var/www/compass_web/v2/ diff --git a/install/prepare.sh b/install/prepare.sh index e10b2189..ce599bd5 100755 --- a/install/prepare.sh +++ b/install/prepare.sh @@ -183,7 +183,7 @@ fi source `which virtualenvwrapper.sh` if ! lsvirtualenv |grep compass-core>/dev/null; then - mkvirtualenv compass-core + mkvirtualenv --system-site-packages compass-core fi cd $COMPASSDIR workon compass-core @@ -238,6 +238,9 @@ download $CHEF_SRV chef-server || exit $? download "$CENTOS_IMAGE_SOURCE" ${CENTOS_IMAGE_NAME}-${CENTOS_IMAGE_ARCH}.iso || exit $? download "$UBUNTU_IMAGE_SOURCE" ${UBUNTU_IMAGE_NAME}-${UBUNTU_IMAGE_ARCH}.iso || exit $? +# download local repo +download -f https://s3-us-west-1.amazonaws.com/compass-local-repo/local_repo.tar.gz local_repo.tar.gz || exit $? + # Install net-snmp echo "install snmp config" if [[ ! -e /etc/snmp ]]; then diff --git a/install/setup_env.sh b/install/setup_env.sh index be277dac..36ce72fc 100755 --- a/install/setup_env.sh +++ b/install/setup_env.sh @@ -9,5 +9,6 @@ NAMESERVER_DOMAINS=\${NAMESERVER_DOMAINS:-$NAMESERVER_DOMAINS} NEXTSERVER=\${NEXTSERVER:-$NEXTSERVER} IP_START=\${IP_START:-$IP_START} IP_END=\${IP_END:-$IP_END} +LOCAL_REPO=\${LOCAL_REPO:-$LOCAL_REPO} EOF chmod ugo+x $SCRIPT_DIR/env.conf diff --git a/regtest/regtest.conf b/regtest/regtest.conf index 682425ff..1e80d441 100644 --- a/regtest/regtest.conf +++ b/regtest/regtest.conf @@ -28,8 +28,8 @@ export DEFAULT_ROLES=${DEFAULT_ROLES:-'allinone-compute'} export NAMESERVERS=${NAMESERVERS:-$IPADDR} export NTP_SERVER=${NTP_SERVER:-$IPADDR} export GATEWAY=${GATEWAY:-$IPADDR} -export PROXY=${PROXY:-http://${IPADDR}:3128} -export IGNORE_PROXY=${IGNORE_PROXY:-"127.0.0.1,localhost,${IPADDR},$HOSTNAME"} +export PROXY=${PROXY:-} +export IGNORE_PROXY=${IGNORE_PROXY:-} export DOMAIN=${DOMAIN:-'ods.com'} export SEARCH_PATH=${SEARCH_PATH:-${DOMAIN}} @@ -37,7 +37,7 @@ export HOME_PERCENTAGE=${HOME_PERCENTAGE:-'5'} export TMP_PERCENTAGE=${TMP_PERCENTAGE:-'5'} export VAR_PERCENTAGE=${VAR_PERCENTAGE:-'10'} export PARTITION=${PARTITION:-"/home=${HOME_PERCENTAGE}%,/tmp=${TMP_PERCENTAGE}%,/var=${VAR_PERCENTAGE}%"} -export LOCAL_REPO_URL=${LOCAL_REPO_URL:-} +export LOCAL_REPO_URL=${LOCAL_REPO_URL:-"http://$IPADDR"} export OS_CONFIG_FILENAME=${OS_CONFIG_FILENAME:-} export PACKAGE_CONFIG_FILENAME=${PACKAGE_CONFIG_FILENAME:-} diff --git a/regtest/regtest10.conf b/regtest/regtest10.conf index a7008feb..8778b336 100644 --- a/regtest/regtest10.conf +++ b/regtest/regtest10.conf @@ -15,3 +15,5 @@ export PUBLIC_IP_START=${PUBLIC_IP_START:-'172.16.3.90'} export STORAGE_IP_START=${STORAGE_IP_START:-'172.16.4.90'} export REGTEST_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source ${REGTEST_DIR}/regtest.conf +export PROXY=${PROXY:-"http://${IPADDR}:3128"} +export LOCAL_REPO_URL='' diff --git a/regtest/regtest7.conf b/regtest/regtest7.conf index 23d21d6f..cdee6bb4 100644 --- a/regtest/regtest7.conf +++ b/regtest/regtest7.conf @@ -11,3 +11,5 @@ export PUBLIC_IP_START=${PUBLIC_IP_START:-'172.16.3.82'} export STORAGE_IP_START=${STORAGE_IP_START:-'172.16.4.82'} export REGTEST_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source ${REGTEST_DIR}/regtest6.conf +export PROXY=${PROXY:-"http://${IPADDR}:3128"} +export LOCAL_REPO_URL='' diff --git a/regtest/regtest8.conf b/regtest/regtest8.conf index 4367b685..e6cbeb81 100644 --- a/regtest/regtest8.conf +++ b/regtest/regtest8.conf @@ -11,3 +11,5 @@ export PUBLIC_IP_START=${PUBLIC_IP_START:-'172.16.3.84'} export STORAGE_IP_START=${STORAGE_IP_START:-'172.16.4.84'} export REGTEST_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source ${REGTEST_DIR}/regtest.conf +export PROXY=${PROXY:-"http://${IPADDR}:3128"} +export LOCAL_REPO_URL='' diff --git a/regtest/regtest9.conf b/regtest/regtest9.conf index 4b029d99..de31898a 100644 --- a/regtest/regtest9.conf +++ b/regtest/regtest9.conf @@ -15,3 +15,5 @@ export PUBLIC_IP_START=${PUBLIC_IP_START:-'172.16.3.86'} export STORAGE_IP_START=${STORAGE_IP_START:-'172.16.4.86'} export REGTEST_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source ${REGTEST_DIR}/regtest.conf +export PROXY=${PROXY:-"http://${IPADDR}:3128"} +export LOCAL_REPO_URL='' diff --git a/requirements.txt b/requirements.txt index 874c723d..2b1f112a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +amqplib argparse celery Markdown==2.4.1 @@ -17,6 +18,7 @@ MySQL-python netaddr paramiko==1.7.5 PyChef +python-daemon SQLAlchemy>=0.9.0 simplejson requests