diff --git a/README.md b/README.md index 7ad0c5a..d55b45f 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,42 @@ worker_server.py is a worker server that loads and runs task_plugins. Each task_plugin is a zuul gearman worker that implements, handles, executes a job, uploads/post-processes the logs and sends back the results to zuul. -plugins +Plugins ------- **gate_real_db_upgrade**: -Runs the db_sync migrations on each dataset available in the datasets subdir. \ No newline at end of file +Runs the db_sync migrations on each dataset available in the datasets subdir. + +Installation +------------ + +* boot a fresh Ubuntu image +* setup ssh authentication for your admin team +* apt-get update; apt-get dist-upgrade +* adduser th +* apt-get install vim git python-pip python-setuptools python-keystoneclient virtualenvwrapper python-eventlet python-numpy python-mysqldb python-git python-gitdb python-netaddr python-pkg-resources libxml2-dev libxml2-utils libxslt-dev git-review libxml2-dev libxml2-utils libxslt-dev libmysqlclient-dev pep8 postgresql-server-dev-9.1 python2.7-dev python-coverage python-netaddr +* pip install -U pip +* apt-get purge python-pip +* cd /home/th; git clone http://github.com/stackforge/turbo-hipster +* apply any patches you need +* python setup.py install +* cp turbo_hipster/task_plugins/gate_real_db_upgrade/*.sh /usr/local/lib/python2.7/dist-packages/turbo_hipster/task_plugins/gate_real_db_upgrade/ +* cp -R etc/* /etc/ +* mkdir /var/lib/turbo-hipster +* chown -R th.th /var/lib/turbo-hipster +* mkdir /var/log/turbo-hipster +* chown -R th.th /var/log/turbo-hipster +* install your chosen MySQL-like database engine (percona, maria, mysql) +* mysql -u root --password=$1 -e "create user 'nova'@'localhost' identified by 'tester';" +* mysql -u root --password=$1 -e "create user 'nova'@'172.16.0.2' identified by 'tester';" +* mysql -u root --password=$1 -e "grant all privileges on *.* to 'nova'@'localhost' with grant option;" +* mysql -u root --password=$1 -e "grant all privileges on *.* to 'nova'@'172.16.0.2' with grant option;" +* /etc/rc.local +* rsync the datasets from the master +* logrotate -f /etc/logrotate.conf +* chmod -R ugo+r /var/log/* +* chmod ugo+rx /var/log/mysql +* mkdir /var/cache/pip +* chmod -R ugo+rwx /var/cache/pip +* touch /var/log/mysql/slow-queries.log +* /etc/init.d/turbo-hipster start \ No newline at end of file diff --git a/etc/init.d/turbo-hipster b/etc/init.d/turbo-hipster index 45cdc3d..4cc16d1 100755 --- a/etc/init.d/turbo-hipster +++ b/etc/init.d/turbo-hipster @@ -19,7 +19,7 @@ DAEMON=/usr/local/bin/turbo-hipster PIDFILE=/var/run/$NAME/$NAME.pid DAEMON_ARGS="--background -c /etc/turbo-hipster/config.json -p $PIDFILE" SCRIPTNAME=/etc/init.d/$NAME -USER=turbo-hipster +USER=th # Exit if the package is not installed [ -x "$DAEMON" ] || exit 0 diff --git a/etc/logrotate.d/mysql-server b/etc/logrotate.d/mysql-server new file mode 100644 index 0000000..5f66109 --- /dev/null +++ b/etc/logrotate.d/mysql-server @@ -0,0 +1,27 @@ +# - I put everything in one block and added sharedscripts, so that mysql gets +# flush-logs'd only once. +# Else the binary logs would automatically increase by n times every day. +# - The error log is obsolete, messages go to syslog now. +/var/log/mysql.log /var/log/mysql/mysql.log /var/log/mysql/mysql-slow.log /var/log/mysql/error.log { + daily + rotate 7 + missingok + create 644 mysql adm + compress + sharedscripts + postrotate + test -x /usr/bin/mysqladmin || exit 0 + # If this fails, check debian.conf! + MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf" + if [ -z "`$MYADMIN ping 2>/dev/null`" ]; then + # Really no mysqld or rather a missing debian-sys-maint user? + # If this occurs and is not a error please report a bug. + #if ps cax | grep -q mysqld; then + if killall -q -s0 -umysql mysqld; then + exit 1 + fi + else + $MYADMIN flush-logs + fi + endscript +} \ No newline at end of file diff --git a/etc/logrotate.d/rsyslog b/etc/logrotate.d/rsyslog new file mode 100644 index 0000000..3d00952 --- /dev/null +++ b/etc/logrotate.d/rsyslog @@ -0,0 +1,37 @@ +/var/log/syslog +{ + rotate 7 + daily + missingok + notifempty + delaycompress + compress + postrotate + reload rsyslog >/dev/null 2>&1 || true + endscript +} + +/var/log/mail.info +/var/log/mail.warn +/var/log/mail.err +/var/log/mail.log +/var/log/daemon.log +/var/log/kern.log +/var/log/auth.log +/var/log/user.log +/var/log/lpr.log +/var/log/cron.log +/var/log/debug +/var/log/messages +{ + rotate 4 + weekly + missingok + notifempty + compress + delaycompress + sharedscripts + postrotate + reload rsyslog >/dev/null 2>&1 || true + endscript +} \ No newline at end of file diff --git a/etc/mysql/my.cnf b/etc/mysql/my.cnf new file mode 100644 index 0000000..ce34161 --- /dev/null +++ b/etc/mysql/my.cnf @@ -0,0 +1,127 @@ +# +# The MySQL database server configuration file. +# +# You can copy this to one of: +# - "/etc/mysql/my.cnf" to set global options, +# - "~/.my.cnf" to set user-specific options. +# +# One can use all long options that the program supports. +# Run program with --help to get a list of available options and with +# --print-defaults to see which it would actually understand and use. +# +# For explanations see +# http://dev.mysql.com/doc/mysql/en/server-system-variables.html + +# This will be passed to all mysql clients +# It has been reported that passwords should be enclosed with ticks/quotes +# escpecially if they contain "#" chars... +# Remember to edit /etc/mysql/debian.cnf when changing the socket location. +[client] +port = 3306 +socket = /var/run/mysqld/mysqld.sock + +# Here is entries for some specific programs +# The following values assume you have at least 32M ram + +# This was formally known as [safe_mysqld]. Both versions are currently parsed. +[mysqld_safe] +socket = /var/run/mysqld/mysqld.sock +nice = 0 + +[mysqld] +# +# * Basic Settings +# +user = mysql +pid-file = /var/run/mysqld/mysqld.pid +socket = /var/run/mysqld/mysqld.sock +port = 3306 +basedir = /usr +datadir = /var/lib/mysql +tmpdir = /tmp +lc-messages-dir = /usr/share/mysql +skip-external-locking +# +# Instead of skip-networking the default is now to listen only on +# localhost which is more compatible and is not less secure. +bind-address = 0.0.0.0 +# +# * Fine Tuning +# +key_buffer = 16M +max_allowed_packet = 16M +thread_stack = 192K +thread_cache_size = 8 +# This replaces the startup script and checks MyISAM tables if needed +# the first time they are touched +myisam-recover = BACKUP +#max_connections = 100 +#table_cache = 64 +#thread_concurrency = 10 +# +# * Query Cache Configuration +# +query_cache_limit = 1M +query_cache_size = 16M +# +# * Logging and Replication +# +# Both location gets rotated by the cronjob. +# Be aware that this log type is a performance killer. +# As of 5.1 you can enable the log at runtime! +#general_log_file = /var/log/mysql/mysql.log +#general_log = 1 +# +# Error log - should be very few entries. +# +log_error = /var/log/mysql/error.log +# +# Here you can see queries with especially long duration +#log_slow_queries = /var/log/mysql/mysql-slow.log +#long_query_time = 2 +#log-queries-not-using-indexes +# +# The following can be used as easy to replay backup logs or for replication. +# note: if you are setting up a replication slave, see README.Debian about +# other settings you may need to change. +#server-id = 1 +#log_bin = /var/log/mysql/mysql-bin.log +expire_logs_days = 10 +max_binlog_size = 100M +#binlog_do_db = include_database_name +#binlog_ignore_db = include_database_name +# +# * InnoDB +# +# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. +# Read the manual for more InnoDB related options. There are many! +# +# * Security Features +# +# Read the manual, too, if you want chroot! +# chroot = /var/lib/mysql/ +# +# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". +# +# ssl-ca=/etc/mysql/cacert.pem +# ssl-cert=/etc/mysql/server-cert.pem +# ssl-key=/etc/mysql/server-key.pem + + + +[mysqldump] +quick +quote-names +max_allowed_packet = 16M + +[mysql] +#no-auto-rehash # faster start of mysql but no tab completition + +[isamchk] +key_buffer = 16M + +# +# * IMPORTANT: Additional settings that can override those from this file! +# The files must end with '.cnf', otherwise they'll be ignored. +# +!includedir /etc/mysql/conf.d/ diff --git a/etc/rc.local b/etc/rc.local new file mode 100644 index 0000000..db2b33a --- /dev/null +++ b/etc/rc.local @@ -0,0 +1,7 @@ +ip netns add nonet +ip link add veth0 type veth peer name veth1 +ifconfig veth0 172.16.0.1/24 up +ip link set veth1 netns nonet +ip netns exec nonet ifconfig veth1 172.16.0.2/24 up +/sbin/iptables -A INPUT -p tcp --dport 3306 -i eth0 -j DROP +/sbin/iptables -A INPUT -p tcp --dport 3306 -i eth1 -j DROP diff --git a/etc/sudoers b/etc/sudoers new file mode 100644 index 0000000..d75d601 --- /dev/null +++ b/etc/sudoers @@ -0,0 +1,33 @@ +# +# This file MUST be edited with the 'visudo' command as root. +# +# Please consider adding local content in /etc/sudoers.d/ instead of +# directly modifying this file. +# +# See the man page for details on how to write a sudoers file. +# +Defaults env_reset +Defaults mail_badpass +Defaults secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +# Host alias specification + +# User alias specification + +# Cmnd alias specification + +# User privilege specification +root ALL=(ALL:ALL) ALL + +# Members of the admin group may gain root privileges +%admin ALL=(ALL) ALL + +# Allow members of group sudo to execute any command +%sudo ALL=(ALL:ALL) ALL + +# Turbo Hipster +th ALL=(root) NOPASSWD: /sbin/ip netns exec nonet * + +# See sudoers(5) for more information on "#include" directives: + +#includedir /etc/sudoers.d diff --git a/etc/turbo-hipster/config.json b/etc/turbo-hipster/config.json index 325a373..ef02532 100644 --- a/etc/turbo-hipster/config.json +++ b/etc/turbo-hipster/config.json @@ -1,7 +1,7 @@ { "zuul_server": { - "git_url": "/var/lib/zuul/git/", - "gearman_host": "localhost", + "git_url": "http://119.9.13.90/p/", + "gearman_host": "119.9.13.90", "gearman_port": 4730 }, "debug_log": "/var/log/turbo-hipster/debug.log", @@ -11,13 +11,38 @@ "plugins": [ { "name": "gate_real_db_upgrade", - "function": "build:gate-real-db-upgrade_nova_mysql", - "datasets_dir": "/var/lib/turbo-hipster/datasets" + "datasets_dir": "/var/lib/turbo-hipster/datasets_devstack_131007", + "function": "build:gate-real-db-upgrade_nova_mysql_devstack_131007" + }, { + "name": "gate_real_db_upgrade", + "datasets_dir": "/var/lib/turbo-hipster/datasets_devstack_150", + "function": "build:gate-real-db-upgrade_nova_mysql_devstack_150" + }, { + "name": "gate_real_db_upgrade", + "datasets_dir": "/var/lib/turbo-hipster/datasets_trivial_500", + "function": "build:gate-real-db-upgrade_nova_mysql_trivial_500" + }, { + "name": "gate_real_db_upgrade", + "datasets_dir": "/var/lib/turbo-hipster/datasets_trivial_6000", + "function": "build:gate-real-db-upgrade_nova_mysql_trivial_6000" + }, { + "name": "gate_real_db_upgrade", + "datasets_dir": "/var/lib/turbo-hipster/datasets_user_001", + "function": "build:gate-real-db-upgrade_nova_mysql_user_001" + }, { + "name": "gate_real_db_upgrade", + "datasets_dir": "/var/lib/turbo-hipster/datasets_user_002", + "function": "build:gate-real-db-upgrade_nova_mysql_user_002" } ], "publish_logs": { - "type": "local", - "path": "/var/www/results/", - "prepend_url": "http://localhost/results/" + "type": "swift", + "authurl": "https://identity.api.rackspacecloud.com/v2.0/", + "tenant": " ", + "user": "XXXXXX", + "password": "XXXXXX", + "container": "XXXXXX", + "region": "SYD", + "prepend_url": "http://www.rcbops.com/turbo_hipster/results/" } -} \ No newline at end of file +} diff --git a/turbo_hipster/task_plugins/gate_real_db_upgrade/nova_mysql_migrations.sh b/turbo_hipster/task_plugins/gate_real_db_upgrade/nova_mysql_migrations.sh index 06bc12a..3e0b29b 100755 --- a/turbo_hipster/task_plugins/gate_real_db_upgrade/nova_mysql_migrations.sh +++ b/turbo_hipster/task_plugins/gate_real_db_upgrade/nova_mysql_migrations.sh @@ -136,7 +136,8 @@ cd $3 echo "Setting up virtual env" source ~/.bashrc source /etc/bash_completion.d/virtualenvwrapper -VENV_PATH=~/.virtualenvs/$1 +export WORKON_HOME=/var/lib/turbo-hipster/envs +VENV_PATH=$WORKON_HOME/$1 rm -rf $VENV_PATH mkvirtualenv $1 toggleglobalsitepackages