x/turbo-hipster
Code Issues Proposed changes

More worker build documentation.

Browse Source 
Also a small tweak to how to setup venvs to put them somewhere
more obvious.

Change-Id: I140de9aad4e7609edf0d1ab8888e3d224bb3606b
This commit is contained in:
Michael Still 2013-12-25 11:19:28 +11:00
parent 26be80f7bb
commit 9e814216d1
9 changed files with 303 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
README.md
View File

@ -8,8 +8,42 @@ worker_server.py is a worker server that loads and runs task_plugins.
Each task_plugin is a zuul gearman worker that implements, handles, executes a
job, uploads/post-processes the logs and sends back the results to zuul.
plugins
Plugins
-------
**gate_real_db_upgrade**:
Runs the db_sync migrations on each dataset available in the datasets subdir.
Runs the db_sync migrations on each dataset available in the datasets subdir.
Installation
------------
* boot a fresh Ubuntu image
* setup ssh authentication for your admin team
* apt-get update; apt-get dist-upgrade
* adduser th
* apt-get install vim git python-pip python-setuptools python-keystoneclient virtualenvwrapper python-eventlet python-numpy python-mysqldb python-git python-gitdb python-netaddr python-pkg-resources libxml2-dev libxml2-utils libxslt-dev git-review libxml2-dev libxml2-utils libxslt-dev libmysqlclient-dev pep8 postgresql-server-dev-9.1 python2.7-dev python-coverage python-netaddr
* pip install -U pip
* apt-get purge python-pip
* cd /home/th; git clone http://github.com/stackforge/turbo-hipster
* apply any patches you need
* python setup.py install
* cp turbo_hipster/task_plugins/gate_real_db_upgrade/*.sh /usr/local/lib/python2.7/dist-packages/turbo_hipster/task_plugins/gate_real_db_upgrade/
* cp -R etc/* /etc/
* mkdir /var/lib/turbo-hipster
* chown -R th.th /var/lib/turbo-hipster
* mkdir /var/log/turbo-hipster
* chown -R th.th /var/log/turbo-hipster
* install your chosen MySQL-like database engine (percona, maria, mysql)
* mysql -u root --password=$1 -e "create user 'nova'@'localhost' identified by 'tester';"
* mysql -u root --password=$1 -e "create user 'nova'@'172.16.0.2' identified by 'tester';"
* mysql -u root --password=$1 -e "grant all privileges on *.* to 'nova'@'localhost' with grant option;"
* mysql -u root --password=$1 -e "grant all privileges on *.* to 'nova'@'172.16.0.2' with grant option;"
* /etc/rc.local
* rsync the datasets from the master
* logrotate -f /etc/logrotate.conf
* chmod -R ugo+r /var/log/*
* chmod ugo+rx /var/log/mysql
* mkdir /var/cache/pip
* chmod -R ugo+rwx /var/cache/pip
* touch /var/log/mysql/slow-queries.log
* /etc/init.d/turbo-hipster start

2
etc/init.d/turbo-hipster
View File

@ -19,7 +19,7 @@ DAEMON=/usr/local/bin/turbo-hipster
PIDFILE=/var/run/$NAME/$NAME.pid
DAEMON_ARGS="--background -c /etc/turbo-hipster/config.json -p $PIDFILE"
SCRIPTNAME=/etc/init.d/$NAME
USER=turbo-hipster
USER=th
# Exit if the package is not installed
[ -x "$DAEMON" ] || exit 0

27
etc/logrotate.d/mysql-server Normal file
View File

@ -0,0 +1,27 @@
# - I put everything in one block and added sharedscripts, so that mysql gets
# flush-logs'd only once.
# Else the binary logs would automatically increase by n times every day.
# - The error log is obsolete, messages go to syslog now.
/var/log/mysql.log /var/log/mysql/mysql.log /var/log/mysql/mysql-slow.log /var/log/mysql/error.log {
daily
rotate 7
missingok
create 644 mysql adm
compress
sharedscripts
postrotate
test -x /usr/bin/mysqladmin || exit 0
# If this fails, check debian.conf!
MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf"
if [ -z "`$MYADMIN ping 2>/dev/null`" ]; then
# Really no mysqld or rather a missing debian-sys-maint user?
# If this occurs and is not a error please report a bug.
#if ps cax | grep -q mysqld; then
if killall -q -s0 -umysql mysqld; then
exit 1
fi
else
$MYADMIN flush-logs
fi
endscript
}

37
etc/logrotate.d/rsyslog Normal file
View File

@ -0,0 +1,37 @@
/var/log/syslog
{
rotate 7
daily
missingok
notifempty
delaycompress
compress
postrotate
reload rsyslog >/dev/null 2>&1 || true
endscript
}
/var/log/mail.info
/var/log/mail.warn
/var/log/mail.err
/var/log/mail.log
/var/log/daemon.log
/var/log/kern.log
/var/log/auth.log
/var/log/user.log
/var/log/lpr.log
/var/log/cron.log
/var/log/debug
/var/log/messages
{
rotate 4
weekly
missingok
notifempty
compress
delaycompress
sharedscripts
postrotate
reload rsyslog >/dev/null 2>&1 || true
endscript
}

127
etc/mysql/my.cnf Normal file
View File

@ -0,0 +1,127 @@
#
# The MySQL database server configuration file.
#
# You can copy this to one of:
# - "/etc/mysql/my.cnf" to set global options,
# - "~/.my.cnf" to set user-specific options.
#
# One can use all long options that the program supports.
# Run program with --help to get a list of available options and with
# --print-defaults to see which it would actually understand and use.
#
# For explanations see
# http://dev.mysql.com/doc/mysql/en/server-system-variables.html
# This will be passed to all mysql clients
# It has been reported that passwords should be enclosed with ticks/quotes
# escpecially if they contain "#" chars...
# Remember to edit /etc/mysql/debian.cnf when changing the socket location.
[client]
port = 3306
socket = /var/run/mysqld/mysqld.sock
# Here is entries for some specific programs
# The following values assume you have at least 32M ram
# This was formally known as [safe_mysqld]. Both versions are currently parsed.
[mysqld_safe]
socket = /var/run/mysqld/mysqld.sock
nice = 0
[mysqld]
#
# * Basic Settings
#
user = mysql
pid-file = /var/run/mysqld/mysqld.pid
socket = /var/run/mysqld/mysqld.sock
port = 3306
basedir = /usr
datadir = /var/lib/mysql
tmpdir = /tmp
lc-messages-dir = /usr/share/mysql
skip-external-locking
#
# Instead of skip-networking the default is now to listen only on
# localhost which is more compatible and is not less secure.
bind-address = 0.0.0.0
#
# * Fine Tuning
#
key_buffer = 16M
max_allowed_packet = 16M
thread_stack = 192K
thread_cache_size = 8
# This replaces the startup script and checks MyISAM tables if needed
# the first time they are touched
myisam-recover = BACKUP
#max_connections = 100
#table_cache = 64
#thread_concurrency = 10
#
# * Query Cache Configuration
#
query_cache_limit = 1M
query_cache_size = 16M
#
# * Logging and Replication
#
# Both location gets rotated by the cronjob.
# Be aware that this log type is a performance killer.
# As of 5.1 you can enable the log at runtime!
#general_log_file = /var/log/mysql/mysql.log
#general_log = 1
#
# Error log - should be very few entries.
#
log_error = /var/log/mysql/error.log
#
# Here you can see queries with especially long duration
#log_slow_queries = /var/log/mysql/mysql-slow.log
#long_query_time = 2
#log-queries-not-using-indexes
#
# The following can be used as easy to replay backup logs or for replication.
# note: if you are setting up a replication slave, see README.Debian about
# other settings you may need to change.
#server-id = 1
#log_bin = /var/log/mysql/mysql-bin.log
expire_logs_days = 10
max_binlog_size = 100M
#binlog_do_db = include_database_name
#binlog_ignore_db = include_database_name
#
# * InnoDB
#
# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.
# Read the manual for more InnoDB related options. There are many!
#
# * Security Features
#
# Read the manual, too, if you want chroot!
# chroot = /var/lib/mysql/
#
# For generating SSL certificates I recommend the OpenSSL GUI "tinyca".
#
# ssl-ca=/etc/mysql/cacert.pem
# ssl-cert=/etc/mysql/server-cert.pem
# ssl-key=/etc/mysql/server-key.pem
[mysqldump]
quick
quote-names
max_allowed_packet = 16M
[mysql]
#no-auto-rehash # faster start of mysql but no tab completition
[isamchk]
key_buffer = 16M
#
# * IMPORTANT: Additional settings that can override those from this file!
# The files must end with '.cnf', otherwise they'll be ignored.
#
!includedir /etc/mysql/conf.d/

7
etc/rc.local Normal file
View File

@ -0,0 +1,7 @@
ip netns add nonet
ip link add veth0 type veth peer name veth1
ifconfig veth0 172.16.0.1/24 up
ip link set veth1 netns nonet
ip netns exec nonet ifconfig veth1 172.16.0.2/24 up
/sbin/iptables -A INPUT -p tcp --dport 3306 -i eth0 -j DROP
/sbin/iptables -A INPUT -p tcp --dport 3306 -i eth1 -j DROP

33
etc/sudoers Normal file
View File

@ -0,0 +1,33 @@
#
# This file MUST be edited with the 'visudo' command as root.
#
# Please consider adding local content in /etc/sudoers.d/ instead of
# directly modifying this file.
#
# See the man page for details on how to write a sudoers file.
#
Defaults env_reset
Defaults mail_badpass
Defaults secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
# Host alias specification
# User alias specification
# Cmnd alias specification
# User privilege specification
root ALL=(ALL:ALL) ALL
# Members of the admin group may gain root privileges
%admin ALL=(ALL) ALL
# Allow members of group sudo to execute any command
%sudo ALL=(ALL:ALL) ALL
# Turbo Hipster
th ALL=(root) NOPASSWD: /sbin/ip netns exec nonet *
# See sudoers(5) for more information on "#include" directives:
#includedir /etc/sudoers.d

41
etc/turbo-hipster/config.json
View File

@ -1,7 +1,7 @@
{
"zuul_server": {
"git_url": "/var/lib/zuul/git/",
"gearman_host": "localhost",
"git_url": "http://119.9.13.90/p/",
"gearman_host": "119.9.13.90",
"gearman_port": 4730
},
"debug_log": "/var/log/turbo-hipster/debug.log",
@ -11,13 +11,38 @@
"plugins": [
{
"name": "gate_real_db_upgrade",
"function": "build:gate-real-db-upgrade_nova_mysql",
"datasets_dir": "/var/lib/turbo-hipster/datasets"
"datasets_dir": "/var/lib/turbo-hipster/datasets_devstack_131007",
"function": "build:gate-real-db-upgrade_nova_mysql_devstack_131007"
}, {
"name": "gate_real_db_upgrade",
"datasets_dir": "/var/lib/turbo-hipster/datasets_devstack_150",
"function": "build:gate-real-db-upgrade_nova_mysql_devstack_150"
}, {
"name": "gate_real_db_upgrade",
"datasets_dir": "/var/lib/turbo-hipster/datasets_trivial_500",
"function": "build:gate-real-db-upgrade_nova_mysql_trivial_500"
}, {
"name": "gate_real_db_upgrade",
"datasets_dir": "/var/lib/turbo-hipster/datasets_trivial_6000",
"function": "build:gate-real-db-upgrade_nova_mysql_trivial_6000"
}, {
"name": "gate_real_db_upgrade",
"datasets_dir": "/var/lib/turbo-hipster/datasets_user_001",
"function": "build:gate-real-db-upgrade_nova_mysql_user_001"
}, {
"name": "gate_real_db_upgrade",
"datasets_dir": "/var/lib/turbo-hipster/datasets_user_002",
"function": "build:gate-real-db-upgrade_nova_mysql_user_002"
}
],
"publish_logs": {
"type": "local",
"path": "/var/www/results/",
"prepend_url": "http://localhost/results/"
"type": "swift",
"authurl": "https://identity.api.rackspacecloud.com/v2.0/",
"tenant": " ",
"user": "XXXXXX",
"password": "XXXXXX",
"container": "XXXXXX",
"region": "SYD",
"prepend_url": "http://www.rcbops.com/turbo_hipster/results/"
}
}
}

3
turbo_hipster/task_plugins/gate_real_db_upgrade/nova_mysql_migrations.sh
View File

@ -136,7 +136,8 @@ cd $3
echo "Setting up virtual env"
source ~/.bashrc
source /etc/bash_completion.d/virtualenvwrapper
VENV_PATH=~/.virtualenvs/$1
export WORKON_HOME=/var/lib/turbo-hipster/envs
VENV_PATH=$WORKON_HOME/$1
rm -rf $VENV_PATH
mkvirtualenv $1
toggleglobalsitepackages