More worker build documentation.

Also a small tweak to how to setup venvs to put them somewhere
more obvious.

Change-Id: I140de9aad4e7609edf0d1ab8888e3d224bb3606b

README.md

@@ -8,8 +8,42 @@ worker_server.py is a worker server that loads and runs task_plugins.
 Each task_plugin is a zuul gearman worker that implements, handles, executes a
 job, uploads/post-processes the logs and sends back the results to zuul.
 
-plugins
+Plugins
 -------
 
 **gate_real_db_upgrade**:
 Runs the db_sync migrations on each dataset available in the datasets subdir.
+
+Installation
+------------
+
+* boot a fresh Ubuntu image
+* setup ssh authentication for your admin team
+* apt-get update; apt-get dist-upgrade
+* adduser th
+* apt-get install vim git python-pip python-setuptools python-keystoneclient virtualenvwrapper python-eventlet python-numpy python-mysqldb python-git python-gitdb python-netaddr python-pkg-resources libxml2-dev libxml2-utils libxslt-dev git-review libxml2-dev libxml2-utils libxslt-dev libmysqlclient-dev pep8 postgresql-server-dev-9.1 python2.7-dev python-coverage python-netaddr
+* pip install -U pip 
+* apt-get purge python-pip
+* cd /home/th; git clone http://github.com/stackforge/turbo-hipster
+* apply any patches you need
+* python setup.py install
+* cp turbo_hipster/task_plugins/gate_real_db_upgrade/*.sh /usr/local/lib/python2.7/dist-packages/turbo_hipster/task_plugins/gate_real_db_upgrade/
+* cp -R etc/* /etc/
+* mkdir /var/lib/turbo-hipster
+* chown -R th.th /var/lib/turbo-hipster
+* mkdir /var/log/turbo-hipster
+* chown -R th.th /var/log/turbo-hipster
+* install your chosen MySQL-like database engine (percona, maria, mysql)
+* mysql -u root --password=$1 -e "create user 'nova'@'localhost' identified by 'tester';"
+* mysql -u root --password=$1 -e "create user 'nova'@'172.16.0.2' identified by 'tester';"
+* mysql -u root --password=$1 -e "grant all privileges on *.* to 'nova'@'localhost' with grant option;"
+* mysql -u root --password=$1 -e "grant all privileges on *.* to 'nova'@'172.16.0.2' with grant option;"
+* /etc/rc.local
+* rsync the datasets from the master
+* logrotate -f /etc/logrotate.conf
+* chmod -R ugo+r /var/log/*
+* chmod ugo+rx /var/log/mysql
+* mkdir /var/cache/pip
+* chmod -R ugo+rwx /var/cache/pip
+* touch /var/log/mysql/slow-queries.log
+* /etc/init.d/turbo-hipster start

etc/init.d/turbo-hipster

@@ -19,7 +19,7 @@ DAEMON=/usr/local/bin/turbo-hipster
 PIDFILE=/var/run/$NAME/$NAME.pid
 DAEMON_ARGS="--background -c /etc/turbo-hipster/config.json -p $PIDFILE"
 SCRIPTNAME=/etc/init.d/$NAME
-USER=turbo-hipster
+USER=th
 
 # Exit if the package is not installed
 [ -x "$DAEMON" ] || exit 0

etc/logrotate.d/mysql-server

@@ -0,0 +1,27 @@
+# - I put everything in one block and added sharedscripts, so that mysql gets 
+#   flush-logs'd only once.
+#   Else the binary logs would automatically increase by n times every day.
+# - The error log is obsolete, messages go to syslog now.
+/var/log/mysql.log /var/log/mysql/mysql.log /var/log/mysql/mysql-slow.log /var/log/mysql/error.log {
+        daily
+        rotate 7
+        missingok
+        create 644 mysql adm
+        compress
+        sharedscripts
+        postrotate
+                test -x /usr/bin/mysqladmin || exit 0
+                # If this fails, check debian.conf! 
+                MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf"
+                if [ -z "`$MYADMIN ping 2>/dev/null`" ]; then
+                  # Really no mysqld or rather a missing debian-sys-maint user?
+                  # If this occurs and is not a error please report a bug.
+                  #if ps cax | grep -q mysqld; then
+                  if killall -q -s0 -umysql mysqld; then
+                    exit 1
+                  fi 
+                else
+                  $MYADMIN flush-logs
+                fi
+        endscript
+}

etc/logrotate.d/rsyslog

@@ -0,0 +1,37 @@
+/var/log/syslog
+{
+        rotate 7
+        daily
+        missingok
+        notifempty
+        delaycompress
+        compress
+        postrotate
+                reload rsyslog >/dev/null 2>&1 || true
+        endscript
+}
+
+/var/log/mail.info
+/var/log/mail.warn
+/var/log/mail.err
+/var/log/mail.log
+/var/log/daemon.log
+/var/log/kern.log
+/var/log/auth.log
+/var/log/user.log
+/var/log/lpr.log
+/var/log/cron.log
+/var/log/debug
+/var/log/messages
+{
+        rotate 4
+        weekly
+        missingok
+        notifempty
+        compress
+        delaycompress
+        sharedscripts
+        postrotate
+                reload rsyslog >/dev/null 2>&1 || true
+        endscript
+}

etc/mysql/my.cnf

@@ -0,0 +1,127 @@
+#
+# The MySQL database server configuration file.
+#
+# You can copy this to one of:
+# - "/etc/mysql/my.cnf" to set global options,
+# - "~/.my.cnf" to set user-specific options.
+# 
+# One can use all long options that the program supports.
+# Run program with --help to get a list of available options and with
+# --print-defaults to see which it would actually understand and use.
+#
+# For explanations see
+# http://dev.mysql.com/doc/mysql/en/server-system-variables.html
+
+# This will be passed to all mysql clients
+# It has been reported that passwords should be enclosed with ticks/quotes
+# escpecially if they contain "#" chars...
+# Remember to edit /etc/mysql/debian.cnf when changing the socket location.
+[client]
+port            = 3306
+socket          = /var/run/mysqld/mysqld.sock
+
+# Here is entries for some specific programs
+# The following values assume you have at least 32M ram
+
+# This was formally known as [safe_mysqld]. Both versions are currently parsed.
+[mysqld_safe]
+socket          = /var/run/mysqld/mysqld.sock
+nice            = 0
+
+[mysqld]
+#
+# * Basic Settings
+#
+user            = mysql
+pid-file        = /var/run/mysqld/mysqld.pid
+socket          = /var/run/mysqld/mysqld.sock
+port            = 3306
+basedir         = /usr
+datadir         = /var/lib/mysql
+tmpdir          = /tmp
+lc-messages-dir = /usr/share/mysql
+skip-external-locking
+#
+# Instead of skip-networking the default is now to listen only on
+# localhost which is more compatible and is not less secure.
+bind-address            = 0.0.0.0
+#
+# * Fine Tuning
+#
+key_buffer              = 16M
+max_allowed_packet      = 16M
+thread_stack            = 192K
+thread_cache_size       = 8
+# This replaces the startup script and checks MyISAM tables if needed
+# the first time they are touched
+myisam-recover         = BACKUP
+#max_connections        = 100
+#table_cache            = 64
+#thread_concurrency     = 10
+#
+# * Query Cache Configuration
+#
+query_cache_limit       = 1M
+query_cache_size        = 16M
+#
+# * Logging and Replication
+#
+# Both location gets rotated by the cronjob.
+# Be aware that this log type is a performance killer.
+# As of 5.1 you can enable the log at runtime!
+#general_log_file        = /var/log/mysql/mysql.log
+#general_log             = 1
+#
+# Error log - should be very few entries.
+#
+log_error = /var/log/mysql/error.log
+#
+# Here you can see queries with especially long duration
+#log_slow_queries       = /var/log/mysql/mysql-slow.log
+#long_query_time = 2
+#log-queries-not-using-indexes
+#
+# The following can be used as easy to replay backup logs or for replication.
+# note: if you are setting up a replication slave, see README.Debian about
+#       other settings you may need to change.
+#server-id              = 1
+#log_bin                        = /var/log/mysql/mysql-bin.log
+expire_logs_days        = 10
+max_binlog_size         = 100M
+#binlog_do_db           = include_database_name
+#binlog_ignore_db       = include_database_name
+#
+# * InnoDB
+#
+# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.
+# Read the manual for more InnoDB related options. There are many!
+#
+# * Security Features
+#
+# Read the manual, too, if you want chroot!
+# chroot = /var/lib/mysql/
+#
+# For generating SSL certificates I recommend the OpenSSL GUI "tinyca".
+#
+# ssl-ca=/etc/mysql/cacert.pem
+# ssl-cert=/etc/mysql/server-cert.pem
+# ssl-key=/etc/mysql/server-key.pem
+
+
+
+[mysqldump]
+quick
+quote-names
+max_allowed_packet      = 16M
+
+[mysql]
+#no-auto-rehash # faster start of mysql but no tab completition
+
+[isamchk]
+key_buffer              = 16M
+
+#
+# * IMPORTANT: Additional settings that can override those from this file!
+#   The files must end with '.cnf', otherwise they'll be ignored.
+#
+!includedir /etc/mysql/conf.d/

etc/rc.local

@@ -0,0 +1,7 @@
+ip netns add nonet
+ip link add veth0 type veth peer name veth1
+ifconfig veth0 172.16.0.1/24 up
+ip link set veth1 netns nonet
+ip netns exec nonet ifconfig veth1 172.16.0.2/24 up
+/sbin/iptables -A INPUT -p tcp --dport 3306 -i eth0 -j DROP
+/sbin/iptables -A INPUT -p tcp --dport 3306 -i eth1 -j DROP

etc/sudoers

@@ -0,0 +1,33 @@
+#
+# This file MUST be edited with the 'visudo' command as root.
+#
+# Please consider adding local content in /etc/sudoers.d/ instead of
+# directly modifying this file.
+#
+# See the man page for details on how to write a sudoers file.
+#
+Defaults	env_reset
+Defaults	mail_badpass
+Defaults	secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+# Host alias specification
+
+# User alias specification
+
+# Cmnd alias specification
+
+# User privilege specification
+root	ALL=(ALL:ALL) ALL
+
+# Members of the admin group may gain root privileges
+%admin ALL=(ALL) ALL
+
+# Allow members of group sudo to execute any command
+%sudo	ALL=(ALL:ALL) ALL
+
+# Turbo Hipster
+th ALL=(root) NOPASSWD: /sbin/ip netns exec nonet *
+
+# See sudoers(5) for more information on "#include" directives:
+
+#includedir /etc/sudoers.d

etc/turbo-hipster/config.json

@@ -1,7 +1,7 @@
 {
     "zuul_server": {
-        "git_url": "/var/lib/zuul/git/",
+        "git_url": "http://119.9.13.90/p/",
-        "gearman_host": "localhost",
+        "gearman_host": "119.9.13.90",
         "gearman_port": 4730
     },
     "debug_log": "/var/log/turbo-hipster/debug.log",
@@ -11,13 +11,38 @@
     "plugins": [
         {
             "name": "gate_real_db_upgrade",
-            "function": "build:gate-real-db-upgrade_nova_mysql",
+            "datasets_dir": "/var/lib/turbo-hipster/datasets_devstack_131007",
-            "datasets_dir": "/var/lib/turbo-hipster/datasets"
+            "function": "build:gate-real-db-upgrade_nova_mysql_devstack_131007"
+        }, {
+            "name": "gate_real_db_upgrade",
+            "datasets_dir": "/var/lib/turbo-hipster/datasets_devstack_150",
+            "function": "build:gate-real-db-upgrade_nova_mysql_devstack_150"
+        }, {
+            "name": "gate_real_db_upgrade",
+            "datasets_dir": "/var/lib/turbo-hipster/datasets_trivial_500",
+            "function": "build:gate-real-db-upgrade_nova_mysql_trivial_500"
+        }, {
+            "name": "gate_real_db_upgrade",
+            "datasets_dir": "/var/lib/turbo-hipster/datasets_trivial_6000",
+            "function": "build:gate-real-db-upgrade_nova_mysql_trivial_6000"
+        }, {
+            "name": "gate_real_db_upgrade",
+            "datasets_dir": "/var/lib/turbo-hipster/datasets_user_001",
+            "function": "build:gate-real-db-upgrade_nova_mysql_user_001"
+        }, {
+            "name": "gate_real_db_upgrade",
+            "datasets_dir": "/var/lib/turbo-hipster/datasets_user_002",
+            "function": "build:gate-real-db-upgrade_nova_mysql_user_002"
         }
     ],
     "publish_logs": {
-        "type": "local",
+        "type": "swift",
-        "path": "/var/www/results/",
+        "authurl": "https://identity.api.rackspacecloud.com/v2.0/",
-        "prepend_url": "http://localhost/results/"
+        "tenant": " ",
+        "user": "XXXXXX",
+        "password": "XXXXXX",
+        "container": "XXXXXX",
+        "region": "SYD",
+        "prepend_url": "http://www.rcbops.com/turbo_hipster/results/"
     }
 }

turbo_hipster/task_plugins/gate_real_db_upgrade/nova_mysql_migrations.sh

@@ -136,7 +136,8 @@ cd $3
 echo "Setting up virtual env"
 source ~/.bashrc
 source /etc/bash_completion.d/virtualenvwrapper
-VENV_PATH=~/.virtualenvs/$1
+export WORKON_HOME=/var/lib/turbo-hipster/envs
+VENV_PATH=$WORKON_HOME/$1
 rm -rf $VENV_PATH
 mkvirtualenv $1
 toggleglobalsitepackages