Change the config format to yaml

Yaml is the format that openstack-infra uses, it's easier to
read and not nearly as ugly as a json file.

It has been decided that we will replace the json formatted config
file with a yaml one. This change completely removes json config
support from turbo hipster and replaces it with yaml. As such the
old config.json has also been replaced by a yaml version.

Change-Id: Ic4c62281ee3796e5f118e441722377046e850135

doc/source/installation.rst

@@ -8,7 +8,7 @@ Installation
 
 Turbo-hipster is installed directly into your Python ``site-packages``
 directory, and is then run as a service. It is managed using a configuration
-file, which is in json format.
+file, which is in yaml format.
 
 Installing turbo-hipster
 ------------------------
@@ -19,12 +19,12 @@ directory:
  $ sudo python setup.py install
 
 2. Copy the configuration file to a convenient location. By default,
-turbo-hipster will look in ``/etc/turbo-hipster/config.json``:
+turbo-hipster will look in ``/etc/turbo-hipster/config.yaml``:
 
  $ cp -R etc/turbo-hipster /etc/
 
-3. The turbo-hipster configuration file is in json format. Open the
-``config.json`` configuration file in your preferred editor and modify it
+3. The turbo-hipster configuration file is in yaml format. Open the
+``config.yaml`` configuration file in your preferred editor and modify it
 for your environment::
 
   **zuul_server**
@@ -113,4 +113,4 @@ editor, and edit it to ensure it is writable by ``other``:
  Debug logging must be configured for turbo-hipster, as it uses the Python
  logging framework to capture log messages from the task plugin code.
  To configure debug logging, set the ``debug_log`` configuration
- setting in the ``config.json`` configuration file.
+ setting in the ``config.yaml`` configuration file.

doc/source/running.rst

@@ -61,7 +61,7 @@ with turbo-hipster. They must contain a ``task.py`` file with a ``Runner``
 class.
 
 Once you have created a turbo-hipster plugin, you need to configure it in
-the ``config.json`` configuration file.
+the ``config.yaml`` configuration file.
 
 .. FIXME More config information required here
 
@@ -86,7 +86,7 @@ turbo-hipster's configuration file.
 
  $ cp /my/dataset.json /var/lib/turbo-hipster/datasets/
 
-3. Open the ``/etc/turbo-hipster/config.json`` file in your preferred
+3. Open the ``/etc/turbo-hipster/config.yaml`` file in your preferred
 editor, locate the plugins section, and add this line::
 
   **plugins**

etc/turbo-hipster/config.json

@@ -1,46 +0,0 @@
-{
-    "zuul_server": {
-        "gerrit_site": "http://review.openstack.org",
-        "zuul_site": "http://119.9.13.90",
-        "git_origin": "git://git.openstack.org/",
-        "gearman_host": "119.9.13.90",
-        "gearman_port": 4730
-    },
-    "debug_log": "/var/log/turbo-hipster/debug.log",
-    "jobs_working_dir": "/var/lib/turbo-hipster/jobs",
-    "git_working_dir": "/var/lib/turbo-hipster/git",
-    "pip_download_cache": "/var/cache/pip",
-    "plugins": [
-        {
-            "name": "gate_real_db_upgrade",
-            "datasets_dir": "/var/lib/turbo-hipster/datasets_devstack_131007",
-            "function": "build:gate-real-db-upgrade_nova_mysql_devstack_131007"
-        }, {
-            "name": "gate_real_db_upgrade",
-            "datasets_dir": "/var/lib/turbo-hipster/datasets_trivial_500",
-            "function": "build:gate-real-db-upgrade_nova_mysql_trivial_500"
-        }, {
-            "name": "gate_real_db_upgrade",
-            "datasets_dir": "/var/lib/turbo-hipster/datasets_trivial_6000",
-            "function": "build:gate-real-db-upgrade_nova_mysql_trivial_6000"
-        }, {
-            "name": "gate_real_db_upgrade",
-            "datasets_dir": "/var/lib/turbo-hipster/datasets_user_001",
-            "function": "build:gate-real-db-upgrade_nova_mysql_user_001"
-        }, {
-            "name": "gate_real_db_upgrade",
-            "datasets_dir": "/var/lib/turbo-hipster/datasets_user_002",
-            "function": "build:gate-real-db-upgrade_nova_mysql_user_002"
-        }
-    ],
-    "publish_logs": {
-        "type": "swift",
-        "authurl": "https://identity.api.rackspacecloud.com/v2.0/",
-        "tenant": " ",
-        "user": "XXXXXX",
-        "password": "XXXXXX",
-        "container": "XXXXXX",
-        "region": "SYD",
-        "prepend_url": "http://www.rcbops.com/turbo_hipster/results/"
-    }
-}

etc/turbo-hipster/config.yaml

@@ -0,0 +1,46 @@
+zuul_server:
+  gerrit_site: http://review.openstack.org
+  zuul_site: http://119.9.13.90
+  git_origin: git://git.openstack.org/
+  gearman_host: 119.9.13.90
+  gearman_port: 4730
+
+debug_log: /var/log/turbo-hipster/debug.log
+jobs_working_dir: /var/lib/turbo-hipster/jobs
+git_working_dir: /var/lib/turbo-hipster/git
+pip_download_cache: /var/cache/pip
+
+plugins:
+  - name: gate_real_db_upgrade
+    datasets_dir: /var/lib/turbo-hipster/datasets_devstack_131007
+    function: build:gate-real-db-upgrade_nova_mysql_devstack_131007
+
+  - name: gate_real_db_upgrade
+    datasets_dir: /var/lib/turbo-hipster/datasets_devstack_150
+    function: build:gate-real-db-upgrade_nova_mysql_devstack_150
+
+  - name: gate_real_db_upgrade
+    datasets_dir: /var/lib/turbo-hipster/datasets_trivial_500
+    function: build:gate-real-db-upgrade_nova_mysql_trivial_500
+
+  - name: gate_real_db_upgrade
+    datasets_dir: /var/lib/turbo-hipster/datasets_trivial_6000
+    function: build:gate-real-db-upgrade_nova_mysql_trivial_6000
+
+  - name: gate_real_db_upgrade
+    datasets_dir: /var/lib/turbo-hipster/datasets_user_001
+    function: build:gate-real-db-upgrade_nova_mysql_user_001
+
+  - name: gate_real_db_upgrade
+    datasets_dir: /var/lib/turbo-hipster/datasets_user_002
+    function: build:gate-real-db-upgrade_nova_mysql_user_002
+
+publish_logs:
+  type: swift
+  authurl: https://identity.api.rackspacecloud.com/v2.0/
+  tenant: XXXX
+  user: XXXXXX
+  password: XXXXXX
+  container: XXXXXX
+  region: SYD
+  prepend_url: http://www.rcbops.com/turbo_hipster/results/

turbo_hipster/cmd/analyse_historical.py

@@ -23,6 +23,7 @@ import MySQLdb
 import os
 import re
 import sys
+import yaml
 
 import swiftclient
 
@@ -33,12 +34,12 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('-c', '--config',
                         default=
-                        '/etc/turbo-hipster/config.json',
-                        help='Path to json config file.')
+                        '/etc/turbo-hipster/config.yaml',
+                        help='Path to yaml config file.')
     args = parser.parse_args()
 
     with open(args.config, 'r') as config_stream:
-        config = json.load(config_stream)
+        config = yaml.safe_load(config_stream)
     swift_config = config['publish_logs']
 
     log = logging.getLogger(__name__)

turbo_hipster/cmd/report_historical.py

@@ -20,6 +20,7 @@ import math
 import MySQLdb
 import os
 import sys
+import yaml
 
 
 def main():
@@ -29,8 +30,8 @@ def main():
 
 
 def process_dataset(dataset):
-    with open('/etc/turbo-hipster/config.json', 'r') as config_stream:
-        config = json.load(config_stream)
+    with open('/etc/turbo-hipster/config.yaml', 'r') as config_stream:
+        config = yaml.safe_load(config_stream)
     db = MySQLdb.connect(host=config['results']['host'],
                          port=config['results'].get('port', 3306),
                          user=config['results']['username'],

turbo_hipster/cmd/server.py

@@ -18,10 +18,10 @@
 import argparse
 import daemon
 import extras
-import json
 import os
 import signal
 import sys
+import yaml
 
 from turbo_hipster import worker_server
 
@@ -33,7 +33,7 @@ PID_FILE_MODULE = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
 def main(args):
 
     with open(args.config, 'r') as config_stream:
-        config = json.load(config_stream)
+        config = yaml.safe_load(config_stream)
 
     server = worker_server.Server(config)
 
@@ -59,8 +59,8 @@ if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('-c', '--config',
                         default=
-                        '/etc/turbo-hipster/config.json',
-                        help='Path to json config file.')
+                        '/etc/turbo-hipster/config.yaml',
+                        help='Path to yaml config file.')
     parser.add_argument('-b', '--background', action='store_true',
                         help='Run as a daemon in the background.')
     parser.add_argument('-p', '--pidfile',