diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 101a8fa..eb01021 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -77,6 +77,10 @@ for your environment:: to use a script to authenticate against a swift account or to use *laughing_spice* to format the logs etc. + **conf_d** + A path of a directory containing pieces of json confiuration. + This is helpful when you want different plugins to add extra + or even modify the default configuration. 4. Create a turbo-hipster user: diff --git a/requirements.txt b/requirements.txt index 7c53f67..d64b484 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ sphinxcontrib-seqdiag mysql-python requests +PyYAML diff --git a/tests/etc/conf.d/extra.yaml b/tests/etc/conf.d/extra.yaml new file mode 100644 index 0000000..44fe974 --- /dev/null +++ b/tests/etc/conf.d/extra.yaml @@ -0,0 +1 @@ +extra_configuration: testing123 diff --git a/tests/etc/config.json b/tests/etc/config.json deleted file mode 100644 index 587b110..0000000 --- a/tests/etc/config.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "zuul_server": { - "git_url": "/home/josh/var/lib/zuul/git/", - "gearman_host": "localhost", - "gearman_port": 0 - }, - "debug_log": "/home/josh/var/log/turbo-hipster/debug.log", - "jobs_working_dir": "/home/josh/var/lib/turbo-hipster/jobs", - "git_working_dir": "/home/josh/var/lib/turbo-hipster/git", - "pip_download_cache": "/home/josh/var/cache/pip", - "plugins": [ - { - "name": "gate_real_db_upgrade", - "datasets_dir": "/home/josh/var/lib/turbo-hipster/datasets", - "job": "gate-real-db-upgrade_nova_mysql" - } - ], - "publish_logs": - { - "type": "local", - "path": "/home/josh/var/www/results/", - "prepend_url": "http://localhost/results/" - } -} \ No newline at end of file diff --git a/tests/etc/config.yaml b/tests/etc/config.yaml new file mode 100644 index 0000000..0e332ee --- /dev/null +++ b/tests/etc/config.yaml @@ -0,0 +1,18 @@ +zuul_server: + git_url: /home/josh/var/lib/zuul/git/ + gearman_host: localhost + gearman_port: 0 + +debug_log: /home/josh/var/log/turbo-hipster/debug.log +jobs_working_dir: /home/josh/var/lib/turbo-hipster/jobs +git_working_dir: /home/josh/var/lib/turbo-hipster/git +pip_download_cache: /home/josh/var/cache/pip +plugins: + - name: gate_real_db_upgrade + datasets_dir": /home/josh/var/lib/turbo-hipster/datasets + job: gate-real-db-upgrade_nova_mysql + +publish_logs: + type: local + path: /home/josh/var/www/results/ + prepend_url: http://localhost/results/ diff --git a/tests/test_worker_manager.py b/tests/test_worker_manager.py index eefbcbe..ae2d5b5 100644 --- a/tests/test_worker_manager.py +++ b/tests/test_worker_manager.py @@ -15,16 +15,16 @@ # under the License. -import json import os import testtools import time +import yaml from fakes import FakeZuulManager, FakeGearmanServer,\ FakeRealDbUpgradeRunner CONFIG_DIR = os.path.join(os.path.dirname(__file__), 'etc') -with open(os.path.join(CONFIG_DIR, 'config.json'), 'r') as config_stream: - CONFIG = json.load(config_stream) +with open(os.path.join(CONFIG_DIR, 'config.yaml'), 'r') as config_stream: + CONFIG = yaml.safe_load(config_stream) class TestZuulManager(testtools.TestCase): diff --git a/tests/test_worker_server.py b/tests/test_worker_server.py new file mode 100644 index 0000000..200a7b3 --- /dev/null +++ b/tests/test_worker_server.py @@ -0,0 +1,52 @@ +#!/usr/bin/python2 +# +# Copyright 2014 Rackspace Australia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import os +import testtools +import yaml + +from turbo_hipster import worker_server + +CONFIG_DIR = os.path.join(os.path.dirname(__file__), 'etc') +with open(os.path.join(CONFIG_DIR, 'config.yaml'), 'r') as config_stream: + CONFIG = yaml.safe_load(config_stream) + +CONF_D_DIR = os.path.join(CONFIG_DIR, "conf.d") + + +class TestServerManager(testtools.TestCase): + def setUp(self): + super(TestServerManager, self).setUp() + self.config = CONFIG + + def tearDown(self): + super(TestServerManager, self).tearDown() + + def test_confd_configuration(self): + """ Check that the server can load in other configuration from a + conf.d directory """ + + def pass_function(*args, **kargs): + pass + + self.config["conf_d"] = CONF_D_DIR + + worker_server.Server.setup_logging = pass_function + serv = worker_server.Server(self.config) + serv_config = serv.config + self.assertIn("extra_configuration", serv_config) + self.assertEquals("testing123", serv_config["extra_configuration"]) diff --git a/turbo_hipster/worker_server.py b/turbo_hipster/worker_server.py index d2edf18..a0f3cf2 100755 --- a/turbo_hipster/worker_server.py +++ b/turbo_hipster/worker_server.py @@ -21,8 +21,10 @@ task_plugins. """ import logging import os import threading +import yaml import worker_manager +from os.path import join, isdir, isfile class Server(threading.Thread): @@ -34,6 +36,13 @@ class Server(threading.Thread): super(Server, self).__init__() self._stop = threading.Event() self.config = config + + # Load extra configuration first + # NOTE(Mattoliverau): debug_log might be specified in + # a conf.d snippet. + if 'conf_d' in self.config: + self.load_extra_configuration() + # Python logging output file. self.debug_log = self.config['debug_log'] self.setup_logging() @@ -51,6 +60,24 @@ class Server(threading.Thread): self.tasks = {} self.load_plugins() + def load_extra_configuration(self): + if isdir(self.config["conf_d"]): + extra_configs = (join(self.config["conf_d"], item) + for item in os.listdir(self.config["conf_d"]) + if isfile(join(self.config["conf_d"], item))) + for conf in extra_configs: + try: + with open(conf, 'r') as config_stream: + extra_config = yaml.safe_load(config_stream) + self.config.update(extra_config) + except: + self.log.warn("Failed to load extra configuration: '%s'" % + (conf)) + continue + else: + self.log.warn("conf_d parameter '%s' isn't a directory" % + (self.config["conf_d"])) + def setup_logging(self): if not self.debug_log: raise Exception('Debug log not configured')