From 82773ab56c28cfbe07a9c9595b5f01977c6ff8a3 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 7 Nov 2016 17:25:08 +1100 Subject: [PATCH] Add image upload filters for nodepool This adds loggers and handlers for each provider in the nodepool config file for separate image upload logs on the nodepool host. This doesn't regenerate the file yet. It won't do anything until the dependent change to nodepool starts generating upload logs with logger names that match these. Change-Id: Ic80e532891f039e0f835847ddfbae38f1ce3cd8a --- tools/nodepool_log_config.py | 85 +++++++++++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 11 deletions(-) diff --git a/tools/nodepool_log_config.py b/tools/nodepool_log_config.py index 0f9df66309..9d06498090 100755 --- a/tools/nodepool_log_config.py +++ b/tools/nodepool_log_config.py @@ -34,7 +34,7 @@ CONFIG_FILE = '../../project-config/nodepool/nodepool.yaml' LOGGING_CONFIG_FILE = MODULES_PATH + '/nodepool-builder.logging.conf.erb' LOG_DIR = '/var/log/nodepool' IMAGE_LOG_DIR = '<%= @image_log_document_root %>' - +UPLOAD_LOG_DIR = '<%= @upload_log_document_root %>' _BASIC_FILE = """ @@ -107,9 +107,10 @@ args=('%(image_log_dir)s/image.log', 'H', 8, 30,) format=%%(asctime)s %%(levelname)s %%(name)s: %%(message)s datefmt= -# ==== individual image loggers ==== +# ==== loggers and handlers ==== -%(image_loggers_and_handlers)s""" +%(loggers_and_handlers)s +""" _IMAGE_HANDLER = """ [handler_%(title)s] @@ -127,25 +128,50 @@ qualname=nodepool.image.build.%(qualname)s propagate=0 """ +_UPLOAD_HANDLER = """ +[handler_%(title)s] +level=DEBUG +class=logging.handlers.TimedRotatingFileHandler +formatter=simple +args=('%(upload_log_dir)s/%(filename)s', 'H', 8, 30,) +""" -def _get_providers_and_images(config_file): +_UPLOAD_LOGGER = """ +[logger_%(title)s] +level=DEBUG +handlers=%(handler)s +qualname=nodepool.image.upload.%(qualname)s +propagate=0 +""" + + +def _get_providers_and_images(config): ret = [] - config = yaml.load(config_file) for provider in config['providers']: for image in provider['images']: if 'diskimage' not in image: ret.append((provider['name'], image['name'])) - logging.debug("Added %d providers & images" % len(ret)) + logging.debug("Added snapshot image: %s:%s" % ( + provider['name'], image['name'])) # diskimages have a special provider if 'diskimages' in config: for diskimage in config['diskimages']: ret.append(('dib', diskimage['name'])) + logging.debug("Added dib image: %s" % diskimage['name']) return ret -def _generate_logger_and_handler(image_log_dir, provider, image): +def _get_upload_providers(config): + ret = [] + for provider in config['providers']: + ret.append(provider['name']) + logging.debug("Added upload provider: %s" % provider['name']) + return ret + + +def _generate_image_logger_and_handler(image_log_dir, provider, image): handler = _IMAGE_HANDLER % { 'image_log_dir': image_log_dir, 'title': '%s_%s' % (provider, image), @@ -167,7 +193,29 @@ def _generate_logger_and_handler(image_log_dir, provider, image): } -def generate_log_config(config, log_dir, image_log_dir, output): +def _generate_upload_logger_and_handler(upload_log_dir, provider): + handler = _UPLOAD_HANDLER % { + 'upload_log_dir': upload_log_dir, + 'title': '%s_upload' % (provider), + 'filename': '%s.log' % (provider), + } + + logger = _UPLOAD_LOGGER % { + 'title': '%s_upload' % (provider), + 'handler': '%s_upload' % (provider), + 'qualname': '%s' % (provider), + } + + return { + 'handler_title': '%s_upload' % (provider), + 'logger_title': '%s_upload' % (provider), + 'handler': handler, + 'logger': logger, + } + + +def generate_log_config(config, log_dir, image_log_dir, + upload_log_dir, output): """Generate a sample logging file @@ -199,16 +247,26 @@ def generate_log_config(config, log_dir, image_log_dir, output): :param config: input config file :param log_dir: directory for main log file :param image_log_dir: directory for image build logs + :param upload_log_dir: directory for upload logs :param output: open file handle to output sample configuration to """ loggers_and_handlers = [] logging.debug("Reading config file %s" % config.name) + config = yaml.load(config) + + # image loggers for (provider, image) in _get_providers_and_images(config): loggers_and_handlers.append( - _generate_logger_and_handler(image_log_dir, provider, image)) + _generate_image_logger_and_handler(image_log_dir, provider, image)) + # provider loggers (for upload) + for provider in _get_upload_providers(config): + loggers_and_handlers.append( + _generate_upload_logger_and_handler(upload_log_dir, provider)) + + # output all loggers and handlers logger_titles = [] handler_titles = [] image_loggers_and_handlers = "" @@ -217,6 +275,7 @@ def generate_log_config(config, log_dir, image_log_dir, output): handler_titles.append(item['handler_title']) image_loggers_and_handlers += item['logger'] + item['handler'] + # Final output final_output = _BASIC_FILE % { 'log_dir': log_dir, 'image_log_dir': image_log_dir, @@ -228,7 +287,7 @@ def generate_log_config(config, log_dir, image_log_dir, output): break_long_words=False, break_on_hyphens=False, subsequent_indent=' ')), - 'image_loggers_and_handlers': image_loggers_and_handlers, + 'loggers_and_handlers': image_loggers_and_handlers, } logging.debug("Writing output to %s" % output.name) @@ -255,7 +314,10 @@ def main(): "(default: %s)" % LOG_DIR) parser.add_argument('-i', '--image-log-dir', default=IMAGE_LOG_DIR, help="Output directory for image logs " - "(default: %s)" % IMAGE_LOG_DIR) + "(default: %s)" % IMAGE_LOG_DIR.replace('%', '%%')) + parser.add_argument('-u', '--upload-log-dir', default=UPLOAD_LOG_DIR, + help="Output directory for upload logs " + "(default: %s)" % UPLOAD_LOG_DIR.replace('%', '%%')) args = parser.parse_args() logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) @@ -263,6 +325,7 @@ def main(): generate_log_config(args.config, args.log_dir, args.image_log_dir, + args.upload_log_dir, args.output)