diff --git a/requirements.txt b/requirements.txt index bc6960a..ce72fbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ python-openstackclient>=0.4.1 python-keystoneclient>=3.10.0 scp>=0.8.0 tabulate>=0.7.3 +fluent-logger>=0.5.2 # Workaround for pip install failed on RHEL/CentOS functools32>=3.2.3 diff --git a/test-requirements.txt b/test-requirements.txt index 8ae84af..c28bb94 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,4 +14,4 @@ oslotest>=1.10.0 # Apache-2.0 testrepository>=0.0.18 testscenarios>=0.4 testtools>=1.4.0 - +pytest>=3.0.2 \ No newline at end of file diff --git a/vmtp/tests/__init__.py b/test/__init__.py similarity index 100% rename from vmtp/tests/__init__.py rename to test/__init__.py diff --git a/vmtp/tests/test_vmtp.py b/test/test_vmtp.py similarity index 56% rename from vmtp/tests/test_vmtp.py rename to test/test_vmtp.py index e166c10..2ff9800 100644 --- a/vmtp/tests/test_vmtp.py +++ b/test/test_vmtp.py @@ -19,10 +19,23 @@ test_vmtp Tests for `vmtp` module. """ -from vmtp.tests import base +import logging +from vmtp.fluentd import FluentLogHandler +import vmtp.log -class TestVmtp(base.TestCase): +def setup_module(module): + vmtp.log.setup(product_name="test") - def test_something(self): - pass + +def test_fluentd(): + logger = logging.getLogger('fluent-logger') + handler = FluentLogHandler('vmtp', fluentd_port=7081) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + logger.info('test') + logger.warning('test %d', 100) + try: + raise Exception("test") + except Exception: + logger.exception("got exception") diff --git a/tox.ini b/tox.ini index 1686237..9c5be6e 100644 --- a/tox.ini +++ b/tox.ini @@ -10,10 +10,10 @@ setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = python setup.py testr --slowest --testr-args='{posargs}' +commands = py.test -q -s --basetemp={envtmpdir} {posargs} [testenv:pep8] -commands = flake8 +commands = flake8 {toxinidir} [testenv:venv] commands = {posargs} diff --git a/vmtp/cfg.default.yaml b/vmtp/cfg.default.yaml index 9488da4..b457318 100644 --- a/vmtp/cfg.default.yaml +++ b/vmtp/cfg.default.yaml @@ -211,3 +211,17 @@ vmtp_db: "client_db" ######################################## vmtp_collection: "pns_web_entry" +# When enabled, all logs will be sent to a fluentd server at the requested IP and port +# The fluentd "tag" and "label" fields for every message will be set to "nfvbench" +fluentd: + # by default (logging_tag is empty) nfvbench log messages are not sent to fluentd + # to enable logging to fluents, specify a valid fluentd tag name to be used for the + # log records + logging_tag: + + # IP address of the server, defaults to loopback + ip: 127.0.0.1 + + # port # to use, by default, use the default fluentd forward port + port: 24224 + diff --git a/vmtp/fluentd.py b/vmtp/fluentd.py new file mode 100644 index 0000000..bdfd8f7 --- /dev/null +++ b/vmtp/fluentd.py @@ -0,0 +1,49 @@ +# Copyright 2017 Cisco Systems, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from datetime import datetime +from fluent import sender +import logging + + +class FluentLogHandler(logging.Handler): + '''This is a minimalist log handler for use with Fluentd + + Needs to be attached to a logger using the addHandler method. + It only picks up from every record: + - the formatted message (no timestamp and no level) + - the level name + - the runlogdate (to tie multiple run-related logs together) + The timestamp is retrieved by the fluentd library. + ''' + + def __init__(self, tag, fluentd_ip='127.0.0.1', fluentd_port=24224): + logging.Handler.__init__(self) + self.tag = tag + self.formatter = logging.Formatter('%(message)s') + self.sender = sender.FluentSender(self.tag, port=fluentd_port) + self.start_new_run() + + def start_new_run(self): + '''Delimitate a new run in the stream of records with a new timestamp + ''' + self.runlogdate = str(datetime.now()) + + def emit(self, record): + data = { + "runlogdate": self.runlogdate, + "loglevel": record.levelname, + "message": self.formatter.format(record) + } + self.sender.emit(None, data) diff --git a/vmtp/tests/base.py b/vmtp/tests/base.py deleted file mode 100644 index 1c30cdb..0000000 --- a/vmtp/tests/base.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2010-2011 OpenStack Foundation -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslotest import base - - -class TestCase(base.BaseTestCase): - - """Test case base class for all unit tests.""" diff --git a/vmtp/vmtp.py b/vmtp/vmtp.py index ac33af5..5284ede 100755 --- a/vmtp/vmtp.py +++ b/vmtp/vmtp.py @@ -30,6 +30,7 @@ import compute from config import config_load from config import config_loads import credentials +from fluentd import FluentLogHandler from glanceclient.v2 import client as glanceclient import iperf_tool from keystoneclient import client as keystoneclient @@ -49,7 +50,7 @@ import sshutils flow_num = 0 return_code = 0 - +fluent_logger = None class FlowPrinter(object): @staticmethod @@ -189,7 +190,7 @@ class VmtpTest(object): self.instance_access.public_key_file = pub_key self.instance_access.private_key_file = priv_key else: - LOG.error('Default keypair ~/.ssh/id_rsa[.pub] does not exist. Please ' + LOG.error('Default id ~/.ssh/id_rsa[.pub] does not exist. Please ' 'either create one in your home directory, or specify your ' 'keypair information in the config file before running VMTP.') sys.exit(1) @@ -458,7 +459,10 @@ class VmtpTest(object): def run(self): error_flag = False - + if fluent_logger: + # take a snapshot of the current time for this new run + # so that all subsequent logs can relate to this run + fluent_logger.start_new_run() try: self.setup() self.measure_vm_flows() @@ -1141,6 +1145,7 @@ def merge_opts_to_configs(opts): def run_vmtp(opts): + global fluent_logger '''Run VMTP :param opts: Parameters that to be passed to VMTP in type argparse.Namespace(). See: http://vmtp.readthedocs.org/en/latest/usage.html#running-vmtp-as-a-library @@ -1156,6 +1161,14 @@ def run_vmtp(opts): opts.__setattr__(key, value) config = merge_opts_to_configs(opts) + # setup the fluent logger as soon as possible right after the config plugin is called + if config.fluentd.logging_tag: + fluent_logger = FluentLogHandler(config.fluentd.logging_tag, + fluentd_ip=config.fluentd.ip, + fluentd_port=config.fluentd.port) + LOG.addHandler(fluent_logger) + else: + fluent_logger = None rescol = ResultsCollector() # Run the native host tests if specified by user