Add PEP8 check and fix related issues

- Add PEP8 section to tox.ini
- Add hacking to requirements to enforce OpenStack style requirements
- Change setup.py to use PBR
- Add setup.cfg
- Fix formatting issues flagged by flake8 check
- Add copyright notices to all remaining files
- Update .gitignore file
- Bump version number

Change-Id: If32d332d3b7800f66fe6ad0f815f178bda739036

.gitignore

@@ -28,3 +28,8 @@ nosetests.xml
 
 # Translations
 *.mo
+
+# IDE Project Files
+*.project
+*.pydev*
+*.idea

bin/test-distiller.py

@@ -3,8 +3,6 @@
 #
 # Copyright © 2014 Rackspace Hosting.
 #
-# Author: Monsyne Dragon <mdragon@rackspace.com>
-#
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
@@ -27,8 +25,8 @@ import argparse
 import json
 import sys
 
-from stackdistiller import distiller
 from stackdistiller import condenser
+from stackdistiller import distiller
 
 
 class TestCondenser(condenser.CondenserBase):
@@ -81,28 +79,32 @@ def test_data(args):
                 yield n
 
 
-parser = argparse.ArgumentParser(description="Test Distiller configuration")
+parser = argparse.ArgumentParser(
-parser.add_argument('-c', '--config',
+    description="Test Distiller configuration")
-                    default='event_definitions.yaml',
+parser.add_argument(
-                    help='Name of event definitions file '
+    '-c', '--config',
-                         'to test (Default: %(default)s)')
+    default='event_definitions.yaml',
-parser.add_argument('-l', '--list', action='store_true',
+    help='Name of event definitions file '
-                    help='Test data files contain JSON list of notifications.'
+         'to test (Default: %(default)s)')
-                    ' (By default data files should contain a single '
+parser.add_argument(
-                    'notification.)')
+    '-l', '--list', action='store_true',
-parser.add_argument('-d', '--add_default_definition', action='store_true',
+    help='Test data files contain JSON list of notifications.'
-                    help='Add default event definition. Normally, '
+         ' (By default data files should contain a single '
-                    'notifications are dropped if there is no event '
+         'notification.)')
-                    'definition for their event_type. Setting this adds a '
+parser.add_argument(
-                    '"catchall" that converts unknown notifications to Events'
+    '-d', '--add_default_definition', action='store_true',
-                    ' with a few basic traits.')
+    help='Add default event definition. Normally, '
-parser.add_argument('-o', '--output',  type=argparse.FileType('w'),
+         'notifications are dropped if there is no event '
+         'definition for their event_type. Setting this adds a '
+         '"catchall" that converts unknown notifications to Events'
+         ' with a few basic traits.')
+parser.add_argument('-o', '--output', type=argparse.FileType('w'),
                     default=sys.stdout, help="Output file. Default stdout")
-parser.add_argument('test_data', nargs='*', metavar='JSON_FILE',
+parser.add_argument(
-                    help="Test notifications in JSON format. Defaults to stdin")
+    'test_data', nargs='*', metavar='JSON_FILE',
+    help="Test notifications in JSON format. Defaults to stdin")
 args = parser.parse_args()
 
-
 config = distiller.load_config(args.config)
 
 out = args.output
@@ -115,7 +117,7 @@ drops = 0
 cond = TestCondenser()
 for notification in notifications:
     cond.clear()
-    nct +=1
+    nct += 1
     if dist.to_event(notification, cond) is None:
         out.write("Dropped notification: %s\n" %
                   notification['message_id'])

requirements.txt

@@ -1 +1,6 @@
--e .
+hacking>=0.10.0,<0.11
+enum34>=1.0
+iso8601>=0.1.10
+jsonpath-rw>=1.2.0, < 2.0
+PyYAML>=3.1.0
+six>=1.5.2

setup.cfg

@@ -0,0 +1,26 @@
+[metadata]
+description-file = README.md
+name = stackdistiller
+version = 0.12
+author = Monsyne Dragon
+author_email = mdragon@rackspace.com
+summary = A data extraction and transformation library for OpenStack notifications
+license = Apache-2
+keywords =
+    OpenStack
+    notifications
+    events
+    extraction
+    transformation
+classifiers =
+    Development Status :: 3 - Alpha
+    License :: OSI Approved :: Apache Software License
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python :: 2.6
+    Programming Language :: Python :: 2.7
+
+home-page = https://github.com/stackforge/stacktach-stackdistiller
+
+[files]
+packages =
+    stackdistiller

setup.py

@@ -1,37 +1,8 @@
-import os
+#!/usr/bin/env python
-from setuptools import setup, find_packages
 
-
+from setuptools import setup
-def read(fname):
-    return open(os.path.join(os.path.dirname(__file__), fname)).read()
 
 setup(
-    name='stackdistiller',
+    setup_requires=['pbr'],
-    version='0.11',
+    pbr=True,
-    author='Monsyne Dragon',
-    author_email='mdragon@rackspace.com',
-    description=("A data extraction and transformation library for "
-                 "OpenStack notifications"),
-    license='Apache License (2.0)',
-    keywords='OpenStack notifications events extraction transformation',
-    packages=find_packages(exclude=['tests']),
-    classifiers=[
-        'Development Status :: 3 - Alpha',
-        'License :: OSI Approved :: Apache Software License',
-        'Operating System :: POSIX :: Linux',
-        'Programming Language :: Python :: 2.6',
-        'Programming Language :: Python :: 2.7',
-    ],
-    url='https://github.com/stackforge/stacktach-stackdistiller',
-    scripts=['bin/test-distiller.py'],
-    long_description=read('README.md'),
-    install_requires=[
-        "enum34 >= 1.0",
-        "iso8601 >= 0.1.10",
-        "jsonpath-rw >= 1.2.0, < 2.0",
-        "PyYAML >= 3.1.0",
-        "six >= 1.5.2",
-    ],
-
-    zip_safe=False
 )

stackdistiller/condenser.py

@@ -2,8 +2,6 @@
 #
 # Copyright © 2014 Rackspace Hosting.
 #
-# Author: Monsyne Dragon <mdragon@rackspace.com>
-#
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
@@ -22,18 +20,23 @@ import six
 
 @six.add_metaclass(abc.ABCMeta)
 class CondenserBase(object):
-    """Base class for Condenser objects that collect data extracted from a
+    """Base class for Condenser objects
-       Notification by the Distiller, and format it into a usefull datastructure.
 
-       A simple Condenser may just colect all the traits received into a dictionary.
+     Collect data extracted from a Notification by the Distiller, and
-       More complex ones may build collections of application or ORM model objects,
+     format it into a useful data structure.
-       or XML document trees.
 
-       Condensers also have hooks for verification logic, to check that all needed
+     A simple Condenser may just colect all the traits received into
-       traits are present."""
+     a dictionary. More complex ones may build collections of application
+     or ORM model objects, or XML document trees.
+
+     Condensers also have hooks for verification logic, to check that
+     all needed traits are present.
+     """
 
     def __init__(self, **kw):
-        """Setup the condenser. A new instance of the condenser is passed to the
+        """Set up the condenser.
+
+        A new instance of the condenser is passed to the
         distiller for each notification extracted.
 
         :param kw: keyword parameters for condenser.
@@ -43,7 +46,9 @@ class CondenserBase(object):
 
     @abc.abstractmethod
     def add_trait(self, name, trait_type, value):
-        """Add a trait to the Event datastructure being built by this
+        """Add a trait
+
+        Add a trait to the Event data structure being built by this
         condenser. The distiller will call this for each extracted trait.
 
         :param name: (string) name of the trait
@@ -54,7 +59,9 @@ class CondenserBase(object):
 
     @abc.abstractmethod
     def add_envelope_info(self, event_type, message_id, when):
-        """Add the metadata for this event, extracted from the notification's
+        """Add the metadata for this event
+
+        Add metadata extracted from the notification's
         envelope. The distiller will call this once.
 
         :param event_type: (string) Type of event, as a dotted string such as
@@ -66,14 +73,14 @@ class CondenserBase(object):
 
     @abc.abstractmethod
     def get_event(self):
-        """Return the Event datastructure constructed by this condenser."""
+        """Return the Event data structure constructed by this condenser."""
 
     @abc.abstractmethod
     def clear(self):
         """Clear condenser state."""
 
     def validate(self):
-        """Check Event against whatever validation logic this condenser may have
+        """Check Event against whatever validation logic this condenser has
 
         :returns:   (bool) True if valid.
 
@@ -83,6 +90,7 @@ class CondenserBase(object):
 
 class DictionaryCondenser(CondenserBase):
     """Return event data as a simple python dictionary"""
+
     def __init__(self, **kw):
         self.clear()
         super(DictionaryCondenser, self).__init__(**kw)

stackdistiller/distiller.py

@@ -2,8 +2,6 @@
 #
 # Copyright © 2013 Rackspace Hosting.
 #
-# Author: Monsyne Dragon <mdragon@rackspace.com>
-#
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
@@ -20,7 +18,6 @@ import collections
 import datetime
 import fnmatch
 import logging
-import os
 
 from enum import Enum
 import iso8601
@@ -60,13 +57,13 @@ def load_config(filename):
         if hasattr(err, 'problem_mark'):
             mark = err.problem_mark
             errmsg = ("Invalid YAML syntax in Event Definitions file "
-                        "%(file)s at line: %(line)s, column: %(column)s."
+                      "%(file)s at line: %(line)s, column: %(column)s."
                       % dict(file=filename,
                              line=mark.line + 1,
                              column=mark.column + 1))
         else:
             errmsg = ("YAML error reading Event Definitions file "
-                        "%(file)s"
+                      "%(file)s"
                       % dict(file=filename))
         logger.error(errmsg)
         raise
@@ -100,7 +97,6 @@ Trait = collections.namedtuple('Trait', ('name', 'trait_type', 'value'))
 
 
 class TraitDefinition(object):
-
     def __init__(self, name, trait_cfg, plugin_map):
         self.cfg = trait_cfg
         self.name = name
@@ -110,8 +106,8 @@ class TraitDefinition(object):
             type_name = trait_cfg.get('type', 'text')
         except AttributeError as e:
             raise EventDefinitionException(
-                    "Unable to get type for '%s'" % trait_cfg,
+                "Unable to get type for '%s'" % trait_cfg,
-                    self.cfg)
+                self.cfg)
 
         if 'plugin' in trait_cfg:
             plugin_cfg = trait_cfg['plugin']
@@ -124,7 +120,7 @@ class TraitDefinition(object):
                 except KeyError:
                     raise EventDefinitionException(
                         'Plugin specified, but no plugin name supplied for '
-                          'trait %s' % name, self.cfg)
+                        'trait %s' % name, self.cfg)
                 plugin_params = plugin_cfg.get('parameters')
                 if plugin_params is None:
                     plugin_params = {}
@@ -133,8 +129,8 @@ class TraitDefinition(object):
             except KeyError:
                 raise EventDefinitionException(
                     'No plugin named %(plugin)s available for '
-                      'trait %(trait)s' % dict(plugin=plugin_name,
+                    'trait %(trait)s' % dict(plugin=plugin_name,
-                                                trait=name), self.cfg)
+                                             trait=name), self.cfg)
             self.plugin = plugin_class(**plugin_params)
         else:
             self.plugin = None
@@ -142,7 +138,7 @@ class TraitDefinition(object):
         if 'fields' not in trait_cfg:
             raise EventDefinitionException(
                 "Required field in trait definition not specified: "
-                  "'%s'" % 'fields',
+                "'%s'" % 'fields',
                 self.cfg)
 
         fields = trait_cfg['fields']
@@ -157,7 +153,7 @@ class TraitDefinition(object):
         except Exception as e:
             raise EventDefinitionException(
                 "Parse error in JSONPath specification "
-                  "'%(jsonpath)s' for %(trait)s: %(err)s"
+                "'%(jsonpath)s' for %(trait)s: %(err)s"
                 % dict(jsonpath=fields, trait=name, err=e), self.cfg)
         try:
             self.trait_type = Datatype[type_name]
@@ -196,7 +192,6 @@ class TraitDefinition(object):
 
 
 class EventDefinition(object):
-
     DEFAULT_TRAITS = dict(
         service=dict(type='text', fields='publisher_id'),
         request_id=dict(type='text', fields='_context_request_id'),
@@ -262,8 +257,8 @@ class EventDefinition(object):
 
     @staticmethod
     def _extract_when(body):
-        """Extract the generated datetime from the notification.
+        """Extract the generated datetime from the notification."""
-        """
+
         # NOTE: I am keeping the logic the same as it was in openstack
         # code, However, *ALL* notifications should have a 'timestamp'
         # field, it's part of the notification envelope spec. If this was

stackdistiller/trait_plugins.py

@@ -2,8 +2,6 @@
 #
 # Copyright © 2013 Rackspace Hosting.
 #
-# Author: Monsyne Dragon <mdragon@rackspace.com>
-#
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
@@ -22,9 +20,7 @@ import six
 
 @six.add_metaclass(abc.ABCMeta)
 class TraitPluginBase(object):
-    """Base class for plugins that convert notification fields to
+    """Base class for plugins that convert notification fields to Traits"""
-       Trait values.
-    """
 
     def __init__(self, **kw):
         """Setup the trait plugin.

tests/test_distiller.py

@@ -2,13 +2,11 @@
 #
 # Copyright © 2013 Rackspace Hosting.
 #
-# Author: Monsyne Dragon <mdragon@rackspace.com>
-#
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
@@ -18,7 +16,7 @@
 
 import datetime
 
-#for Python2.6 compatability.
+# for Python2.6 compatability.
 import unittest2 as unittest
 
 import iso8601
@@ -50,13 +48,14 @@ class TestCondenser(object):
 
 class DistillerTestBase(unittest.TestCase):
     def _create_test_notification(self, event_type, message_id, **kw):
-        return dict(event_type=event_type,
+        return dict(
-                    message_id=message_id,
+            event_type=event_type,
-                    priority="INFO",
+            message_id=message_id,
-                    publisher_id="compute.host-1-2-3",
+            priority="INFO",
-                    timestamp="2013-08-08 21:06:37.803826",
+            publisher_id="compute.host-1-2-3",
-                    payload=kw,
+            timestamp="2013-08-08 21:06:37.803826",
-                    )
+            payload=kw,
+        )
 
     def assertIsValidEvent(self, event, notification):
         self.assertIsNot(
@@ -113,7 +112,6 @@ class DistillerTestBase(unittest.TestCase):
 
 
 class TestTraitDefinition(DistillerTestBase):
-
     def setUp(self):
         super(TestTraitDefinition, self).setUp()
         self.n1 = self._create_test_notification(
@@ -126,8 +124,8 @@ class TestTraitDefinition(DistillerTestBase):
             host='host-1-2-3',
             bogus_date='',
             image_meta=dict(
-                        disk_gb='20',
+                disk_gb='20',
-                        thing='whatzit'),
+                thing='whatzit'),
             foobar=50)
 
         self.test_plugin_class = mock.MagicMock(name='mock_test_plugin')
@@ -244,14 +242,14 @@ class TestTraitDefinition(DistillerTestBase):
 
     def test_to_trait_multiple_different_nesting(self):
         cfg = dict(type='int', fields=['payload.foobar',
-                   'payload.image_meta.disk_gb'])
+                                       'payload.image_meta.disk_gb'])
         tdef = distiller.TraitDefinition('test_trait', cfg,
                                          self.fake_plugin_map)
         t = tdef.to_trait(self.n1)
         self.assertEqual(50, t.value)
 
         cfg = dict(type='int', fields=['payload.image_meta.disk_gb',
-                   'payload.foobar'])
+                                       'payload.foobar'])
         tdef = distiller.TraitDefinition('test_trait', cfg,
                                          self.fake_plugin_map)
         t = tdef.to_trait(self.n1)
@@ -322,7 +320,7 @@ class TestTraitDefinition(DistillerTestBase):
             jsonpath_rw.parse('(payload.test)|(payload.other)'))
 
     def test_invalid_path_config(self):
-        #test invalid jsonpath...
+        # test invalid jsonpath...
         cfg = dict(fields='payload.bogus(')
         self.assertRaises(distiller.EventDefinitionException,
                           distiller.TraitDefinition,
@@ -331,7 +329,7 @@ class TestTraitDefinition(DistillerTestBase):
                           self.fake_plugin_map)
 
     def test_invalid_plugin_config(self):
-        #test invalid jsonpath...
+        # test invalid jsonpath...
         cfg = dict(fields='payload.test', plugin=dict(bogus="true"))
         self.assertRaises(distiller.EventDefinitionException,
                           distiller.TraitDefinition,
@@ -340,7 +338,7 @@ class TestTraitDefinition(DistillerTestBase):
                           self.fake_plugin_map)
 
     def test_unknown_plugin(self):
-        #test invalid jsonpath...
+        # test invalid jsonpath...
         cfg = dict(fields='payload.test', plugin=dict(name='bogus'))
         self.assertRaises(distiller.EventDefinitionException,
                           distiller.TraitDefinition,
@@ -366,7 +364,7 @@ class TestTraitDefinition(DistillerTestBase):
         self.assertEqual(distiller.Datatype.datetime, t.trait_type)
 
     def test_invalid_type_config(self):
-        #test invalid jsonpath...
+        # test invalid jsonpath...
         cfg = dict(type='bogus', fields='payload.test')
         self.assertRaises(distiller.EventDefinitionException,
                           distiller.TraitDefinition,
@@ -376,7 +374,6 @@ class TestTraitDefinition(DistillerTestBase):
 
 
 class TestEventDefinition(DistillerTestBase):
-
     def setUp(self):
         super(TestEventDefinition, self).setUp()
 
@@ -419,11 +416,13 @@ class TestEventDefinition(DistillerTestBase):
         e = edef.to_event(self.test_notification1, self.condenser)
         self.assertTrue(e is self.condenser)
         self.assertEqual('test.thing', e.event_type)
-        self.assertEqual(datetime.datetime(2013, 8, 8, 21, 6, 37, 803826, iso8601.iso8601.UTC),
+        self.assertEqual(datetime.datetime(2013, 8, 8, 21, 6, 37, 803826,
+                                           iso8601.iso8601.UTC),
                          e.when)
 
         self.assertHasDefaultTraits(e)
-        self.assertHasTrait(e, 'host', value='host-1-2-3', trait_type=trait_type)
+        self.assertHasTrait(e, 'host', value='host-1-2-3',
+                            trait_type=trait_type)
         self.assertHasTrait(e, 'instance_id',
                             value='uuid-for-instance-0001',
                             trait_type=trait_type)
@@ -609,24 +608,25 @@ class TestEventDefinition(DistillerTestBase):
 
 
 class TestDistiller(DistillerTestBase):
-
     def setUp(self):
         super(TestDistiller, self).setUp()
 
-        self.valid_event_def1 = [{
+        self.valid_event_def1 = [
-            'event_type': 'compute.instance.create.*',
+            {
-            'traits': {
+                'event_type': 'compute.instance.create.*',
-                'instance_id': {
+                'traits': {
-                    'type': 'text',
+                    'instance_id': {
-                    'fields': ['payload.instance_uuid',
+                        'type': 'text',
-                               'payload.instance_id'],
+                        'fields': ['payload.instance_uuid',
+                                   'payload.instance_id'],
+                    },
+                    'host': {
+                        'type': 'text',
+                        'fields': 'payload.host',
+                    },
                 },
-                'host': {
+            }
-                    'type': 'text',
+        ]
-                    'fields': 'payload.host',
-                },
-            },
-        }]
 
         self.test_notification1 = self._create_test_notification(
             "compute.instance.create.start",
@@ -645,10 +645,7 @@ class TestDistiller(DistillerTestBase):
         # test a malformed notification
         now = datetime.datetime.utcnow().replace(tzinfo=iso8601.iso8601.UTC)
         mock_utcnow.return_value = now
-        c = distiller.Distiller(
+        c = distiller.Distiller([], self.fake_plugin_map, catchall=True)
-            [],
-            self.fake_plugin_map,
-            catchall=True)
         message = {'event_type': "foo",
                    'message_id': "abc",
                    'publisher_id': "1"}
@@ -674,7 +671,8 @@ class TestDistiller(DistillerTestBase):
         e = c.to_event(self.test_notification2, TestCondenser())
         self.assertIsValidEvent(e, self.test_notification2)
         self.assertEqual(1, len(e.traits),
-            "Wrong number of traits %s: %s" % (len(e.traits), e.traits))
+                         "Wrong number of traits %s: %s" % (
+                             len(e.traits), e.traits))
         self.assertHasDefaultTraits(e)
         self.assertDoesNotHaveTrait(e, 'instance_id')
         self.assertDoesNotHaveTrait(e, 'host')
@@ -696,10 +694,7 @@ class TestDistiller(DistillerTestBase):
         self.assertIsNotValidEvent(e, self.test_notification2)
 
     def test_distiller_empty_cfg_with_catchall(self):
-        c = distiller.Distiller(
+        c = distiller.Distiller([], self.fake_plugin_map, catchall=True)
-            [],
-            self.fake_plugin_map,
-            catchall=True)
         self.assertEqual(1, len(c.definitions))
         e = c.to_event(self.test_notification1, TestCondenser())
         self.assertIsValidEvent(e, self.test_notification1)
@@ -712,10 +707,7 @@ class TestDistiller(DistillerTestBase):
         self.assertHasDefaultTraits(e)
 
     def test_distiller_empty_cfg_without_catchall(self):
-        c = distiller.Distiller(
+        c = distiller.Distiller([], self.fake_plugin_map, catchall=False)
-            [],
-            self.fake_plugin_map,
-            catchall=False)
         self.assertEqual(0, len(c.definitions))
         e = c.to_event(self.test_notification1, TestCondenser())
         self.assertIsNotValidEvent(e, self.test_notification1)

tests/test_trait_plugins.py

@@ -2,8 +2,6 @@
 #
 # Copyright © 2013 Rackspace Hosting.
 #
-# Author: Monsyne Dragon <mdragon@rackspace.com>
-#
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
@@ -16,14 +14,13 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-#for Python2.6 compatability.
+# for Python2.6 compatability.
 import unittest2 as unittest
 
 from stackdistiller import trait_plugins
 
 
 class TestSplitterPlugin(unittest.TestCase):
-
     def setUp(self):
         super(TestSplitterPlugin, self).setUp()
         self.pclass = trait_plugins.SplitterTraitPlugin
@@ -70,7 +67,6 @@ class TestSplitterPlugin(unittest.TestCase):
 
 
 class TestBitfieldPlugin(unittest.TestCase):
-
     def setUp(self):
         super(TestBitfieldPlugin, self).setUp()
         self.pclass = trait_plugins.BitfieldTraitPlugin

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py26,py27
+envlist = py26,py27,pep8
 
 [testenv]
 deps = 
@@ -13,3 +13,11 @@ commands =
 
 sitepackages = False
 
+[testenv:pep8]
+commands =
+    flake8
+
+[flake8]
+ignore =
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg
+show-source = True