Add PEP8 check and fix related issues

- Add PEP8 section to tox.ini
- Add hacking to requirements to enforce OpenStack style requirements
- Fix formatting issues flagged by flake8 check
- Add copyright notices to all remaining files
- Update .gitignore file

Change-Id: I7e9a0203ddf2002c08ac96800fe30c1c46ebba88

.gitignore

@@ -26,6 +26,7 @@ pip-log.txt
 .coverage
 .tox
 nosetests.xml
+test_temp/*
 
 # Translations
 *.mo
@@ -34,3 +35,8 @@ nosetests.xml
 .mr.developer.cfg
 .project
 .pydevproject
+
+# IDE Project Files
+*.project
+*.pydev*
+*.idea

requirements.txt

@@ -1,3 +1,4 @@
+hacking>=0.10.0,<0.11
 notigen
 notification_utils
 python-dateutil

shoebox/archive.py

@@ -30,7 +30,9 @@ class Archive(object):
 
 class ArchiveWriter(Archive):
     """The active Archive for appending.
+
     """
+
     def __init__(self, filename):
         super(ArchiveWriter, self).__init__(filename)
         self._open_file(filename)
@@ -47,7 +49,9 @@ class ArchiveWriter(Archive):
 
 class ArchiveReader(Archive):
     """The active Archive for consuming.
+
     """
+
     def __init__(self, filename):
         super(ArchiveReader, self).__init__(filename)
         self._open_file(filename)

shoebox/disk_storage.py

@@ -1,6 +1,18 @@
-import calendar
-import datetime
-import json
+# Copyright (c) 2014 Dark Secret Software Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import struct
 
 
@@ -61,7 +73,7 @@ class Version1(Version0):
     # *s = raw data
 
     # EXAMPLE
-    #--------
+    # --------
     # With above Event and Metadata
     #
     # Header schema: "iii"
@@ -142,7 +154,7 @@ class Version1(Version0):
         offset += struct.calcsize(lengths_schema)
         key_values = struct.unpack_from(key_value_schema, metadata_bytes,
                                         offset=offset)
-        metadata = dict((key_values[n], key_values[n+1])
+        metadata = dict((key_values[n], key_values[n + 1])
                         for n in range(len(key_values))[::2])
 
         raw = file_handle.read(header[1])
@@ -156,6 +168,7 @@ class Version1(Version0):
 VERSIONS = {1: Version1()}
 CURRENT_VERSION = 1
 
+
 def get_version_handler(version=CURRENT_VERSION):
     global VERSIONS
 

shoebox/handlers.py

@@ -34,6 +34,7 @@ class ArchiveCallback(object):
 
     def on_close(self, filename):
         """Called when an Archive is closed.
+
            If you move/change the file/name return the
            new location so subsequent callbacks will
            have the right location.
@@ -50,8 +51,10 @@ class CallbackList(ArchiveCallback):
         callback_str_list = [x.strip() for x in callback_str.split(",")]
         self.callbacks = [simport.load(c)(**self.config)
                           for c in callback_str_list]
+
     # TODO(Sandy): Need some exception handling around these.
     # The failure of one shouldn't stop processing.
+
     def on_open(self, filename):
         for c in self.callbacks:
             c.on_open(filename)
@@ -63,6 +66,7 @@ class CallbackList(ArchiveCallback):
 
 class ChangeExtensionCallback(ArchiveCallback):
     """filename.dat becomes filename.dat.done"""
+
     def __init__(self, **kwargs):
         super(ChangeExtensionCallback, self).__init__(**kwargs)
         self.new_extension = kwargs.get('new_extension', '.done')

shoebox/roll_manager.py

@@ -64,7 +64,7 @@ class RollManager(object):
 class ReadingRollManager(RollManager):
     def __init__(self, filename_template, directory=".",
                  destination_directory=".",
-                 archive_class = archive.ArchiveReader,
+                 archive_class=archive.ArchiveReader,
                  archive_callback=None, roll_size_mb=1000):
         super(ReadingRollManager, self).__init__(
             filename_template,
@@ -114,7 +114,9 @@ class WritingRollManager(RollManager):
         self.roll_checker = roll_checker
 
     def write(self, metadata, payload):
-        """metadata is string:string dict.
+        """Write metadata
+
+        metadata is string:string dict.
         payload must be encoded as string.
         """
         a = self.get_active_archive()
@@ -143,11 +145,15 @@ class WritingRollManager(RollManager):
 
 
 class WritingJSONRollManager(object):
-    """No archiver. No roll checker. Just write 1 file line per json payload.
+    """Simple event archival.
+
+    No archiver. No roll checker. Just write 1 file line per json payload.
     Once the file gets big enough, gzip the file and move
     into the destination_directory.
     Expects an external tool like rsync to move the file.
-       A SHA-256 of the payload may be included in the archive filename."""
+    A SHA-256 of the payload may be included in the archive filename.
+    """
+
     def __init__(self, *args, **kwargs):
         self.filename_template = args[0]
         self.directory = kwargs.get('directory', '.')
@@ -193,7 +199,7 @@ class WritingJSONRollManager(object):
                  self._get_time() >= (self.start_time + self.roll_after)))
 
     def _get_file_sha(self, filename):
-        block_size=2**20
+        block_size = 2 ** 20
         sha256 = hashlib.sha256()
         # no context manager, just to keep testing simple.
         f = open(filename, "r")

test/integration/test_json_tarball.py

@@ -1,8 +1,21 @@
+# Copyright (c) 2014 Dark Secret Software Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import datetime
 import gzip
-import hashlib
 import json
-import mock
 import os
 import shutil
 import unittest
@@ -39,7 +52,8 @@ class TestDirectory(unittest.TestCase):
             if events:
                 for event in events:
                     metadata = {}
-                    json_event = json.dumps(event,
+                    json_event = json.dumps(
+                        event,
                         cls=notification_utils.DateTimeEncoder)
                     manager.write(metadata, json_event)
                     msg_id = event['message_id']
@@ -49,7 +63,7 @@ class TestDirectory(unittest.TestCase):
         manager.close()
         manager._archive_working_files()
 
-        print "Starting entries:", len(entries)
+        print("Starting entries:", len(entries))
 
         actual = len(entries)
 
@@ -68,8 +82,8 @@ class TestDirectory(unittest.TestCase):
 
             num = len(file_content) - 1
             total += num
-            print "In %s: %d of %d Remaining: %d" % (f, num, actual,
-                                                     actual - total)
+            print("In %s: %d of %d Remaining: %d"
+                  % (f, num, actual, actual - total))
 
         if actual != total:
             raise Exception("Num generated != actual")

test/integration/test_rolling.py

@@ -1,6 +1,20 @@
+# Copyright (c) 2014 Dark Secret Software Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import datetime
 import json
-import mock
 import os
 import shutil
 import unittest
@@ -8,7 +22,6 @@ import unittest
 import notification_utils
 import notigen
 
-from shoebox import disk_storage
 from shoebox import roll_checker
 from shoebox import roll_manager
 
@@ -64,13 +77,15 @@ class TestSizeRolling(unittest.TestCase):
             events = g.generate(now)
             if events:
                 for event in events:
-                    metadata = {'event': event['event_type'],
+                    metadata = {
+                        'event': event['event_type'],
                         'request_id': event['_context_request_id'],
                         'generated': str(event['timestamp']),
-                                'uuid': event.get('payload', {}
-                                                  ).get("instance_id", ""),
+                        'uuid': event.get('payload', {}).get(
+                            "instance_id", ""),
                     }
-                    json_event = json.dumps(event,
+                    json_event = json.dumps(
+                        event,
                         cls=notification_utils.DateTimeEncoder)
                     manager.write(metadata, json_event)
                     entries.append((metadata, json_event))

test/test_callbacks.py

@@ -1,5 +1,19 @@
-import unittest
+# Copyright (c) 2014 Dark Secret Software Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
+import unittest
 
 from shoebox import handlers
 
@@ -26,4 +40,3 @@ class TestCallbackList(unittest.TestCase):
         self.assertTrue(isinstance(c.callbacks[1],
                                    handlers.ChangeExtensionCallback))
         self.assertTrue(isinstance(c.callbacks[2], BlahCallback))
-

test/test_disk_storage.py

@@ -1,11 +1,23 @@
-import datetime
-import mock
+# Copyright (c) 2014 Dark Secret Software Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import json
+import mock
 import struct
 import unittest
 
-import dateutil.tz
-
 from shoebox import disk_storage
 
 
@@ -24,8 +36,10 @@ class TestVersion0(unittest.TestCase):
 
     def test_load_preamble(self):
         file_handle = mock.Mock()
-        file_handle.read.return_value = struct.pack("ih",
-                          disk_storage.BOR_MAGIC_NUMBER, 99)
+        file_handle.read.return_value = struct.pack(
+            "ih",
+            disk_storage.BOR_MAGIC_NUMBER,
+            99)
         self.assertEqual(99, self.v0.load_preamble(file_handle))
 
 
@@ -76,7 +90,7 @@ class TestVersion1(unittest.TestCase):
         blocks = blocks[1:]  # Remove preamble
 
         # break the EOR marker
-        print len(blocks[0])
+        print(len(blocks[0]))
         newblock = blocks[0][:8] + '\x00\x00\x01\x02'
         blocks = list(blocks)
         blocks[0] = newblock
@@ -110,8 +124,10 @@ class TestHelpers(unittest.TestCase):
 
     def test_unpack_notification(self):
         file_handle = mock.Mock()
-        file_handle.read.return_value = struct.pack("ih",
-                          disk_storage.BOR_MAGIC_NUMBER, 99)
+        file_handle.read.return_value = struct.pack(
+            "ih",
+            disk_storage.BOR_MAGIC_NUMBER,
+            99)
 
         with mock.patch('shoebox.disk_storage.get_version_handler') as h:
             fake_handler = mock.Mock()

test/test_roll_checker.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2014 Dark Secret Software Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import datetime
 import mock
 import unittest
@@ -33,7 +48,7 @@ class TestRollChecker(unittest.TestCase):
             self.assertFalse(x.check(None))
 
         with mock.patch.object(notification_utils, 'now') as dt:
-            dt.return_value = now + one_hour - datetime.timedelta(seconds = 1)
+            dt.return_value = now + one_hour - datetime.timedelta(seconds=1)
             self.assertFalse(x.check(None))
 
     def test_size_roll_checker_end(self):

test/test_roll_manager.py

@@ -1,10 +1,23 @@
+# Copyright (c) 2014 Dark Secret Software Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import datetime
-import mock
-import os
 import unittest
 
+import mock
 import notification_utils
-
 from shoebox import archive
 from shoebox import roll_checker
 from shoebox import roll_manager
@@ -53,7 +66,7 @@ class TestWritingRollManager(unittest.TestCase):
 
     def test_correct_archiver(self):
         x = roll_manager.WritingRollManager("foo", None)
-        print x.archive_class
+        print(x.archive_class)
         self.assertEqual(x.archive_class, archive.ArchiveWriter)
 
     def test_get_active_archive(self):
@@ -63,8 +76,8 @@ class TestWritingRollManager(unittest.TestCase):
         x = roll_manager.WritingRollManager(filename_template, checker,
                                             archive_callback=callback,
                                             archive_class=FakeArchive)
-        with mock.patch("shoebox.archive.ArchiveWriter._open_file") as of:
-            arc = x.get_active_archive()
+        with mock.patch("shoebox.archive.ArchiveWriter._open_file"):
+            x.get_active_archive()
             self.assertTrue(checker.start.called)
             self.assertTrue(callback.on_open.called)
 
@@ -124,10 +137,11 @@ class TestJSONRollManager(unittest.TestCase):
     def test_should_roll(self, awf):
         rm = roll_manager.WritingJSONRollManager("template.foo")
         rm.roll_size_mb = 10
-        self.assertFalse(rm._should_roll(9*1048576))
-        self.assertTrue(rm._should_roll(10*1048576))
+        self.assertFalse(rm._should_roll(9 * 1048576))
+        self.assertTrue(rm._should_roll(10 * 1048576))
 
-        rm = roll_manager.WritingJSONRollManager("template.foo", roll_minutes=10)
+        rm = roll_manager.WritingJSONRollManager("template.foo",
+                                                 roll_minutes=10)
         self.assertFalse(rm._should_roll(0))
         self.assertFalse(rm._should_roll(1))
         with mock.patch.object(rm, "_get_time") as gt:

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py26, py27
+envlist = py26, py27, pep8
 
 [testenv]
 deps = 
@@ -12,3 +12,12 @@ deps =
     simport
 
 commands = nosetests -d -v --with-coverage --cover-inclusive --cover-package shoebox []
+
+[testenv:pep8]
+commands =
+    flake8
+
+[flake8]
+ignore =
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg
+show-source = True