improved logging in log processors

swift/stats/access_processor.py

@@ -17,7 +17,7 @@ import collections
 from urllib import unquote
 import copy
 
-from swift.common.utils import split_path
+from swift.common.utils import split_path, get_logger
 
 month_map = '_ Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split()
 
@@ -34,6 +34,7 @@ class AccessLogProcessor(object):
                             conf.get('service_ips', '').split(',')\
                             if x.strip()]
         self.warn_percent = float(conf.get('warn_percent', '0.8'))
+        self.logger = get_logger(conf)
 
     def log_line_parser(self, raw_log):
         '''given a raw access log line, return a dict of the good parts'''
@@ -58,9 +59,12 @@ class AccessLogProcessor(object):
             headers,
             processing_time) = (unquote(x) for x in raw_log[16:].split(' '))
         except ValueError:
+            self.logger.debug('Bad line data: %s' % repr(raw_log))
             return {}
         if server != self.server_name:
             # incorrect server name in log line
+            self.logger.debug('Bad server name: found "%s" expected "%s"' \
+                               % (server, self.server_name))
             return {}
         (version,
         account,
@@ -185,8 +189,8 @@ class AccessLogProcessor(object):
             hourly_aggr_info[aggr_key] = d
         if bad_lines > (total_lines * self.warn_percent):
             name = '/'.join([account, container, object_name])
-            print >>sys.stderr, 'I found a bunch of bad lines in %s '\
-                        '(%d bad, %d total)' % (name, bad_lines, total_lines)
+            self.logger.warning('I found a bunch of bad lines in %s '\
+                        '(%d bad, %d total)' % (name, bad_lines, total_lines))
         return hourly_aggr_info
 
     def keylist_mapping(self):

swift/stats/log_processor.py

@@ -58,6 +58,7 @@ class LogProcessor(object):
             module = __import__(import_target, fromlist=[import_target])
             klass = getattr(module, class_name)
             self.plugins[plugin_name]['instance'] = klass(plugin_conf)
+            self.logger.debug('Loaded plugin "%s"' % plugin_name)
 
     @property
     def internal_proxy(self):
@@ -74,6 +75,11 @@ class LogProcessor(object):
         return self._internal_proxy
 
     def process_one_file(self, plugin_name, account, container, object_name):
+        self.logger.info('Processing %s/%s/%s with plugin "%s"' % (account,
+                                                               container,
+                                                               object_name,
+                                                               plugin_name)
+                                                               )
         # get an iter of the object data
         compressed = object_name.endswith('.gz')
         stream = self.get_object_data(account, container, object_name,
@@ -185,6 +191,10 @@ class LogProcessor(object):
                     try:
                         chunk = d.decompress(chunk)
                     except zlib.error:
+                        self.logger.debug('Bad compressed data for %s/%s/%s' %
+                                                                (swift_account,
+                                                                container_name,
+                                                                object_name))
                         raise BadFileDownload()  # bad compressed data
                 parts = chunk.split('\n')
                 parts[0] = last_part + parts[0]

swift/stats/stats_processor.py

@@ -13,12 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from swift.common.utils import get_logger
 
 class StatsLogProcessor(object):
     """Transform account storage stat logs"""
 
     def __init__(self, conf):
-        pass
+        self.logger = get_logger(conf)
 
     def process(self, obj_stream, account, container, object_name):
         '''generate hourly groupings of data from one stats log file'''
@@ -34,6 +35,7 @@ class StatsLogProcessor(object):
                 bytes_used) = line.split(',')
             except (IndexError, ValueError):
                 # bad line data
+                self.logger.debug('Bad line data: %s' % repr(line))
                 continue
             account = account.strip('"')
             container_count = int(container_count.strip('"'))