updated stats binaries to be DRY compliant

bin/swift-account-stats-logger

@@ -14,11 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import signal
 import sys
-import time
-from ConfigParser import ConfigParser
 
 from swift.account_stats import AccountStat
 from swift.common import utils
@@ -27,55 +23,5 @@ if __name__ == '__main__':
     if len(sys.argv) < 2:
         print "Usage: swift-account-stats-logger CONFIG_FILE"
         sys.exit()
-
+    stats_conf = utils.readconf(sys.argv[1], 'log-processor-stats')
-    c = ConfigParser()
+    stats = AccountStat(stats_conf).run(once=True)
-    if not c.read(sys.argv[1]):
-        print "Unable to read config file."
-        sys.exit(1)
-
-    if c.has_section('log-processor-stats'):
-        stats_conf = dict(c.items('log-processor-stats'))
-    else:
-        print "Unable to find log-processor-stats config section in %s." % \
-                sys.argv[1]
-        sys.exit(1)
-
-    # reference this from the account stats conf
-
-    target_dir = stats.conf.get('log_dir', '/var/log/swift')
-    account_server_conf_loc = stats_conf.get('account_server_conf',
-                                             '/etc/swift/account-server.conf')
-    filename_format = stats.conf['source_filename_format']
-    try:
-        c = ConfigParser()
-        c.read(account_server_conf_loc)
-        account_server_conf = dict(c.items('account-server'))
-    except:
-        print "Unable to load account server conf from %s" % account_server_conf_loc
-        sys.exit(1)
-
-    utils.drop_privileges(account_server_conf.get('user', 'swift'))
-
-    try:
-        os.setsid()
-    except OSError:
-        pass
-
-    logger = utils.get_logger(stats_conf, 'swift-account-stats-logger')
-
-    def kill_children(*args):
-        signal.signal(signal.SIGTERM, signal.SIG_IGN)
-        os.killpg(0, signal.SIGTERM)
-        sys.exit()
-
-    signal.signal(signal.SIGTERM, kill_children)
-
-    stats = AccountStat(filename_format,
-                        target_dir,
-                        account_server_conf,
-                        logger)
-    logger.info("Gathering account stats")
-    start = time.time()
-    stats.find_and_process()
-    logger.info("Gathering account stats complete (%0.2f minutes)" %
-        ((time.time()-start)/60))

bin/swift-log-uploader

@@ -14,70 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import signal
 import sys
 import time
-from ConfigParser import ConfigParser
 
 from swift.stats.log_uploader import LogUploader
-from swift.common.utils import get_logger
+from swift.common import utils
 
 if __name__ == '__main__':
     if len(sys.argv) < 3:
         print "Usage: swift-log-uploader CONFIG_FILE plugin"
         sys.exit()
-
+    uploader_conf = utils.readconf(sys.argv[1], 'log-processor')
-    c = ConfigParser()
-    if not c.read(sys.argv[1]):
-        print "Unable to read config file."
-        sys.exit(1)
-
-    if c.has_section('log-processor'):
-        parser_conf = dict(c.items('log-processor'))
-    else:
-        print "Unable to find log-processor config section in %s." % sys.argv[1]
-        sys.exit(1)
-    
     plugin = sys.argv[2]
     section_name = 'log-processor-%s' % plugin
-    if c.has_section(section_name):
+    plugin_conf = utils.readconf(sys.argv[1], section_name)
-        uploader_conf.update(dict(c.items(section_name)))
+    uploader_conf.update(plugin_conf)
-    else:
+    uploader = LogUploader(uploader_conf, plugin).run(once=True)
-        print "Unable to find %s config section in %s." % (section_name,
-                                                           sys.argv[1])
-        sys.exit(1)
-
-    try:
-        os.setsid()
-    except OSError:
-        pass
-
-    logger = get_logger(uploader_conf, 'swift-log-uploader')
-
-    def kill_children(*args):
-        signal.signal(signal.SIGTERM, signal.SIG_IGN)
-        os.killpg(0, signal.SIGTERM)
-        sys.exit()
-
-    signal.signal(signal.SIGTERM, kill_children)
-
-    log_dir = uploader_conf.get('log_dir', '/var/log/swift/')
-    swift_account = uploader_conf['swift_account']
-    container_name = uploader_conf['container_name']
-    source_filename_format = uploader_conf['source_filename_format']
-    proxy_server_conf_loc = uploader_conf.get('proxy_server_conf',
-                                              '/etc/swift/proxy-server.conf')
-    try:
-        c = ConfigParser()
-        c.read(proxy_server_conf_loc)
-        proxy_server_conf = dict(c.items('proxy-server'))
-    except:
-        proxy_server_conf = None
-    uploader = LogUploader(log_dir, swift_account, container_name,
-        source_filename_format, proxy_server_conf, logger)
-    logger.info("Uploading logs")
-    start = time.time()
-    uploader.upload_all_logs()
-    logger.info("Uploading logs complete (%0.2f minutes)" %
-        ((time.time()-start)/60))

swift/stats/account_stats.py

@@ -19,16 +19,30 @@ import time
 from swift.account.server import DATADIR as account_server_data_dir
 from swift.common.db import AccountBroker
 from swift.common.internal_proxy import InternalProxy
-from swift.common.utils import renamer
+from swift.common.utils import renamer, get_logger
+from swift.common.daemon import Daemon
 
-class AccountStat(object):
+class AccountStat(Daemon):
-    def __init__(self, filename_format, target_dir, server_conf, logger):
+    def __init__(self, stats_conf):
+        super(self, AccountStat).__init__(stats_conf)
+        target_dir = stats_conf.get('log_dir', '/var/log/swift')
+        account_server_conf_loc = stats_conf.get('account_server_conf',
+                                             '/etc/swift/account-server.conf')
+        server_conf = utils.readconf(account_server_conf_loc, 'account-server')
+        filename_format = stats_conf['source_filename_format']
         self.filename_format = filename_format
         self.target_dir = target_dir
         self.devices = server_conf.get('devices', '/srv/node')
         self.mount_check = server_conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')
-        self.logger = logger
+        self.logger = get_logger(stats_conf, 'swift-account-stats-logger')
+
+    def run_once(self):
+        self.logger.info("Gathering account stats")
+        start = time.time()
+        self.find_and_process()
+        self.logger.info("Gathering account stats complete (%0.2f minutes)" %
+            ((time.time()-start)/60))
 
     def find_and_process(self):
         src_filename = time.strftime(self.filename_format)

swift/stats/log_uploader.py

@@ -21,8 +21,10 @@ import gzip
 import glob
 
 from swift.common.internal_proxy import InternalProxy
+from swift.common.daemon import Daemon
+from swift.common import utils
 
-class LogUploader(object):
+class LogUploader(Daemon):
     '''
     Given a local directory, a swift account, and a container name, LogParser
     will upload all files in the local directory to the given account/container.
@@ -38,8 +40,16 @@ class LogUploader(object):
     the object uploads.
     '''
 
-    def __init__(self, log_dir, swift_account, container_name, filename_format,
+    def __init__(self, uploader_conf, plugin_name):
-            proxy_server_conf, logger):
+        super(self, LogUploader).__init__(uploader_conf)
+        log_dir = uploader_conf.get('log_dir', '/var/log/swift/')
+        swift_account = uploader_conf['swift_account']
+        container_name = uploader_conf['container_name']
+        source_filename_format = uploader_conf['source_filename_format']
+        proxy_server_conf_loc = uploader_conf.get('proxy_server_conf',
+                                            '/etc/swift/proxy-server.conf')
+        proxy_server_conf = utils.readconf(proxy_server_conf_loc,
+                                            'proxy-server')
         if not log_dir.endswith('/'):
             log_dir = log_dir + '/'
         self.log_dir = log_dir
@@ -47,7 +57,15 @@ class LogUploader(object):
         self.container_name = container_name
         self.filename_format = filename_format
         self.internal_proxy = InternalProxy(proxy_server_conf, logger)
-        self.logger = logger
+        log_name = 'swift-log-uploader-%s' % plugin_name
+        self.logger = utils.get_logger(uploader_conf, plugin_name)
+
+    def run_once(self):
+        self.logger.info("Uploading logs")
+        start = time.time()
+        self.upload_all_logs()
+        self.logger.info("Uploading logs complete (%0.2f minutes)" %
+            ((time.time()-start)/60))
 
     def upload_all_logs(self):
         i = [(c,self.filename_format.index(c)) for c in '%Y %m %d %H'.split()]