diff --git a/bin/swift-account-stats-logger b/bin/swift-account-stats-logger index 050545af9a..fc7b128270 100644 --- a/bin/swift-account-stats-logger +++ b/bin/swift-account-stats-logger @@ -14,11 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import signal import sys -import time -from ConfigParser import ConfigParser from swift.account_stats import AccountStat from swift.common import utils @@ -27,55 +23,5 @@ if __name__ == '__main__': if len(sys.argv) < 2: print "Usage: swift-account-stats-logger CONFIG_FILE" sys.exit() - - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - - if c.has_section('log-processor-stats'): - stats_conf = dict(c.items('log-processor-stats')) - else: - print "Unable to find log-processor-stats config section in %s." % \ - sys.argv[1] - sys.exit(1) - - # reference this from the account stats conf - - target_dir = stats.conf.get('log_dir', '/var/log/swift') - account_server_conf_loc = stats_conf.get('account_server_conf', - '/etc/swift/account-server.conf') - filename_format = stats.conf['source_filename_format'] - try: - c = ConfigParser() - c.read(account_server_conf_loc) - account_server_conf = dict(c.items('account-server')) - except: - print "Unable to load account server conf from %s" % account_server_conf_loc - sys.exit(1) - - utils.drop_privileges(account_server_conf.get('user', 'swift')) - - try: - os.setsid() - except OSError: - pass - - logger = utils.get_logger(stats_conf, 'swift-account-stats-logger') - - def kill_children(*args): - signal.signal(signal.SIGTERM, signal.SIG_IGN) - os.killpg(0, signal.SIGTERM) - sys.exit() - - signal.signal(signal.SIGTERM, kill_children) - - stats = AccountStat(filename_format, - target_dir, - account_server_conf, - logger) - logger.info("Gathering account stats") - start = time.time() - stats.find_and_process() - logger.info("Gathering account stats complete (%0.2f minutes)" % - ((time.time()-start)/60)) + stats_conf = utils.readconf(sys.argv[1], 'log-processor-stats') + stats = AccountStat(stats_conf).run(once=True) diff --git a/bin/swift-log-uploader b/bin/swift-log-uploader index 4ca9c17795..83c8f67303 100755 --- a/bin/swift-log-uploader +++ b/bin/swift-log-uploader @@ -14,70 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import signal import sys import time -from ConfigParser import ConfigParser from swift.stats.log_uploader import LogUploader -from swift.common.utils import get_logger +from swift.common import utils if __name__ == '__main__': if len(sys.argv) < 3: print "Usage: swift-log-uploader CONFIG_FILE plugin" sys.exit() - - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - - if c.has_section('log-processor'): - parser_conf = dict(c.items('log-processor')) - else: - print "Unable to find log-processor config section in %s." % sys.argv[1] - sys.exit(1) - + uploader_conf = utils.readconf(sys.argv[1], 'log-processor') plugin = sys.argv[2] section_name = 'log-processor-%s' % plugin - if c.has_section(section_name): - uploader_conf.update(dict(c.items(section_name))) - else: - print "Unable to find %s config section in %s." % (section_name, - sys.argv[1]) - sys.exit(1) - - try: - os.setsid() - except OSError: - pass - - logger = get_logger(uploader_conf, 'swift-log-uploader') - - def kill_children(*args): - signal.signal(signal.SIGTERM, signal.SIG_IGN) - os.killpg(0, signal.SIGTERM) - sys.exit() - - signal.signal(signal.SIGTERM, kill_children) - - log_dir = uploader_conf.get('log_dir', '/var/log/swift/') - swift_account = uploader_conf['swift_account'] - container_name = uploader_conf['container_name'] - source_filename_format = uploader_conf['source_filename_format'] - proxy_server_conf_loc = uploader_conf.get('proxy_server_conf', - '/etc/swift/proxy-server.conf') - try: - c = ConfigParser() - c.read(proxy_server_conf_loc) - proxy_server_conf = dict(c.items('proxy-server')) - except: - proxy_server_conf = None - uploader = LogUploader(log_dir, swift_account, container_name, - source_filename_format, proxy_server_conf, logger) - logger.info("Uploading logs") - start = time.time() - uploader.upload_all_logs() - logger.info("Uploading logs complete (%0.2f minutes)" % - ((time.time()-start)/60)) + plugin_conf = utils.readconf(sys.argv[1], section_name) + uploader_conf.update(plugin_conf) + uploader = LogUploader(uploader_conf, plugin).run(once=True) diff --git a/swift/stats/account_stats.py b/swift/stats/account_stats.py index 1624807cfe..a514aec2e8 100644 --- a/swift/stats/account_stats.py +++ b/swift/stats/account_stats.py @@ -19,16 +19,30 @@ import time from swift.account.server import DATADIR as account_server_data_dir from swift.common.db import AccountBroker from swift.common.internal_proxy import InternalProxy -from swift.common.utils import renamer +from swift.common.utils import renamer, get_logger +from swift.common.daemon import Daemon -class AccountStat(object): - def __init__(self, filename_format, target_dir, server_conf, logger): +class AccountStat(Daemon): + def __init__(self, stats_conf): + super(self, AccountStat).__init__(stats_conf) + target_dir = stats_conf.get('log_dir', '/var/log/swift') + account_server_conf_loc = stats_conf.get('account_server_conf', + '/etc/swift/account-server.conf') + server_conf = utils.readconf(account_server_conf_loc, 'account-server') + filename_format = stats_conf['source_filename_format'] self.filename_format = filename_format self.target_dir = target_dir self.devices = server_conf.get('devices', '/srv/node') self.mount_check = server_conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.logger = logger + self.logger = get_logger(stats_conf, 'swift-account-stats-logger') + + def run_once(self): + self.logger.info("Gathering account stats") + start = time.time() + self.find_and_process() + self.logger.info("Gathering account stats complete (%0.2f minutes)" % + ((time.time()-start)/60)) def find_and_process(self): src_filename = time.strftime(self.filename_format) diff --git a/swift/stats/log_uploader.py b/swift/stats/log_uploader.py index 8c45d64db5..6634611767 100644 --- a/swift/stats/log_uploader.py +++ b/swift/stats/log_uploader.py @@ -21,8 +21,10 @@ import gzip import glob from swift.common.internal_proxy import InternalProxy +from swift.common.daemon import Daemon +from swift.common import utils -class LogUploader(object): +class LogUploader(Daemon): ''' Given a local directory, a swift account, and a container name, LogParser will upload all files in the local directory to the given account/container. @@ -38,8 +40,16 @@ class LogUploader(object): the object uploads. ''' - def __init__(self, log_dir, swift_account, container_name, filename_format, - proxy_server_conf, logger): + def __init__(self, uploader_conf, plugin_name): + super(self, LogUploader).__init__(uploader_conf) + log_dir = uploader_conf.get('log_dir', '/var/log/swift/') + swift_account = uploader_conf['swift_account'] + container_name = uploader_conf['container_name'] + source_filename_format = uploader_conf['source_filename_format'] + proxy_server_conf_loc = uploader_conf.get('proxy_server_conf', + '/etc/swift/proxy-server.conf') + proxy_server_conf = utils.readconf(proxy_server_conf_loc, + 'proxy-server') if not log_dir.endswith('/'): log_dir = log_dir + '/' self.log_dir = log_dir @@ -47,7 +57,15 @@ class LogUploader(object): self.container_name = container_name self.filename_format = filename_format self.internal_proxy = InternalProxy(proxy_server_conf, logger) - self.logger = logger + log_name = 'swift-log-uploader-%s' % plugin_name + self.logger = utils.get_logger(uploader_conf, plugin_name) + + def run_once(self): + self.logger.info("Uploading logs") + start = time.time() + self.upload_all_logs() + self.logger.info("Uploading logs complete (%0.2f minutes)" % + ((time.time()-start)/60)) def upload_all_logs(self): i = [(c,self.filename_format.index(c)) for c in '%Y %m %d %H'.split()]