openstack/swift
Code Issues Proposed changes

updated stats binaries to be DRY compliant

Browse Source
This commit is contained in:
John Dickinson 2010-09-07 09:53:26 -05:00
parent 1f3d05d486
commit 560243f2f0
4 changed files with 47 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
bin/swift-account-stats-logger
View File

@ -14,11 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import signal
import sys
import time
from ConfigParser import ConfigParser
from swift.account_stats import AccountStat
from swift.common import utils
@ -27,55 +23,5 @@ if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: swift-account-stats-logger CONFIG_FILE"
sys.exit()
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
if c.has_section('log-processor-stats'):
stats_conf = dict(c.items('log-processor-stats'))
else:
print "Unable to find log-processor-stats config section in %s." % \
sys.argv[1]
sys.exit(1)
# reference this from the account stats conf
target_dir = stats.conf.get('log_dir', '/var/log/swift')
account_server_conf_loc = stats_conf.get('account_server_conf',
'/etc/swift/account-server.conf')
filename_format = stats.conf['source_filename_format']
try:
c = ConfigParser()
c.read(account_server_conf_loc)
account_server_conf = dict(c.items('account-server'))
except:
print "Unable to load account server conf from %s" % account_server_conf_loc
sys.exit(1)
utils.drop_privileges(account_server_conf.get('user', 'swift'))
try:
os.setsid()
except OSError:
pass
logger = utils.get_logger(stats_conf, 'swift-account-stats-logger')
def kill_children(*args):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
os.killpg(0, signal.SIGTERM)
sys.exit()
signal.signal(signal.SIGTERM, kill_children)
stats = AccountStat(filename_format,
target_dir,
account_server_conf,
logger)
logger.info("Gathering account stats")
start = time.time()
stats.find_and_process()
logger.info("Gathering account stats complete (%0.2f minutes)" %
((time.time()-start)/60))
stats_conf = utils.readconf(sys.argv[1], 'log-processor-stats')
stats = AccountStat(stats_conf).run(once=True)

61
bin/swift-log-uploader
View File

@ -14,70 +14,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import signal
import sys
import time
from ConfigParser import ConfigParser
from swift.stats.log_uploader import LogUploader
from swift.common.utils import get_logger
from swift.common import utils
if __name__ == '__main__':
if len(sys.argv) < 3:
print "Usage: swift-log-uploader CONFIG_FILE plugin"
sys.exit()
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
if c.has_section('log-processor'):
parser_conf = dict(c.items('log-processor'))
else:
print "Unable to find log-processor config section in %s." % sys.argv[1]
sys.exit(1)
uploader_conf = utils.readconf(sys.argv[1], 'log-processor')
plugin = sys.argv[2]
section_name = 'log-processor-%s' % plugin
if c.has_section(section_name):
uploader_conf.update(dict(c.items(section_name)))
else:
print "Unable to find %s config section in %s." % (section_name,
sys.argv[1])
sys.exit(1)
try:
os.setsid()
except OSError:
pass
logger = get_logger(uploader_conf, 'swift-log-uploader')
def kill_children(*args):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
os.killpg(0, signal.SIGTERM)
sys.exit()
signal.signal(signal.SIGTERM, kill_children)
log_dir = uploader_conf.get('log_dir', '/var/log/swift/')
swift_account = uploader_conf['swift_account']
container_name = uploader_conf['container_name']
source_filename_format = uploader_conf['source_filename_format']
proxy_server_conf_loc = uploader_conf.get('proxy_server_conf',
'/etc/swift/proxy-server.conf')
try:
c = ConfigParser()
c.read(proxy_server_conf_loc)
proxy_server_conf = dict(c.items('proxy-server'))
except:
proxy_server_conf = None
uploader = LogUploader(log_dir, swift_account, container_name,
source_filename_format, proxy_server_conf, logger)
logger.info("Uploading logs")
start = time.time()
uploader.upload_all_logs()
logger.info("Uploading logs complete (%0.2f minutes)" %
((time.time()-start)/60))
plugin_conf = utils.readconf(sys.argv[1], section_name)
uploader_conf.update(plugin_conf)
uploader = LogUploader(uploader_conf, plugin).run(once=True)

22
swift/stats/account_stats.py
View File

@ -19,16 +19,30 @@ import time
from swift.account.server import DATADIR as account_server_data_dir
from swift.common.db import AccountBroker
from swift.common.internal_proxy import InternalProxy
from swift.common.utils import renamer
from swift.common.utils import renamer, get_logger
from swift.common.daemon import Daemon
class AccountStat(object):
def __init__(self, filename_format, target_dir, server_conf, logger):
class AccountStat(Daemon):
def __init__(self, stats_conf):
super(self, AccountStat).__init__(stats_conf)
target_dir = stats_conf.get('log_dir', '/var/log/swift')
account_server_conf_loc = stats_conf.get('account_server_conf',
'/etc/swift/account-server.conf')
server_conf = utils.readconf(account_server_conf_loc, 'account-server')
filename_format = stats_conf['source_filename_format']
self.filename_format = filename_format
self.target_dir = target_dir
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.logger = logger
self.logger = get_logger(stats_conf, 'swift-account-stats-logger')
def run_once(self):
self.logger.info("Gathering account stats")
start = time.time()
self.find_and_process()
self.logger.info("Gathering account stats complete (%0.2f minutes)" %
((time.time()-start)/60))
def find_and_process(self):
src_filename = time.strftime(self.filename_format)

26
swift/stats/log_uploader.py
View File

@ -21,8 +21,10 @@ import gzip
import glob
from swift.common.internal_proxy import InternalProxy
from swift.common.daemon import Daemon
from swift.common import utils
class LogUploader(object):
class LogUploader(Daemon):
'''
Given a local directory, a swift account, and a container name, LogParser
will upload all files in the local directory to the given account/container.
@ -38,8 +40,16 @@ class LogUploader(object):
the object uploads.
'''
def __init__(self, log_dir, swift_account, container_name, filename_format,
proxy_server_conf, logger):
def __init__(self, uploader_conf, plugin_name):
super(self, LogUploader).__init__(uploader_conf)
log_dir = uploader_conf.get('log_dir', '/var/log/swift/')
swift_account = uploader_conf['swift_account']
container_name = uploader_conf['container_name']
source_filename_format = uploader_conf['source_filename_format']
proxy_server_conf_loc = uploader_conf.get('proxy_server_conf',
'/etc/swift/proxy-server.conf')
proxy_server_conf = utils.readconf(proxy_server_conf_loc,
'proxy-server')
if not log_dir.endswith('/'):
log_dir = log_dir + '/'
self.log_dir = log_dir
@ -47,7 +57,15 @@ class LogUploader(object):
self.container_name = container_name
self.filename_format = filename_format
self.internal_proxy = InternalProxy(proxy_server_conf, logger)
self.logger = logger
log_name = 'swift-log-uploader-%s' % plugin_name
self.logger = utils.get_logger(uploader_conf, plugin_name)
def run_once(self):
self.logger.info("Uploading logs")
start = time.time()
self.upload_all_logs()
self.logger.info("Uploading logs complete (%0.2f minutes)" %
((time.time()-start)/60))
def upload_all_logs(self):
i = [(c,self.filename_format.index(c)) for c in '%Y %m %d %H'.split()]