documentation clarification and pep8 fixes
This commit is contained in:
parent
b44296d146
commit
880f5af2ff
@ -4,7 +4,7 @@ Swift stats system
|
||||
|
||||
The swift stats system is composed of three parts parts: log creation, log
|
||||
uploading, and log processing. The system handles two types of logs (access
|
||||
and storage stats), but it can be extended to handle other types of logs.
|
||||
and account stats), but it can be extended to handle other types of logs.
|
||||
|
||||
---------
|
||||
Log Types
|
||||
@ -20,21 +20,21 @@ is made on August 4, 2010 at 12:37 gets logged in a file named 2010080412.
|
||||
This allows easy log rotation and easy per-hour log processing.
|
||||
|
||||
******************
|
||||
Storage stats logs
|
||||
Account stats logs
|
||||
******************
|
||||
|
||||
Storage logs (also referred to as stats logs) are generated by a stats system
|
||||
process. swift-account-stats-logger runs on each account server (via cron) and
|
||||
walks the filesystem looking for account databases. When an account database
|
||||
is found, the logger selects the account hash, bytes_used, container_count,
|
||||
and object_count. These values are then written out as one line in a csv file.
|
||||
One csv file is produced for every run of swift-account-stats-logger. This
|
||||
means that, system wide, one csv file is produced for every storage node.
|
||||
Rackspace runs the account stats logger every hour. Therefore, in a cluster of
|
||||
ten account servers, ten csv files are produced every hour. Also, every
|
||||
account will have one entry for every replica in the system. On average, there
|
||||
will be three copies of each account in the aggregate of all account stat csv
|
||||
files created in one system-wide run.
|
||||
Account stats logs are generated by a stats system process.
|
||||
swift-account-stats-logger runs on each account server (via cron) and walks
|
||||
the filesystem looking for account databases. When an account database is
|
||||
found, the logger selects the account hash, bytes_used, container_count, and
|
||||
object_count. These values are then written out as one line in a csv file. One
|
||||
csv file is produced for every run of swift-account-stats-logger. This means
|
||||
that, system wide, one csv file is produced for every storage node. Rackspace
|
||||
runs the account stats logger every hour. Therefore, in a cluster of ten
|
||||
account servers, ten csv files are produced every hour. Also, every account
|
||||
will have one entry for every replica in the system. On average, there will be
|
||||
three copies of each account in the aggregate of all account stat csv files
|
||||
created in one system-wide run.
|
||||
|
||||
----------------------
|
||||
Log Processing plugins
|
||||
|
@ -16,6 +16,7 @@
|
||||
import zlib
|
||||
import struct
|
||||
|
||||
|
||||
class CompressingFileReader(object):
|
||||
'''
|
||||
Wraps a file object and provides a read method that returns gzip'd data.
|
||||
@ -35,6 +36,7 @@ class CompressingFileReader(object):
|
||||
:param file_obj: File object to read from
|
||||
:param compresslevel: compression level
|
||||
'''
|
||||
|
||||
def __init__(self, file_obj, compresslevel=9):
|
||||
self._f = file_obj
|
||||
self._compressor = zlib.compressobj(compresslevel,
|
||||
|
@ -20,13 +20,27 @@ from json import loads as json_loads
|
||||
from swift.common.compressing_file_reader import CompressingFileReader
|
||||
from swift.proxy.server import BaseApplication
|
||||
|
||||
|
||||
class MemcacheStub(object):
|
||||
def get(self, *a, **kw): return None
|
||||
def set(self, *a, **kw): return None
|
||||
def incr(self, *a, **kw): return 0
|
||||
def delete(self, *a, **kw): return None
|
||||
def set_multi(self, *a, **kw): return None
|
||||
def get_multi(self, *a, **kw): return []
|
||||
|
||||
def get(self, *a, **kw):
|
||||
return None
|
||||
|
||||
def set(self, *a, **kw):
|
||||
return None
|
||||
|
||||
def incr(self, *a, **kw):
|
||||
return 0
|
||||
|
||||
def delete(self, *a, **kw):
|
||||
return None
|
||||
|
||||
def set_multi(self, *a, **kw):
|
||||
return None
|
||||
|
||||
def get_multi(self, *a, **kw):
|
||||
return []
|
||||
|
||||
|
||||
class InternalProxy(object):
|
||||
"""
|
||||
@ -38,6 +52,7 @@ class InternalProxy(object):
|
||||
:param logger: logger to log requests to
|
||||
:param retries: number of times to retry each request
|
||||
"""
|
||||
|
||||
def __init__(self, proxy_server_conf=None, logger=None, retries=0):
|
||||
self.upload_app = BaseApplication(proxy_server_conf,
|
||||
memcache=MemcacheStub(),
|
||||
@ -56,6 +71,7 @@ class InternalProxy(object):
|
||||
:param object_name: name of object being uploaded
|
||||
:param compress: if True, compresses object as it is uploaded
|
||||
:param content_type: content-type of object
|
||||
:param etag: etag for object to check successful upload
|
||||
:returns: True if successful, False otherwise
|
||||
"""
|
||||
log_create_pattern = '/v1/%s/%s/%s' % (account, container, object_name)
|
||||
@ -72,7 +88,8 @@ class InternalProxy(object):
|
||||
if hasattr(source_file, 'read'):
|
||||
compressed_file = CompressingFileReader(source_file)
|
||||
else:
|
||||
compressed_file = CompressingFileReader(open(source_file, 'rb'))
|
||||
compressed_file = CompressingFileReader(
|
||||
open(source_file, 'rb'))
|
||||
req.body_file = compressed_file
|
||||
else:
|
||||
if not hasattr(source_file, 'read'):
|
||||
|
@ -552,6 +552,7 @@ def cache_from_env(env):
|
||||
"""
|
||||
return item_from_env(env, 'swift.cache')
|
||||
|
||||
|
||||
def readconf(conf, section_name, log_name=None):
|
||||
"""
|
||||
Read config file and return config items as a dict
|
||||
@ -571,7 +572,8 @@ def readconf(conf, section_name, log_name=None):
|
||||
if c.has_section(section_name):
|
||||
conf = dict(c.items(section_name))
|
||||
else:
|
||||
print "Unable to find %s config section in %s" % (section_name, conf)
|
||||
print "Unable to find %s config section in %s" % (section_name,
|
||||
conf)
|
||||
sys.exit(1)
|
||||
if "log_name" not in conf:
|
||||
if log_name is not None:
|
||||
|
Loading…
Reference in New Issue
Block a user