added account stats logger to stats system

bin/swift-account-stats-logger.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python
+# Copyright (c) 2010 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import signal
+import sys
+import time
+from ConfigParser import ConfigParser
+
+from swift.account_stats import AccountStat
+from swift.common import utils
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print "Usage: swift-account-stats-logger CONFIG_FILE"
+        sys.exit()
+
+    c = ConfigParser()
+    if not c.read(sys.argv[1]):
+        print "Unable to read config file."
+        sys.exit(1)
+
+    if c.has_section('log-processor-stats'):
+        stats_conf = dict(c.items('log-processor-stats'))
+    else:
+        print "Unable to find log-processor-stats config section in %s." % \
+                sys.argv[1]
+        sys.exit(1)
+
+    # reference this from the account stats conf
+
+    target_dir = stats.conf.get('log_dir', '/var/log/swift')
+    account_server_conf_loc = stats_conf.get('account_server_conf',
+                                             '/etc/swift/account-server.conf')
+    filename_format = stats.conf['source_filename_format']
+    try:
+        c = ConfigParser()
+        c.read(account_server_conf_loc)
+        account_server_conf = dict(c.items('account-server'))
+    except:
+        print "Unable to load account server conf from %s" % account_server_conf_loc
+        sys.exit(1)
+
+    utils.drop_privileges(account_server_conf.get('user', 'swift'))
+
+    try:
+        os.setsid()
+    except OSError:
+        pass
+
+    logger = utils.get_logger(stats_conf, 'swift-account-stats-logger')
+
+    def kill_children(*args):
+        signal.signal(signal.SIGTERM, signal.SIG_IGN)
+        os.killpg(0, signal.SIGTERM)
+        sys.exit()
+
+    signal.signal(signal.SIGTERM, kill_children)
+
+    stats = AccountStat(filename_format,
+                        target_dir,
+                        account_server_conf,
+                        logger)
+    logger.info("Gathering account stats")
+    start = time.time()
+    stats.find_and_process()
+    logger.info("Gathering account stats complete (%0.2f minutes)" %
+        ((time.time()-start)/60))

etc/log-processing.conf-sample

@@ -23,4 +23,5 @@ class_path = swift.stats.access_processor
 swift_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31
 container_name = account_stats
 source_filename_format = %Y%m%d%H*
-class_path = swift.stats.stats_processor
+class_path = swift.stats.stats_processor
+# account_server_conf = /etc/swift/account-server.conf

swift/stats/account_stats.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2010 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+
+from swift.account.server import DATADIR as account_server_data_dir
+from swift.common.db import AccountBroker
+from swift.common.internal_proxy import InternalProxy
+from swift.common.utils import renamer
+
+class AccountStat(object):
+    def __init__(self, filename_format, target_dir, server_conf, logger):
+        self.filename_format = filename_format
+        self.target_dir = target_dir
+        self.devices = server_conf.get('devices', '/srv/node')
+        self.mount_check = server_conf.get('mount_check', 'true').lower() in \
+                              ('true', 't', '1', 'on', 'yes', 'y')
+        self.logger = logger
+
+    def find_and_process(self):
+        src_filename = time.strftime(self.filename_format)
+        tmp_filename = os.path.join('/tmp', src_filename)
+        with open(tmp_filename, 'wb') as statfile:
+            #statfile.write('Account Name, Container Count, Object Count, Bytes Used, Created At\n')
+            for device in os.listdir(self.devices):
+                if self.mount_check and \
+                        not os.path.ismount(os.path.join(self.devices, device)):
+                    self.logger.error("Device %s is not mounted, skipping." %
+                        device)
+                    continue
+                accounts = os.path.join(self.devices,
+                                        device,
+                                        account_server_data_dir)
+                if not os.path.exists(accounts):
+                    self.logger.debug("Path %s does not exist, skipping." %
+                        accounts)
+                    continue
+                for root, dirs, files in os.walk(accounts, topdown=False):
+                    for filename in files:
+                        if filename.endswith('.db'):
+                            broker = AccountBroker(os.path.join(root, filename))
+                            if not broker.is_deleted():
+                                account_name,
+                                created_at,
+                                _, _,
+                                container_count,
+                                object_count,
+                                bytes_used,
+                                _, _ = broker.get_info()
+                                line_data = '"%s",%d,%d,%d,%s\n' % (account_name,
+                                                                    container_count,
+                                                                    object_count,
+                                                                    bytes_used,
+                                                                    created_at)
+                                statfile.write(line_data)
+        renamer(tmp_filename, os.path.join(self.target_dir, src_filename))