openstack/swift
Code Issues Proposed changes

rename files and mas unit tests

Browse Source
This commit is contained in:
David Goetz 2011-05-11 13:02:39 -07:00
parent a1321fb17d
commit 91a42f3576
5 changed files with 47 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
bin/swift-account-stats-logger
View File

@ -14,13 +14,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from swift.stats.db_stats import AccountStat
from swift.stats.db_stats_collector import AccountStatsCollector
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
conf_file, options = parse_options()
# currently AccountStat only supports run_once
# currently AccountStatsCollector only supports run_once
options['once'] = True
run_daemon(AccountStat, conf_file, section_name='log-processor-stats',
run_daemon(AccountStatsCollector, conf_file,
section_name='log-processor-stats',
log_name="account-stats", **options)

6
bin/swift-container-stats-logger
View File

@ -14,14 +14,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from swift.stats.db_stats import ContainerStat
from swift.stats.db_stats_collector import ContainerStatsContainer
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
conf_file, options = parse_options()
# currently ContainerStat only supports run_once
# currently ContainerStatsContainer only supports run_once
options['once'] = True
run_daemon(ContainerStat, conf_file,
run_daemon(ContainerStatsContainer, conf_file,
section_name='log-processor-container-stats',
log_name="container-stats", **options)

29
swift/stats/db_stats.py → swift/stats/db_stats_collector.py
View File

@ -28,6 +28,7 @@ from swift.common.utils import renamer, get_logger, readconf, mkdirs, \
from swift.common.constraints import check_mount
from swift.common.daemon import Daemon
class DatabaseStatsCollector(Daemon):
"""
Extract storage stats from account databases on the account
@ -75,9 +76,7 @@ class DatabaseStatsCollector(Daemon):
self.logger.error(
_("Device %s is not mounted, skipping.") % device)
continue
db_dir = os.path.join(self.devices,
device,
self.data_dir)
db_dir = os.path.join(self.devices, device, self.data_dir)
if not os.path.exists(db_dir):
self.logger.debug(
_("Path %s does not exist, skipping.") % db_dir)
@ -93,21 +92,20 @@ class DatabaseStatsCollector(Daemon):
src_filename += hasher.hexdigest()
renamer(tmp_filename, os.path.join(self.target_dir, src_filename))
shutil.rmtree(working_dir, ignore_errors=True)
finally:
# clean up temp file, remove_file ignores errors
remove_file(tmp_filename)
shutil.rmtree(working_dir, ignore_errors=True)
class AccountStat(DatabaseStatsCollector):
class AccountStatsCollector(DatabaseStatsCollector):
"""
Extract storage stats from account databases on the account
storage nodes
"""
def __init__(self, stats_conf):
super(AccountStat, self).__init__(stats_conf, 'account',
account_server_data_dir,
'stats-%Y%m%d%H_')
super(AccountStatsCollector, self).__init__(stats_conf, 'account',
account_server_data_dir,
'stats-%Y%m%d%H_')
def get_data(self, db_path):
"""
@ -124,15 +122,17 @@ class AccountStat(DatabaseStatsCollector):
info['bytes_used'])
return line_data
class ContainerStat(DatabaseStatsCollector):
class ContainerStatsCollector(DatabaseStatsCollector):
"""
Extract storage stats from container databases on the container
storage nodes
"""
def __init__(self, stats_conf):
super(ContainerStat, self).__init__(stats_conf, 'container',
container_server_data_dir,
'container-stats-%Y%m%d%H_')
super(ContainerStatsCollector, self).__init__(stats_conf, 'container',
container_server_data_dir,
'container-stats-%Y%m%d%H_')
def get_data(self, db_path):
"""
@ -151,4 +151,3 @@ class ContainerStat(DatabaseStatsCollector):
info['object_count'],
info['bytes_used'])
return line_data

4
swift/stats/log_uploader.py
View File

@ -116,8 +116,8 @@ class LogUploader(Daemon):
filename2match[full_path] = match.groupdict()
else:
self.logger.debug(_('%(filename)s does not match '
'%(pattern)s') % {'filename': filename,
'pattern': self.filename_pattern})
'%(pattern)s') % {'filename': filename,
'pattern': self.filename_pattern})
return filename2match
def upload_all_logs(self):

39
test/unit/stats/test_db_stats.py → test/unit/stats/test_db_stats_collector.py
View File

@ -19,34 +19,36 @@ import os
import time
import uuid
from shutil import rmtree
from swift.stats import db_stats
from swift.stats import db_stats_collector
from tempfile import mkdtemp
from test.unit import FakeLogger
from swift.common.db import AccountBroker, ContainerBroker
from swift.common.utils import mkdirs
class TestDbStats(unittest.TestCase):
def setUp(self):
self._was_logger = db_stats.get_logger
db_stats.get_logger = FakeLogger
self._was_logger = db_stats_collector.get_logger
db_stats_collector.get_logger = FakeLogger
self.testdir = os.path.join(mkdtemp(), 'tmp_test_db_stats')
self.devices = os.path.join(self.testdir, 'node')
rmtree(self.testdir, ignore_errors=1)
mkdirs(os.path.join(self.devices, 'sda'))
self.accounts = os.path.join(self.devices, 'sda', 'accounts')
self.containers= os.path.join(self.devices, 'sda', 'containers')
self.log_dir= '%s/log' % self.testdir
self.containers = os.path.join(self.devices, 'sda', 'containers')
self.log_dir = '%s/log' % self.testdir
self.conf = dict(devices=self.devices,
log_dir=self.log_dir,
mount_check='false')
def tearDown(self):
db_stats.get_logger = self._was_logger
db_stats_collector.get_logger = self._was_logger
rmtree(self.testdir)
def test_account_stat_get_data(self):
stat = db_stats.AccountStat(self.conf)
stat = db_stats_collector.AccountStatsCollector(self.conf)
account_db = AccountBroker("%s/acc.db" % self.accounts,
account='test_acc')
account_db.initialize()
@ -56,7 +58,7 @@ class TestDbStats(unittest.TestCase):
self.assertEquals('''"test_acc",1,10,1000\n''', info)
def test_container_stat_get_data(self):
stat = db_stats.ContainerStat(self.conf)
stat = db_stats_collector.ContainerStatsCollector(self.conf)
container_db = ContainerBroker("%s/con.db" % self.containers,
account='test_acc', container='test_con')
container_db.initialize()
@ -65,7 +67,7 @@ class TestDbStats(unittest.TestCase):
self.assertEquals('''"test_acc","test_con",1,10\n''', info)
def _gen_account_stat(self):
stat = db_stats.AccountStat(self.conf)
stat = db_stats_collector.AccountStatsCollector(self.conf)
output_data = set()
for i in range(10):
account_db = AccountBroker("%s/stats-201001010%s-%s.db" %
@ -82,15 +84,16 @@ class TestDbStats(unittest.TestCase):
return stat, output_data
def _gen_container_stat(self):
stat = db_stats.ContainerStat(self.conf)
stat = db_stats_collector.ContainerStatsCollector(self.conf)
output_data = set()
for i in range(10):
account_db = ContainerBroker(
"%s/container-stats-201001010%s-%s.db" % (self.containers, i,
uuid.uuid4().hex),
"%s/container-stats-201001010%s-%s.db" % (self.containers, i,
uuid.uuid4().hex),
account='test_acc_%s' % i, container='test_con')
account_db.initialize()
account_db.put_object('test_obj', time.time(), 10, 'text', 'faketag')
account_db.put_object('test_obj', time.time(), 10, 'text',
'faketag')
# this will "commit" the data
account_db.get_info()
output_data.add('''"test_acc_%s","test_con",1,10''' % i),
@ -138,9 +141,15 @@ class TestDbStats(unittest.TestCase):
self.assertEquals(len(stat.logger.log_dict['debug']), 1)
def test_not_implemented(self):
db_stat = db_stats.DatabaseStatsCollector(self.conf, 'account',
'test_dir', 'stats-%Y%m%d%H_')
db_stat = db_stats_collector.DatabaseStatsCollector(self.conf,
'account', 'test_dir', 'stats-%Y%m%d%H_')
self.assertRaises(Exception, db_stat.get_data)
def test_not_not_mounted(self):
self.conf['mount_check'] = 'true'
stat, output_data = self._gen_account_stat()
stat.run_once()
self.assertEquals(len(stat.logger.log_dict['error']), 1)
if __name__ == '__main__':
unittest.main()