From 7750cc72f870c5d1a1beaad55cf62cec0146d25f Mon Sep 17 00:00:00 2001 From: Anton Beloglazov Date: Mon, 15 Oct 2012 11:36:36 +1100 Subject: [PATCH] Added a DB cleaner module and db_cleaner_interval config option --- neat.conf | 4 ++ neat/config.py | 1 + tests/locals/test_manager.py | 135 ++++------------------------------- 3 files changed, 20 insertions(+), 120 deletions(-) diff --git a/neat.conf b/neat.conf index d571d4d..58845b7 100644 --- a/neat.conf +++ b/neat.conf @@ -51,6 +51,10 @@ global_manager_host = controller # The port of the REST web service exposed by the global manager global_manager_port = 60080 +# The time interval between subsequent invocations of the database +# cleaner in seconds +db_cleaner_interval = 7200 + # The directory used by the data collector to store the data on the # resource usage by the VMs running on the host local_data_directory = /var/lib/neat diff --git a/neat/config.py b/neat/config.py index d0700af..8ff6a80 100644 --- a/neat/config.py +++ b/neat/config.py @@ -48,6 +48,7 @@ REQUIRED_FIELDS = [ 'compute_hosts', 'global_manager_host', 'global_manager_port', + 'db_cleaner_interval', 'local_data_directory', 'local_manager_interval', 'data_collector_interval', diff --git a/tests/locals/test_manager.py b/tests/locals/test_manager.py index 3ecc74d..a96e597 100644 --- a/tests/locals/test_manager.py +++ b/tests/locals/test_manager.py @@ -15,19 +15,16 @@ from mocktest import * from pyqcy import * -import shutil -import libvirt -from hashlib import sha1 +import datetime -import neat.locals.manager as manager +import neat.globals.db_cleaner as cleaner import neat.common as common -import neat.locals.collector as collector import logging logging.disable(logging.CRITICAL) -class LocalManager(TestCase): +class DbCleaner(TestCase): @qc(10) def start( @@ -39,128 +36,26 @@ class LocalManager(TestCase): config = { 'log_directory': 'dir', 'log_level': 2, - 'local_manager_interval': str(time_interval)} - paths = [manager.DEFAILT_CONFIG_PATH, manager.CONFIG_PATH] - fields = manager.REQUIRED_FIELDS - expect(manager).read_and_validate_config(paths, fields). \ + 'db_cleaner_interval': str(time_interval)} + paths = [cleaner.DEFAILT_CONFIG_PATH, cleaner.CONFIG_PATH] + fields = cleaner.REQUIRED_FIELDS + expect(cleaner).read_and_validate_config(paths, fields). \ and_return(config).once() - expect(common).init_logging('dir', 'local-manager.log', 2).once() - expect(common).start(manager.init_state, - manager.execute, + expect(common).init_logging('dir', 'db-cleaner.log', 2).once() + expect(common).start(cleaner.init_state, + cleaner.execute, config, time_interval).and_return(state).once() - assert manager.start() == state + assert cleaner.start() == state @qc(1) def init_state(): with MockTransaction: - vir_connection = mock('virConnect') db = mock('db') - mhz = mock('mhz') - expect(libvirt).openReadOnly(None). \ - and_return(vir_connection).once() - expect(manager).init_db('db'). \ + expect(cleaner).init_db('db'). \ and_return(db).once() - expect(common).physical_cpu_mhz_total(vir_connection). \ - and_return(mhz) - expect(vir_connection).getHostname().and_return('host').once() config = {'sql_connection': 'db', - 'os_admin_user': 'user', - 'os_admin_password': 'password'} - state = manager.init_state(config) - assert state['previous_time'] == 0 - assert state['vir_connection'] == vir_connection + 'db_cleaner_interval': 7200} + state = cleaner.init_state(config) assert state['db'] == db - assert state['physical_cpu_mhz_total'] == mhz - assert state['hostname'] == 'host' - assert state['hashed_username'] == sha1('user').hexdigest() - assert state['hashed_password'] == sha1('password').hexdigest() - - @qc(1) - def get_local_data( - data=dict_( - keys=str_(of='abc123-', min_length=36, max_length=36), - values=list_(of=int_(min=1, max=3000), - min_length=0, max_length=10), - min_length=0, max_length=5 - ) - ): - path = os.path.join(os.path.dirname(__file__), - '..', 'resources', 'vms', 'tmp') - shutil.rmtree(path, True) - os.mkdir(path) - collector.write_data_locally(path, data, 10) - - assert manager.get_local_data(path) == data - shutil.rmtree(path) - - @qc(10) - def cleanup_vm_data( - data=dict_( - keys=str_(of='abc123-', min_length=36, max_length=36), - values=list_(of=int_(min=1, max=3000), - min_length=0, max_length=10), - min_length=0, max_length=5 - ) - ): - original_data = dict(data) - uuids = data.keys() - if data: - n = random.randrange(len(data)) - for _ in range(n): - uuid = random.choice(uuids) - del data[uuid] - uuids.remove(uuid) - - assert manager.cleanup_vm_data(original_data, uuids) == data - - @qc(10) - def get_ram( - data=dict_( - keys=str_(of='abc123-', min_length=36, max_length=36), - values=int_(min=1, max=100), - min_length=0, max_length=10 - ) - ): - with MockTransaction: - def mock_get_max_ram(vir_connection, uuid): - return data[uuid] - - connection = libvirt.virConnect() - when(manager).get_max_ram(connection, any_string). \ - then_call(mock_get_max_ram) - - assert manager.get_ram(connection, data.keys()) == data - - @qc(10) - def get_max_ram( - uuid=str_(of='abc123-', min_length=36, max_length=36), - x=int_(min=0) - ): - with MockTransaction: - connection = libvirt.virConnect() - domain = mock('domain') - expect(connection).lookupByUUIDString(uuid). \ - and_return(domain).once() - expect(domain).maxMemory().and_return(x).once() - assert manager.get_max_ram(connection, uuid) == int(x / 1024) - - @qc(1) - def get_max_ram_none( - uuid=str_(of='abc123-', min_length=36, max_length=36) - ): - with MockTransaction: - def raise_libvirt_error(): - raise libvirt.libvirtError(None) - connection = libvirt.virConnect() - expect(connection).lookupByUUIDString(uuid). \ - and_call(lambda _: raise_libvirt_error()) - assert manager.get_max_ram(connection, uuid) is None - - def test_vm_mhz_to_percentage(self): - self.assertEqual(manager.vm_mhz_to_percentage( - {'a': [100, 200, 300], - 'b': [100, 300, 200], - 'c': [100, 100, 700]}, - 3000), - [0.1, 0.2, 0.4]) + assert state['time_delta'] == datetime.timedelta(0, 7200)