From 762821bb84a0a8bd13178156e1ed61b034a9c311 Mon Sep 17 00:00:00 2001 From: Anton Beloglazov Date: Mon, 22 Oct 2012 11:58:57 +1100 Subject: [PATCH] Added a host_resource_usage DB table and a few query functions --- neat/db.py | 77 +++++++++++++++++++++++++++++++++++++--- neat/db_utils.py | 11 ++++-- neat/locals/collector.py | 2 +- tests/test_db.py | 52 +++++++++++++++++++++++++-- tests/test_db_utils.py | 4 +++ 5 files changed, 136 insertions(+), 10 deletions(-) diff --git a/neat/db.py b/neat/db.py index 58eadd0..e48b1d9 100644 --- a/neat/db.py +++ b/neat/db.py @@ -29,17 +29,19 @@ class Database(object): @contract(connection=Connection, hosts=Table, + host_resource_usage=Table, vms=Table, vm_resource_usage=Table, vm_migrations=Table, host_states=Table, host_overload=Table) - def __init__(self, connection, hosts, vms, vm_resource_usage, - vm_migrations, host_states, host_overload): + def __init__(self, connection, hosts, host_resource_usage, vms, + vm_resource_usage, vm_migrations, host_states, host_overload): """ Initialize the database. :param connection: A database connection table. :param hosts: The hosts table. + :param host_resource_usage: The host_resource_usage table. :param vms: The vms table. :param vm_resource_usage: The vm_resource_usage table. :param vm_migrations: The vm_migrations table. @@ -48,6 +50,7 @@ class Database(object): """ self.connection = connection self.hosts = hosts + self.host_resource_usage = host_resource_usage self.vms = vms self.vm_resource_usage = vm_resource_usage self.vm_migrations = vm_migrations @@ -122,7 +125,7 @@ class Database(object): return row['id'] @contract - def insert_cpu_mhz(self, data): + def insert_vm_cpu_mhz(self, data): """ Insert a set of CPU MHz values for a set of VMs. :param data: A dictionary of VM UUIDs and CPU MHz values. @@ -175,6 +178,69 @@ class Database(object): ram=ram)) return row['id'] + @contract + def insert_host_cpu_mhz(self, hostname, cpu_mhz): + """ Insert a CPU MHz value for a host. + + :param hostname: A host name. + :type hostname: str + + :param cpu_mhz: The CPU usage of the host in MHz. + :type cpu_mhz: int + """ + self.host_resource_usage.insert().execute( + host_id=self.select_host_id(hostname), + cpu_mhz=cpu_mhz) + + @contract + def select_cpu_mhz_for_host(self, hostname, n): + """ Select n last values of CPU MHz for a host. + + :param hostname: A host name. + :type hostname: str + + :param n: The number of last values to select. + :type n: int,>0 + + :return: The list of n last CPU Mhz values. + :rtype: list(int) + """ + sel = select([self.host_resource_usage.c.cpu_mhz]). \ + where(and_( + self.hosts.c.id == self.host_resource_usage.c.host_id, + self.hosts.c.hostname == hostname)). \ + order_by(self.host_resource_usage.c.id.desc()). \ + limit(n) + res = self.connection.execute(sel).fetchall() + return list(reversed([int(x[0]) for x in res])) + + @contract + def select_last_cpu_mhz_for_hosts(self): + """ Select the last value of CPU MHz for all the hosts. + + :return: A dict of host names to the last CPU MHz values. + :rtype: dict(str: int) + """ + hru1 = self.host_resource_usage + hru2 = self.host_resource_usage.alias() + sel = select([hru1.c.host_id, hru1.c.cpu_mhz], from_obj=[ + hru1.outerjoin(hru2, and_( + hru1.c.host_id == hru2.c.host_id, + hru1.c.id < hru2.c.id))]). \ + where(hru2.c.id == None) + hosts_cpu_mhz = dict(self.connection.execute(sel).fetchall()) + + sel = select([self.hosts.c.id, self.hosts.c.hostname]) + hosts_names = dict(self.connection.execute(sel).fetchall()) + + hosts_last_mhz = {} + for id, hostname in hosts_names.items(): + if id in hosts_cpu_mhz: + hosts_last_mhz[str(hostname)] = int(hosts_cpu_mhz[id]) + else: + hosts_last_mhz[str(hostname)] = 0 + return hosts_last_mhz + @contract def select_host_characteristics(self): """ Select the characteristics of all the hosts. @@ -204,7 +270,10 @@ class Database(object): """ sel = select([self.hosts.c.id]). \ where(self.hosts.c.hostname == hostname) - return int(self.connection.execute(sel).fetchone()['id']) + row = self.connection.execute(sel).fetchone() + if not row: + raise LookupError('No host found for hostname: %s', hostname) + return int(row['id']) @contract def select_host_ids(self): diff --git a/neat/db_utils.py b/neat/db_utils.py index ad8efd9..b8c9d46 100644 --- a/neat/db_utils.py +++ b/neat/db_utils.py @@ -45,6 +45,13 @@ def init_db(sql_connection): Column('cpu_cores', Integer, nullable=False), Column('ram', Integer, nullable=False)) + host_resource_usage = \ + Table('host_resource_usage', metadata, + Column('id', Integer, primary_key=True), + Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), + Column('timestamp', DateTime, default=func.now()), + Column('cpu_mhz', Integer, nullable=False)) + vms = Table('vms', metadata, Column('id', Integer, primary_key=True), Column('uuid', String(36), nullable=False)) @@ -79,8 +86,8 @@ def init_db(sql_connection): metadata.create_all() connection = engine.connect() - db = Database(connection, hosts, vms, vm_resource_usage, - vm_migrations, host_states, host_overload) + db = Database(connection, hosts, host_resource_usage, vms, + vm_resource_usage, vm_migrations, host_states, host_overload) log.debug('Initialized a DB connection to %s', sql_connection) return db diff --git a/neat/locals/collector.py b/neat/locals/collector.py index c5ab552..575e4a2 100644 --- a/neat/locals/collector.py +++ b/neat/locals/collector.py @@ -458,7 +458,7 @@ def append_data_remotely(db, data): :param data: A map of VM UUIDs onto the corresponing CPU MHz values. :type data: dict(str : int) """ - db.insert_cpu_mhz(data) + db.insert_vm_cpu_mhz(data) @contract diff --git a/tests/test_db.py b/tests/test_db.py index 8a2fe1f..ce89c3c 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -62,7 +62,7 @@ class Db(TestCase): res = {} for uuid, data in vms.items(): for value in data: - db.insert_cpu_mhz({uuid: value}) + db.insert_vm_cpu_mhz({uuid: value}) if data: res[uuid] = data[-1] assert db.select_last_cpu_mhz_for_vms() == res @@ -79,7 +79,7 @@ class Db(TestCase): assert db.select_vm_id(uuid2) == vm_id + 1 @qc(10) - def insert_cpu_mhz( + def insert_vm_cpu_mhz( vms=dict_( keys=str_(of='abc123-', min_length=36, max_length=36), values=tuple_(int_(min=1, max=3000), @@ -104,7 +104,7 @@ class Db(TestCase): if initial_data: db.vm_resource_usage.insert().execute(initial_data) - db.insert_cpu_mhz(data_to_submit) + db.insert_vm_cpu_mhz(data_to_submit) for uuid, data in final_data.items(): assert db.select_cpu_mhz_for_vm(uuid, 11) == data @@ -130,6 +130,52 @@ class Db(TestCase): assert host['cpu_cores'] == 8 assert host['ram'] == 8000 + @qc(10) + def select_cpu_mhz_for_host( + hostname=str_(of='abc123', min_length=5, max_length=10), + cpu_mhz=list_(of=int_(min=0, max=3000), min_length=0, max_length=10), + n=int_(min=1, max=10) + ): + db = db_utils.init_db('sqlite:///:memory:') + host_id = db.update_host(hostname, 1, 1, 1) + for mhz in cpu_mhz: + db.host_resource_usage.insert().execute( + host_id=host_id, + cpu_mhz=mhz) + assert db.select_cpu_mhz_for_host(hostname, n) == cpu_mhz[-n:] + + @qc(10) + def select_last_cpu_mhz_for_hosts( + hosts=dict_( + keys=str_(of='abc123', min_length=5, max_length=10), + values=list_(of=int_(min=1, max=3000), + min_length=0, max_length=10), + min_length=0, max_length=3 + ) + ): + db = db_utils.init_db('sqlite:///:memory:') + res = {} + for hostname, data in hosts.items(): + db.update_host(hostname, 1, 1, 1) + for value in data: + db.insert_host_cpu_mhz(hostname, value) + if data: + res[hostname] = data[-1] + else: + res[hostname] = 0 + assert db.select_last_cpu_mhz_for_hosts() == res + + @qc(10) + def insert_host_cpu_mhz( + hostname=str_(of='abc123', min_length=5, max_length=10), + cpu_mhz=list_(of=int_(min=0, max=3000), min_length=1, max_length=10) + ): + db = db_utils.init_db('sqlite:///:memory:') + db.update_host(hostname, 1, 1, 1) + for value in cpu_mhz: + db.insert_host_cpu_mhz(hostname, value) + assert db.select_cpu_mhz_for_host(hostname, len(cpu_mhz)) == cpu_mhz + @qc(1) def select_host_characteristics(): db = db_utils.init_db('sqlite:///:memory:') diff --git a/tests/test_db_utils.py b/tests/test_db_utils.py index 93a0554..ac2275f 100644 --- a/tests/test_db_utils.py +++ b/tests/test_db_utils.py @@ -36,6 +36,10 @@ class DbUtils(TestCase): assert isinstance(db.host_states, Table) assert db.hosts.c.keys() == \ ['id', 'hostname', 'cpu_mhz', 'cpu_cores', 'ram'] + assert db.host_resource_usage.c.keys() == \ + ['id', 'host_id', 'timestamp', 'cpu_mhz'] + assert list(db.host_resource_usage.foreign_keys)[0].target_fullname \ + == 'hosts.id' assert db.vms.c.keys() == \ ['id', 'uuid'] assert db.vm_resource_usage.c.keys() == \