x/terracotta
Code Issues Proposed changes

Added a host_resource_usage DB table and a few query functions

Browse Source
This commit is contained in:
Anton Beloglazov 2012-10-22 11:58:57 +11:00
parent 9b9f139e82
commit 762821bb84
5 changed files with 136 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
neat/db.py
View File

@ -29,17 +29,19 @@ class Database(object):
@contract(connection=Connection,
hosts=Table,
host_resource_usage=Table,
vms=Table,
vm_resource_usage=Table,
vm_migrations=Table,
host_states=Table,
host_overload=Table)
def __init__(self, connection, hosts, vms, vm_resource_usage,
vm_migrations, host_states, host_overload):
def __init__(self, connection, hosts, host_resource_usage, vms,
vm_resource_usage, vm_migrations, host_states, host_overload):
""" Initialize the database.
:param connection: A database connection table.
:param hosts: The hosts table.
:param host_resource_usage: The host_resource_usage table.
:param vms: The vms table.
:param vm_resource_usage: The vm_resource_usage table.
:param vm_migrations: The vm_migrations table.
@ -48,6 +50,7 @@ class Database(object):
"""
self.connection = connection
self.hosts = hosts
self.host_resource_usage = host_resource_usage
self.vms = vms
self.vm_resource_usage = vm_resource_usage
self.vm_migrations = vm_migrations
@ -122,7 +125,7 @@ class Database(object):
return row['id']
@contract
def insert_cpu_mhz(self, data):
def insert_vm_cpu_mhz(self, data):
""" Insert a set of CPU MHz values for a set of VMs.
:param data: A dictionary of VM UUIDs and CPU MHz values.
@ -175,6 +178,69 @@ class Database(object):
ram=ram))
return row['id']
@contract
def insert_host_cpu_mhz(self, hostname, cpu_mhz):
""" Insert a CPU MHz value for a host.
:param hostname: A host name.
:type hostname: str
:param cpu_mhz: The CPU usage of the host in MHz.
:type cpu_mhz: int
"""
self.host_resource_usage.insert().execute(
host_id=self.select_host_id(hostname),
cpu_mhz=cpu_mhz)
@contract
def select_cpu_mhz_for_host(self, hostname, n):
""" Select n last values of CPU MHz for a host.
:param hostname: A host name.
:type hostname: str
:param n: The number of last values to select.
:type n: int,>0
:return: The list of n last CPU Mhz values.
:rtype: list(int)
"""
sel = select([self.host_resource_usage.c.cpu_mhz]). \
where(and_(
self.hosts.c.id == self.host_resource_usage.c.host_id,
self.hosts.c.hostname == hostname)). \
order_by(self.host_resource_usage.c.id.desc()). \
limit(n)
res = self.connection.execute(sel).fetchall()
return list(reversed([int(x[0]) for x in res]))
@contract
def select_last_cpu_mhz_for_hosts(self):
""" Select the last value of CPU MHz for all the hosts.
:return: A dict of host names to the last CPU MHz values.
:rtype: dict(str: int)
"""
hru1 = self.host_resource_usage
hru2 = self.host_resource_usage.alias()
sel = select([hru1.c.host_id, hru1.c.cpu_mhz], from_obj=[
hru1.outerjoin(hru2, and_(
hru1.c.host_id == hru2.c.host_id,
hru1.c.id < hru2.c.id))]). \
where(hru2.c.id == None)
hosts_cpu_mhz = dict(self.connection.execute(sel).fetchall())
sel = select([self.hosts.c.id, self.hosts.c.hostname])
hosts_names = dict(self.connection.execute(sel).fetchall())
hosts_last_mhz = {}
for id, hostname in hosts_names.items():
if id in hosts_cpu_mhz:
hosts_last_mhz[str(hostname)] = int(hosts_cpu_mhz[id])
else:
hosts_last_mhz[str(hostname)] = 0
return hosts_last_mhz
@contract
def select_host_characteristics(self):
""" Select the characteristics of all the hosts.
@ -204,7 +270,10 @@ class Database(object):
"""
sel = select([self.hosts.c.id]). \
where(self.hosts.c.hostname == hostname)
return int(self.connection.execute(sel).fetchone()['id'])
row = self.connection.execute(sel).fetchone()
if not row:
raise LookupError('No host found for hostname: %s', hostname)
return int(row['id'])
@contract
def select_host_ids(self):

11
neat/db_utils.py
View File

@ -45,6 +45,13 @@ def init_db(sql_connection):
Column('cpu_cores', Integer, nullable=False),
Column('ram', Integer, nullable=False))
host_resource_usage = \
Table('host_resource_usage', metadata,
Column('id', Integer, primary_key=True),
Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False),
Column('timestamp', DateTime, default=func.now()),
Column('cpu_mhz', Integer, nullable=False))
vms = Table('vms', metadata,
Column('id', Integer, primary_key=True),
Column('uuid', String(36), nullable=False))
@ -79,8 +86,8 @@ def init_db(sql_connection):
metadata.create_all()
connection = engine.connect()
db = Database(connection, hosts, vms, vm_resource_usage,
vm_migrations, host_states, host_overload)
db = Database(connection, hosts, host_resource_usage, vms,
vm_resource_usage, vm_migrations, host_states, host_overload)
log.debug('Initialized a DB connection to %s', sql_connection)
return db

2
neat/locals/collector.py
View File

@ -458,7 +458,7 @@ def append_data_remotely(db, data):
:param data: A map of VM UUIDs onto the corresponing CPU MHz values.
:type data: dict(str : int)
"""
db.insert_cpu_mhz(data)
db.insert_vm_cpu_mhz(data)
@contract

52
tests/test_db.py
View File

@ -62,7 +62,7 @@ class Db(TestCase):
res = {}
for uuid, data in vms.items():
for value in data:
db.insert_cpu_mhz({uuid: value})
db.insert_vm_cpu_mhz({uuid: value})
if data:
res[uuid] = data[-1]
assert db.select_last_cpu_mhz_for_vms() == res
@ -79,7 +79,7 @@ class Db(TestCase):
assert db.select_vm_id(uuid2) == vm_id + 1
@qc(10)
def insert_cpu_mhz(
def insert_vm_cpu_mhz(
vms=dict_(
keys=str_(of='abc123-', min_length=36, max_length=36),
values=tuple_(int_(min=1, max=3000),
@ -104,7 +104,7 @@ class Db(TestCase):
if initial_data:
db.vm_resource_usage.insert().execute(initial_data)
db.insert_cpu_mhz(data_to_submit)
db.insert_vm_cpu_mhz(data_to_submit)
for uuid, data in final_data.items():
assert db.select_cpu_mhz_for_vm(uuid, 11) == data
@ -130,6 +130,52 @@ class Db(TestCase):
assert host['cpu_cores'] == 8
assert host['ram'] == 8000
@qc(10)
def select_cpu_mhz_for_host(
hostname=str_(of='abc123', min_length=5, max_length=10),
cpu_mhz=list_(of=int_(min=0, max=3000), min_length=0, max_length=10),
n=int_(min=1, max=10)
):
db = db_utils.init_db('sqlite:///:memory:')
host_id = db.update_host(hostname, 1, 1, 1)
for mhz in cpu_mhz:
db.host_resource_usage.insert().execute(
host_id=host_id,
cpu_mhz=mhz)
assert db.select_cpu_mhz_for_host(hostname, n) == cpu_mhz[-n:]
@qc(10)
def select_last_cpu_mhz_for_hosts(
hosts=dict_(
keys=str_(of='abc123', min_length=5, max_length=10),
values=list_(of=int_(min=1, max=3000),
min_length=0, max_length=10),
min_length=0, max_length=3
)
):
db = db_utils.init_db('sqlite:///:memory:')
res = {}
for hostname, data in hosts.items():
db.update_host(hostname, 1, 1, 1)
for value in data:
db.insert_host_cpu_mhz(hostname, value)
if data:
res[hostname] = data[-1]
else:
res[hostname] = 0
assert db.select_last_cpu_mhz_for_hosts() == res
@qc(10)
def insert_host_cpu_mhz(
hostname=str_(of='abc123', min_length=5, max_length=10),
cpu_mhz=list_(of=int_(min=0, max=3000), min_length=1, max_length=10)
):
db = db_utils.init_db('sqlite:///:memory:')
db.update_host(hostname, 1, 1, 1)
for value in cpu_mhz:
db.insert_host_cpu_mhz(hostname, value)
assert db.select_cpu_mhz_for_host(hostname, len(cpu_mhz)) == cpu_mhz
@qc(1)
def select_host_characteristics():
db = db_utils.init_db('sqlite:///:memory:')

4
tests/test_db_utils.py
View File

@ -36,6 +36,10 @@ class DbUtils(TestCase):
assert isinstance(db.host_states, Table)
assert db.hosts.c.keys() == \
['id', 'hostname', 'cpu_mhz', 'cpu_cores', 'ram']
assert db.host_resource_usage.c.keys() == \
['id', 'host_id', 'timestamp', 'cpu_mhz']
assert list(db.host_resource_usage.foreign_keys)[0].target_fullname \
== 'hosts.id'
assert db.vms.c.keys() == \
['id', 'uuid']
assert db.vm_resource_usage.c.keys() == \