x/terracotta
Code Issues Proposed changes

Added scripts for printing real-time system statistics

Browse Source
This commit is contained in:
Anton Beloglazov 2013-01-18 11:55:00 +11:00
parent 6f51d51cce
commit 9e95b59eda
5 changed files with 395 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
stats.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/sh
if [ $# -ne 2 ]
then
echo "You must specify 2 arguments: start and end time as YYYY-MM-DD HH:MM:SS"
exit 1
fi
./utils/overload-time-fraction.py root $MYSQL_ROOT_PASSWORD "$1" "$2"
./utils/idle-time-fraction.py root $MYSQL_ROOT_PASSWORD "$1" "$2"
./utils/vm-migrations.py root $MYSQL_ROOT_PASSWORD "$1" "$2"

196
utils/db.py Normal file
View File

@ -0,0 +1,196 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from contracts import contract
import datetime
from sqlalchemy import *
from sqlalchemy.engine.base import Connection
class Database(object):
""" A class representing the database, where fields are tables.
"""
@contract(connection=Connection,
hosts=Table,
host_resource_usage=Table,
vms=Table,
vm_resource_usage=Table,
vm_migrations=Table,
host_states=Table,
host_overload=Table)
def __init__(self, connection, hosts, host_resource_usage, vms,
vm_resource_usage, vm_migrations, host_states, host_overload):
""" Initialize the database.
:param connection: A database connection table.
:param hosts: The hosts table.
:param host_resource_usage: The host_resource_usage table.
:param vms: The vms table.
:param vm_resource_usage: The vm_resource_usage table.
:param vm_migrations: The vm_migrations table.
:param host_states: The host_states table.
:param host_overload: The host_overload table.
"""
self.connection = connection
self.hosts = hosts
self.host_resource_usage = host_resource_usage
self.vms = vms
self.vm_resource_usage = vm_resource_usage
self.vm_migrations = vm_migrations
self.host_states = host_states
self.host_overload = host_overload
@contract
def select_host_ids(self):
""" Select the IDs of all the hosts.
:return: A dict of host names to IDs.
:rtype: dict(str: int)
"""
return dict((str(x[1]), int(x[0]))
for x in self.hosts.select().execute().fetchall())
@contract
def select_host_states(self, host_id, start_time, end_time):
""" Select the states of a host.
:param start_time: The start time to select host states.
:type start_time: *
:param end_time: The end time to select host states.
:type end_time: *
:return: A list of timestamps and host states.
:rtype: list(tuple(*, int))
"""
hs = self.host_states
sel = select([hs.c.timestamp, hs.c.state]). \
where(and_(hs.c.host_id == host_id,
hs.c.timestamp >= start_time,
hs.c.timestamp <= end_time)). \
order_by(hs.c.id.asc())
return [(x[0], int(x[1]))
for x in self.connection.execute(sel).fetchall()]
@contract
def select_host_overload(self, host_id, start_time, end_time):
""" Select the overload of a host.
:param start_time: The start time to select host overload.
:type start_time: *
:param end_time: The end time to select host states.
:type end_time: *
:return: A list of timestamps and overloads.
:rtype: list(tuple(*, int))
"""
ho = self.host_overload
sel = select([ho.c.timestamp, ho.c.overload]). \
where(and_(ho.c.host_id == host_id,
ho.c.timestamp >= start_time,
ho.c.timestamp <= end_time)). \
order_by(ho.c.id.asc())
return [(x[0], int(x[1]))
for x in self.connection.execute(sel).fetchall()]
@contract
def select_vm_migrations(self, start_time, end_time):
""" Select VM migrations.
:param start_time: The start time to select data.
:type start_time: *
:param end_time: The end time to select data.
:type end_time: *
:return: A list of timestamps and VM IDs.
:rtype: list(tuple(*, int))
"""
vm = self.vm_migrations
sel = select([vm.c.timestamp, vm.c.vm_id]). \
where(and_(vm.c.timestamp >= start_time,
vm.c.timestamp <= end_time)). \
order_by(vm.c.id.asc())
return [(x[0], int(x[1]))
for x in self.connection.execute(sel).fetchall()]
@contract
def init_db(sql_connection):
""" Initialize the database.
:param sql_connection: A database connection URL.
:type sql_connection: str
:return: The initialized database.
:rtype: *
"""
engine = create_engine(sql_connection) # 'sqlite:///:memory:'
metadata = MetaData()
metadata.bind = engine
hosts = Table('hosts', metadata,
Column('id', Integer, primary_key=True),
Column('hostname', String(255), nullable=False),
Column('cpu_mhz', Integer, nullable=False),
Column('cpu_cores', Integer, nullable=False),
Column('ram', Integer, nullable=False))
host_resource_usage = \
Table('host_resource_usage', metadata,
Column('id', Integer, primary_key=True),
Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False),
Column('timestamp', DateTime, default=func.now()),
Column('cpu_mhz', Integer, nullable=False))
vms = Table('vms', metadata,
Column('id', Integer, primary_key=True),
Column('uuid', String(36), nullable=False))
vm_resource_usage = \
Table('vm_resource_usage', metadata,
Column('id', Integer, primary_key=True),
Column('vm_id', Integer, ForeignKey('vms.id'), nullable=False),
Column('timestamp', DateTime, default=func.now()),
Column('cpu_mhz', Integer, nullable=False))
vm_migrations = \
Table('vm_migrations', metadata,
Column('id', Integer, primary_key=True),
Column('vm_id', Integer, ForeignKey('vms.id'), nullable=False),
Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False),
Column('timestamp', DateTime, default=func.now()))
host_states = \
Table('host_states', metadata,
Column('id', Integer, primary_key=True),
Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False),
Column('timestamp', DateTime, default=func.now()),
Column('state', Integer, nullable=False))
host_overload = \
Table('host_overload', metadata,
Column('id', Integer, primary_key=True),
Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False),
Column('timestamp', DateTime, default=func.now()),
Column('overload', Integer, nullable=False))
metadata.create_all()
connection = engine.connect()
db = Database(connection, hosts, host_resource_usage, vms,
vm_resource_usage, vm_migrations, host_states, host_overload)
return db

66
utils/idle-time-fraction.py Executable file
View File

@ -0,0 +1,66 @@
#!/usr/bin/python2
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import os
import random
import shutil
import time
from datetime import datetime
from db import init_db
if len(sys.argv) < 5:
print 'You must specify 4 arguments:'
print '1. The MySQL DB user name'
print '2. The MySQL DB password'
print '3. The start datetime in the format: %Y-%m-%d %H:%M:%S'
print '4. The finish datetime in the format: %Y-%m-%d %H:%M:%S'
sys.exit(1)
db = init_db('mysql://' + sys.argv[1] + ':' + sys.argv[2] + '@localhost/spe')
start_time = datetime.fromtimestamp(
time.mktime(time.strptime(sys.argv[3], '%Y-%m-%d %H:%M:%S')))
finish_time = datetime.fromtimestamp(
time.mktime(time.strptime(sys.argv[4], '%Y-%m-%d %H:%M:%S')))
#print "Start time: " + str(start_time)
#print "Finish time: " + str(finish_time)
def total_seconds(delta):
return (delta.microseconds +
(delta.seconds + delta.days * 24 * 3600) * 1000000) / 1000000
total_time = 0
total_idle_time = 0
for hostname, host_id in db.select_host_ids().items():
prev_timestamp = start_time
prev_state = 1
states = {0: [], 1: []}
for timestamp, state in db.select_host_states(host_id, start_time, finish_time):
if prev_timestamp:
states[prev_state].append(total_seconds(timestamp - prev_timestamp))
prev_timestamp = timestamp
prev_state = state
states[prev_state].append(total_seconds(finish_time - prev_timestamp))
#print states
off_time = sum(states[0])
on_time = sum(states[1])
total_time += off_time + on_time
total_idle_time += off_time
print "Total time: " + str(total_time)
print "Total idle time: " + str(total_idle_time)
print "Idle time fraction: " + str(float(total_idle_time) / total_time)

81
utils/overload-time-fraction.py Executable file
View File

@ -0,0 +1,81 @@
#!/usr/bin/python2
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import os
import random
import shutil
import time
from datetime import datetime
from db import init_db
if len(sys.argv) < 5:
print 'You must specify 4 arguments:'
print '1. The MySQL DB user name'
print '2. The MySQL DB password'
print '3. The start datetime in the format: %Y-%m-%d %H:%M:%S'
print '4. The finish datetime in the format: %Y-%m-%d %H:%M:%S'
sys.exit(1)
db = init_db('mysql://' + sys.argv[1] + ':' + sys.argv[2] + '@localhost/neat')
start_time = datetime.fromtimestamp(
time.mktime(time.strptime(sys.argv[3], '%Y-%m-%d %H:%M:%S')))
finish_time = datetime.fromtimestamp(
time.mktime(time.strptime(sys.argv[4], '%Y-%m-%d %H:%M:%S')))
#print "Start time: " + str(start_time)
#print "Finish time: " + str(finish_time)
def total_seconds(delta):
return (delta.microseconds +
(delta.seconds + delta.days * 24 * 3600) * 1000000) / 1000000
total_idle_time = 0
for hostname, host_id in db.select_host_ids().items():
prev_timestamp = start_time
prev_state = 1
states = {0: [], 1: []}
for timestamp, state in db.select_host_states(host_id, start_time, finish_time):
if prev_timestamp:
states[prev_state].append(total_seconds(timestamp - prev_timestamp))
prev_timestamp = timestamp
prev_state = state
states[prev_state].append(total_seconds(finish_time - prev_timestamp))
#print states
off_time = sum(states[0])
total_idle_time += off_time
total_time = 0
total_overload_time = 0
for hostname, host_id in db.select_host_ids().items():
prev_timestamp = start_time
prev_state = 0
states = {0: [], 1: []}
for timestamp, state in db.select_host_overload(host_id, start_time, finish_time):
if prev_timestamp:
states[prev_state].append(total_seconds(timestamp - prev_timestamp))
prev_timestamp = timestamp
prev_state = state
states[prev_state].append(total_seconds(finish_time - prev_timestamp))
#print states
nonoverload_time = sum(states[0])
overload_time = sum(states[1])
total_time += nonoverload_time + overload_time
total_overload_time += overload_time
print "Total time: " + str(total_time)
print "Overload time: " + str(total_overload_time)
print "Overload time fraction: " + str(float(total_overload_time) / (total_time - total_idle_time))

41
utils/vm-migrations.py Executable file
View File

@ -0,0 +1,41 @@
#!/usr/bin/python2
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import os
import random
import shutil
import time
from datetime import datetime
from db import init_db
if len(sys.argv) < 5:
print 'You must specify 4 arguments:'
print '1. The MySQL DB user name'
print '2. The MySQL DB password'
print '3. The start datetime in the format: %Y-%m-%d %H:%M:%S'
print '4. The finish datetime in the format: %Y-%m-%d %H:%M:%S'
sys.exit(1)
db = init_db('mysql://' + sys.argv[1] + ':' + sys.argv[2] + '@localhost/spe')
start_time = datetime.fromtimestamp(
time.mktime(time.strptime(sys.argv[3], '%Y-%m-%d %H:%M:%S')))
finish_time = datetime.fromtimestamp(
time.mktime(time.strptime(sys.argv[4], '%Y-%m-%d %H:%M:%S')))
#print "Start time: " + str(start_time)
#print "Finish time: " + str(finish_time)
print "VM migrations: " + str(len(db.select_vm_migrations(start_time, finish_time)))