diff --git a/stats.sh b/stats.sh new file mode 100755 index 0000000..237d09c --- /dev/null +++ b/stats.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +if [ $# -ne 2 ] +then + echo "You must specify 2 arguments: start and end time as YYYY-MM-DD HH:MM:SS" + exit 1 +fi + +./utils/overload-time-fraction.py root $MYSQL_ROOT_PASSWORD "$1" "$2" +./utils/idle-time-fraction.py root $MYSQL_ROOT_PASSWORD "$1" "$2" +./utils/vm-migrations.py root $MYSQL_ROOT_PASSWORD "$1" "$2" diff --git a/utils/db.py b/utils/db.py new file mode 100644 index 0000000..bdf1e18 --- /dev/null +++ b/utils/db.py @@ -0,0 +1,196 @@ +# Copyright 2012 Anton Beloglazov +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contracts import contract +import datetime +from sqlalchemy import * +from sqlalchemy.engine.base import Connection + + +class Database(object): + """ A class representing the database, where fields are tables. + """ + + @contract(connection=Connection, + hosts=Table, + host_resource_usage=Table, + vms=Table, + vm_resource_usage=Table, + vm_migrations=Table, + host_states=Table, + host_overload=Table) + def __init__(self, connection, hosts, host_resource_usage, vms, + vm_resource_usage, vm_migrations, host_states, host_overload): + """ Initialize the database. + + :param connection: A database connection table. + :param hosts: The hosts table. + :param host_resource_usage: The host_resource_usage table. + :param vms: The vms table. + :param vm_resource_usage: The vm_resource_usage table. + :param vm_migrations: The vm_migrations table. + :param host_states: The host_states table. + :param host_overload: The host_overload table. + """ + self.connection = connection + self.hosts = hosts + self.host_resource_usage = host_resource_usage + self.vms = vms + self.vm_resource_usage = vm_resource_usage + self.vm_migrations = vm_migrations + self.host_states = host_states + self.host_overload = host_overload + + @contract + def select_host_ids(self): + """ Select the IDs of all the hosts. + + :return: A dict of host names to IDs. + :rtype: dict(str: int) + """ + return dict((str(x[1]), int(x[0])) + for x in self.hosts.select().execute().fetchall()) + + @contract + def select_host_states(self, host_id, start_time, end_time): + """ Select the states of a host. + + :param start_time: The start time to select host states. + :type start_time: * + + :param end_time: The end time to select host states. + :type end_time: * + + :return: A list of timestamps and host states. + :rtype: list(tuple(*, int)) + """ + hs = self.host_states + sel = select([hs.c.timestamp, hs.c.state]). \ + where(and_(hs.c.host_id == host_id, + hs.c.timestamp >= start_time, + hs.c.timestamp <= end_time)). \ + order_by(hs.c.id.asc()) + return [(x[0], int(x[1])) + for x in self.connection.execute(sel).fetchall()] + + @contract + def select_host_overload(self, host_id, start_time, end_time): + """ Select the overload of a host. + + :param start_time: The start time to select host overload. + :type start_time: * + + :param end_time: The end time to select host states. + :type end_time: * + + :return: A list of timestamps and overloads. + :rtype: list(tuple(*, int)) + """ + ho = self.host_overload + sel = select([ho.c.timestamp, ho.c.overload]). \ + where(and_(ho.c.host_id == host_id, + ho.c.timestamp >= start_time, + ho.c.timestamp <= end_time)). \ + order_by(ho.c.id.asc()) + return [(x[0], int(x[1])) + for x in self.connection.execute(sel).fetchall()] + + @contract + def select_vm_migrations(self, start_time, end_time): + """ Select VM migrations. + + :param start_time: The start time to select data. + :type start_time: * + + :param end_time: The end time to select data. + :type end_time: * + + :return: A list of timestamps and VM IDs. + :rtype: list(tuple(*, int)) + """ + vm = self.vm_migrations + sel = select([vm.c.timestamp, vm.c.vm_id]). \ + where(and_(vm.c.timestamp >= start_time, + vm.c.timestamp <= end_time)). \ + order_by(vm.c.id.asc()) + return [(x[0], int(x[1])) + for x in self.connection.execute(sel).fetchall()] + + +@contract +def init_db(sql_connection): + """ Initialize the database. + + :param sql_connection: A database connection URL. + :type sql_connection: str + + :return: The initialized database. + :rtype: * + """ + engine = create_engine(sql_connection) # 'sqlite:///:memory:' + metadata = MetaData() + metadata.bind = engine + + hosts = Table('hosts', metadata, + Column('id', Integer, primary_key=True), + Column('hostname', String(255), nullable=False), + Column('cpu_mhz', Integer, nullable=False), + Column('cpu_cores', Integer, nullable=False), + Column('ram', Integer, nullable=False)) + + host_resource_usage = \ + Table('host_resource_usage', metadata, + Column('id', Integer, primary_key=True), + Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), + Column('timestamp', DateTime, default=func.now()), + Column('cpu_mhz', Integer, nullable=False)) + + vms = Table('vms', metadata, + Column('id', Integer, primary_key=True), + Column('uuid', String(36), nullable=False)) + + vm_resource_usage = \ + Table('vm_resource_usage', metadata, + Column('id', Integer, primary_key=True), + Column('vm_id', Integer, ForeignKey('vms.id'), nullable=False), + Column('timestamp', DateTime, default=func.now()), + Column('cpu_mhz', Integer, nullable=False)) + + vm_migrations = \ + Table('vm_migrations', metadata, + Column('id', Integer, primary_key=True), + Column('vm_id', Integer, ForeignKey('vms.id'), nullable=False), + Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), + Column('timestamp', DateTime, default=func.now())) + + host_states = \ + Table('host_states', metadata, + Column('id', Integer, primary_key=True), + Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), + Column('timestamp', DateTime, default=func.now()), + Column('state', Integer, nullable=False)) + + host_overload = \ + Table('host_overload', metadata, + Column('id', Integer, primary_key=True), + Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), + Column('timestamp', DateTime, default=func.now()), + Column('overload', Integer, nullable=False)) + + metadata.create_all() + connection = engine.connect() + db = Database(connection, hosts, host_resource_usage, vms, + vm_resource_usage, vm_migrations, host_states, host_overload) + + return db diff --git a/utils/idle-time-fraction.py b/utils/idle-time-fraction.py new file mode 100755 index 0000000..9967109 --- /dev/null +++ b/utils/idle-time-fraction.py @@ -0,0 +1,66 @@ +#!/usr/bin/python2 + +# Copyright 2012 Anton Beloglazov +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import random +import shutil +import time +from datetime import datetime +from db import init_db + +if len(sys.argv) < 5: + print 'You must specify 4 arguments:' + print '1. The MySQL DB user name' + print '2. The MySQL DB password' + print '3. The start datetime in the format: %Y-%m-%d %H:%M:%S' + print '4. The finish datetime in the format: %Y-%m-%d %H:%M:%S' + sys.exit(1) + +db = init_db('mysql://' + sys.argv[1] + ':' + sys.argv[2] + '@localhost/spe') +start_time = datetime.fromtimestamp( + time.mktime(time.strptime(sys.argv[3], '%Y-%m-%d %H:%M:%S'))) +finish_time = datetime.fromtimestamp( + time.mktime(time.strptime(sys.argv[4], '%Y-%m-%d %H:%M:%S'))) + +#print "Start time: " + str(start_time) +#print "Finish time: " + str(finish_time) + +def total_seconds(delta): + return (delta.microseconds + + (delta.seconds + delta.days * 24 * 3600) * 1000000) / 1000000 + +total_time = 0 +total_idle_time = 0 +for hostname, host_id in db.select_host_ids().items(): + prev_timestamp = start_time + prev_state = 1 + states = {0: [], 1: []} + for timestamp, state in db.select_host_states(host_id, start_time, finish_time): + if prev_timestamp: + states[prev_state].append(total_seconds(timestamp - prev_timestamp)) + prev_timestamp = timestamp + prev_state = state + states[prev_state].append(total_seconds(finish_time - prev_timestamp)) + #print states + off_time = sum(states[0]) + on_time = sum(states[1]) + total_time += off_time + on_time + total_idle_time += off_time + +print "Total time: " + str(total_time) +print "Total idle time: " + str(total_idle_time) +print "Idle time fraction: " + str(float(total_idle_time) / total_time) diff --git a/utils/overload-time-fraction.py b/utils/overload-time-fraction.py new file mode 100755 index 0000000..9d3a584 --- /dev/null +++ b/utils/overload-time-fraction.py @@ -0,0 +1,81 @@ +#!/usr/bin/python2 + +# Copyright 2012 Anton Beloglazov +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import random +import shutil +import time +from datetime import datetime +from db import init_db + +if len(sys.argv) < 5: + print 'You must specify 4 arguments:' + print '1. The MySQL DB user name' + print '2. The MySQL DB password' + print '3. The start datetime in the format: %Y-%m-%d %H:%M:%S' + print '4. The finish datetime in the format: %Y-%m-%d %H:%M:%S' + sys.exit(1) + +db = init_db('mysql://' + sys.argv[1] + ':' + sys.argv[2] + '@localhost/neat') +start_time = datetime.fromtimestamp( + time.mktime(time.strptime(sys.argv[3], '%Y-%m-%d %H:%M:%S'))) +finish_time = datetime.fromtimestamp( + time.mktime(time.strptime(sys.argv[4], '%Y-%m-%d %H:%M:%S'))) + +#print "Start time: " + str(start_time) +#print "Finish time: " + str(finish_time) + +def total_seconds(delta): + return (delta.microseconds + + (delta.seconds + delta.days * 24 * 3600) * 1000000) / 1000000 + +total_idle_time = 0 +for hostname, host_id in db.select_host_ids().items(): + prev_timestamp = start_time + prev_state = 1 + states = {0: [], 1: []} + for timestamp, state in db.select_host_states(host_id, start_time, finish_time): + if prev_timestamp: + states[prev_state].append(total_seconds(timestamp - prev_timestamp)) + prev_timestamp = timestamp + prev_state = state + states[prev_state].append(total_seconds(finish_time - prev_timestamp)) + #print states + off_time = sum(states[0]) + total_idle_time += off_time + +total_time = 0 +total_overload_time = 0 +for hostname, host_id in db.select_host_ids().items(): + prev_timestamp = start_time + prev_state = 0 + states = {0: [], 1: []} + for timestamp, state in db.select_host_overload(host_id, start_time, finish_time): + if prev_timestamp: + states[prev_state].append(total_seconds(timestamp - prev_timestamp)) + prev_timestamp = timestamp + prev_state = state + states[prev_state].append(total_seconds(finish_time - prev_timestamp)) + #print states + nonoverload_time = sum(states[0]) + overload_time = sum(states[1]) + total_time += nonoverload_time + overload_time + total_overload_time += overload_time + +print "Total time: " + str(total_time) +print "Overload time: " + str(total_overload_time) +print "Overload time fraction: " + str(float(total_overload_time) / (total_time - total_idle_time)) diff --git a/utils/vm-migrations.py b/utils/vm-migrations.py new file mode 100755 index 0000000..6618ae2 --- /dev/null +++ b/utils/vm-migrations.py @@ -0,0 +1,41 @@ +#!/usr/bin/python2 + +# Copyright 2012 Anton Beloglazov +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import random +import shutil +import time +from datetime import datetime +from db import init_db + +if len(sys.argv) < 5: + print 'You must specify 4 arguments:' + print '1. The MySQL DB user name' + print '2. The MySQL DB password' + print '3. The start datetime in the format: %Y-%m-%d %H:%M:%S' + print '4. The finish datetime in the format: %Y-%m-%d %H:%M:%S' + sys.exit(1) + +db = init_db('mysql://' + sys.argv[1] + ':' + sys.argv[2] + '@localhost/spe') +start_time = datetime.fromtimestamp( + time.mktime(time.strptime(sys.argv[3], '%Y-%m-%d %H:%M:%S'))) +finish_time = datetime.fromtimestamp( + time.mktime(time.strptime(sys.argv[4], '%Y-%m-%d %H:%M:%S'))) + +#print "Start time: " + str(start_time) +#print "Finish time: " + str(finish_time) +print "VM migrations: " + str(len(db.select_vm_migrations(start_time, finish_time)))