x/terracotta
Code Issues Proposed changes

Merge branch 'master' of ssh://github.com/beloglazov/openstack-neat

Browse Source 
Conflicts:
	neat/common.py
	tests/test_common.py
This commit is contained in:
Anton Beloglazov 2012-09-14 10:08:15 +10:00
commit 49448a81fe
20 changed files with 2105 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
neat/common.py
View File

@ -110,3 +110,21 @@ def physical_cpu_mhz_total(vir_connection):
:rtype: int
"""
return physical_cpu_count(vir_connection) * physical_cpu_mhz(vir_connection)
@contract
def frange(start, end, step):
""" A range generator for floats.
:param start: The starting value.
:type start: number
:param end: The end value.
:type end: number
:param step: The step.
:type step: number
"""
while start <= end:
yield start
start += step

10
neat/config.py
View File

@ -47,9 +47,13 @@ REQUIRED_FIELDS = [
'compute_password',
'sleep_command',
'algorithm_underload_detection',
'algorithm_underload_detection_parameters',
'algorithm_overload_detection',
'algorithm_overload_detection_parameters',
'algorithm_vm_selection',
'algorithm_vm_selection_parameters',
'algorithm_vm_placement',
'algorithm_vm_placement_parameters',
]
@ -61,7 +65,7 @@ def read_config(paths):
:type paths: list(str)
:return: A dictionary of the configuration options.
:rtype: dict(str: *)
:rtype: dict(str: str)
"""
configParser = ConfigParser.ConfigParser()
for path in paths:
@ -74,7 +78,7 @@ def validate_config(config, required_fields):
""" Check that the config contains all the required fields.
:param config: A config dictionary to check.
:type config: dict(str: *)
:type config: dict(str: str)
:param required_fields: A list of required fields.
:type required_fields: list(str)
@ -99,7 +103,7 @@ def read_and_validate_config(paths, required_fields):
:type required_fields: list(str)
:return: A dictionary of the configuration options.
:rtype: dict(str: *)
:rtype: dict(str: str)
"""
config = read_config(paths)
if not validate_config(config, required_fields):

2
neat/contracts_extra.py
View File

@ -14,11 +14,13 @@
from contracts import new_contract
import collections
import libvirt
import sqlalchemy
import neat.db
new_contract('deque', collections.deque)
new_contract('function', lambda x: hasattr(x, '__call__'))
new_contract('virConnect', libvirt.virConnect)
new_contract('virDomain', libvirt.virDomain)

0
neat/local/overload/__init__.py Normal file
View File

0
neat/local/overload/mhod/__init__.py Normal file
View File

97
neat/local/overload/mhod/bruteforce.py Normal file
View File

@ -0,0 +1,97 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Functions for solving NLP problems using the bruteforce method.
"""
from contracts import contract
from neat.contracts_extra import *
import nlp
from neat.common import frange
@contract
def solve2(objective, constraint, step, limit):
""" Solve a maximization problem for 2 states.
:param objective: The objective function.
:type objective: function
:param constraint: A tuple representing the constraint.
:type constraint: tuple(function, function, number)
:param step: The step size.
:type step: number,>0
:param limit: The maximum value of the variables.
:type limit: number,>0
:return: The problem solution.
:rtype: list(number)
"""
res_best = 0
solution = []
for x in frange(0, limit, step):
for y in frange(0, limit, step):
try:
res = objective(x, y)
if res > res_best and \
constraint[1](constraint[0](x, y), constraint[2]):
res_best = res
solution = [x, y]
except ZeroDivisionError:
pass
return solution
@contract
def optimize(step, limit, otf, migration_time, ls, p, state_vector,
time_in_states, time_in_state_n):
""" Solve a MHOD optimization problem.
:param step: The step size for the bruteforce algorithm.
:type step: number,>0
:param limit: The maximum value of the variables.
:type limit: number,>0
:param otf: The OTF parameter.
:type otf: number,>=0,<=1
:param migration_time: The VM migration time in seconds.
:type migration_time: int,>=0
:param ls: L functions.
:type ls: list(function)
:param p: A matrix of transition probabilities.
:type p: list(list(number))
:param state_vector: A state vector.
:type state_vector: list(int)
:param time_in_states: The total time in all the states so far.
:type time_in_states: number,>=0
:param time_in_state_n: The total time in the state N so far.
:type time_in_state_n: number,>=0
:return: The solution of the problem.
:rtype: list(number)
"""
objective = nlp.build_objective(ls, state_vector, p)
constraint = nlp.build_constraint(otf, migration_time, ls, state_vector,
p, time_in_states, time_in_state_n)
return solve2(objective, constraint, step, limit)

214
neat/local/overload/mhod/core.py Normal file
View File

@ -0,0 +1,214 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" This is the main module of the MHOD algorithm.
"""
from contracts import contract
from neat.contracts_extra import *
import neat.local.overload.mhod.multisize_estimation as estimation
import neat.local.overload.mhod.bruteforce as bruteforce
from neat.local.overload.mhod.l_2_states import ls
@contract
def init_state(window_sizes, number_of_states):
""" Initialize the state dictionary of the MHOD algorithm.
:param window_sizes: The required window sizes.
:type window_sizes: list(int)
:param number_of_states: The number of states.
:type number_of_states: int,>0
:return: The initialization state dictionary.
:rtype: dict(str: *)
"""
state = {}
state['previous_state'] = 0
state['request_windows'] = estimation.init_request_windows(number_of_states)
state['estimate_windows'] = estimation.init_deque_structure(window_sizes, number_of_states)
state['variances'] = estimation.init_variances(window_sizes, number_of_states)
state['acceptable_variances'] = estimation.init_variances(window_sizes, number_of_states)
return state
@contract
def execute(state_config, otf, window_sizes, bruteforce_step,
time_step, migration_time, utilization, state):
""" The MHOD algorithm returning a decision of whether the host is overloaded.
:param state_config: The state configuration.
:type state_config: list(float)
:param otf: The OTF parameter.
:type otf: int,>0
:param window_sizes: A list of window sizes.
:type window_sizes: list(int)
:param bruteforce_step: The step of the bruteforce algorithm.
:type bruteforce_step: float
:param time_step: The lenght of a time frame in seconds.
:type time_step: int
:param migration_time: The VM migration time in seconds.
:type migration_time: int
:param utilization: The history of the host's CPU utilization.
:type utilization: list(float)
:param state: The state of the algorithm.
:type state: dict
:return: The updated state and decision of the algorithm.
:rtype: tuple(dict, bool)
"""
total_time = len(utilization)
max_window_size = max(window_sizes)
state_vector = build_state_vector(state_config, utilization)
state = current_state(state_vector)
selected_windows = estimation.select_window(state['variances'],
state['acceptable_variances'],
window_sizes)
p = estimation.select_best_estimates(state['estimate_windows'],
selected_windows)
state['request_windows'] = estimation.update_request_windows(state['request_windows'],
max_window_size,
state['previous_state'],
state)
state['estimate_windows'] = estimation.update_estimate_windows(state['estimate_windows'],
state['request_windows'],
state['previous_state'])
state['variances'] = estimation.update_variances(state['variances'],
state['estimate_windows'],
state['previous_state'])
state['acceptable_variances'] = estimation.update_acceptable_variances(state['acceptable_variances'],
state['estimate_windows'],
state['previous_state'])
state['previous_state'] = state
if len(utilization) >= 30:
state_history = utilization_to_states(state_config, utilization)
time_in_states = total_time
time_in_state_n = get_time_in_state_n(state_config, state_history)
tmp = set(p[state])
if len(tmp) != 1 or tmp[0] != 0:
policy = bruteforce.optimize(step, 1.0, otf, (migration_time / time_step), ls,
p, state_vector, time_in_states, time_in_state_n)
return issue_command_deterministic(policy)
return false
@contract
def build_state_vector(state_config, utilization):
""" Build the current state PMF corresponding to the utilization history and state config.
:param state_config: The state configuration.
:type state_config: list(float)
:param utilization: The history of the host's CPU utilization.
:type utilization: list(float)
:return: The current state vector.
:rtype: list(int)
"""
state = utilization_to_state(state_config, utilization[-1])
return [int(state == x) for x in range(len(state_config) + 1)]
@contract
def utilization_to_state(state_config, utilization):
""" Transform a utilization value into the corresponding state.
:param state_config: The state configuration.
:type state_config: list(float)
:param utilization: A utilization value.
:type utilization: number,>=0
:return: The state corresponding to the utilization value.
:rtype: int
"""
prev = -1
for state, threshold in enumerate(state_config):
if utilization >= prev and utilization < threshold:
return state
prev = state
return prev + 1
@contract
def current_state(state_vector):
""" Get the current state corresponding to the state probability vector.
:param state_vector: The state PMF vector.
:type state_vector: list(int)
:return: The current state.
:rtype: int,>=0
"""
return state_vector.index(1)
@contract
def utilization_to_states(state_config, utilization):
""" Get the state history corresponding to the utilization history.
Adds the 0 state to the beginning to simulate the first transition.
(map (partial utilization-to-state state-config) utilization))
:param state_config: The state configuration.
:type state_config: list(float)
:param utilization: The history of the host's CPU utilization.
:type utilization: list(float)
:return: The state history.
:rtype: list(int)
"""
return [utilization_to_state(state_config, x) for x in utilization]
@contract
def get_time_in_state_n(state_config, state_history):
""" Get the number of time steps the system has been in the state N.
:param state_config: The state configuration.
:type state_config: list(float)
:param state_history: The state history.
:type state_history: list(int)
:return: The total time the system has been in the state N.
:rtype: int
"""
return state_history.count(len(state_config))
@contract
def issue_command_deterministic(policy):
""" Issue a migration command according to the policy PMF p.
:param policy: A policy PMF.
:type policy: list(number)
:return: A migration command.
:rtype: bool
"""
return len(policy) == 0

82
neat/local/overload/mhod/l_2_states.py Normal file
View File

@ -0,0 +1,82 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" L functions for the 2 state configuration of the MHOD algorithm.
"""
from contracts import contract
from neat.contracts_extra import *
@contract
def l0(p_initial, p_matrix, m):
""" Compute the L0 function.
:param p_initial: The initial state distribution.
:type p_initial: list(number)
:param p_matrix: A matrix of transition probabilities.
:type p_matrix: list(list(number))
:param m: The m values.
:type m: list(number)
:return: The value of the L0 function.
:rtype: number
"""
p0 = p_initial[0]
p1 = p_initial[1]
p00 = p_matrix[0][0]
p01 = p_matrix[0][1]
p10 = p_matrix[1][0]
p11 = p_matrix[1][1]
m0 = m[0]
m1 = m[1]
return ((p0 * (-1 * m1 * p11 + p11 - 1) + (m1 * p1 - p1) * p10) /
(p00 * (m1 * (p11 - m0 * p11) - p11 + m0 * (p11 - 1) + 1) -
m1 * p11 + p11 + (m1 * (m0 * p01 - p01) - m0 * p01 + p01) *
p10 - 1))
@contract
def l1(p_initial, p_matrix, m):
""" Compute the L1 function.
:param p_initial: The initial state distribution.
:type p_initial: list(number)
:param p_matrix: A matrix of transition probabilities.
:type p_matrix: list(list(number))
:param m: The m values.
:type m: list(number)
:return: The value of the L1 function.
:rtype: number
"""
p0 = p_initial[0]
p1 = p_initial[1]
p00 = p_matrix[0][0]
p01 = p_matrix[0][1]
p10 = p_matrix[1][0]
p11 = p_matrix[1][1]
m0 = m[0]
m1 = m[1]
return (-1 * (p00 * (m0 * p1 - p1) + p1 + p0 * (p01 - m0 * p01)) /
(p00 * (m1 * (p11 - m0 * p11) - p11 + m0 * (p11 - 1) + 1) -
m1 * p11 + p11 + (m1 * (m0 * p01 - p01) - m0 * p01 + p01) *
p10 - 1))
ls = [l0, l1]

324
neat/local/overload/mhod/multisize_estimation.py Normal file
View File

@ -0,0 +1,324 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Multisize sliding window workload estimation functions.
"""
from contracts import contract
from neat.contracts_extra import *
from itertools import islice
from collections import deque
@contract
def mean(data, window_size):
""" Get the data mean according to the window size.
:param data: A list of values.
:type data: list(number)
:param window_size: A window size.
:type window_size: int,>0
:return: The mean value.
:rtype: float
"""
return float(sum(data)) / window_size
@contract
def variance(data, window_size):
""" Get the data variance according to the window size.
:param data: A list of values.
:type data: list(number)
:param window_size: A window size.
:type window_size: int,>0
:return: The variance value.
:rtype: float
"""
m = mean(data, window_size)
return float(sum((x - m) ** 2 for x in data)) / (window_size - 1)
@contract
def acceptable_variance(probability, window_size):
""" Get the acceptable variance.
:param probability: The probability to use.
:type probability: number,>=0,<=1
:param window_size: A window size.
:type window_size: int,>0
:return: The acceptable variance.
:rtype: float
"""
return float(probability * (1 - probability)) / window_size
@contract
def estimate_probability(data, window_size, state):
""" Get the estimated probability.
:param data: A list of data values.
:type data: list(number)
:param window_size: The window size.
:type window_size: int,>0
:param state: The current state.
:type state: int,>=0
:return: The estimated probability.
:rtype: float,>=0
"""
return float(data.count(state)) / window_size
@contract
def update_request_windows(request_windows, previous_state, current_state):
""" Update and return the updated request windows.
:param request_windows: The previous request windows.
:type request_windows: list(deque)
:param previous_state: The previous state.
:type previous_state: int,>=0
:param current_state: The current state.
:type current_state: int,>=0
:return: The updated request windows.
:rtype: list(deque)
"""
request_windows[previous_state].append(current_state)
return request_windows
@contract
def update_estimate_windows(estimate_windows, request_windows, previous_state):
""" Update and return the updated estimate windows.
:param estimate_windows: The previous estimate windows.
:type estimate_windows: list(list(dict))
:param request_windows: The current request windows.
:type request_windows: list(deque)
:param previous_state: The previous state.
:type previous_state: int,>=0
:return: The updated estimate windows.
:rtype: list(list(dict))
"""
request_window = request_windows[previous_state]
for state, estimate_window in enumerate(estimate_windows[previous_state]):
for window_size, estimates in estimate_window.items():
slice_from = len(request_window) - window_size
if slice_from < 0:
slice_from = 0
estimates.append(
estimate_probability(
list(islice(request_window, slice_from, None)),
window_size, state))
return estimate_windows
@contract
def update_variances(variances, estimate_windows, previous_state):
""" Updated and return the updated variances.
:param variances: The previous variances.
:type variances: list(list(dict))
:param estimate_windows: The current estimate windows.
:type estimate_windows: list(list(dict))
:param previous_state: The previous state.
:type previous_state: int,>=0
:return: The updated variances.
:rtype: list(list(dict))
"""
estimate_window = estimate_windows[previous_state]
for state, variance_map in enumerate(variances[previous_state]):
for window_size in variance_map:
estimates = estimate_window[state][window_size]
if len(estimates) < window_size:
variance_map[window_size] = 1.0
else:
variance_map[window_size] = variance(list(estimates), window_size)
return variances
@contract
def update_acceptable_variances(acceptable_variances, estimate_windows, previous_state):
""" Update and return the updated acceptable variances.
:param acceptable_variances: The previous acceptable variances.
:type acceptable_variances: list(list(dict))
:param estimate_windows: The current estimate windows.
:type estimate_windows: list(list(dict))
:param previous_state: The previous state.
:type previous_state: int,>=0
:return: The updated acceptable variances.
:rtype: list(list(dict))
"""
estimate_window = estimate_windows[previous_state]
for state, acceptable_variance_map in enumerate(acceptable_variances[previous_state]):
for window_size in acceptable_variance_map:
estimates = estimate_window[state][window_size]
acceptable_variance_map[window_size] = acceptable_variance(estimates[-1], window_size)
return acceptable_variances
@contract
def select_window(variances, acceptable_variances, window_sizes):
""" Select window sizes according to the acceptable variances.
:param variances: The variances.
:type variances: list(list(dict))
:param acceptable_variances: The acceptable variances.
:type acceptable_variances: list(list(dict))
:param window_sizes: The available window sizes.
:type window_sizes: list(int)
:return: The selected window sizes.
:rtype: list(list(int))
"""
n = len(variances)
selected_windows = []
for i in range(n):
selected_windows.append([])
for j in range(n):
selected_size = window_sizes[0]
for window_size in window_sizes:
if variances[i][j][window_size] > acceptable_variances[i][j][window_size]:
break
selected_size = window_size
selected_windows[i].append(selected_size)
return selected_windows
@contract
def select_best_estimates(estimate_windows, selected_windows):
""" Select the best estimates according to the selected windows.
:param estimate_windows: The estimate windows.
:type estimate_windows: list(list(dict))
:param selected_windows: The selected window sizes.
:type selected_windows: list(list(int))
:return: The selected best estimates.
:rtype: list(list(number))
"""
n = len(estimate_windows)
selected_estimates = []
for i in range(n):
selected_estimates.append([])
for j in range(n):
estimates = estimate_windows[i][j][selected_windows[i][j]]
if estimates:
selected_estimates[i].append(estimates[-1])
else:
selected_estimates[i].append(0.0)
return selected_estimates
@contract
def init_request_windows(number_of_states):
""" Initialize a request window data structure.
:param number_of_states: The number of states.
:type number_of_states: int,>0
:return: The initialized request windows data structure.
:rtype: list(deque)
"""
return number_of_states * [deque()]
@contract
def init_variances(window_sizes, number_of_states):
""" Initialize a variances data structure.
:param window_sizes: The required window sizes.
:type window_sizes: list(int)
:param number_of_states: The number of states.
:type number_of_states: int,>0
:return: The initialized variances data structure.
:rtype: list(list(dict))
"""
data = dict(zip(window_sizes, len(window_sizes) * [1.0]))
variances = []
for i in range(number_of_states):
variances.append([])
for j in range(number_of_states):
variances[i].append(dict(data))
return variances
@contract
def init_deque_structure(window_sizes, number_of_states):
""" Initialize a 3 level deque data structure.
:param window_sizes: The required window sizes.
:type window_sizes: list(int)
:param number_of_states: The number of states.
:type number_of_states: int,>0
:return: The initialized 3 level deque data structure.
:rtype: list(list(dict))
"""
data = dict((size, deque([], size)) for size in window_sizes)
structure = []
for i in range(number_of_states):
structure.append([])
for j in range(number_of_states):
structure[i].append(dict(data))
return structure
@contract
def init_selected_window_sizes(window_sizes, number_of_states):
""" Initialize a selected window sizes data structure.
:param window_sizes: The required window sizes.
:type window_sizes: list(int)
:param number_of_states: The number of states.
:type number_of_states: int,>0
:return: The initialized selected window sizes data structure.
:rtype: list(list(int))
"""
structure = []
for i in range(number_of_states):
structure.append([])
for j in range(number_of_states):
structure[i].append(window_sizes[0])
return structure

76
neat/local/overload/mhod/nlp.py Normal file
View File

@ -0,0 +1,76 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Functions for defing the NLP problem of the MHOD algorithm.
"""
from contracts import contract
from neat.contracts_extra import *
import operator
@contract
def build_objective(ls, state_vector, p):
""" Creates an objective function, which is a sum of the L functions.
:param ls: A list of L functions.
:type ls: list(function)
:param state-vector: A state vector.
:type state-vector: list(int)
:param p: A matrix of transition probabilities.
:type p: list(list(number))
:return: An objective function.
:rtype: function
"""
def objective(*m):
return sum(l(state_vector, p, m) for l in ls)
return objective
@contract
def build_constraint(otf, migration_time, ls, state_vector, p, time_in_states, time_in_state_n):
""" Creates a constraint for the optimization problem from the L functions.
:param otf: The OTF parameter.
:type otf: float
:param migration_time: The VM migration time in seconds.
:type migration_time: int
:param ls: A list of L functions.
:type ls: list(function)
:param state-vector: A state vector.
:type state-vector: list(int)
:param p: A matrix of transition probabilities.
:type p: list(list(number))
:param time_in_states: The total time on all the states in seconds.
:type time_in_states: int
:param time_in_state_n: The total time in the state N in seconds.
:type time_in_state_n: int
:return: The created constraint.
:rtype: tuple(function, function, number)
"""
def constraint(*m):
return float(migration_time + time_in_state_n + ls[-1](state_vector, p, m)) / \
(migration_time + time_in_states + sum(l(state_vector, p, m) for l in ls))
return (constraint, operator.le, otf)

132
neat/local/overload/otf.py Normal file
View File

@ -0,0 +1,132 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" OTF threshold based algorithms.
"""
from contracts import contract
from neat.contracts_extra import *
@contract
def init_state():
""" Initialize the state dictionary of the OTF algorithm.
:return: The initialization state dictionary.
:rtype: dict(str: *)
"""
return {}
@contract
def init_otf(threshold, time_step, migration_time):
""" Initialize the OTF threshold algorithm.
"""
migration_time_normalized = migration_time / time_step
return lambda state, utilization: otf(threshold, time_step, migration_time)
@contract
def init_otf_migration_time(threshold, time_step, migration_time):
""" Initialize the OTF threshold algorithm.
"""
migration_time_normalized = migration_time / time_step
return lambda state, utilization: otf(threshold, time_step, migration_time)
@contract
def otf(threshold, utilization):
""" The OTF threshold algorithm.
:param threshold: The threshold on the OTF value.
:type threshold: float,>=0
:param utilization: The history of the host's CPU utilization.
:type utilization: list(float)
:return: The updated state and decision of the algorithm.
:rtype: bool
"""
overloading_steps = len([1 for x in utilization if x >= 1])
return overloading_steps / len(utilization) > threshold
@contract
def otf_limit(threshold, limit, utilization):
""" The OTF threshold algorithm with limiting the minimum utilization values.
:param threshold: The threshold on the OTF value.
:type threshold: float,>=0
:param limit: The minimum number of values in the utilization history.
:type limit: int,>=0
:param utilization: The history of the host's CPU utilization.
:type utilization: list(float)
:return: The updated state and decision of the algorithm.
:rtype: bool
"""
cnt = len(utilization)
if cnt < limit:
return False
overloading_steps = len([1 for x in utilization if x >= 1])
return overloading_steps / cnt > threshold
@contract
def otf_migration_time(threshold, migration_time, utilization):
""" The OTF threshold algorithm considering the migration time.
:param threshold: The threshold on the OTF value.
:type threshold: float,>=0
:param migration_time: The VM migration time in time steps..
:type migration_time: int,>=0
:param utilization: The history of the host's CPU utilization.
:type utilization: list(float)
:return: The updated state and decision of the algorithm.
:rtype: bool
"""
overloading_steps = len([1 for x in utilization if x >= 1])
return (migration_time + overloading_steps) / \
(migration_time + len(utilization)) > threshold
@contract
def otf_limit_migration_time(threshold, limit, migration_time, utilization):
""" The OTF threshold algorithm with limiting and migration time.
:param threshold: The threshold on the OTF value.
:type threshold: float,>=0
:param limit: The minimum number of values in the utilization history.
:type limit: int,>=0
:param migration_time: The VM migration time in time steps.
:type migration_time: int,>=0
:param utilization: The history of the host's CPU utilization.
:type utilization: list(float)
:return: The updated state and decision of the algorithm.
:rtype: bool
"""
cnt = len(utilization)
if cnt < limit:
return False
return (migration_time + overloading_steps) / \
(migration_time + cnt) > threshold

0
tests/local/overload/__init__.py Normal file
View File

0
tests/local/overload/mhod/__init__.py Normal file
View File

71
tests/local/overload/mhod/test_bruteforce.py Normal file
View File

@ -0,0 +1,71 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from mocktest import *
from pyqcy import *
from operator import le
import neat.local.overload.mhod.bruteforce as b
import neat.local.overload.mhod.nlp as nlp
class Bruteforce(TestCase):
def test_solve2(self):
def fn1(x, y):
return x + y
def fn2(x, y):
return 2 * x + y
def fn3(x, y):
return x - y
def fn4(x, y):
return x / y
self.assertEqual([round(x, 1) for x in b.solve2(fn1, (fn1, le, 10), 0.1, 1.0)],
[1.0, 1.0])
self.assertEqual([round(x, 1) for x in b.solve2(fn1, (fn1, le, 0.5), 0.1, 1.0)],
[0.0, 0.5])
self.assertEqual([round(x, 1) for x in b.solve2(fn2, (fn1, le, 0.5), 0.1, 1.0)],
[0.5, 0.0])
self.assertEqual([round(x, 1) for x in b.solve2(fn3, (fn3, le, 10), 0.1, 1.0)],
[1.0, 0.0])
self.assertEqual([round(x, 1) for x in b.solve2(fn4, (fn4, le, 10), 0.1, 1.0)],
[1.0, 0.1])
def test_optimize(self):
with MockTransaction:
step = 0.1
limit = 1
otf = 0.3
migration_time = 20
ls = [lambda x: x, lambda x: x]
p = [[0, 1]]
state_vector = [0, 1]
time_in_states = 10
time_in_state_n = 5
objective = mock('objective')
constraint = mock('constraint')
solution = [1, 2, 3]
expect(nlp).build_objective(ls, state_vector, p).and_return(objective).once()
expect(nlp).build_constraint(otf, migration_time, ls, state_vector,
p, time_in_states, time_in_state_n). \
and_return(constraint).once()
expect(b).solve2(objective, constraint, step, limit).and_return(solution).once()
self.assertEqual(b.optimize(step, limit, otf, migration_time, ls,
p, state_vector, time_in_states, time_in_state_n),
solution)

96
tests/local/overload/mhod/test_core.py Normal file
View File

@ -0,0 +1,96 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from mocktest import *
from pyqcy import *
import neat.local.overload.mhod.core as c
class Core(TestCase):
def test_init_state(self):
state = c.init_state([20, 40], 2)
self.assertEquals(state['previous_state'], 0)
self.assertTrue('request_windows' in state)
self.assertTrue('estimate_windows' in state)
self.assertTrue('variances' in state)
self.assertTrue('acceptable_variances' in state)
def test_utilization_to_state(self):
state_config = [0.4, 0.7]
self.assertEqual(c.utilization_to_state(state_config, 0.0), 0)
self.assertEqual(c.utilization_to_state(state_config, 0.1), 0)
self.assertEqual(c.utilization_to_state(state_config, 0.2), 0)
self.assertEqual(c.utilization_to_state(state_config, 0.3), 0)
self.assertEqual(c.utilization_to_state(state_config, 0.4), 1)
self.assertEqual(c.utilization_to_state(state_config, 0.5), 1)
self.assertEqual(c.utilization_to_state(state_config, 0.6), 1)
self.assertEqual(c.utilization_to_state(state_config, 0.7), 2)
self.assertEqual(c.utilization_to_state(state_config, 0.8), 2)
self.assertEqual(c.utilization_to_state(state_config, 0.9), 2)
self.assertEqual(c.utilization_to_state(state_config, 1.0), 2)
self.assertEqual(c.utilization_to_state(state_config, 1.1), 2)
self.assertEqual(c.utilization_to_state([1.0], 0.0), 0)
self.assertEqual(c.utilization_to_state([1.0], 0.1), 0)
self.assertEqual(c.utilization_to_state([1.0], 0.2), 0)
self.assertEqual(c.utilization_to_state([1.0], 0.3), 0)
self.assertEqual(c.utilization_to_state([1.0], 0.4), 0)
self.assertEqual(c.utilization_to_state([1.0], 0.5), 0)
self.assertEqual(c.utilization_to_state([1.0], 0.6), 0)
self.assertEqual(c.utilization_to_state([1.0], 0.7), 0)
self.assertEqual(c.utilization_to_state([1.0], 0.8), 0)
self.assertEqual(c.utilization_to_state([1.0], 0.9), 0)
self.assertEqual(c.utilization_to_state([1.0], 1.0), 1)
self.assertEqual(c.utilization_to_state([1.0], 1.1), 1)
def test_build_state_vector(self):
state_config = [0.4, 0.7]
self.assertEqual(c.build_state_vector(state_config, [0.0, 0.1]), [1, 0, 0])
self.assertEqual(c.build_state_vector(state_config, [0.0, 0.2]), [1, 0, 0])
self.assertEqual(c.build_state_vector(state_config, [0.0, 0.3]), [1, 0, 0])
self.assertEqual(c.build_state_vector(state_config, [0.0, 0.4]), [0, 1, 0])
self.assertEqual(c.build_state_vector(state_config, [0.0, 0.5]), [0, 1, 0])
self.assertEqual(c.build_state_vector(state_config, [0.0, 0.6]), [0, 1, 0])
self.assertEqual(c.build_state_vector(state_config, [0.0, 0.7]), [0, 0, 1])
self.assertEqual(c.build_state_vector(state_config, [0.0, 0.8]), [0, 0, 1])
self.assertEqual(c.build_state_vector(state_config, [0.0, 0.9]), [0, 0, 1])
self.assertEqual(c.build_state_vector(state_config, [0.0, 1.0]), [0, 0, 1])
def test_current_state(self):
self.assertEqual(c.current_state([1, 0, 0]), 0)
self.assertEqual(c.current_state([0, 1, 0]), 1)
self.assertEqual(c.current_state([0, 0, 1]), 2)
def test_utilization_to_states(self):
state_config = [0.4, 0.7]
data = [0.25, 0.30, 0.62, 0.59, 0.67, 0.73, 0.85, 0.97, 0.73, 0.68, 0.69,
0.52, 0.51, 0.25, 0.38, 0.46, 0.52, 0.55, 0.58, 0.65, 0.70]
states = [0, 0, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 2]
self.assertEqual(c.utilization_to_states(state_config, data), states)
state_config = [1.0]
data = [0.5, 0.5, 1.0, 1.0, 0.5]
states = [0, 0, 1, 1, 0]
self.assertEqual(c.utilization_to_states(state_config, data), states)
def test_get_time_in_state_n(self):
state_config = [0.4, 0.7]
states = [0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 2]
self.assertEqual(c.get_time_in_state_n(state_config, states), 5)
def test_issue_command_deterministic(self):
self.assertEqual(c.issue_command_deterministic([1]), False)
self.assertEqual(c.issue_command_deterministic([]), True)

37
tests/local/overload/mhod/test_l_2_states.py Normal file
View File

@ -0,0 +1,37 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from mocktest import *
from pyqcy import *
import neat.local.overload.mhod.l_2_states as l
class L2States(TestCase):
def test_l0(self):
p = [[0.4, 0.6],
[0.9, 0.1]]
p0 = [1, 0]
self.assertAlmostEqual(l.l0(p0, p, [0.2, 0.8]), 1.690, 3)
self.assertAlmostEqual(l.l0(p0, p, [0.62, 0.38]), 1.404, 3)
def test_l1(self):
p = [[0.4, 0.6],
[0.9, 0.1]]
p0 = [1, 0]
self.assertAlmostEqual(l.l1(p0, p, [0.2, 0.8]), 0.828, 3)
self.assertAlmostEqual(l.l1(p0, p, [0.62, 0.38]), 0.341, 3)

803
tests/local/overload/mhod/test_multisize_estimation.py Normal file
View File

@ -0,0 +1,803 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from mocktest import *
from pyqcy import *
from collections import deque
from copy import deepcopy
import neat.local.overload.mhod.multisize_estimation as m
def c(data):
return deepcopy(data)
class Multisize(TestCase):
def test_mean(self):
self.assertEqual(m.mean([], 100), 0.0)
self.assertEqual(m.mean([0], 100), 0.0)
self.assertEqual(m.mean([0, 0], 100), 0.0)
self.assertEqual(m.mean([1, 1], 100), 0.02)
self.assertEqual(m.mean([0, 1], 100), 0.01)
self.assertEqual(m.mean([1, 2, 3, 4, 5], 100), 0.15)
def test_variance(self):
self.assertEqual(m.variance([], 100), 0.0)
self.assertEqual(m.variance([0], 100), 0.0)
self.assertEqual(m.variance([0, 0], 100), 0.0)
self.assertAlmostEqual(m.variance([1, 1], 100), 0.0194020202)
self.assertAlmostEqual(m.variance([0, 1], 100), 0.0099010101)
self.assertAlmostEqual(m.variance([1, 2, 3, 4, 5], 100), 0.5112373737)
self.assertAlmostEqual(m.variance([0, 0, 0, 1], 100), 0.0099030303)
def test_acceptable_variance(self):
self.assertAlmostEqual(m.acceptable_variance(0.2, 5), 0.032, 3)
self.assertAlmostEqual(m.acceptable_variance(0.6, 15), 0.016, 3)
def test_estimate_probability(self):
self.assertEqual(
m.estimate_probability([0, 0, 1, 1, 0, 0, 0, 0, 0, 0], 100, 0), 0.08)
self.assertEqual(
m.estimate_probability([0, 0, 1, 1, 0, 0, 0, 0, 0, 0], 100, 1), 0.02)
self.assertEqual(
m.estimate_probability([1, 1, 0, 0, 1, 1, 1, 1, 1, 1], 200, 0), 0.01)
self.assertEqual(
m.estimate_probability([1, 1, 0, 0, 1, 1, 1, 1, 1, 1], 200, 1), 0.04)
def test_update_request_windows(self):
max_window_size = 4
windows = [deque([0, 0], max_window_size),
deque([1, 1], max_window_size)]
self.assertEqual(m.update_request_windows(c(windows), 0, 0), [deque([0, 0, 0]),
deque([1, 1])])
self.assertEqual(m.update_request_windows(c(windows), 0, 1), [deque([0, 0, 1]),
deque([1, 1])])
self.assertEqual(m.update_request_windows(c(windows), 1, 0), [deque([0, 0]),
deque([1, 1, 0])])
self.assertEqual(m.update_request_windows(c(windows), 1, 1), [deque([0, 0]),
deque([1, 1, 1])])
max_window_size = 2
windows = [deque([0, 0], max_window_size),
deque([1, 1], max_window_size)]
self.assertEqual(m.update_request_windows(c(windows), 0, 0), [deque([0, 0]),
deque([1, 1])])
self.assertEqual(m.update_request_windows(c(windows), 0, 1), [deque([0, 1]),
deque([1, 1])])
self.assertEqual(m.update_request_windows(c(windows), 1, 0), [deque([0, 0]),
deque([1, 0])])
self.assertEqual(m.update_request_windows(c(windows), 1, 1), [deque([0, 0]),
deque([1, 1])])
max_window_size = 4
windows = [deque([0, 0], max_window_size),
deque([1, 1], max_window_size),
deque([2, 2], max_window_size)]
self.assertEqual(m.update_request_windows(c(windows), 0, 0), [deque([0, 0, 0]),
deque([1, 1]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 0, 1), [deque([0, 0, 1]),
deque([1, 1]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 0, 2), [deque([0, 0, 2]),
deque([1, 1]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 1, 0), [deque([0, 0]),
deque([1, 1, 0]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 1, 1), [deque([0, 0]),
deque([1, 1, 1]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 1, 2), [deque([0, 0]),
deque([1, 1, 2]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 2, 0), [deque([0, 0]),
deque([1, 1]),
deque([2, 2, 0])])
self.assertEqual(m.update_request_windows(c(windows), 2, 1), [deque([0, 0]),
deque([1, 1]),
deque([2, 2, 1])])
self.assertEqual(m.update_request_windows(c(windows), 2, 2), [deque([0, 0]),
deque([1, 1]),
deque([2, 2, 2])])
max_window_size = 2
windows = [deque([0, 0], max_window_size),
deque([1, 1], max_window_size),
deque([2, 2], max_window_size)]
self.assertEqual(m.update_request_windows(c(windows), 0, 0), [deque([0, 0]),
deque([1, 1]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 0, 1), [deque([0, 1]),
deque([1, 1]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 0, 2), [deque([0, 2]),
deque([1, 1]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 1, 0), [deque([0, 0]),
deque([1, 0]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 1, 1), [deque([0, 0]),
deque([1, 1]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 1, 2), [deque([0, 0]),
deque([1, 2]),
deque([2, 2])])
self.assertEqual(m.update_request_windows(c(windows), 2, 0), [deque([0, 0]),
deque([1, 1]),
deque([2, 0])])
self.assertEqual(m.update_request_windows(c(windows), 2, 1), [deque([0, 0]),
deque([1, 1]),
deque([2, 1])])
self.assertEqual(m.update_request_windows(c(windows), 2, 2), [deque([0, 0]),
deque([1, 1]),
deque([2, 2])])
def test_update_estimate_windows(self):
req_win = [deque([1, 0, 0, 0]),
deque([1, 0, 1, 0])]
est_win = [[{2: deque([0, 0], 2),
4: deque([0, 0], 4)},
{2: deque([0, 0], 2),
4: deque([0, 0], 4)}],
[{2: deque([0, 0], 2),
4: deque([0, 0], 4)},
{2: deque([0, 0], 2),
4: deque([0, 0], 4)}]]
self.assertEqual(m.update_estimate_windows(c(est_win), c(req_win), 0),
[[{2: deque([0, 1.0]),
4: deque([0, 0, 0.75])},
{2: deque([0, 0.0]),
4: deque([0, 0, 0.25])}],
[{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])}]])
self.assertEqual(m.update_estimate_windows(c(est_win), c(req_win), 1),
[[{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])}],
[{2: deque([0, 0.5]),
4: deque([0, 0, 0.5])},
{2: deque([0, 0.5]),
4: deque([0, 0, 0.5])}]])
req_win = [deque([1, 0, 2, 0]),
deque([1, 0, 1, 0]),
deque([2, 2, 1, 0])]
est_win = [[{2: deque([0, 0], 2),
4: deque([0, 0], 4)},
{2: deque([0, 0], 2),
4: deque([0, 0], 4)},
{2: deque([0, 0], 2),
4: deque([0, 0], 4)}],
[{2: deque([0, 0], 2),
4: deque([0, 0], 4)},
{2: deque([0, 0], 2),
4: deque([0, 0], 4)},
{2: deque([0, 0], 2),
4: deque([0, 0], 4)}],
[{2: deque([0, 0], 2),
4: deque([0, 0], 4)},
{2: deque([0, 0], 2),
4: deque([0, 0], 4)},
{2: deque([0, 0], 2),
4: deque([0, 0], 4)}]]
self.assertEqual(m.update_estimate_windows(c(est_win), c(req_win), 0),
[[{2: deque([0, 0.5]),
4: deque([0, 0, 0.5])},
{2: deque([0, 0.0]),
4: deque([0, 0, 0.25])},
{2: deque([0, 0.5]),
4: deque([0, 0, 0.25])}],
[{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])}],
[{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])}]])
self.assertEqual(m.update_estimate_windows(c(est_win), c(req_win), 1),
[[{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])}],
[{2: deque([0, 0.5]),
4: deque([0, 0, 0.5])},
{2: deque([0, 0.5]),
4: deque([0, 0, 0.5])},
{2: deque([0, 0.0]),
4: deque([0, 0, 0.0])}],
[{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])}]])
self.assertEqual(m.update_estimate_windows(c(est_win), c(req_win), 2),
[[{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])}],
[{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])},
{2: deque([0, 0]),
4: deque([0, 0])}],
[{2: deque([0, 0.5]),
4: deque([0, 0, 0.25])},
{2: deque([0, 0.5]),
4: deque([0, 0, 0.25])},
{2: deque([0, 0.0]),
4: deque([0, 0, 0.5])}]])
def test_update_variances(self):
est_win = [[{2: deque([0, 0.5], 2),
4: deque([1, 0, 0, 0], 4)},
{2: deque([1.0, 0.5], 2),
4: deque([0, 1, 1, 1], 4)}],
[{2: deque([0.5, 0.25], 2),
4: deque([0.25, 0.25, 0.5, 0.5], 4)},
{2: deque([0.5, 0.75], 2),
4: deque([0.75, 0.75, 0.5, 0.5], 4)}]]
variances = [[{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0}]]
self.assertEqual(m.update_variances(c(variances), c(est_win), 0),
[[{2: 0.125,
4: 0.25},
{2: 0.125,
4: 0.25}],
[{2: 0,
4: 0},
{2: 0,
4: 0}]])
self.assertEqual(m.update_variances(c(variances), c(est_win), 1),
[[{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0.03125,
4: 0.020833333333333332},
{2: 0.03125,
4: 0.020833333333333332}]])
self.assertEqual(m.update_variances(
m.update_variances(c(variances), c(est_win), 0), c(est_win), 0),
[[{2: 0.125,
4: 0.25},
{2: 0.125,
4: 0.25}],
[{2: 0,
4: 0},
{2: 0,
4: 0}]])
est_win = [[{2: deque([0, 0], 2),
4: deque([1, 0, 0, 0], 4)},
{2: deque([1, 1], 2),
4: deque([0, 0, 1, 1], 4)},
{2: deque([0, 0], 2),
4: deque([0, 1, 0, 0], 4)}],
[{2: deque([0.5, 0.25], 2),
4: deque([0.25, 0.05, 0.5, 0.25], 4)},
{2: deque([0.25, 0.5], 2),
4: deque([0.4, 0.55, 0.25, 0.5], 4)},
{2: deque([0.25, 0.25], 2),
4: deque([0.35, 0.4, 0.25, 0.25], 4)}],
[{2: deque([1, 0], 2),
4: deque([1, 0, 1, 0], 4)},
{2: deque([0, 1], 2),
4: deque([0, 0, 0, 1], 4)},
{2: deque([0, 0], 2),
4: deque([0, 1, 0, 0], 4)}]]
variances = [[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}]]
self.assertEqual(m.update_variances(c(variances), c(est_win), 0),
[[{2: 0.0,
4: 0.25},
{2: 0.0,
4: 0.3333333333333333},
{2: 0.0,
4: 0.25}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}]])
self.assertEqual(m.update_variances(c(variances), c(est_win), 1),
[[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0.03125,
4: 0.03395833333333333},
{2: 0.03125,
4: 0.0175},
{2: 0.0,
4: 0.005625000000000001}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}]])
self.assertEqual(m.update_variances(c(variances), c(est_win), 2),
[[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0.5,
4: 0.3333333333333333},
{2: 0.5,
4: 0.25},
{2: 0.0,
4: 0.25}]])
def test_update_acceptable_variances(self):
est_win = [[{2: deque([0, 0.5], 2),
4: deque([1, 0, 0, 0], 4)},
{2: deque([1.0, 0.5], 2),
4: deque([0, 1, 1, 1], 4)}],
[{2: deque([0.5, 0.25], 2),
4: deque([0.25, 0.25, 0.5, 0.5], 4)},
{2: deque([0.5, 0.75], 2),
4: deque([0.75, 0.75, 0.5, 0.5], 4)}]]
acc_variances = [[{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0}]]
self.assertEqual(m.update_acceptable_variances(c(acc_variances),
c(est_win), 0),
[[{2: 0.125,
4: 0.0},
{2: 0.125,
4: 0.0}],
[{2: 0,
4: 0},
{2: 0,
4: 0}]])
self.assertEqual(m.update_acceptable_variances(c(acc_variances),
c(est_win), 1),
[[{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0.09375,
4: 0.0625},
{2: 0.09375,
4: 0.0625}]])
self.assertEqual(m.update_acceptable_variances(
m.update_acceptable_variances(c(acc_variances), c(est_win), 0),
c(est_win), 0),
[[{2: 0.125,
4: 0.0},
{2: 0.125,
4: 0.0}],
[{2: 0,
4: 0},
{2: 0,
4: 0}]])
est_win = [[{2: deque([0, 0], 2),
4: deque([1, 0, 0, 0], 4)},
{2: deque([1, 1], 2),
4: deque([0, 0, 1, 1], 4)},
{2: deque([0, 0], 2),
4: deque([0, 1, 0, 0], 4)}],
[{2: deque([0.5, 0.25], 2),
4: deque([0.25, 0.05, 0.5, 0.25], 4)},
{2: deque([0.25, 0.5], 2),
4: deque([0.4, 0.55, 0.25, 0.5], 4)},
{2: deque([0.25, 0.25], 2),
4: deque([0.35, 0.4, 0.25, 0.25], 4)}],
[{2: deque([1, 0], 2),
4: deque([1, 0, 1, 0], 4)},
{2: deque([0, 1], 2),
4: deque([0, 0, 0, 1], 4)},
{2: deque([0, 0], 2),
4: deque([0, 1, 0, 0], 4)}]]
acc_variances = [[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}]]
self.assertEqual(m.update_acceptable_variances(c(acc_variances),
c(est_win), 0),
[[{2: 0.0,
4: 0.0},
{2: 0.0,
4: 0.0},
{2: 0.0,
4: 0.0}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}]])
self.assertEqual(m.update_acceptable_variances(c(acc_variances),
c(est_win), 1),
[[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0.09375,
4: 0.046875},
{2: 0.125,
4: 0.0625},
{2: 0.09375,
4: 0.046875}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}]])
self.assertEqual(m.update_acceptable_variances(c(acc_variances),
c(est_win), 2),
[[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0},
{2: 0,
4: 0}],
[{2: 0.0,
4: 0.0},
{2: 0.0,
4: 0.0},
{2: 0.0,
4: 0.0}]])
def test_select_window(self):
variances = [[{2: 0.2,
4: 0.9},
{2: 0.2,
4: 0.6}],
[{2: 0.2,
4: 0},
{2: 0.2,
4: 0.8}]]
acc_variances = [[{2: 0.1,
4: 0.5},
{2: 0.4,
4: 0.5}],
[{2: 0.4,
4: 0.5},
{2: 0.1,
4: 0.5}]]
window_sizes = [2, 4]
self.assertEqual(m.select_window(variances, acc_variances, window_sizes),
[[2, 2],
[4, 2]])
variances = [[{2: 0,
4: 0.9},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0.8}]]
acc_variances = [[{2: 0.5,
4: 0.5},
{2: 0.6,
4: 0.5}],
[{2: 0.7,
4: 0.5},
{2: 0.4,
4: 0.5}]]
window_sizes = [2, 4]
self.assertEqual(m.select_window(variances, acc_variances, window_sizes),
[[2, 4],
[4, 2]])
variances = [[{2: 0,
4: 0.9},
{2: 0,
4: 0},
{2: 0,
4: 1.0}],
[{2: 0,
4: 0},
{2: 0,
4: 0.8},
{2: 0,
4: 0}],
[{2: 0,
4: 0},
{2: 0,
4: 0.8},
{2: 0.5,
4: 0}]]
acc_variances = [[{2: 0.5,
4: 0.9},
{2: 0.6,
4: 0.9},
{2: 0.6,
4: 0.9}],
[{2: 0.7,
4: 0.9},
{2: 0.4,
4: 0.9},
{2: 0.4,
4: 0.9}],
[{2: 0.7,
4: 0.9},
{2: 0.4,
4: 0.5},
{2: 0.4,
4: 0.9}]]
window_sizes = [2, 4]
self.assertEqual(m.select_window(variances, acc_variances, window_sizes),
[[4, 4, 2],
[4, 4, 4],
[4, 2, 2]])
def test_select_best_estimates(self):
est_win = [[{2: deque([0, 0], 2),
4: deque([1, 0, 0, 0], 4)},
{2: deque([1, 1], 2),
4: deque([0, 0, 1, 1], 4)},
{2: deque([0, 0], 2),
4: deque([0, 1, 0, 0], 4)}],
[{2: deque([0.5, 0.25], 2),
4: deque([0.25, 0.05, 0.5, 0.25], 4)},
{2: deque([0.25, 0.5], 2),
4: deque([0.4, 0.55, 0.25, 0.6], 4)},
{2: deque([0.25, 0.25], 2),
4: deque([0.35, 0.4, 0.25, 0.15], 4)}],
[{2: deque([1, 0], 2),
4: deque([1, 0, 1, 0], 4)},
{2: deque([0, 1], 2),
4: deque([0, 0, 0, 0.2], 4)},
{2: deque([0, 0], 2),
4: deque([0, 1, 0, 0], 4)}]]
selected_windows1 = [[2, 4, 2],
[2, 2, 4],
[4, 2, 2]]
selected_windows2 = [[4, 4, 4],
[2, 2, 2],
[2, 4, 2]]
self.assertEqual(m.select_best_estimates(c(est_win), selected_windows1),
[[0, 1, 0],
[0.25, 0.5, 0.15],
[0, 1, 0]])
self.assertEqual(m.select_best_estimates(c(est_win), selected_windows2),
[[0, 1, 0],
[0.25, 0.5, 0.25],
[0, 0.2, 0]])
est_win = [[{2: deque(),
4: deque()},
{2: deque(),
4: deque()}],
[{2: deque(),
4: deque()},
{2: deque(),
4: deque()}]]
self.assertEqual(m.select_best_estimates(c(est_win), [[2, 4], [4, 2]]),
[[0.0, 0.0],
[0.0, 0.0]])
self.assertEqual(m.select_best_estimates(c(est_win), [[2, 2], [4, 4]]),
[[0.0, 0.0],
[0.0, 0.0]])
def test_init_request_windows(self):
self.assertEqual(m.init_request_windows(1), [deque()])
self.assertEqual(m.init_request_windows(2), [deque(),
deque()])
self.assertEqual(m.init_request_windows(3), [deque(),
deque(),
deque()])
def test_init_variances(self):
self.assertEqual(m.init_variances([2, 4], 1), [[{2: 1.0,
4: 1.0}]])
self.assertEqual(m.init_variances([2, 4], 2), [[{2: 1.0,
4: 1.0},
{2: 1.0,
4: 1.0}],
[{2: 1.0,
4: 1.0},
{2: 1.0,
4: 1.0}]])
self.assertEqual(m.init_variances([2, 4], 3), [[{2: 1.0,
4: 1.0},
{2: 1.0,
4: 1.0},
{2: 1.0,
4: 1.0}],
[{2: 1.0,
4: 1.0},
{2: 1.0,
4: 1.0},
{2: 1.0,
4: 1.0}],
[{2: 1.0,
4: 1.0},
{2: 1.0,
4: 1.0},
{2: 1.0,
4: 1.0}]])
def test_init_3_level_structure(self):
structure = m.init_deque_structure([2, 4], 1)
self.assertEqual(structure, [[{2: deque(),
4: deque()}]])
self.assertEqual(structure[0][0][2].maxlen, 2)
self.assertEqual(structure[0][0][4].maxlen, 4)
structure = m.init_deque_structure([2, 4], 2)
self.assertEqual(structure, [[{2: deque(),
4: deque()},
{2: deque(),
4: deque()}],
[{2: deque(),
4: deque()},
{2: deque(),
4: deque()}]])
self.assertEqual(structure[0][0][2].maxlen, 2)
self.assertEqual(structure[0][0][4].maxlen, 4)
self.assertEqual(structure[0][1][2].maxlen, 2)
self.assertEqual(structure[0][1][4].maxlen, 4)
self.assertEqual(structure[1][0][2].maxlen, 2)
self.assertEqual(structure[1][0][4].maxlen, 4)
self.assertEqual(structure[1][1][2].maxlen, 2)
self.assertEqual(structure[1][1][4].maxlen, 4)
structure = m.init_deque_structure([2, 4], 3)
self.assertEqual(structure, [[{2: deque(),
4: deque()},
{2: deque(),
4: deque()},
{2: deque(),
4: deque()}],
[{2: deque(),
4: deque()},
{2: deque(),
4: deque()},
{2: deque(),
4: deque()}],
[{2: deque(),
4: deque()},
{2: deque(),
4: deque()},
{2: deque(),
4: deque()}]])
self.assertEqual(structure[0][0][2].maxlen, 2)
self.assertEqual(structure[0][0][4].maxlen, 4)
self.assertEqual(structure[0][1][2].maxlen, 2)
self.assertEqual(structure[0][1][4].maxlen, 4)
self.assertEqual(structure[0][2][2].maxlen, 2)
self.assertEqual(structure[0][2][4].maxlen, 4)
self.assertEqual(structure[1][0][2].maxlen, 2)
self.assertEqual(structure[1][0][4].maxlen, 4)
self.assertEqual(structure[1][1][2].maxlen, 2)
self.assertEqual(structure[1][1][4].maxlen, 4)
self.assertEqual(structure[1][2][2].maxlen, 2)
self.assertEqual(structure[1][2][4].maxlen, 4)
self.assertEqual(structure[2][0][2].maxlen, 2)
self.assertEqual(structure[2][0][4].maxlen, 4)
self.assertEqual(structure[2][1][2].maxlen, 2)
self.assertEqual(structure[2][1][4].maxlen, 4)
self.assertEqual(structure[2][2][2].maxlen, 2)
self.assertEqual(structure[2][2][4].maxlen, 4)
def test_init_selected_window_sizes(self):
self.assertEqual(m.init_selected_window_sizes([2, 4], 1), [[2]])
self.assertEqual(m.init_selected_window_sizes([2, 4], 2), [[2, 2],
[2, 2]])
self.assertEqual(m.init_selected_window_sizes([2, 4], 3), [[2, 2, 2],
[2, 2, 2],
[2, 2, 2]])

64
tests/local/overload/mhod/test_nlp.py Normal file
View File

@ -0,0 +1,64 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from mocktest import *
from pyqcy import *
import operator
import neat.local.overload.mhod.nlp as nlp
class Nlp(TestCase):
def test_build_objective(self):
with MockTransaction:
state_vector = [1, 0]
p = [[-0.1, 0.1],
[0.3, -0.3]]
m1 = mock('m1')
m2 = mock('m2')
m = (m1, m2)
container = mock('function container')
expect(container).l0(state_vector, p, m).and_return(2).once()
expect(container).l1(state_vector, p, m).and_return(3).once()
ls = [container.l0, container.l1]
objective = nlp.build_objective(ls, state_vector, p)
self.assertTrue(hasattr(objective, '__call__'))
self.assertEqual(objective(m1, m2), 5)
def test_build_constraint(self):
with MockTransaction:
otf = 0.05
migration_time = 20
state_vector = [1, 0]
p = [[-0.1, 0.1],
[0.3, -0.3]]
m1 = mock('m1')
m2 = mock('m2')
m = (m1, m2)
container = mock('function container')
expect(container).l0(state_vector, p, m).and_return(2).once()
expect(container).l1(state_vector, p, m).and_return(3).exactly(2).times()
ls = [container.l0, container.l1]
constraint = nlp.build_constraint(otf, migration_time, ls, state_vector, p, 0, 0)
self.assertTrue(hasattr(constraint[0], '__call__'))
self.assertIs(constraint[1], operator.le)
self.assertEqual(constraint[2], otf)
self.assertEqual(constraint[0](m1, m2),
float(migration_time + 3) / (migration_time + 5))

76
tests/local/overload/test_otf.py Normal file
View File

@ -0,0 +1,76 @@
# Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from mocktest import *
from pyqcy import *
import neat.local.overload.otf as otf
class Otf(TestCase):
pass
# (def host {:mips 3000})
# (def time-step 300) ;in seconds
# (def migration-time 20)
# (def vm1 {:mips 2000
# :utilization [0.5 0.6 0.4]})
# (def vm2 {:mips 2000
# :utilization [0.2 0.3 0.6]})
# (def vms (list vm1 vm2))
# (fact
# (otf 0.5 0 0 host [{:mips 3000
# :utilization [0.9 0.8 1.1 1.2 1.3]}]) => true
# (otf 0.5 0 0 host [{:mips 3000
# :utilization [0.9 0.8 1.1 1.2 0.3]}]) => false)
# def test_init_state(self):
# state = c.init_state([20, 40], 2)
# self.assertEquals(state['previous_state'], 0)
# self.assertTrue('request_windows' in state)
# self.assertTrue('estimate_windows' in state)
# self.assertTrue('variances' in state)
# self.assertTrue('acceptable_variances' in state)
# def test_utilization_to_state(self):
# state_config = [0.4, 0.7]
# self.assertEqual(c.utilization_to_state(state_config, 0.0), 0)
# self.assertEqual(c.utilization_to_state(state_config, 0.1), 0)
# self.assertEqual(c.utilization_to_state(state_config, 0.2), 0)
# self.assertEqual(c.utilization_to_state(state_config, 0.3), 0)
# self.assertEqual(c.utilization_to_state(state_config, 0.4), 1)
# self.assertEqual(c.utilization_to_state(state_config, 0.5), 1)
# self.assertEqual(c.utilization_to_state(state_config, 0.6), 1)
# self.assertEqual(c.utilization_to_state(state_config, 0.7), 2)
# self.assertEqual(c.utilization_to_state(state_config, 0.8), 2)
# self.assertEqual(c.utilization_to_state(state_config, 0.9), 2)
# self.assertEqual(c.utilization_to_state(state_config, 1.0), 2)
# self.assertEqual(c.utilization_to_state(state_config, 1.1), 2)
# self.assertEqual(c.utilization_to_state([1.0], 0.0), 0)
# self.assertEqual(c.utilization_to_state([1.0], 0.1), 0)
# self.assertEqual(c.utilization_to_state([1.0], 0.2), 0)
# self.assertEqual(c.utilization_to_state([1.0], 0.3), 0)
# self.assertEqual(c.utilization_to_state([1.0], 0.4), 0)
# self.assertEqual(c.utilization_to_state([1.0], 0.5), 0)
# self.assertEqual(c.utilization_to_state([1.0], 0.6), 0)
# self.assertEqual(c.utilization_to_state([1.0], 0.7), 0)
# self.assertEqual(c.utilization_to_state([1.0], 0.8), 0)
# self.assertEqual(c.utilization_to_state([1.0], 0.9), 0)
# self.assertEqual(c.utilization_to_state([1.0], 1.0), 1)
# self.assertEqual(c.utilization_to_state([1.0], 1.1), 1)

6
tests/test_common.py
View File

@ -65,3 +65,9 @@ class Common(TestCase):
expect(common).physical_cpu_count(connection).and_return(x).once()
expect(common).physical_cpu_mhz(connection).and_return(y).once()
assert common.physical_cpu_mhz_total(connection) == x * y
def test_frange(self):
self.assertEqual([round(x, 1) for x in common.frange(0, 1.0, 0.5)],
[0.0, 0.5, 1.0])
self.assertEqual([round(x, 1) for x in common.frange(0, 1.0, 0.2)],
[0.0, 0.2, 0.4, 0.6, 0.8, 1.0])