From c402b72d2f8fa01ecc8bd6a609543eea719868f5 Mon Sep 17 00:00:00 2001 From: LingxianKong Date: Tue, 5 May 2015 12:10:48 +0800 Subject: [PATCH] Remove contract package dependency Change-Id: I16f8430289447575b4c22b809a7a7da6521f2db8 --- .testr.conf | 2 +- terracotta/common.py | 55 -------- terracotta/globals/db_cleaner.py | 42 ++---- .../globals/vm_placement/bin_packing.py | 60 ++------ terracotta/locals/overload/mhod/bruteforce.py | 46 +------ terracotta/locals/overload/mhod/core.py | 130 +++++------------- terracotta/locals/overload/mhod/l_2_states.py | 24 +--- .../overload/mhod/multisize_estimation.py | 110 +-------------- terracotta/locals/overload/mhod/nlp.py | 30 ---- terracotta/locals/overload/otf.py | 55 ++------ terracotta/locals/overload/statistics.py | 125 ----------------- terracotta/locals/overload/trivial.py | 45 ------ terracotta/locals/underload/trivial.py | 45 ------ terracotta/locals/vm_selection/algorithms.py | 59 -------- terracotta/rpc.py | 12 +- terracotta/tests/functional/__init__.py | 0 terracotta/tests/unit/__init__.py | 0 17 files changed, 91 insertions(+), 749 deletions(-) create mode 100644 terracotta/tests/functional/__init__.py create mode 100644 terracotta/tests/unit/__init__.py diff --git a/.testr.conf b/.testr.conf index bb49f15..fd2c04b 100644 --- a/.testr.conf +++ b/.testr.conf @@ -3,7 +3,7 @@ test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} \ OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \ - ${PYTHON:-python} -m subunit.run discover -t ./ ./mistral/tests/unit $LISTOPT $IDOPTION + ${PYTHON:-python} -m subunit.run discover -t ./ ./terracotta/tests/unit $LISTOPT $IDOPTION test_id_option=--load-list $IDFILE test_list_option=--list diff --git a/terracotta/common.py b/terracotta/common.py index a8cab1b..57408d9 100644 --- a/terracotta/common.py +++ b/terracotta/common.py @@ -22,108 +22,71 @@ import re import subprocess import time -from contracts import contract -from terracotta.contracts_primitive import * -from terracotta.contracts_extra import * - - -@contract def build_local_vm_path(local_data_directory): """ Build the path to the local VM data directory. :param local_data_directory: The base local data path. - :type local_data_directory: str - :return: The path to the local VM data directory. - :rtype: str """ return os.path.join(local_data_directory, 'vms') -@contract def build_local_host_path(local_data_directory): """ Build the path to the local host data file. :param local_data_directory: The base local data path. - :type local_data_directory: str - :return: The path to the local host data file. - :rtype: str """ return os.path.join(local_data_directory, 'host') -@contract def physical_cpu_count(vir_connection): """ Get the number of physical CPUs using libvirt. :param vir_connection: A libvirt connection object. - :type vir_connection: virConnect - :return: The number of physical CPUs. - :rtype: int """ return vir_connection.getInfo()[2] -@contract def physical_cpu_mhz(vir_connection): """ Get the CPU frequency in MHz using libvirt. :param vir_connection: A libvirt connection object. - :type vir_connection: virConnect - :return: The CPU frequency in MHz. - :rtype: int """ return vir_connection.getInfo()[3] -@contract def physical_cpu_mhz_total(vir_connection): """ Get the sum of the core CPU frequencies in MHz using libvirt. :param vir_connection: A libvirt connection object. - :type vir_connection: virConnect - :return: The total CPU frequency in MHz. - :rtype: int """ return physical_cpu_count(vir_connection) * \ physical_cpu_mhz(vir_connection) -@contract def frange(start, end, step): """ A range generator for floats. :param start: The starting value. - :type start: number - :param end: The end value. - :type end: number - :param step: The step. - :type step: number """ while start <= end: yield start start += step -@contract def call_function_by_name(name, args): """ Call a function specified by a fully qualified name. :param name: A fully qualified name of a function. - :type name: str - :param args: A list of positional arguments of the function. - :type args: list - :return: The return value of the function call. - :rtype: * """ fragments = name.split('.') module = '.'.join(fragments[:-1]) @@ -133,58 +96,40 @@ def call_function_by_name(name, args): return getattr(m, function)(*args) -@contract def parse_parameters(params): """ Parse algorithm parameters from the config file. :param params: JSON encoded parameters. - :type params: str - :return: A dict of parameters. - :rtype: dict(str: *) """ return dict((str(k), v) for k, v in json.loads(params).items()) -@contract def parse_compute_hosts(compute_hosts): """ Transform a coma-separated list of host names into a list. :param compute_hosts: A coma-separated list of host names. - :type compute_hosts: str - :return: A list of host names. - :rtype: list(str) """ return filter(None, re.split('[^a-zA-Z0-9\-_]+', compute_hosts)) -@contract def calculate_migration_time(vms, bandwidth): """ Calculate the mean migration time from VM RAM usage data. :param vms: A map of VM UUIDs to the corresponding maximum RAM in MB. - :type vms: dict(str: long) - :param bandwidth: The network bandwidth in MB/s. - :type bandwidth: float,>0 - :return: The mean VM migration time in seconds. - :rtype: float """ return float(numpy.mean(vms.values()) / bandwidth) -@contract def execute_on_hosts(hosts, commands): """ Execute Shell command on hosts over SSH. :param hosts: A list of host names. - :type hosts: list(str) - :param commands: A list of Shell commands. - :type commands: list(str) """ commands_merged = '' for command in commands: diff --git a/terracotta/globals/db_cleaner.py b/terracotta/globals/db_cleaner.py index f6cb55a..b3b3161 100644 --- a/terracotta/globals/db_cleaner.py +++ b/terracotta/globals/db_cleaner.py @@ -1,4 +1,5 @@ # Copyright 2012 Anton Beloglazov +# Copyright 2015 Huawei Technologies Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,26 +20,23 @@ by VMs stored in the database. This is requried to avoid excess growth of the database size. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * - import datetime -import neat.common as common -from neat.config import * -from neat.db_utils import * +from oslo_config import cfg +from oslo_log import log as logging -import logging -log = logging.getLogger(__name__) +import terracotta.common as common +from terracotta.config import * +from terracotta.utils.db_utils import * + + +LOG = logging.getLogger(__name__) -@contract def start(): """ Start the database cleaner loop. :return: The final state. - :rtype: dict(str: *) """ config = read_and_validate_config([DEFAULT_CONFIG_PATH, CONFIG_PATH], REQUIRED_FIELDS) @@ -49,9 +47,8 @@ def start(): int(config['log_level'])) interval = config['db_cleaner_interval'] - if log.isEnabledFor(logging.INFO): - log.info('Starting the database cleaner, ' + - 'iterations every %s seconds', interval) + LOG.info('Starting the database cleaner, ' + + 'iterations every %s seconds', interval) return common.start( init_state, execute, @@ -59,15 +56,11 @@ def start(): int(interval)) -@contract def init_state(config): """ Initialize a dict for storing the state of the database cleaner. :param config: A config dictionary. - :type config: dict(str: *) - :return: A dictionary containing the initial state of the database cleaner. - :rtype: dict """ return { 'db': init_db(config['sql_connection']), @@ -75,33 +68,24 @@ def init_state(config): seconds=int(config['db_cleaner_interval']))} -@contract def execute(config, state): """ Execute an iteration of the database cleaner. :param config: A config dictionary. - :type config: dict(str: *) - :param state: A state dictionary. - :type state: dict(str: *) - :return: The updated state dictionary. - :rtype: dict(str: *) """ datetime_threshold = today() - state['time_delta'] state['db'].cleanup_vm_resource_usage(datetime_threshold) state['db'].cleanup_host_resource_usage(datetime_threshold) - if log.isEnabledFor(logging.INFO): - log.info('Cleaned up data older than %s', - datetime_threshold.strftime('%Y-%m-%d %H:%M:%S')) + LOG.info('Cleaned up data older than %s', + datetime_threshold.strftime('%Y-%m-%d %H:%M:%S')) return state -@contract def today(): """ Return the today's datetime. :return: A datetime object representing current date and time. - :rtype: datetime """ return datetime.datetime.today() diff --git a/terracotta/globals/vm_placement/bin_packing.py b/terracotta/globals/vm_placement/bin_packing.py index 325a157..83bed9c 100644 --- a/terracotta/globals/vm_placement/bin_packing.py +++ b/terracotta/globals/vm_placement/bin_packing.py @@ -1,4 +1,5 @@ # Copyright 2012 Anton Beloglazov +# Copyright 2015 Huawei Technologies Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,29 +16,19 @@ """ Bin Packing based VM placement algorithms. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * - -import logging -log = logging.getLogger(__name__) +from oslo_log import log as logging + + +LOG = logging.getLogger(__name__) -@contract def best_fit_decreasing_factory(time_step, migration_time, params): """ Creates the Best Fit Decreasing (BFD) heuristic for VM placement. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the BFD algorithm. - :rtype: function """ return lambda hosts_cpu_usage, hosts_cpu_total, \ hosts_ram_usage, hosts_ram_total, \ @@ -60,64 +51,39 @@ def best_fit_decreasing_factory(time_step, migration_time, params): {}) -@contract def get_available_resources(threshold, usage, total): """ Get a map of the available resource capacity. :param threshold: A threshold on the maximum allowed resource usage. - :type threshold: float,>=0 - :param usage: A map of hosts to the resource usage. - :type usage: dict(str: number) - :param total: A map of hosts to the total resource capacity. - :type total: dict(str: number) - :return: A map of hosts to the available resource capacity. - :rtype: dict(str: int) """ return dict((host, int(threshold * total[host] - resource)) for host, resource in usage.items()) -@contract def best_fit_decreasing(last_n_vm_cpu, hosts_cpu, hosts_ram, inactive_hosts_cpu, inactive_hosts_ram, vms_cpu, vms_ram): """ The Best Fit Decreasing (BFD) heuristic for placing VMs on hosts. :param last_n_vm_cpu: The last n VM CPU usage values to average. - :type last_n_vm_cpu: int - :param hosts_cpu: A map of host names and their available CPU in MHz. - :type hosts_cpu: dict(str: int) - :param hosts_ram: A map of host names and their available RAM in MB. - :type hosts_ram: dict(str: int) - :param inactive_hosts_cpu: A map of inactive hosts and available CPU MHz. - :type inactive_hosts_cpu: dict(str: int) - :param inactive_hosts_ram: A map of inactive hosts and available RAM MB. - :type inactive_hosts_ram: dict(str: int) - :param vms_cpu: A map of VM UUID and their CPU utilization in MHz. - :type vms_cpu: dict(str: list(int)) - :param vms_ram: A map of VM UUID and their RAM usage in MB. - :type vms_ram: dict(str: int) - :return: A map of VM UUIDs to host names, or {} if cannot be solved. - :rtype: dict(str: str) """ - if log.isEnabledFor(logging.DEBUG): - log.debug('last_n_vm_cpu: %s', str(last_n_vm_cpu)) - log.debug('hosts_cpu: %s', str(hosts_cpu)) - log.debug('hosts_ram: %s', str(hosts_ram)) - log.debug('inactive_hosts_cpu: %s', str(inactive_hosts_cpu)) - log.debug('inactive_hosts_ram: %s', str(inactive_hosts_ram)) - log.debug('vms_cpu: %s', str(vms_cpu)) - log.debug('vms_ram: %s', str(vms_ram)) + LOG.debug('last_n_vm_cpu: %s', str(last_n_vm_cpu)) + LOG.debug('hosts_cpu: %s', str(hosts_cpu)) + LOG.debug('hosts_ram: %s', str(hosts_ram)) + LOG.debug('inactive_hosts_cpu: %s', str(inactive_hosts_cpu)) + LOG.debug('inactive_hosts_ram: %s', str(inactive_hosts_ram)) + LOG.debug('vms_cpu: %s', str(vms_cpu)) + LOG.debug('vms_ram: %s', str(vms_ram)) vms_tmp = [] for vm, cpu in vms_cpu.items(): if cpu: @@ -126,7 +92,7 @@ def best_fit_decreasing(last_n_vm_cpu, hosts_cpu, hosts_ram, vms_ram[vm], vm)) else: - log.warning('No CPU data for VM: %s - skipping', vm) + LOG.warning('No CPU data for VM: %s - skipping', vm) vms = sorted(vms_tmp, reverse=True) hosts = sorted(((v, hosts_ram[k], k) diff --git a/terracotta/locals/overload/mhod/bruteforce.py b/terracotta/locals/overload/mhod/bruteforce.py index 814573e..a1cd611 100644 --- a/terracotta/locals/overload/mhod/bruteforce.py +++ b/terracotta/locals/overload/mhod/bruteforce.py @@ -1,4 +1,5 @@ # Copyright 2012 Anton Beloglazov +# Copyright 2015 Huawei Technologies Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,35 +16,18 @@ """ Functions for solving NLP problems using the bruteforce method. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * - import nlp -from neat.common import frange - -import logging -log = logging.getLogger(__name__) +from terracotta.common import frange -@contract def solve2(objective, constraint, step, limit): """ Solve a maximization problem for 2 states. :param objective: The objective function. - :type objective: function - :param constraint: A tuple representing the constraint. - :type constraint: tuple(function, function, number) - :param step: The step size. - :type step: number,>0 - :param limit: The maximum value of the variables. - :type limit: number,>0 - :return: The problem solution. - :rtype: list(number) """ res_best = 0 solution = [] @@ -52,48 +36,28 @@ def solve2(objective, constraint, step, limit): try: res = objective(x, y) if res > res_best and \ - constraint[1](constraint[0](x, y), constraint[2]): - res_best = res - solution = [x, y] + constraint[1](constraint[0](x, y), constraint[2]): + res_best = res + solution = [x, y] except ZeroDivisionError: pass return solution -@contract def optimize(step, limit, otf, migration_time, ls, p, state_vector, time_in_states, time_in_state_n): """ Solve a MHOD optimization problem. :param step: The step size for the bruteforce algorithm. - :type step: number,>0 - :param limit: The maximum value of the variables. - :type limit: number,>0 - :param otf: The OTF parameter. - :type otf: number,>=0,<=1 - :param migration_time: The VM migration time in time steps. - :type migration_time: float,>=0 - :param ls: L functions. - :type ls: list(function) - :param p: A matrix of transition probabilities. - :type p: list(list(number)) - :param state_vector: A state vector. - :type state_vector: list(int) - :param time_in_states: The total time in all the states in time steps. - :type time_in_states: number,>=0 - :param time_in_state_n: The total time in the state N in time steps. - :type time_in_state_n: number,>=0 - :return: The solution of the problem. - :rtype: list(number) """ objective = nlp.build_objective(ls, state_vector, p) constraint = nlp.build_constraint(otf, migration_time, ls, state_vector, diff --git a/terracotta/locals/overload/mhod/core.py b/terracotta/locals/overload/mhod/core.py index 5131176..8381dd6 100644 --- a/terracotta/locals/overload/mhod/core.py +++ b/terracotta/locals/overload/mhod/core.py @@ -1,4 +1,5 @@ # Copyright 2012 Anton Beloglazov +# Copyright 2015 Huawei Technologies Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,34 +16,25 @@ """ This is the main module of the MHOD algorithm. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * +from oslo_log import log as logging -import neat.locals.overload.mhod.multisize_estimation as estimation -import neat.locals.overload.mhod.bruteforce as bruteforce -from neat.locals.overload.mhod.l_2_states import ls - -import logging -log = logging.getLogger(__name__) +import terracotta.locals.overload.mhod.multisize_estimation as estimation +import terracotta.locals.overload.mhod.bruteforce as bruteforce +from terracotta.locals.overload.mhod.l_2_states import ls + + +LOG = logging.getLogger(__name__) -@contract def mhod_factory(time_step, migration_time, params): """ Creates the MHOD algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the MHOD algorithm. - :rtype: function """ + def mhod_wrapper(utilization, state=None): if not state: state = init_state(params['history_size'], @@ -57,24 +49,17 @@ def mhod_factory(time_step, migration_time, params): migration_time, utilization, state) + return mhod_wrapper -@contract def init_state(history_size, window_sizes, number_of_states): """ Initialize the state dictionary of the MHOD algorithm. :param history_size: The number of last system states to store. - :type history_size: int,>0 - :param window_sizes: The required window sizes. - :type window_sizes: list(int) - :param number_of_states: The number of states. - :type number_of_states: int,>0 - :return: The initialization state dictionary. - :rtype: dict(str: *) """ return { 'previous_state': 0, @@ -91,47 +76,29 @@ def init_state(history_size, window_sizes, number_of_states): window_sizes, number_of_states)} -@contract def mhod(state_config, otf, window_sizes, bruteforce_step, learning_steps, time_step, migration_time, utilization, state): """ The MHOD algorithm returning whether the host is overloaded. :param state_config: The state configuration. - :type state_config: list(float) - :param otf: The OTF parameter. - :type otf: float,>0 - :param window_sizes: A list of window sizes. - :type window_sizes: list(int) - :param bruteforce_step: The step of the bruteforce algorithm. - :type bruteforce_step: float - :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param utilization: The history of the host's CPU utilization. - :type utilization: list(float) - :param state: The state of the algorithm. - :type state: dict - :return: The updated state and decision of the algorithm. - :rtype: tuple(bool, dict) """ utilization_length = len(utilization) -# if utilization_length == state['time_in_states'] and \ -# utilization == state['previous_utilization']: -# # No new utilization values -# return False, state + # if utilization_length == state['time_in_states'] and \ + # utilization == state['previous_utilization']: + # # No new utilization values + # return False, state number_of_states = len(state_config) + 1 previous_state = 0 -# state['previous_utilization'] = utilization + # state['previous_utilization'] = utilization state['request_windows'] = estimation.init_request_windows( number_of_states, max(window_sizes)) state['estimate_windows'] = estimation.init_deque_structure( @@ -141,7 +108,8 @@ def mhod(state_config, otf, window_sizes, bruteforce_step, learning_steps, state['acceptable_variances'] = estimation.init_variances( window_sizes, number_of_states) - for i, current_state in enumerate(utilization_to_states(state_config, utilization)): + for i, current_state in enumerate( + utilization_to_states(state_config, utilization)): state['request_windows'] = estimation.update_request_windows( state['request_windows'], previous_state, @@ -176,73 +144,59 @@ def mhod(state_config, otf, window_sizes, bruteforce_step, learning_steps, state['previous_state'] = current_state state_n = len(state_config) -# if utilization_length > state['time_in_states'] + 1: -# for s in utilization_to_states( -# state_config, -# utilization[-(utilization_length - state['time_in_states']):]): -# state['time_in_states'] += 1 -# if s == state_n: -# state['time_in_state_n'] += 1 -# else: + # if utilization_length > state['time_in_states'] + 1: + # for s in utilization_to_states( + # state_config, + # utilization[-(utilization_length - state['time_in_states']):]): + # state['time_in_states'] += 1 + # if s == state_n: + # state['time_in_state_n'] += 1 + # else: state['time_in_states'] += 1 if current_state == state_n: state['time_in_state_n'] += 1 - if log.isEnabledFor(logging.DEBUG): - log.debug('MHOD utilization:' + str(utilization)) - log.debug('MHOD time_in_states:' + str(state['time_in_states'])) - log.debug('MHOD time_in_state_n:' + str(state['time_in_state_n'])) - log.debug('MHOD p:' + str(p)) - log.debug('MHOD current_state:' + str(current_state)) - log.debug('MHOD p[current_state]:' + str(p[current_state])) + LOG.debug('MHOD utilization:' + str(utilization)) + LOG.debug('MHOD time_in_states:' + str(state['time_in_states'])) + LOG.debug('MHOD time_in_state_n:' + str(state['time_in_state_n'])) + LOG.debug('MHOD p:' + str(p)) + LOG.debug('MHOD current_state:' + str(current_state)) + LOG.debug('MHOD p[current_state]:' + str(p[current_state])) if utilization_length >= learning_steps: if current_state == state_n and p[state_n][state_n] > 0: # if p[current_state][state_n] > 0: policy = bruteforce.optimize( bruteforce_step, 1.0, otf, (migration_time / time_step), ls, p, - state_vector, state['time_in_states'], state['time_in_state_n']) + state_vector, state['time_in_states'], + state['time_in_state_n']) # This is saved for testing purposes state['policy'] = policy - if log.isEnabledFor(logging.DEBUG): - log.debug('MHOD policy:' + str(policy)) + LOG.debug('MHOD policy:' + str(policy)) command = issue_command_deterministic(policy) - if log.isEnabledFor(logging.DEBUG): - log.debug('MHOD command:' + str(command)) + LOG.debug('MHOD command:' + str(command)) return command, state return False, state -@contract def build_state_vector(state_config, utilization): """ Build the current state PMF corresponding to the utilization history and state config. :param state_config: The state configuration. - :type state_config: list(float) - :param utilization: The history of the host's CPU utilization. - :type utilization: list(float) - :return: The current state vector. - :rtype: list(int) """ state = utilization_to_state(state_config, utilization[-1]) return [int(state == x) for x in range(len(state_config) + 1)] -@contract def utilization_to_state(state_config, utilization): """ Transform a utilization value into the corresponding state. :param state_config: The state configuration. - :type state_config: list(float) - :param utilization: A utilization value. - :type utilization: number,>=0 - :return: The state corresponding to the utilization value. - :rtype: int """ prev = -1 for state, threshold in enumerate(state_config): @@ -252,20 +206,15 @@ def utilization_to_state(state_config, utilization): return prev + 1 -@contract def get_current_state(state_vector): """ Get the current state corresponding to the state probability vector. :param state_vector: The state PMF vector. - :type state_vector: list(int) - :return: The current state. - :rtype: int,>=0 """ return state_vector.index(1) -@contract def utilization_to_states(state_config, utilization): """ Get the state history corresponding to the utilization history. @@ -274,25 +223,16 @@ def utilization_to_states(state_config, utilization): (map (partial utilization-to-state state-config) utilization)) :param state_config: The state configuration. - :type state_config: list(float) - :param utilization: The history of the host's CPU utilization. - :type utilization: list(float) - :return: The state history. - :rtype: list(int) """ return [utilization_to_state(state_config, x) for x in utilization] -@contract def issue_command_deterministic(policy): """ Issue a migration command according to the policy PMF p. :param policy: A policy PMF. - :type policy: list(number) - :return: A migration command. - :rtype: bool """ return len(policy) == 0 diff --git a/terracotta/locals/overload/mhod/l_2_states.py b/terracotta/locals/overload/mhod/l_2_states.py index bede123..9872967 100644 --- a/terracotta/locals/overload/mhod/l_2_states.py +++ b/terracotta/locals/overload/mhod/l_2_states.py @@ -1,4 +1,5 @@ # Copyright 2012 Anton Beloglazov +# Copyright 2015 Huawei Technologies Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,29 +16,14 @@ """ L functions for the 2 state configuration of the MHOD algorithm. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * -import logging -log = logging.getLogger(__name__) - - -@contract def l0(p_initial, p_matrix, m): """ Compute the L0 function. :param p_initial: The initial state distribution. - :type p_initial: list(number) - :param p_matrix: A matrix of transition probabilities. - :type p_matrix: list(list(number)) - :param m: The m values. - :type m: list(number) - :return: The value of the L0 function. - :rtype: number """ p0 = p_initial[0] p1 = p_initial[1] @@ -53,21 +39,13 @@ def l0(p_initial, p_matrix, m): p10 - 1)) -@contract def l1(p_initial, p_matrix, m): """ Compute the L1 function. :param p_initial: The initial state distribution. - :type p_initial: list(number) - :param p_matrix: A matrix of transition probabilities. - :type p_matrix: list(list(number)) - :param m: The m values. - :type m: list(number) - :return: The value of the L1 function. - :rtype: number """ p0 = p_initial[0] p1 = p_initial[1] diff --git a/terracotta/locals/overload/mhod/multisize_estimation.py b/terracotta/locals/overload/mhod/multisize_estimation.py index e132532..fb9d9b7 100644 --- a/terracotta/locals/overload/mhod/multisize_estimation.py +++ b/terracotta/locals/overload/mhod/multisize_estimation.py @@ -15,121 +15,72 @@ """ Multisize sliding window workload estimation functions. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * - from itertools import islice from collections import deque -import logging -log = logging.getLogger(__name__) - -@contract def mean(data, window_size): """ Get the data mean according to the window size. :param data: A list of values. - :type data: list(number) - :param window_size: A window size. - :type window_size: int,>0 - :return: The mean value. - :rtype: float """ return float(sum(data)) / window_size -@contract def variance(data, window_size): """ Get the data variance according to the window size. :param data: A list of values. - :type data: list(number) - :param window_size: A window size. - :type window_size: int,>0 - :return: The variance value. - :rtype: float """ m = mean(data, window_size) return float(sum((x - m) ** 2 for x in data)) / (window_size - 1) -@contract def acceptable_variance(probability, window_size): """ Get the acceptable variance. :param probability: The probability to use. - :type probability: number,>=0,<=1 - :param window_size: A window size. - :type window_size: int,>0 - :return: The acceptable variance. - :rtype: float """ return float(probability * (1 - probability)) / window_size -@contract def estimate_probability(data, window_size, state): """ Get the estimated probability. :param data: A list of data values. - :type data: list(number) - :param window_size: The window size. - :type window_size: int,>0 - :param state: The current state. - :type state: int,>=0 - :return: The estimated probability. - :rtype: float,>=0 """ return float(data.count(state)) / window_size -@contract def update_request_windows(request_windows, previous_state, current_state): """ Update and return the updated request windows. :param request_windows: The previous request windows. - :type request_windows: list(deque) - :param previous_state: The previous state. - :type previous_state: int,>=0 - :param current_state: The current state. - :type current_state: int,>=0 - :return: The updated request windows. - :rtype: list(deque) """ request_windows[previous_state].append(current_state) return request_windows -@contract def update_estimate_windows(estimate_windows, request_windows, previous_state): """ Update and return the updated estimate windows. :param estimate_windows: The previous estimate windows. - :type estimate_windows: list(list(dict)) - :param request_windows: The current request windows. - :type request_windows: list(deque) - :param previous_state: The previous state. - :type previous_state: int,>=0 - :return: The updated estimate windows. - :rtype: list(list(dict)) """ request_window = request_windows[previous_state] state_estimate_windows = estimate_windows[previous_state] @@ -145,21 +96,13 @@ def update_estimate_windows(estimate_windows, request_windows, return estimate_windows -@contract def update_variances(variances, estimate_windows, previous_state): """ Updated and return the updated variances. :param variances: The previous variances. - :type variances: list(list(dict)) - :param estimate_windows: The current estimate windows. - :type estimate_windows: list(list(dict)) - :param previous_state: The previous state. - :type previous_state: int,>=0 - :return: The updated variances. - :rtype: list(list(dict)) """ estimate_window = estimate_windows[previous_state] for state, variance_map in enumerate(variances[previous_state]): @@ -173,21 +116,14 @@ def update_variances(variances, estimate_windows, previous_state): return variances -@contract -def update_acceptable_variances(acceptable_variances, estimate_windows, previous_state): +def update_acceptable_variances(acceptable_variances, estimate_windows, + previous_state): """ Update and return the updated acceptable variances. :param acceptable_variances: The previous acceptable variances. - :type acceptable_variances: list(list(dict)) - :param estimate_windows: The current estimate windows. - :type estimate_windows: list(list(dict)) - :param previous_state: The previous state. - :type previous_state: int,>=0 - :return: The updated acceptable variances. - :rtype: list(list(dict)) """ estimate_window = estimate_windows[previous_state] state_acc_variances = acceptable_variances[previous_state] @@ -199,21 +135,13 @@ def update_acceptable_variances(acceptable_variances, estimate_windows, previous return acceptable_variances -@contract def select_window(variances, acceptable_variances, window_sizes): """ Select window sizes according to the acceptable variances. :param variances: The variances. - :type variances: list(list(dict)) - :param acceptable_variances: The acceptable variances. - :type acceptable_variances: list(list(dict)) - :param window_sizes: The available window sizes. - :type window_sizes: list(int) - :return: The selected window sizes. - :rtype: list(list(int)) """ n = len(variances) selected_windows = [] @@ -223,25 +151,19 @@ def select_window(variances, acceptable_variances, window_sizes): selected_size = window_sizes[0] for window_size in window_sizes: if variances[i][j][window_size] > \ - acceptable_variances[i][j][window_size]: - break + acceptable_variances[i][j][window_size]: + break selected_size = window_size selected_windows[i].append(selected_size) return selected_windows -@contract def select_best_estimates(estimate_windows, selected_windows): """ Select the best estimates according to the selected windows. :param estimate_windows: The estimate windows. - :type estimate_windows: list(list(dict)) - :param selected_windows: The selected window sizes. - :type selected_windows: list(list(int)) - :return: The selected best estimates. - :rtype: list(list(number)) """ n = len(estimate_windows) selected_estimates = [] @@ -256,35 +178,23 @@ def select_best_estimates(estimate_windows, selected_windows): return selected_estimates -@contract def init_request_windows(number_of_states, max_window_size): """ Initialize a request window data structure. :param number_of_states: The number of states. - :type number_of_states: int,>0 - :param max_window_size: The max size of the request windows. - :type max_window_size: int,>0 - :return: The initialized request windows data structure. - :rtype: list(deque) """ return [deque([], max_window_size) for _ in range(number_of_states)] -@contract def init_variances(window_sizes, number_of_states): """ Initialize a variances data structure. :param window_sizes: The required window sizes. - :type window_sizes: list(int) - :param number_of_states: The number of states. - :type number_of_states: int,>0 - :return: The initialized variances data structure. - :rtype: list(list(dict)) """ variances = [] for i in range(number_of_states): @@ -295,18 +205,12 @@ def init_variances(window_sizes, number_of_states): return variances -@contract def init_deque_structure(window_sizes, number_of_states): """ Initialize a 3 level deque data structure. :param window_sizes: The required window sizes. - :type window_sizes: list(int) - :param number_of_states: The number of states. - :type number_of_states: int,>0 - :return: The initialized 3 level deque data structure. - :rtype: list(list(dict)) """ structure = [] for i in range(number_of_states): @@ -317,18 +221,12 @@ def init_deque_structure(window_sizes, number_of_states): return structure -@contract def init_selected_window_sizes(window_sizes, number_of_states): """ Initialize a selected window sizes data structure. :param window_sizes: The required window sizes. - :type window_sizes: list(int) - :param number_of_states: The number of states. - :type number_of_states: int,>0 - :return: The initialized selected window sizes data structure. - :rtype: list(list(int)) """ structure = [] for i in range(number_of_states): diff --git a/terracotta/locals/overload/mhod/nlp.py b/terracotta/locals/overload/mhod/nlp.py index 0290ecc..018251f 100644 --- a/terracotta/locals/overload/mhod/nlp.py +++ b/terracotta/locals/overload/mhod/nlp.py @@ -15,31 +15,16 @@ """ Functions for defing the NLP problem of the MHOD algorithm. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * - import operator -import logging -log = logging.getLogger(__name__) - -@contract def build_objective(ls, state_vector, p): """ Creates an objective function, which is a sum of the L functions. :param ls: A list of L functions. - :type ls: list(function) - :param state-vector: A state vector. - :type state-vector: list(int) - :param p: A matrix of transition probabilities. - :type p: list(list(number)) - :return: An objective function. - :rtype: function """ def objective(*m): return sum(l(state_vector, p, list(m)) for l in ls) @@ -52,28 +37,13 @@ def build_constraint(otf, migration_time, ls, state_vector, """ Creates an optimization constraint from the L functions. :param otf: The OTF parameter. - :type otf: float - :param migration_time: The VM migration time in time steps. - :type migration_time: float,>=0 - :param ls: A list of L functions. - :type ls: list(function) - :param state-vector: A state vector. - :type state-vector: list(int) - :param p: A matrix of transition probabilities. - :type p: list(list(number)) - :param time_in_states: The total time in all the states in time steps. - :type time_in_states: number,>=0 - :param time_in_state_n: The total time in the state N in time steps. - :type time_in_state_n: number,>=0 - :return: The created constraint. - :rtype: tuple(function, function, number) """ def constraint(*m): m_list = list(m) diff --git a/terracotta/locals/overload/otf.py b/terracotta/locals/overload/otf.py index 4227a47..4fcfd77 100644 --- a/terracotta/locals/overload/otf.py +++ b/terracotta/locals/overload/otf.py @@ -15,29 +15,19 @@ """ OTF threshold based algorithms. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * - -import logging -log = logging.getLogger(__name__) +from oslo_log import log as logging + + +LOG = logging.getLogger(__name__) -@contract def otf_factory(time_step, migration_time, params): """ Creates the OTF algorithm with limiting and migration time. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the OTF algorithm. - :rtype: function """ migration_time_normalized = float(migration_time) / time_step def otf_wrapper(utilization, state=None): @@ -54,48 +44,33 @@ def otf_factory(time_step, migration_time, params): return otf_wrapper -@contract def otf(otf, threshold, limit, migration_time, utilization, state): """ The OTF threshold algorithm with limiting and migration time. :param otf: The threshold on the OTF value. - :type otf: float,>=0 - :param threshold: The utilization overload threshold. - :type threshold: float,>=0 - :param limit: The minimum number of values in the utilization history. - :type limit: int,>=0 - :param migration_time: The VM migration time in time steps. - :type migration_time: float,>=0 - :param utilization: The history of the host's CPU utilization. - :type utilization: list(float) - :param state: The state dictionary. - :type state: dict(str: *) - :return: The decision of the algorithm and updated state. - :rtype: tuple(bool, dict(*: *)) """ state['total'] += 1 overload = (utilization[-1] >= threshold) if overload: state['overload'] += 1 - if log.isEnabledFor(logging.DEBUG): - log.debug('OTF overload:' + str(overload)) - log.debug('OTF overload steps:' + str(state['overload'])) - log.debug('OTF total steps:' + str(state['total'])) - log.debug('OTF:' + str(float(state['overload']) / state['total'])) - log.debug('OTF migration time:' + str(migration_time)) - log.debug('OTF + migration time:' + - str((migration_time + state['overload']) / \ - (migration_time + state['total']))) - log.debug('OTF decision:' + - str(overload and (migration_time + state['overload']) / \ - (migration_time + state['total']) >= otf)) + LOG.debug('OTF overload:' + str(overload)) + LOG.debug('OTF overload steps:' + str(state['overload'])) + LOG.debug('OTF total steps:' + str(state['total'])) + LOG.debug('OTF:' + str(float(state['overload']) / state['total'])) + LOG.debug('OTF migration time:' + str(migration_time)) + LOG.debug('OTF + migration time:' + + str((migration_time + state['overload']) / \ + (migration_time + state['total']))) + LOG.debug('OTF decision:' + + str(overload and (migration_time + state['overload']) / \ + (migration_time + state['total']) >= otf)) if not overload or len(utilization) < limit: decision = False diff --git a/terracotta/locals/overload/statistics.py b/terracotta/locals/overload/statistics.py index db53e21..19960a2 100644 --- a/terracotta/locals/overload/statistics.py +++ b/terracotta/locals/overload/statistics.py @@ -15,33 +15,18 @@ """ Statistics based overload detection algorithms. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * - from numpy import median from scipy.optimize import leastsq import numpy as np -import logging -log = logging.getLogger(__name__) - -@contract def loess_factory(time_step, migration_time, params): """ Creates the Loess based overload detection algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the OTF algorithm. - :rtype: function """ migration_time_normalized = float(migration_time) / time_step return lambda utilization, state=None: \ @@ -53,21 +38,13 @@ def loess_factory(time_step, migration_time, params): {}) -@contract def loess_robust_factory(time_step, migration_time, params): """ Creates the robust Loess based overload detection algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the OTF algorithm. - :rtype: function """ migration_time_normalized = float(migration_time) / time_step return lambda utilization, state=None: \ @@ -79,21 +56,13 @@ def loess_robust_factory(time_step, migration_time, params): {}) -@contract def mad_threshold_factory(time_step, migration_time, params): """ Creates the MAD based utilization threshold algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the static threshold algorithm. - :rtype: function """ return lambda utilization, state=None: \ (mad_threshold(params['threshold'], @@ -102,21 +71,13 @@ def mad_threshold_factory(time_step, migration_time, params): {}) -@contract def iqr_threshold_factory(time_step, migration_time, params): """ Creates the IQR based utilization threshold algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the static threshold algorithm. - :rtype: function """ return lambda utilization, state=None: \ (iqr_threshold(params['threshold'], @@ -125,27 +86,15 @@ def iqr_threshold_factory(time_step, migration_time, params): {}) -@contract def loess(threshold, param, length, migration_time, utilization): """ The Loess based overload detection algorithm. :param threshold: The CPU utilization threshold. - :type threshold: float - :param param: The safety parameter. - :type param: float - :param length: The required length of the utilization history. - :type length: int - :param migration_time: The VM migration time in time steps. - :type migration_time: float - :param utilization: The utilization history to analize. - :type utilization: list(float) - :return: A decision of whether the host is overloaded. - :rtype: bool """ return loess_abstract(loess_parameter_estimates, threshold, @@ -155,27 +104,15 @@ def loess(threshold, param, length, migration_time, utilization): utilization) -@contract def loess_robust(threshold, param, length, migration_time, utilization): """ The robust Loess based overload detection algorithm. :param threshold: The CPU utilization threshold. - :type threshold: float - :param param: The safety parameter. - :type param: float - :param length: The required length of the utilization history. - :type length: int - :param migration_time: The VM migration time in time steps. - :type migration_time: float - :param utilization: The utilization history to analize. - :type utilization: list(float) - :return: A decision of whether the host is overloaded. - :rtype: bool """ return loess_abstract(loess_robust_parameter_estimates, threshold, @@ -185,30 +122,16 @@ def loess_robust(threshold, param, length, migration_time, utilization): utilization) -@contract def loess_abstract(estimator, threshold, param, length, migration_time, utilization): """ The abstract Loess algorithm. :param estimator: A parameter estimation function. - :type estimator: function - :param threshold: The CPU utilization threshold. - :type threshold: float - :param param: The safety parameter. - :type param: float - :param length: The required length of the utilization history. - :type length: int - :param migration_time: The VM migration time in time steps. - :type migration_time: float - :param utilization: The utilization history to analize. - :type utilization: list(float) - :return: A decision of whether the host is overloaded. - :rtype: bool """ if len(utilization) < length: return False @@ -217,92 +140,60 @@ def loess_abstract(estimator, threshold, param, length, migration_time, utilizat return param * prediction >= threshold -@contract def mad_threshold(param, limit, utilization): """ The MAD based threshold algorithm. :param param: The safety parameter. - :type param: float - :param limit: The minimum allowed length of the utilization history. - :type limit: int - :param utilization: The utilization history to analize. - :type utilization: list(float) - :return: A decision of whether the host is overloaded. - :rtype: bool """ return utilization_threshold_abstract(lambda x: 1 - param * mad(x), limit, utilization) -@contract def iqr_threshold(param, limit, utilization): """ The IQR based threshold algorithm. :param param: The safety parameter. - :type param: float - :param limit: The minimum allowed length of the utilization history. - :type limit: int - :param utilization: The utilization history to analize. - :type utilization: list(float) - :return: A decision of whether the host is overloaded. - :rtype: bool """ return utilization_threshold_abstract(lambda x: 1 - param * iqr(x), limit, utilization) -@contract def utilization_threshold_abstract(f, limit, utilization): """ The abstract utilization threshold algorithm. :param f: A function to calculate the utilization threshold. - :type f: function - :param limit: The minimum allowed length of the utilization history. - :type limit: int - :param utilization: The utilization history to analize. - :type utilization: list(float) - :return: A decision of whether the host is overloaded. - :rtype: bool """ if (len(utilization) < limit): return False return f(utilization) <= utilization[-1] -@contract def mad(data): """ Calculate the Median Absolute Deviation from the data. :param data: The data to analyze. - :type data: list(number) - :return: The calculated MAD. - :rtype: float """ data_median = median(data) return float(median([abs(data_median - x) for x in data])) -@contract def iqr(data): """ Calculate the Interquartile Range from the data. :param data: The data to analyze. - :type data: list(number) - :return: The calculated IQR. - :rtype: float """ sorted_data = sorted(data) n = len(data) + 1 @@ -311,15 +202,11 @@ def iqr(data): return float(sorted_data[q3] - sorted_data[q1]) -@contract def loess_parameter_estimates(data): """ Calculate Loess parameter estimates. :param data: A data set. - :type data: list(float) - :return: The parameter estimates. - :rtype: list(float) """ def f(p, x, y, weights): return weights * (y - (p[0] + p[1] * x)) @@ -333,15 +220,11 @@ def loess_parameter_estimates(data): return estimates.tolist() -@contract def loess_robust_parameter_estimates(data): """ Calculate Loess robust parameter estimates. :param data: A data set. - :type data: list(float) - :return: The parameter estimates. - :rtype: list(float) """ def f(p, x, y, weights): return weights * (y - (p[0] + p[1] * x)) @@ -361,15 +244,11 @@ def loess_robust_parameter_estimates(data): return estimates2.tolist() -@contract def tricube_weights(n): """ Generates a list of weights according to the tricube function. :param n: The number of weights to generate. - :type n: int - :return: A list of generated weights. - :rtype: list(float) """ spread = top = float(n - 1) weights = [] @@ -378,15 +257,11 @@ def tricube_weights(n): return [weights[0], weights[0]] + weights -@contract def tricube_bisquare_weights(data): """ Generates a weights according to the tricube bisquare function. :param data: The input data. - :type data: list(float) - :return: A list of generated weights. - :rtype: list(float) """ n = len(data) s6 = 6 * median(map(abs, data)) diff --git a/terracotta/locals/overload/trivial.py b/terracotta/locals/overload/trivial.py index 7825352..8c91059 100644 --- a/terracotta/locals/overload/trivial.py +++ b/terracotta/locals/overload/trivial.py @@ -15,69 +15,38 @@ """ Trivial overload detection algorithms. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * -import logging -log = logging.getLogger(__name__) - - -@contract def never_overloaded_factory(time_step, migration_time, params): """ Creates an algorithm that never considers the host overloaded. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the algorithm. - :rtype: function """ return lambda utilization, state=None: (False, {}) -@contract def threshold_factory(time_step, migration_time, params): """ Creates the static CPU utilization threshold algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the static threshold algorithm. - :rtype: function """ return lambda utilization, state=None: (threshold(params['threshold'], utilization), {}) -@contract def last_n_average_threshold_factory(time_step, migration_time, params): """ Creates the averaging CPU utilization threshold algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the averaging threshold algorithm. - :rtype: function """ return lambda utilization, state=None: ( last_n_average_threshold(params['threshold'], @@ -86,39 +55,25 @@ def last_n_average_threshold_factory(time_step, migration_time, params): {}) -@contract def threshold(threshold, utilization): """ The static CPU utilization threshold algorithm. :param threshold: The threshold on the CPU utilization. - :type threshold: float,>=0 - :param utilization: The history of the host's CPU utilization. - :type utilization: list(float) - :return: The decision of the algorithm. - :rtype: bool """ if utilization: return utilization[-1] > threshold return False -@contract def last_n_average_threshold(threshold, n, utilization): """ The averaging CPU utilization threshold algorithm. :param threshold: The threshold on the CPU utilization. - :type threshold: float,>=0 - :param n: The number of last CPU utilization values to average. - :type n: int,>0 - :param utilization: The history of the host's CPU utilization. - :type utilization: list(float) - :return: The decision of the algorithm. - :rtype: bool """ if utilization: utilization = utilization[-n:] diff --git a/terracotta/locals/underload/trivial.py b/terracotta/locals/underload/trivial.py index 911529e..f779b40 100644 --- a/terracotta/locals/underload/trivial.py +++ b/terracotta/locals/underload/trivial.py @@ -15,68 +15,37 @@ """ Trivial underload detection algorithms. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * -import logging -log = logging.getLogger(__name__) - - -@contract def always_underloaded_factory(time_step, migration_time, params): """ Creates an algorithm that always considers the host underloaded. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the algorithm. - :rtype: function """ return lambda utilization, state=None: (True, {}) -@contract def threshold_factory(time_step, migration_time, params): """ Creates the threshold underload detection algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the OTF algorithm. - :rtype: function """ return lambda utilization, state=None: (threshold(params['threshold'], utilization), {}) -@contract def last_n_average_threshold_factory(time_step, migration_time, params): """ Creates the averaging threshold underload detection algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the averaging underload detection. - :rtype: function """ return lambda utilization, state=None: ( last_n_average_threshold(params['threshold'], @@ -85,7 +54,6 @@ def last_n_average_threshold_factory(time_step, migration_time, params): {}) -@contract def threshold(threshold, utilization): """ Static threshold-based underload detection algorithm. @@ -93,20 +61,14 @@ def threshold(threshold, utilization): CPU utilization is lower than the specified threshold. :param threshold: The static underload CPU utilization threshold. - :type threshold: float,>=0,<=1 - :param utilization: The history of the host's CPU utilization. - :type utilization: list(float) - :return: A decision of whether the host is underloaded. - :rtype: bool """ if utilization: return utilization[-1] <= threshold return False -@contract def last_n_average_threshold(threshold, n, utilization): """ Averaging static threshold-based underload detection algorithm. @@ -114,16 +76,9 @@ def last_n_average_threshold(threshold, n, utilization): the host's CPU utilization is lower than the specified threshold. :param threshold: The static underload CPU utilization threshold. - :type threshold: float,>=0,<=1 - :param n: The number of last values to average. - :type n: int,>0 - :param utilization: The history of the host's CPU utilization. - :type utilization: list(float) - :return: A decision of whether the host is underloaded. - :rtype: bool """ if utilization: utilization = utilization[-n:] diff --git a/terracotta/locals/vm_selection/algorithms.py b/terracotta/locals/vm_selection/algorithms.py index c712762..23c31e6 100644 --- a/terracotta/locals/vm_selection/algorithms.py +++ b/terracotta/locals/vm_selection/algorithms.py @@ -15,91 +15,52 @@ """ VM selection algorithms. """ -from contracts import contract -from neat.contracts_primitive import * -from neat.contracts_extra import * - from random import choice import operator -import logging -log = logging.getLogger(__name__) - -@contract def random_factory(time_step, migration_time, params): """ Creates the random VM selection algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the random VM selection algorithm. - :rtype: function """ return lambda vms_cpu, vms_ram, state=None: ([random(vms_cpu)], {}) -@contract def minimum_utilization_factory(time_step, migration_time, params): """ Creates the minimum utilization VM selection algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the minimum utilization VM selection. - :rtype: function """ return lambda vms_cpu, vms_ram, state=None: \ ([minimum_utilization(vms_cpu)], {}) -@contract def minimum_migration_time_factory(time_step, migration_time, params): """ Creates the minimum migration time VM selection algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the minimum migration time VM selection. - :rtype: function """ return lambda vms_cpu, vms_ram, state=None: \ ([minimum_migration_time(vms_ram)], {}) -@contract def minimum_migration_time_max_cpu_factory(time_step, migration_time, params): """ Creates the minimum migration time / max CPU usage VM selection algorithm. :param time_step: The length of the simulation time step in seconds. - :type time_step: int,>=0 - :param migration_time: The VM migration time in time seconds. - :type migration_time: float,>=0 - :param params: A dictionary containing the algorithm's parameters. - :type params: dict(str: *) - :return: A function implementing the minimum migration time / max CPU VM selection. - :rtype: function """ return lambda vms_cpu, vms_ram, state=None: \ ([minimum_migration_time_max_cpu(params['last_n'], @@ -107,30 +68,22 @@ def minimum_migration_time_max_cpu_factory(time_step, migration_time, params): vms_ram)], {}) -@contract def minimum_migration_time(vms_ram): """ Selects the VM with the minimum RAM usage. :param vms_ram: A map of VM UUID and their RAM usage data. - :type vms_ram: dict(str: number) - :return: A VM to migrate from the host. - :rtype: str """ min_index, min_value = min(enumerate(vms_ram.values()), key=operator.itemgetter(1)) return vms_ram.keys()[min_index] -@contract def minimum_utilization(vms_cpu): """ Selects the VM with the minimum CPU utilization. :param vms_cpu: A map of VM UUID and their CPU utilization histories. - :type vms_cpu: dict(str: list) - :return: A VM to migrate from the host. - :rtype: str """ last_utilization = [x[-1] for x in vms_cpu.values()] min_index, min_value = min(enumerate(last_utilization), @@ -138,34 +91,22 @@ def minimum_utilization(vms_cpu): return vms_cpu.keys()[min_index] -@contract def random(vms_cpu): """ Selects a random VM. :param vms_cpu: A map of VM UUID and their CPU utilization histories. - :type vms_cpu: dict(str: list) - :return: A VM to migrate from the host. - :rtype: str """ return choice(vms_cpu.keys()) -@contract def minimum_migration_time_max_cpu(last_n, vms_cpu, vms_ram): """ Selects the VM with the minimum RAM and maximum CPU usage. :param last_n: The number of last CPU utilization values to average. - :type last_n: int,>0 - :param vms_cpu: A map of VM UUID and their CPU utilization histories. - :type vms_cpu: dict(str: list) - :param vms_ram: A map of VM UUID and their RAM usage data. - :type vms_ram: dict(str: number) - :return: A VM to migrate from the host. - :rtype: str """ min_ram = min(vms_ram.values()) max_cpu = 0 diff --git a/terracotta/rpc.py b/terracotta/rpc.py index 7776f91..f961c7e 100644 --- a/terracotta/rpc.py +++ b/terracotta/rpc.py @@ -12,21 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from oslo.config import cfg -from oslo import messaging +from oslo_config import cfg +from oslo_log import log as logging from oslo_messaging.rpc import client -from mistral import context as auth_ctx -from mistral.engine import base -from mistral import exceptions as exc -from mistral.openstack.common import log as logging -from mistral.workflow import utils as wf_utils +from terracotta import context as auth_ctx +from terracotta import exceptions as exc LOG = logging.getLogger(__name__) _TRANSPORT = None - _ENGINE_CLIENT = None _EXECUTOR_CLIENT = None diff --git a/terracotta/tests/functional/__init__.py b/terracotta/tests/functional/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/terracotta/tests/unit/__init__.py b/terracotta/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29