x/compass-core
Code Issues Proposed changes

add unittests

Browse Source 
Change-Id: I171bb7538b48085beed453e062e188ca32a0cf07
This commit is contained in:
xiaodongwang 2014-03-13 22:35:54 -07:00
parent edf083e303
commit 81c50406c8
13 changed files with 2583 additions and 611 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
compass/config_management/utils/config_filter.py
View File

@ -48,9 +48,10 @@ class ConfigFilter(object):
type(self.allows_), self.allows_))
for i, allow in enumerate(self.allows_):
if not isinstance(allow, str):
if not isinstance(allow, basestring):
raise TypeError(
'allows[%s] type is %s but expected type is str: %s' % (
'allows[%s] type is %s but expected type '
'is str or unicode: %s' % (
i, type(allow), allow))
def _is_denies_valid(self):
@ -61,9 +62,10 @@ class ConfigFilter(object):
type(self.denies_), self.denies_))
for i, deny in enumerate(self.denies_):
if not isinstance(deny, str):
if not isinstance(deny, basestring):
raise TypeError(
'denies[%s] type is %s but expected type is str: %s' % (
'denies[%s] type is %s but expected type '
'is str or unicode: %s' % (
i, type(deny), deny))
def _is_valid(self):

80
compass/config_management/utils/config_merger.py
View File

@ -54,6 +54,7 @@ class ConfigMapping(object):
self.override_ = override
self.override_conditions_ = override_conditions
self.value_ = value
self._is_valid()
def __repr__(self):
return (
@ -69,20 +70,36 @@ class ConfigMapping(object):
def _is_valid_path_list(self):
"""Check path_list are valid."""
if not isinstance(self.path_list_, list):
raise TypeError(
'path_list %s type is %s while expected type is list' % (
self.path_list_, type(self.path_list_)))
for i, path in enumerate(self.path_list_):
if not isinstance(path, str):
if not isinstance(path, basestring):
raise TypeError(
'path_list[%d] type is %s while '
'expected type is str: %s' % (
'expected type is str or unicode: %s' % (
i, type(path), path))
def _is_valid_from_upper_keys(self):
"""Check from_upper_keys are valid."""
if not isinstance(self.from_upper_keys_, dict):
raise TypeError(
'from_upper_keys type is %s while expected is dict',
type(self.from_upper_keys_))
for mapping_key, from_upper_key in self.from_upper_keys_.items():
if not isinstance(from_upper_key, str):
if not isinstance(mapping_key, basestring):
raise TypeError(
'key %s in from_upper_keys type is %s'
'while expected type is str or unicode' % (
mapping_key, type(mapping_key)))
if not isinstance(from_upper_key, basestring):
raise TypeError(
'from_upper_keys[%s] type is %s'
'while expected type is str: %s' % (
'while expected type is str or unicode: %s' % (
mapping_key, type(from_upper_key), from_upper_key))
if '*' in from_upper_key:
@ -92,11 +109,22 @@ class ConfigMapping(object):
def _is_valid_from_lower_keys(self):
"""Check from_lower_keys are valid."""
if not isinstance(self.from_lower_keys_, dict):
raise TypeError(
'from_lower_keys type is %s while expected type is dict',
type(self.from_lower_keys_))
for mapping_key, from_lower_key in self.from_lower_keys_.items():
if not isinstance(from_lower_key, str):
if not isinstance(mapping_key, basestring):
raise TypeError(
'key %s in from_lower_keys type is %s'
'while expected type is str or unicode: %s' % (
mapping_key, type(mapping_key)))
if not isinstance(from_lower_key, basestring):
raise TypeError(
'from_lower_keys[%s] type'
'is %s while expected type is str: %s' % (
'is %s while expected type is str or unicode: %s' % (
mapping_key, type(from_lower_key), from_lower_key))
if '*' in from_lower_key:
@ -119,14 +147,30 @@ class ConfigMapping(object):
def _is_valid_to_key(self):
"""Check to_key is valid."""
if not isinstance(self.to_key_, basestring):
raise TypeError(
'to_key %s type is %s '
'while expected type is [str, unicode]' % (
self.to_key_, type(self.to_key_)))
if '*' in self.to_key_:
raise KeyError('to_key %s contains *' % self.to_key_)
def _is_valid_override_conditions(self):
"""Check override conditions are valid."""
if not isinstance(self.override_conditions_, dict):
raise TypeError(
'override_conditions type is %s while expected type is dict',
type(self.override_conditions_))
override_items = self.override_conditions_.items()
for mapping_key, override_condition in override_items:
if not util.is_instance(override_condition, [str, unicode]):
if not isinstance(mapping_key, basestring):
raise TypeError(
'overrid_conditions key %s type is %s '
'while expected type is [str, unicode]' % (
mapping_key, type(mapping_key)))
if not isinstance(override_condition, basestring):
raise TypeError(
'override_conditions[%s] type is %s '
'while expected type is [str, unicode]: %s' % (
@ -198,10 +242,10 @@ class ConfigMapping(object):
return lower_values
return self.value_(sub_ref, ref_key, lower_sub_refs,
self.to_key_, **sub_configs)
return self.value_(sub_ref, ref_key, lower_sub_refs, self.to_key_,
**sub_configs)
def _get_override(self, ref_key, sub_ref):
def _get_override(self, ref_key, sub_ref, to_key, lower_to_ref):
"""Get override from ref_key, ref from ref_key."""
if not callable(self.override_):
return bool(self.override_)
@ -210,13 +254,13 @@ class ConfigMapping(object):
override_items = self.override_conditions_.items()
for mapping_key, override_condition in override_items:
if override_condition in sub_ref:
override_condition_configs[mapping_key] = \
sub_ref[override_condition]
override_condition_configs[mapping_key] = (
sub_ref[override_condition])
else:
logging.info('%s no override condition %s in %s',
self, override_condition, ref_key)
return self.override_(sub_ref, ref_key,
return self.override_(sub_ref, ref_key, lower_to_ref, to_key,
**override_condition_configs)
def merge(self, upper_ref, lower_refs):
@ -247,7 +291,8 @@ class ConfigMapping(object):
value = values[lower_key]
lower_to_ref = lower_sub_ref.setdefault(self.to_key_)
override = self._get_override(self.to_key_, lower_to_ref)
override = self._get_override(
ref_key, sub_ref, self.to_key_, lower_to_ref)
lower_to_ref.update(value, override)
@ -273,6 +318,13 @@ class ConfigMerger(object):
self.__class__.__name__, type(self.mappings_),
self.mappings_))
for i, mapping in enumerate(self.mappings_):
if not isinstance(mapping, ConfigMapping):
raise TypeError(
'%s mappings[%s] type is %s '
'while expected type is ConfigMapping' % (
self.__class__.__name__, i, type(mapping)))
def merge(self, upper_config, lower_configs):
"""Merge cluster config to host configs.

79
compass/config_management/utils/config_merger_callbacks.py
View File

@ -20,6 +20,7 @@ import copy
import itertools
import logging
import netaddr
import re
from compass.utils import util
@ -214,7 +215,14 @@ def _assign_roles_by_mins(role_bundles, lower_roles, unassigned_hosts,
bundled_mins[bundled_role])
bundled_maxs[bundled_role] = _dec_max_min(
bundled_maxs[bundled_role])
lower_roles[host] = list(roles)
if host not in lower_roles:
lower_roles[host] = list(roles)
elif set(lower_roles[host]) & roles:
duplicated_roles = set(lower_roles[host]) & roles
raise ValueError(
'duplicated roles %s on %s' % (duplicated_roles, host))
else:
lower_roles[host].extend(list(roles))
logging.debug('assigned roles after assigning mins: %s', lower_roles)
logging.debug('unassigned_hosts after assigning mins: %s',
@ -311,10 +319,11 @@ def assign_roles(_upper_ref, _from_key, lower_refs, to_key,
def assign_roles_by_host_numbers(upper_ref, from_key, lower_refs, to_key,
policy_by_host_numbers={}, default={},
**_kwargs):
**kwargs):
"""Assign roles by role assign policy."""
host_numbers = str(len(lower_refs))
policy_kwargs = copy.deepcopy(default)
policy_kwargs = copy.deepcopy(kwargs)
util.merge_dict(policy_kwargs, default)
if host_numbers in policy_by_host_numbers:
util.merge_dict(policy_kwargs, policy_by_host_numbers[host_numbers])
else:
@ -349,10 +358,25 @@ def assign_ips(_upper_ref, _from_key, lower_refs, to_key,
**_kwargs):
"""Assign ips to hosts' configurations."""
if not ip_start or not ip_end:
return {}
raise ValueError(
'ip_start %s or ip_end %s is empty' % (ip_start, ip_end))
if not re.match(r'^\d+\.\d+\.\d+\.\d+$', ip_start):
raise ValueError(
'ip_start %s formmat is not correct' % ip_start)
if not re.match(r'^\d+\.\d+\.\d+\.\d+$', ip_end):
raise ValueError(
'ip_end %s format is not correct' % ip_end)
host_ips = {}
unassigned_hosts = []
ips = netaddr.IPSet(netaddr.IPRange(ip_start, ip_end))
try:
ips = netaddr.IPSet(netaddr.IPRange(ip_start, ip_end))
except Exception:
raise ValueError(
'failed to create ip block [%s, %s]' % (ip_start, ip_end))
for lower_key, lower_ref in lower_refs.items():
ip_addr = lower_ref.get(to_key, '')
if ip_addr:
@ -368,6 +392,11 @@ def assign_ips(_upper_ref, _from_key, lower_refs, to_key,
host = unassigned_hosts.pop(0)
host_ips[host] = str(ip_addr)
if unassigned_hosts:
raise ValueError(
'there is no enough ips to assign to %s: [%s-%s]' % (
unassigned_hosts, ip_start, ip_end))
logging.debug('assign %s: %s', to_key, host_ips)
return host_ips
@ -377,17 +406,34 @@ def assign_from_pattern(_upper_ref, _from_key, lower_refs, to_key,
"""assign to_key by pattern."""
host_values = {}
upper_configs = {}
if set(upper_keys) & set(lower_keys):
raise KeyError(
'overlap between upper_keys %s and lower_keys %s' % (
upper_keys, lower_keys))
for key in upper_keys:
if key not in kwargs:
raise KeyError(
'param %s is missing' % key)
upper_configs[key] = kwargs[key]
for lower_key, _ in lower_refs.items():
group = copy.deepcopy(upper_configs)
for key in lower_keys:
if key not in kwargs:
raise KeyError('param %s is missing' % key)
if not isinstance(kwargs[key], dict):
raise KeyError(
'param %s type is %s while expected type is dict' % (
kwargs[key], type(kwargs[key])))
group[key] = kwargs[key][lower_key]
try:
host_values[lower_key] = pattern % group
except Exception as error:
except KeyError as error:
logging.error('failed to assign %s[%s] = %s %% %s',
lower_key, to_key, pattern, group)
raise error
@ -401,16 +447,29 @@ def assign_noproxy(_upper_ref, _from_key, lower_refs,
hostnames={}, ips={}, **_kwargs):
"""Assign no proxy to hosts."""
no_proxy_list = copy.deepcopy(default)
if not clusterid:
raise KeyError(
'clusterid %s is empty' % clusterid)
for lower_key, _ in lower_refs.items():
if lower_key not in hostnames:
raise KeyError(
'lower_key %s is not in hostnames %s' % (
lower_key, hostnames))
if lower_key not in ips:
raise KeyError(
'lower_key %s is not in ips %s' % (
lower_key, ips))
mapping = {
'clusterid': clusterid,
'hostname': hostnames.get(lower_key, ''),
'ip': ips.get(lower_key, '')
'hostname': hostnames[lower_key],
'ip': ips[lower_key]
}
try:
no_proxy_list.append(noproxy_pattern % mapping)
except Exception as error:
except KeyError as error:
logging.error('failed to assign %s[%s] = %s %% %s',
lower_key, to_key, noproxy_pattern, mapping)
raise error
@ -423,7 +482,7 @@ def assign_noproxy(_upper_ref, _from_key, lower_refs,
return host_no_proxy
def override_if_empty(lower_ref, _ref_key):
def override_if_empty(_upper_ref, _ref_key, lower_ref, _to_key):
"""Override if the configuration value is empty."""
if not lower_ref.config:
return True

10
compass/config_management/utils/config_reference.py
View File

@ -63,11 +63,11 @@ class ConfigReference(object):
:raises: TypeError
"""
if parent and not isinstance(parent, self.__class__):
if parent and not isinstance(parent, ConfigReference):
raise TypeError('parent %s type should be %s'
% (parent, self.__class__.__name__))\
% (parent, ConfigReference))
if parent_key and not util.is_instance(parent_key, [str, unicode]):
if parent_key and not isinstance(parent_key, basestring):
raise TypeError('parent_key %s type should be [str, unicode]'
% parent_key)
@ -90,7 +90,7 @@ class ConfigReference(object):
if config and isinstance(config, dict):
for key, value in config.items():
if not util.is_instance(key, [str, unicode]):
if not isinstance(key, basestring):
msg = 'key type is %s while expected is [str, unicode]: %s'
raise TypeError(msg % (type(key), key))
ConfigReference(value, self, key)
@ -147,7 +147,7 @@ class ConfigReference(object):
parts = []
if util.is_instance(path, [str, unicode]):
if isinstance(path, basestring):
parts = path.split('/')
else:
parts = path

74
compass/config_management/utils/config_translator.py
View File

@ -19,7 +19,6 @@
import logging
from compass.config_management.utils import config_reference
from compass.utils import util
class KeyTranslator(object):
@ -65,8 +64,14 @@ class KeyTranslator(object):
if callable(self.translated_keys_):
return
if not isinstance(self.translated_keys_, list):
raise TypeError(
'translated_keys %s type is %s while expected type is '
'list or callable' % (
self.translated_keys_, type(self.translated_keys_)))
for i, translated_key in enumerate(self.translated_keys_):
if util.is_instance(translated_key, [str, unicode]):
if isinstance(translated_key, basestring):
if '*' in translated_key:
raise KeyError(
'transalted_keys[%d] %s should not contain *' % (
@ -79,8 +84,19 @@ class KeyTranslator(object):
def _is_valid_from_keys(self):
"""Check from keys are valid."""
if not isinstance(self.from_keys_, dict):
raise TypeError(
'from_keys %s type is %s while expected type is dict' % (
self.from_keys_, type(self.from_keys_)))
for mapping_key, from_key in self.from_keys_.items():
if not util.is_instance(from_key, [str, unicode]):
if not isinstance(mapping_key, basestring):
raise TypeError(
'from_keys key %s type is %s while '
'expected type is [str, unicode]' % (
mapping_key, type(mapping_key)))
if not isinstance(from_key, basestring):
raise TypeError(
'from_keys[%s] type is %s while '
'expected type is [str, unicode]: %s' % (
@ -93,8 +109,19 @@ class KeyTranslator(object):
def _is_valid_from_values(self):
"""Check from values are valid."""
if not isinstance(self.from_values_, dict):
raise TypeError(
'from_values %s type is %s while expected type is dict' % (
self.from_values_, type(self.from_values_)))
for mapping_key, from_value in self.from_values_.items():
if not util.is_instance(from_value, [str, unicode]):
if not isinstance(mapping_key, basestring):
raise TypeError(
'from_values key %s type is %s while '
'expected type is [str, unicode]' % (
mapping_key, type(mapping_key)))
if not isinstance(from_value, basestring):
raise TypeError(
'from_values[%s] type is %s while '
'expected type is [str, unicode]: %s' % (
@ -107,9 +134,22 @@ class KeyTranslator(object):
def _is_valid_override_conditions(self):
"""Check override conditions are valid."""
if not isinstance(self.override_conditions_, dict):
raise TypeError(
'override_conditions %s type is %s '
'while expected type is dict' % (
self.override_conditions_,
type(self.override_conditions_)))
override_items = self.override_conditions_.items()
for mapping_key, override_condition in override_items:
if not util.is_instance(override_condition, [str, unicode]):
if not isinstance(mapping_key, basestring):
raise TypeError(
'override_conditions key %s type is %s while '
'expected type is [str, unicode]' % (
mapping_key, type(mapping_key)))
if not isinstance(override_condition, basestring):
raise TypeError(
'override_conditions[%s] type is %s '
'while expected type is [str, unicode]: %s' % (
@ -153,7 +193,7 @@ class KeyTranslator(object):
logging.debug('%s ignore empty translated key', self)
continue
if not util.is_instance(translated_key, [str, unicode]):
if not isinstance(translated_key, basestring):
logging.error(
'%s translated key %s should be [str, unicode]',
self, translated_key)
@ -188,7 +228,7 @@ class KeyTranslator(object):
translated_key, translated_sub_ref):
"""Get override."""
if not callable(self.override_):
return self.override_
return bool(self.override_)
override_condition_configs = {}
override_items = self.override_conditions_.items()
@ -216,6 +256,9 @@ class KeyTranslator(object):
ref_key, sub_ref, translated_key, translated_sub_ref)
if translated_value is None:
logging.debug(
'translated key %s will be ignored '
'since translated value is None', translated_key)
continue
override = self._get_override(
@ -247,16 +290,23 @@ class ConfigTranslator(object):
type(self.mapping_), self.mapping_))
for key, values in self.mapping_.items():
if not isinstance(key, basestring):
raise TypeError(
'mapping key %s type is %s while expected '
'is str or unicode' % (key, type(key)))
if not isinstance(values, list):
msg = 'mapping[%s] type is %s while expected type is list: %s'
raise TypeError(msg % (key, type(values), values))
raise TypeError(
'mapping[%s] type is %s '
'while expected type is list: %s' % (
key, type(values), values))
for i, value in enumerate(values):
if not isinstance(value, KeyTranslator):
msg = (
raise TypeError(
'mapping[%s][%d] type is %s '
'while expected type is KeyTranslator: %s')
raise TypeError(msg % (key, i, type(value), value))
'while expected type is KeyTranslator: %s' % (
key, i, type(value), value))
def translate(self, config):
"""Translate config.

3
compass/db/database.py
View File

@ -17,7 +17,8 @@ import logging
from contextlib import contextmanager
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.orm import scoped_session
from sqlalchemy.orm import sessionmaker
from threading import local
from compass.db import model

300
compass/tests/config_management/utils/1 Normal file
View File

@ -0,0 +1,300 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to set the hosts configs from cluster config.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import copy
import logging
from compass.config_management.utils import config_reference
from compass.utils import util
class ConfigMapping(object):
"""Class to merge cluster config ref to host config ref by path list."""
def __init__(self, path_list, from_upper_keys={},
from_lower_keys={}, to_key='.',
override=False, override_conditions={},
value=None):
"""Constructor
:param path_list: list of path to merge from cluster ref to host refs
:type path_list: list of str
:param from_upper_keys: kwargs from cluster ref for value callback.
:type from_upper_keys: dict of kwargs name to path in cluster ref
:param from_lower_keys: kwargs from host refs for value callback.
:type from_lower_keys: dict of kwargs name to path in host refs.
:param to_key: the path in host refs to be merged to.
:type to_key: str
:param override: if the path in host ref can be overridden.
:type override: callback or bool
:param override_conditions: kwargs from host ref for override callback
:type override_conditions: dict of kwargs name to path in host ref
:param value: the value to be set in host refs.
:type value: callback or any type
"""
self.path_list_ = path_list
self.from_upper_keys_ = from_upper_keys
self.from_lower_keys_ = from_lower_keys
self.to_key_ = to_key
self.override_ = override
self.override_conditions_ = override_conditions
self.value_ = value
self.is_valid()
def __repr__(self):
return (
'%s[path_list=%s,from_upper_keys=%s,'
'from_lower_keys=%s,to_key=%s,override=%s,'
'override_conditions=%s,value=%s]'
) % (
self.__class__.__name__,
self.path_list_, self.from_upper_keys_,
self.from_lower_keys_, self.to_key_,
self.override_, self.override_conditions_,
self.value_)
def _is_valid_path_list(self):
"""Check path_list are valid."""
for i, path in enumerate(self.path_list_):
if not isinstance(path, str):
raise TypeError(
'path_list[%d] type is %s while '
'expected type is str: %s' % (
i, type(path), path))
def _is_valid_from_upper_keys(self):
"""Check from_upper_keys are valid."""
for mapping_key, from_upper_key in self.from_upper_keys_.items():
if not isinstance(from_upper_key, str):
raise TypeError(
'from_upper_keys[%s] type is %s'
'while expected type is str: %s' % (
mapping_key, type(from_upper_key), from_upper_key))
if '*' in from_upper_key:
raise KeyError(
'from_upper_keys[%s] %s contains *' % (
mapping_key, from_upper_key))
def _is_valid_from_lower_keys(self):
"""Check from_lower_keys are valid."""
for mapping_key, from_lower_key in self.from_lower_keys_.items():
if not isinstance(from_lower_key, str):
raise TypeError(
'from_lower_keys[%s] type'
'is %s while expected type is str: %s' % (
mapping_key, type(from_lower_key), from_lower_key))
if '*' in from_lower_key:
raise KeyError(
'from_lower_keys[%s] %s contains *' % (
mapping_key, from_lower_key))
def _is_valid_from_keys(self):
"""Check from keys are valid."""
self._is_valid_from_upper_keys()
self._is_valid_from_lower_keys()
upper_keys = set(self.from_upper_keys_.keys())
lower_keys = set(self.from_lower_keys_.keys())
intersection = upper_keys.intersection(lower_keys)
if intersection:
raise KeyError(
'there is intersection between from_upper_keys %s'
' and from_lower_keys %s: %s' % (
upper_keys, lower_keys, intersection))
def _is_valid_to_key(self):
"""Check to_key is valid."""
if '*' in self.to_key_:
raise KeyError('to_key %s contains *' % self.to_key_)
def _is_valid_override_conditions(self):
"""Check override conditions are valid."""
override_items = self.override_conditions_.items()
for mapping_key, override_condition in override_items:
if not util.is_instance(override_condition, [str, unicode]):
raise TypeError(
'override_conditions[%s] type is %s '
'while expected type is [str, unicode]: %s' % (
mapping_key, type(override_condition),
override_condition))
if '*' in override_condition:
raise KeyError(
'override_conditions[%s] %s contains *' % (
mapping_key, override_condition))
def is_valid(self):
"""Check ConfigMapping instance is valid."""
self._is_valid_path_list()
self._is_valid_from_keys()
self._is_valid_to_key()
self._is_valid_override_conditions()
def _get_upper_sub_refs(self, upper_ref):
"""get sub_refs from upper_ref."""
upper_refs = []
for path in self.path_list_:
upper_refs.extend(upper_ref.ref_items(path))
return upper_refs
def _get_mapping_from_upper_keys(self, ref_key, sub_ref):
"""Get upper config mapping from from_upper_keys."""
sub_configs = {}
for mapping_key, from_upper_key in self.from_upper_keys_.items():
if from_upper_key in sub_ref:
sub_configs[mapping_key] = sub_ref[from_upper_key]
else:
logging.info('%s ignore from_upper_key %s in %s',
self, from_upper_key, ref_key)
return sub_configs
def _get_mapping_from_lower_keys(self, ref_key, lower_sub_refs):
"""Get lower config mapping from from_lower_keys."""
sub_configs = {}
for mapping_key, from_lower_key in self.from_lower_keys_.items():
sub_configs[mapping_key] = {}
for lower_key, lower_sub_ref in lower_sub_refs.items():
for mapping_key, from_lower_key in self.from_lower_keys_.items():
if from_lower_key in lower_sub_ref:
sub_configs[mapping_key][lower_key] = (
lower_sub_ref[from_lower_key])
else:
logging.error(
'%s ignore from_lower_key %s in %s lower_key %s',
self, from_lower_key, ref_key, lower_key)
return sub_configs
def _get_values(self, ref_key, sub_ref, lower_sub_refs, sub_configs):
"""Get values to set to lower configs."""
if self.value_ is None:
lower_values = {}
for lower_key in lower_sub_refs.keys():
lower_values[lower_key] = copy.deepcopy(sub_ref.config)
return lower_values
if not callable(self.value_):
lower_values = {}
for lower_key in lower_sub_refs.keys():
lower_values[lower_key] = copy.deepcopy(self.value_)
return lower_values
return self.value_(sub_ref, ref_key, lower_sub_refs,
self.to_key_, **sub_configs)
def _get_override(self, ref_key, sub_ref):
"""Get override from ref_key, ref from ref_key."""
if not callable(self.override_):
return bool(self.override_)
override_condition_configs = {}
override_items = self.override_conditions_.items()
for mapping_key, override_condition in override_items:
if override_condition in sub_ref:
override_condition_configs[mapping_key] = \
sub_ref[override_condition]
else:
logging.info('%s no override condition %s in %s',
self, override_condition, ref_key)
return self.override_(sub_ref, ref_key,
**override_condition_configs)
def merge(self, upper_ref, lower_refs):
"""merge upper config to lower configs."""
upper_sub_refs = self._get_upper_sub_refs(upper_ref)
for ref_key, sub_ref in upper_sub_refs:
sub_configs = self._get_mapping_from_upper_keys(ref_key, sub_ref)
lower_sub_refs = {}
for lower_key, lower_ref in lower_refs.items():
lower_sub_refs[lower_key] = lower_ref.setdefault(ref_key)
lower_sub_configs = self._get_mapping_from_lower_keys(
ref_key, lower_sub_refs)
util.merge_dict(sub_configs, lower_sub_configs)
values = self._get_values(
ref_key, sub_ref, lower_sub_refs, sub_configs)
logging.debug('%s set values %s to %s',
ref_key, self.to_key_, values)
for lower_key, lower_sub_ref in lower_sub_refs.items():
if lower_key not in values:
logging.error('no key %s in %s', lower_key, values)
continue
value = values[lower_key]
lower_to_ref = lower_sub_ref.setdefault(self.to_key_)
override = self._get_override(self.to_key_, lower_to_ref)
lower_to_ref.update(value, override)
class ConfigMerger(object):
"""Class to merge clsuter config to host configs."""
def __init__(self, mappings):
"""Constructor
:param mappings: list of :class:`ConfigMapping` instance
"""
self.mappings_ = mappings
self.is_valid()
def __repr__(self):
return '%s[mappings=%s]' % (self.__class__.__name__, self.mappings_)
def is_valid(self):
"""Check ConfigMerger instance is valid."""
if not isinstance(self.mappings_, list):
raise TypeError(
'%s mapping type is %s while expect type is list: %s' % (
self.__class__.__name__, type(self.mappings_),
self.mappings_))
def merge(self, upper_config, lower_configs):
"""Merge cluster config to host configs.
:param upper_config: cluster configuration to merge from.
:type upper_config: dict
:param lower_configs: host configurations to merge to.
:type lower_configs: dict of host id to host config as dict
"""
upper_ref = config_reference.ConfigReference(upper_config)
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
for mapping in self.mappings_:
logging.debug('apply merging from the rule %s', mapping)
mapping.merge(upper_ref, lower_refs)
for lower_key, lower_config in lower_configs.items():
lower_configs[lower_key] = config_reference.get_clean_config(
lower_config)
logging.debug('merged upper config\n%s\nto lower configs:\n%s',
upper_config, lower_configs)

77
compass/tests/config_management/utils/test_config_filter.py
View File

@ -40,53 +40,96 @@ class TestConfigFilter(unittest2.TestCase):
def setUp(self):
super(TestConfigFilter, self).setUp()
logsetting.init()
self.config_ = {
'1': '1',
'2': {
'22': '22',
'33': {
'333': '333',
'44': '444'
}
},
'3': {'33': '44'}
}
def tearDown(self):
super(TestConfigFilter, self).tearDown()
def test_allows(self):
def test_init(self):
config_filter.ConfigFilter(
allows=['abc', 'def'], denies=['def', 'ghi'])
config_filter.ConfigFilter(
allows=[u'abc', u'def'], denies=[u'def', u'ghi'])
def test_init_allows(self):
# allows type should be a list of string.
self.assertRaises(
TypeError, config_filter.ConfigFilter,
allows={'abd': 'abc'})
self.assertRaises(
TypeError, config_filter.ConfigFilter,
allows='abc')
self.assertRaises(
TypeError, config_filter.ConfigFilter,
allows=[{'abc': 'bdc'}])
def test_init_denies(self):
# denies type should be a list of string.
self.assertRaises(
TypeError, config_filter.ConfigFilter,
denies={'abd': 'abc'})
self.assertRaises(
TypeError, config_filter.ConfigFilter,
denies='abc')
self.assertRaises(
TypeError, config_filter.ConfigFilter,
denies=[{'abc': 'bdc'}])
def test_allows_asterisks(self):
"""test allows rules."""
config = {'1': '1',
'2': {'22': '22',
'33': {'333': '333',
'44': '444'}},
'3': {'33': '44'}}
# keys in allows will be copied to dest.
# if '*' in allows, all keys will be copied to dest.
allows = ['*', '3', '5']
configfilter = config_filter.ConfigFilter(allows)
filtered_config = configfilter.filter(config)
self.assertEqual(filtered_config, config)
filtered_config = configfilter.filter(self.config_)
self.assertEqual(filtered_config, self.config_)
def test_allows_path(self):
allows = ['/1', '2/22', '5']
expected_config = {'1': '1', '2': {'22': '22'}}
configfilter = config_filter.ConfigFilter(allows)
filtered_config = configfilter.filter(config)
filtered_config = configfilter.filter(self.config_)
self.assertEqual(filtered_config, expected_config)
def test_allows_asterrisks_in_path(self):
allows = ['*/33']
expected_config = {'2': {'33': {'333': '333',
'44': '444'}},
'3': {'33': '44'}}
configfilter = config_filter.ConfigFilter(allows)
filtered_config = configfilter.filter(config)
filtered_config = configfilter.filter(self.config_)
self.assertEqual(filtered_config, expected_config)
def test_denies(self):
"""test denies rules."""
config = {'1': '1', '2': {'22': '22',
'33': {'333': '333',
'44': '444'}},
'3': {'33': '44'}}
# keys in denies list will be removed from filtered config.
denies = ['/1', '2/22', '2/33/333', '5']
expected_config = {'2': {'33': {'44': '444'}}, '3': {'33': '44'}}
configfilter = config_filter.ConfigFilter(denies=denies)
filtered_config = configfilter.filter(config)
filtered_config = configfilter.filter(self.config_)
self.assertEqual(filtered_config, expected_config)
def test_denies_asterisks(self):
denies = ['*']
configfilter = config_filter.ConfigFilter(denies=denies)
filtered_config = configfilter.filter(config)
filtered_config = configfilter.filter(self.config_)
self.assertIsNone(filtered_config)
def tet_deneis_asterisks_in_path(self):
denies = ['*/33']
expected_config = {'1': '1', '2': {'22': '22'}}
configfilter = config_filter.ConfigFilter(denies=denies)
filtered_config = configfilter.filter(config)
filtered_config = configfilter.filter(self.config_)
self.assertEqual(filtered_config, expected_config)

665
compass/tests/config_management/utils/test_config_merger.py
View File

@ -18,7 +18,6 @@
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import functools
import os
import unittest2
@ -30,11 +29,433 @@ reload(setting)
from compass.config_management.utils import config_merger
from compass.config_management.utils import config_merger_callbacks
from compass.config_management.utils import config_reference
from compass.utils import flags
from compass.utils import logsetting
class TestConfigMapping(unittest2.TestCase):
"""test config mapping class."""
def setUp(self):
super(TestConfigMapping, self).setUp()
logsetting.init()
def tearDown(self):
super(TestConfigMapping, self).tearDown()
def test_init(self):
# path_list should be list of string.
config_merger.ConfigMapping(
path_list=['1/2/3', '/4/5/6'])
config_merger.ConfigMapping(
path_list=[u'1/2/3', u'/4/5/6'])
self.assertRaises(
TypeError,
config_merger.ConfigMapping, path_list={'1/2/3': '4'})
self.assertRaises(
TypeError, config_merger.ConfigMapping, path_list='1234')
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=[{'1/2/3': '4'}])
def test_init_from_upper_keys(self):
# from_upper_keys should be dict of string to string.
config_merger.ConfigMapping(
path_list=['1/2/3', '/4/5/6'], from_upper_keys={'4': '4'})
config_merger.ConfigMapping(
path_list=['1/2/3', '/4/5/6'], from_upper_keys={u'4': u'4'})
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
from_upper_keys=['4'])
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
from_upper_keys='4')
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
from_upper_keys={4: '4'})
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
from_upper_keys={'4': 4})
def test_init_from_lower_keys(self):
# from_lower_keys should be dict of string to string.
config_merger.ConfigMapping(
path_list=['1/2/3', '/4/5/6'], from_lower_keys={'4': '4'})
config_merger.ConfigMapping(
path_list=['1/2/3', '/4/5/6'], from_lower_keys={u'4': u'4'})
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
from_lower_keys=['4'])
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
from_lower_keys='4')
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
from_lower_keys={4: '4'})
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
from_lower_keys={'4': 4})
def test_init_overlap_upper_keys_lower_keys(self):
# there should be overlap between from_upper_keys and from_lower_keys.
self.assertRaises(
KeyError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
from_upper_keys={'1': '1', '2': '2'},
from_lower_keys={'1': '1', '3': '3'})
config_merger.ConfigMapping(
path_list=['1/2/3', '/4/5/6'],
from_upper_keys={'1': '1', '2': '2'},
from_lower_keys={'3': '3', '4': '4'})
def test_init_to_key(self):
# to_key type should be string.
config_merger.ConfigMapping(
path_list=['1/2/3', '/4/5/6'],
to_key='hello')
config_merger.ConfigMapping(
path_list=['1/2/3', '/4/5/6'],
to_key=u'hello')
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
to_key=['hello'])
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
to_key=123)
def test_init_to_key_no_asterrisks(self):
# to_key should not contains '*'.
self.assertRaises(
KeyError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
to_key='abc*def')
def test_init_override_conditions(self):
# override_conditions type should be dict of string to string.
config_merger.ConfigMapping(
path_list=['1/2/3', '/4/5/6'],
override_conditions={'hello': 'hi'})
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
override_conditions=['hello', 'hi'])
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
override_conditions='hello')
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
override_conditions={5: 'hi'})
self.assertRaises(
TypeError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
override_conditions={'hello': 5})
def test_init_override_conditions_no_asterrisks(self):
# the value in each override_conditions key value pair
# should not contains '*'.
self.assertRaises(
KeyError, config_merger.ConfigMapping,
path_list=['1/2/3', '/4/5/6'],
override_conditions={'hello': 'hi*hi'})
def test_merge(self):
# the key in path_list will be copied
# from upper config to lower configs.
upper_config = {
'key': 'abc',
'key2': 'def'
}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {
1: {}, 2: {}, 3: {}
}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
merger = config_merger.ConfigMapping(
path_list=['key'])
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs,
{1: {'key': 'abc'}, 2: {'key': 'abc'}, 3: {'key': 'abc'}})
def test_merge_all_matching_keys_copied(self):
# all the keys matching the pattern will be copied
# from upper config to lower configs.
upper_config = {
'key': 'abc',
'key2': 'def'
}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {
1: {}, 2: {}, 3: {}
}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
merger = config_merger.ConfigMapping(
path_list=['key*'])
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs,
{
1: {'key': 'abc', 'key2': 'def'},
2: {'key': 'abc', 'key2': 'def'},
3: {'key': 'abc', 'key2': 'def'}
}
)
def test_merge_value_set_explictly(self):
# key in lower configs are set to expected value.
upper_config = {
'key': 'abc',
'key2': 'def'
}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {
1: {}, 2: {}, 3: {}
}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
merger = config_merger.ConfigMapping(
path_list=['key'], value='def')
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs,
{
1: {'key': 'def'},
2: {'key': 'def'},
3: {'key': 'def'}
}
)
def test_merge_value_set_by_callback(self):
# key in lower config is called a callback to expected value .
upper_config = {
'key': 'abc',
'key2': 'def'
}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {
1: {}, 2: {}, 3: {}
}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
def _merge_value(sub_ref, ref_key, lower_sub_refs, to_key):
values = {}
for lower_key, lower_sub_ref in lower_sub_refs.items():
values[lower_key] = '%s.%s' % (sub_ref.config, lower_key)
return values
merger = config_merger.ConfigMapping(
path_list=['key'], value=_merge_value)
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs,
{
1: {'key': 'abc.1'},
2: {'key': 'abc.2'},
3: {'key': 'abc.3'}
}
)
def test_merge_to_key(self):
# set lower configs expected key from the key in upper config.
upper_config = {
'key': 'abc',
'key2': 'def'
}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {
1: {}, 2: {}, 3: {}
}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
def _merge_value(sub_ref, ref_key, lower_sub_refs, to_key):
values = {}
for lower_key, lower_sub_ref in lower_sub_refs.items():
values[lower_key] = '%s.%s' % (sub_ref.config, lower_key)
return values
merger = config_merger.ConfigMapping(
path_list=['key'], value=_merge_value, to_key='/key2')
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs,
{
1: {'key': None, 'key2': 'abc.1'},
2: {'key': None, 'key2': 'abc.2'},
3: {'key': None, 'key2': 'abc.3'}
}
)
def test_merge_from_upper_and_lower_configs(self):
# set lower configs from some keys of upper config and lower configs.
upper_config = {
'key': 'abc',
'key_prefix': 'A',
'key_suffix': 'B'
}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {
1: {'name': 'hello'}, 2: {'name': 'hi'}, 3: {'name': 'today'}
}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
def _merge_value2(
sub_ref, ref_key, lower_sub_refs, to_key,
prefix='', suffix='', names={}
):
values = {}
for lower_key, lower_sub_ref in lower_sub_refs.items():
values[lower_key] = '%s%s%s' % (
prefix, names.get(lower_key, ''), suffix)
return values
merger = config_merger.ConfigMapping(
path_list=['key'], value=_merge_value2,
from_upper_keys={'prefix': '/key_prefix', 'suffix': '/key_suffix'},
from_lower_keys={'names': '/name'})
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs,
{
1: {'name': 'hello', 'key': 'AhelloB'},
2: {'name': 'hi', 'key': 'AhiB'},
3: {'name': 'today', 'key': 'AtodayB'}
}
)
def test_merge_multikey_to_same_tokey_nooverride(self):
# mutli key in upper config writes the same dest key, the later one
# will be ignored if override is False.
upper_config = {
'key1': 'abc',
'key2': 'bcd'
}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {
1: {}, 2: {}, 3: {}
}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
merger = config_merger.ConfigMapping(
path_list=['key1', 'key2'], to_key='/key', override=False)
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs,
{
1: {'key': 'abc', 'key1': None, 'key2': None},
2: {'key': 'abc', 'key1': None, 'key2': None},
3: {'key': 'abc', 'key1': None, 'key2': None}
}
)
def test_merge_multikey_to_sam_tokey_override(self):
# if multi key in upper config writes the same dest key,
# the later one will override the former one if override is True.
upper_config = {
'key1': 'abc',
'key2': 'bcd'
}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {
1: {}, 2: {}, 3: {}
}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
merger = config_merger.ConfigMapping(
path_list=['key1', 'key2'], to_key='/key', override=True)
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs,
{
1: {'key': 'bcd', 'key1': None, 'key2': None},
2: {'key': 'bcd', 'key1': None, 'key2': None},
3: {'key': 'bcd', 'key1': None, 'key2': None}
}
)
def test_merge_override_callback(self):
# override param can be set from callback.
upper_config = {
'key1': 'abc',
'key2': 'bcd',
'key3': 'def',
'key_prefix': 'b',
'key_suffix': 'd'
}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {
1: {}, 2: {}, 3: {}
}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
def _generate_override(
sub_ref, ref_key, lower_ref, to_key, prefix='', suffix=''
):
return (
sub_ref.config.startswith(prefix) and
sub_ref.config.endswith(suffix)
)
merger = config_merger.ConfigMapping(
path_list=['key1', 'key2', 'key3'], to_key='/key',
override=_generate_override,
override_conditions={
'prefix': '/key_prefix', 'suffix': '/key_suffix'
}
)
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs,
{
1: {'key': 'bcd', 'key1': None, 'key2': None, 'key3': None},
2: {'key': 'bcd', 'key1': None, 'key2': None, 'key3': None},
3: {'key': 'bcd', 'key1': None, 'key2': None, 'key3': None}
}
)
class TestConfigMerger(unittest2.TestCase):
"""test config merger class."""
@ -45,152 +466,110 @@ class TestConfigMerger(unittest2.TestCase):
def tearDown(self):
super(TestConfigMerger, self).tearDown()
def test_init(self):
# mappings should be list of ConfigMapping.
config_merger.ConfigMerger(mappings=[])
config_merger.ConfigMerger(
mappings=[
config_merger.ConfigMapping(
path_list=['1/2/3', '/4/5/6'])
]
)
self.assertRaises(
TypeError, config_merger.ConfigMerger,
mapping={'hello': config_merger.ConfigMapping(path_list=[])})
self.assertRaises(
TypeError, config_merger.ConfigMerger,
mapping=config_merger.ConfigMapping(path_list=[]))
self.assertRaises(
TypeError, config_merger.ConfigMerger,
mapping='config_merger.ConfigMapping(path_list=[])')
self.assertRaises(
TypeError, config_merger.ConfigMerger,
mapping=[{'hello': config_merger.ConfigMapping(path_list=[])}])
self.assertRaises(
TypeError, config_merger.ConfigMerger,
mapping=['config_merger.ConfigMapping(path_list=[])'])
def test_merge(self):
"""test merge."""
upper_config = {
'networking': {
'interfaces': {
'management': {
'ip_start': '192.168.1.1',
'ip_end': '192.168.1.100',
'netmask': '255.255.255.0',
'dns_pattern': (
'%(hostname)s.%(clustername)s.%(search_path)s'),
},
'floating': {
'ip_start': '172.16.0.1',
'ip_end': '172.16.0.100',
'netmask': '0.0.0.0',
'dns_pattern': (
'public-%(hostname)s.%(clustername)s'
'.%(search_path)s'),
},
},
'global': {
'search_path': 'ods.com',
},
},
'clustername': 'cluster1',
'dashboard_roles': ['os-single-controller'],
'role_assign_policy': {
'policy_by_host_numbers': {},
'default': {
'roles': ['os-single-controller', 'os-network',
'os-compute-worker'],
'default_min': 1,
},
},
# if multi ConfigMapping updates the same key and no override is set
# for the later one, the later one will be ignored.
config = {
'key1': 'abc',
'key2': 'bcd'
}
lower_configs = {
1: {
'hostname': 'host1',
},
2: {
'hostname': 'host2',
'networking': {
'interfaces': {
'management': {
'ip': '192.168.1.50',
},
},
},
'roles': ['os-single-controller', 'os-network'],
}
1: {}, 2: {}, 3: {}
}
expected_lower_configs = {
1: {
'networking': {
'interfaces': {
'floating': {
'ip': '172.16.0.1',
'netmask': '0.0.0.0',
'dns_alias': 'public-host1.cluster1.ods.com'
},
'management': {
'ip': '192.168.1.1',
'netmask': '255.255.255.0',
'dns_alias': 'host1.cluster1.ods.com'
}
},
'global': {
'search_path': 'ods.com',
}
},
'hostname': 'host1',
'has_dashboard_roles': False,
'roles': ['os-compute-worker']
},
2: {
'networking': {
'interfaces': {
'floating': {
'ip': '172.16.0.2',
'netmask': '0.0.0.0',
'dns_alias': 'public-host2.cluster1.ods.com'
},
'management': {
'ip': '192.168.1.50',
'netmask': '255.255.255.0',
'dns_alias': 'host2.cluster1.ods.com'
}
},
'global': {
'search_path': 'ods.com',
}
},
'hostname': 'host2',
'has_dashboard_roles': True,
'roles': ['os-single-controller', 'os-network']
}
merger = config_merger.ConfigMerger(
mappings=[
config_merger.ConfigMapping(
path_list=['key1'], to_key='/mkey'),
config_merger.ConfigMapping(
path_list=['key2'], to_key='/mkey')
]
)
merger.merge(config, lower_configs)
self.assertEqual(
lower_configs,
{1: {'mkey': 'abc'}, 2: {'mkey': 'abc'}, 3: {'mkey': 'abc'}}
)
def test_merge_multi_mapping_to_same_tokey(self):
# if multi ConfigMapping updates the same key and override is set
# for the later one, the later one will override the former one.
config = {
'key1': 'abc',
'key2': 'bcd'
}
mappings = [
config_merger.ConfigMapping(
path_list=['/networking/interfaces/*'],
from_upper_keys={'ip_start': 'ip_start', 'ip_end': 'ip_end'},
to_key='ip',
value=config_merger_callbacks.assign_ips
),
config_merger.ConfigMapping(
path_list=['/role_assign_policy'],
from_upper_keys={
'policy_by_host_numbers': 'policy_by_host_numbers',
'default': 'default'},
to_key='/roles',
value=config_merger_callbacks.assign_roles_by_host_numbers
),
config_merger.ConfigMapping(
path_list=['/dashboard_roles'],
from_lower_keys={'lower_values': '/roles'},
to_key='/has_dashboard_roles',
value=config_merger_callbacks.has_intersection
),
config_merger.ConfigMapping(
path_list=[
'/networking/global',
'/networking/interfaces/*/netmask',
'/networking/interfaces/*/nic',
'/networking/interfaces/*/promisc',
'/security/*',
'/partition',
]
),
config_merger.ConfigMapping(
path_list=['/networking/interfaces/*'],
from_upper_keys={'pattern': 'dns_pattern',
'clustername': '/clustername',
'search_path': (
'/networking/global/search_path')},
from_lower_keys={'hostname': '/hostname'},
to_key='dns_alias',
value=functools.partial(
config_merger_callbacks.assign_from_pattern,
upper_keys=['search_path', 'clustername'],
lower_keys=['hostname'])
),
]
merger = config_merger.ConfigMerger(mappings)
merger.merge(upper_config, lower_configs)
self.assertEqual(lower_configs, expected_lower_configs)
lower_configs = {
1: {}, 2: {}, 3: {}
}
merger = config_merger.ConfigMerger(
mappings=[
config_merger.ConfigMapping(
path_list=['key1'], to_key='/mkey'),
config_merger.ConfigMapping(
path_list=['key2'], to_key='/mkey',
override=True)
]
)
merger.merge(config, lower_configs)
self.assertEqual(
lower_configs,
{1: {'mkey': 'bcd'}, 2: {'mkey': 'bcd'}, 3: {'mkey': 'bcd'}}
)
def test_merge_override_callback(self):
# override param can be callback.
config = {
'key1': 'abc',
'key2': 'bcd'
}
lower_configs = {
1: {}, 2: {}, 3: {}
}
def _merge_value(sub_ref, ref_key, lower_sub_refs, to_key):
values = {}
for lower_key, lower_sub_ref in lower_sub_refs.items():
values[lower_key] = None
return values
merger = config_merger.ConfigMerger(
mappings=[
config_merger.ConfigMapping(
path_list=['key1'],
value=_merge_value,
to_key='/mkey')
]
)
merger.merge(config, lower_configs)
self.assertEqual(
lower_configs,
{1: None, 2: None, 3: None}
)
if __name__ == '__main__':

662
compass/tests/config_management/utils/test_config_merger_callbacks.py
View File

@ -41,10 +41,151 @@ class TestAssignRoles(unittest2.TestCase):
def setUp(self):
super(TestAssignRoles, self).setUp()
logsetting.init()
self.roles_ = ['control', 'api', 'compute']
self.maxs_ = {'control': 1, 'api': 1, 'compute': -1}
self.default_min_ = 1
def tearDown(self):
super(TestAssignRoles, self).tearDown()
def test_assign_roles_allinone_roles_empty(self):
"""test assign roles all in one node."""
lower_configs = {
1: {'roles': []},
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_roles(
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=self.maxs_,
default_min=self.default_min_)
self.assertEqual(assigned, {1: ['control', 'api', 'compute']})
def test_assign_roles_allinone_no_roles(self):
lower_configs = {
1: {},
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_roles(
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=self.maxs_, default_min=self.default_min_)
self.assertEqual(assigned, {1: ['control', 'api', 'compute']})
def test_assign_roles_allinone_roles_sorted(self):
lower_configs = {
1: {'roles': ['api', 'control', 'compute']},
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_roles(
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=self.maxs_, default_min=self.default_min_)
self.assertEqual(assigned, {1: ['control', 'api', 'compute']})
def test_assign_roles_allinone_roles_set_additional_roles(self):
lower_configs = {
1: {'roles': ['control', 'api', 'compute', 'mysql']},
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_roles(
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=self.maxs_, default_min=self.default_min_)
self.assertEqual(assigned, {1: ['control', 'api', 'compute']})
def test_assign_roles_allinone_roles_set_less_roles(self):
lower_configs = {
1: {'roles': ['control', 'api']},
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
ValueError, config_merger_callbacks.assign_roles,
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=self.maxs_, default_min=self.default_min_)
def test_assign_roles_allinone_exclusives(self):
exclusives = ['control']
lower_configs = {
1: {'roles': []},
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
ValueError, config_merger_callbacks.assign_roles,
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=self.maxs_, default_min=self.default_min_,
exclusives=exclusives)
def test_assign_roles_allinone_bundles(self):
lower_configs = {
1: {'roles': []},
}
exclusives = ['control']
bundles = [['control', 'api', 'compute']]
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_roles(
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=self.maxs_, default_min=self.default_min_,
exclusives=exclusives, bundles=bundles)
self.assertEqual(assigned, {1: ['control', 'api', 'compute']})
def test_assign_roles_allinone_bundles_noenough_hosts(self):
exclusives = ['control']
bundles = [['control', 'api']]
lower_configs = {
1: {'roles': []},
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
ValueError, config_merger_callbacks.assign_roles,
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=self.maxs_, default_min=self.default_min_,
exclusives=exclusives, bundles=bundles)
def test_assign_roles_allinone_maxes_mins_noenough_hosts(self):
lower_configs = {
1: {'roles': []},
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
maxs = {'control': 1, 'api': 2, 'compute': -1}
mins = {'control': 1, 'api': 2}
default_min = 0
self.assertRaises(
ValueError, config_merger_callbacks.assign_roles,
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=maxs, mins=mins,
default_min=default_min)
def test_assign_roles_allinone_maxes_mins(self):
lower_configs = {
1: {'roles': []},
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
maxs = {'control': 1, 'api': 2, 'compute': -1}
mins = {'control': 1, 'api': 0}
default_min = 0
assigned = config_merger_callbacks.assign_roles(
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=maxs, mins=mins, default_min=default_min)
self.assertEqual(assigned, {1: ['control']})
def test_assign_roles(self):
"""test assign roles."""
lower_configs = {
@ -57,23 +198,22 @@ class TestAssignRoles(unittest2.TestCase):
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
roles = ['control', 'api', 'compute']
maxs = {'control': 1, 'api': 2, 'compute': -1}
default_min = 1
exclusives = ['control']
assigned = config_merger_callbacks.assign_roles(
None, None, lower_refs, 'roles', roles=roles,
maxs=maxs,
default_min=default_min,
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=self.maxs_,
default_min=self.default_min_,
exclusives=exclusives)
self.assertEqual(assigned, {1: ['control'],
2: ['api', 'compute'],
3: ['api'],
3: ['compute'],
4: ['compute'],
5: ['compute']})
def test_assign_roles_multihosts_one_role(self):
default_min = 2
maxs = {'control': 1, 'api': 2, 'compute': 2}
exclusives = ['control']
lower_configs = {
1: {'roles': ['control']},
2: {'roles': ['api', 'compute']},
@ -85,7 +225,7 @@ class TestAssignRoles(unittest2.TestCase):
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_roles(
None, None, lower_refs, 'roles', roles=roles,
None, None, lower_refs, 'roles', roles=self.roles_,
maxs=maxs, default_min=default_min,
exclusives=exclusives)
self.assertEqual(assigned, {1: ['control'],
@ -94,9 +234,10 @@ class TestAssignRoles(unittest2.TestCase):
4: ['api'],
5: ['compute']})
default_min = 1
def test_assign_roles_bundles(self):
roles = ['control', 'api', 'compute', 'mysql']
maxs = {'control': 1, 'api': 2, 'compute': -1, 'mysql': 2}
exclusives = ['control']
bundles = [['control', 'api']]
lower_configs = {
1: {},
@ -110,7 +251,7 @@ class TestAssignRoles(unittest2.TestCase):
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_roles(
None, None, lower_refs, 'roles', roles=roles,
maxs=maxs, default_min=default_min,
maxs=maxs, default_min=self.default_min_,
exclusives=exclusives, bundles=bundles)
self.assertEqual(assigned, {1: ['control', 'api'],
2: ['compute'],
@ -118,6 +259,507 @@ class TestAssignRoles(unittest2.TestCase):
4: ['mysql'],
5: ['compute']})
def test_assign_roles_multi_default_roles(self):
roles = ['control', 'api', 'compute', 'mysql']
maxs = {'control': 1, 'api': 2, 'compute': -1, 'mysql': -2}
exclusives = ['control']
bundles = [['control', 'api']]
lower_configs = {
1: {},
2: {},
3: {},
4: {},
5: {},
6: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_roles(
None, None, lower_refs, 'roles', roles=roles,
maxs=maxs, default_min=self.default_min_,
exclusives=exclusives, bundles=bundles)
self.assertEqual(assigned, {1: ['control', 'api'],
2: ['compute'],
3: ['mysql'],
4: ['mysql'],
5: ['compute'],
6: ['mysql']})
def test_assign_roles_hosts_portion_by_default_roles(self):
roles = ['control', 'api', 'compute', 'mysql']
maxs = {'control': 1, 'api': 2, 'compute': -1, 'mysql': -1}
exclusives = ['control']
bundles = [['control', 'api']]
lower_configs = {
1: {},
2: {},
3: {},
4: {},
5: {},
6: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_roles(
None, None, lower_refs, 'roles', roles=roles,
maxs=maxs, default_min=self.default_min_,
exclusives=exclusives, bundles=bundles)
self.assertEqual(assigned, {1: ['control', 'api'],
2: ['compute'],
3: ['mysql'],
4: ['compute'],
5: ['mysql'],
6: ['compute']})
def test_assign_roles_by_host_number_one_host(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
default = {
'roles': ['control', 'api', 'compute'],
'maxs': {'control': 1, 'api': 1, 'compute': -1},
'default_min': 1,
'exclusives': ['control']
}
policy_by_host_numbers = {
'1': {
'bundles': [['control', 'api', 'compute']]
},
'2': {
'bundles': [['control', 'api']]
},
}
assigned = config_merger_callbacks.assign_roles_by_host_numbers(
None, None, lower_refs, 'roles',
policy_by_host_numbers=policy_by_host_numbers,
default=default)
self.assertEqual(assigned, {1: ['control', 'api', 'compute']})
def test_assign_roles_by_host_number_two_hosts(self):
lower_configs = {
1: {},
2: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
default = {
'roles': ['control', 'api', 'compute'],
'maxs': {'control': 1, 'api': 1, 'compute': -1},
'default_min': 1,
'exclusives': ['control']
}
policy_by_host_numbers = {
'1': {
'bundles': [['control', 'api', 'compute']]
},
'2': {
'bundles': [['control', 'api']]
},
}
assigned = config_merger_callbacks.assign_roles_by_host_numbers(
None, None, lower_refs, 'roles',
policy_by_host_numbers=policy_by_host_numbers,
default=default)
self.assertEqual(assigned, {1: ['control', 'api'], 2: ['compute']})
def test_assign_roles_by_host_number_host_number_not_found(self):
lower_configs = {
1: {},
2: {},
3: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
default = {
'roles': ['control', 'api', 'compute'],
'maxs': {'control': 1, 'api': 1, 'compute': -1},
'default_min': 1,
'exclusives': ['control']
}
policy_by_host_numbers = {
'1': {
'bundles': [['control', 'api', 'compute']]
},
'2': {
'bundles': [['control', 'api']]
},
}
assigned = config_merger_callbacks.assign_roles_by_host_numbers(
None, None, lower_refs, 'roles',
policy_by_host_numbers=policy_by_host_numbers,
default=default)
self.assertEqual(
assigned, {1: ['control'], 2: ['api'], 3: ['compute']})
def test_assign_roles_by_host_number_host_number_host_number_int(self):
default = {
'roles': ['control', 'api', 'compute'],
'maxs': {'control': 1, 'api': 1, 'compute': -1},
'default_min': 1,
'exclusives': ['control']
}
policy_by_host_numbers = {
1: {
'bundles': [['control', 'api', 'compute']]
},
2: {
'bundles': [['control', 'api']]
},
}
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
ValueError, config_merger_callbacks.assign_roles_by_host_numbers,
None, None, lower_refs, 'roles',
policy_by_host_numbers=policy_by_host_numbers,
default=default)
class TestAssignIPs(unittest2.TestCase):
"""test assign ips."""
def setUp(self):
super(TestAssignIPs, self).setUp()
logsetting.init()
def tearDown(self):
super(TestAssignIPs, self).tearDown()
def test_assign_ips_validate(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
# ip_start and ip_end param should be the correct format.
self.assertRaises(
ValueError, config_merger_callbacks.assign_ips,
None, None, lower_refs, 'ip',
ip_start='')
self.assertRaises(
ValueError, config_merger_callbacks.assign_ips,
None, None, lower_refs, 'ip',
ip_start='100')
self.assertRaises(
ValueError, config_merger_callbacks.assign_ips,
None, None, lower_refs, 'ip',
ip_end='')
self.assertRaises(
ValueError, config_merger_callbacks.assign_ips,
None, None, lower_refs, 'ip',
ip_end='100')
def test_assign_ip_ip_start_ip_end_relation(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
ValueError, config_merger_callbacks.assign_ips,
None, None, lower_refs, 'ip',
ip_start='192.168.100.100', ip_end='192.168.100.99')
assigned = config_merger_callbacks.assign_ips(
None, None, lower_refs, 'ip',
ip_start='192.168.100.100', ip_end='192.168.100.100')
self.assertEqual(assigned, {1: '192.168.100.100'})
def test_assign_ips_multi_hosts_noenough_ips(self):
lower_configs = {
1: {}, 2: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
ValueError, config_merger_callbacks.assign_ips,
None, None, lower_refs, 'ip',
ip_start='192.168.100.100', ip_end='192.168.100.100')
def test_assign_ips_multi_hosts(self):
lower_configs = {
1: {}, 2: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_ips(
None, None, lower_refs, 'ip',
ip_start='192.168.100.100', ip_end='192.168.100.101')
self.assertEqual(
assigned, {1: '192.168.100.100', 2: '192.168.100.101'})
class TestAssignFromPattern(unittest2.TestCase):
"""test assign value from pattern."""
def setUp(self):
super(TestAssignFromPattern, self).setUp()
logsetting.init()
def tearDown(self):
super(TestAssignFromPattern, self).tearDown()
def test_pattern(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_from_pattern(
None, None, lower_refs, 'pattern', pattern='hello')
self.assertEqual(assigned, {1: 'hello'})
def test_pattern_upper_keys(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_from_pattern(
None, None, lower_refs, 'pattern',
upper_keys=['clustername'], pattern='%(clustername)s',
clustername='mycluster')
self.assertEqual(assigned, {1: 'mycluster'})
def test_pattern_lower_keys(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_from_pattern(
None, None, lower_refs, 'pattern',
lower_keys=['hostname'], pattern='%(hostname)s',
hostname={1: 'myhost'})
self.assertEqual(assigned, {1: 'myhost'})
def test_pattern_upper_keys_lower_keys(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_from_pattern(
None, None, lower_refs, 'pattern',
upper_keys=['clustername'], lower_keys=['hostname'],
pattern='%(hostname)s.%(clustername)s',
hostname={1: 'myhost'}, clustername='mycluster')
self.assertEqual(assigned, {1: 'myhost.mycluster'})
def test_pattern_upper_keys_lower_keys_overlap(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
KeyError, config_merger_callbacks.assign_from_pattern,
None, None, lower_refs, 'pattern',
upper_keys=['clustername'],
lower_keys=['clustername', 'hostname'],
pattern='%(hostname)s.%(clustername)s',
hostname={1: 'myhost'}, clustername='mycluster')
def test_pattern_extra_keys(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
KeyError, config_merger_callbacks.assign_from_pattern,
None, None, lower_refs, 'pattern',
upper_keys=['clustername', 'clusterid'],
lower_keys=['hostname', 'hostid'],
pattern='%(hostname)s.%(clustername)s',
hostname={1: 'myhost'}, clustername='mycluster')
def test_pattern_lower_key_not_dict(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
KeyError, config_merger_callbacks.assign_from_pattern,
None, None, lower_refs, 'pattern',
upper_keys=['clustername'],
lower_keys=['hostname'],
pattern='%(hostname)s.%(clustername)s',
hostname='myhost', clustername='mycluster')
def test_pattern_extra_kwargs(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_from_pattern(
None, None, lower_refs, 'pattern',
upper_keys=['clustername'],
lower_keys=['hostname'],
pattern='%(hostname)s.%(clustername)s',
hostname={1: 'myhost'}, clustername='mycluster',
hostid={1: 'myhost'}, clusterid=1)
self.assertEqual(assigned, {1: 'myhost.mycluster'})
def test_pattern_extra_key_in_pattern(self):
lower_configs = {
1: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
KeyError, config_merger_callbacks.assign_from_pattern,
None, None, lower_refs, 'pattern',
upper_keys=['clustername'],
lower_keys=['hostname'],
pattern='%(hostid)s.%(clusterid)s',
hostname={1: 'myhost'}, clustername='mycluster')
class TestNoProxy(unittest2.TestCase):
"""test assign noproxy."""
def setUp(self):
super(TestNoProxy, self).setUp()
logsetting.init()
def tearDown(self):
super(TestNoProxy, self).tearDown()
def test_noproxy(self):
lower_configs = {
1: {},
2: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_noproxy(
None, None, lower_refs, 'noproxy',
default=['127.0.0.1', 'compass', '10.145.88.3'],
clusterid=1, noproxy_pattern='%(hostname)s.%(clusterid)s,%(ip)s',
hostnames={1: 'host1', 2: 'host2'},
ips={1: '10.145.88.1', 2: '10.145.88.2'})
self.assertEqual(
assigned, {
1: (
'127.0.0.1,compass,10.145.88.3,'
'host1.1,10.145.88.1,host2.1,10.145.88.2'
),
2: (
'127.0.0.1,compass,10.145.88.3,'
'host1.1,10.145.88.1,host2.1,10.145.88.2'
)
})
def test_noproxy_noclusterid(self):
lower_configs = {
1: {},
2: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
KeyError, config_merger_callbacks.assign_noproxy,
None, None, lower_refs, 'noproxy',
default=['127.0.0.1', 'compass', '10.145.88.3'],
noproxy_pattern='%(hostname)s.%(clusterid)s,%(ip)s',
hostnames={1: 'host1', 2: 'host2'},
ips={1: '10.145.88.1', 2: '10.145.88.2'})
def test_noproxy_nohostname_ips(self):
lower_configs = {
1: {},
2: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
KeyError, config_merger_callbacks.assign_noproxy,
None, None, lower_refs, 'noproxy',
default=['127.0.0.1', 'compass', '10.145.88.3'],
noproxy_pattern='%(hostname)s.%(clusterid)s,%(ip)s',
clusterid=1, hostnames={1: 'host1'},
ips={1: '10.145.88.1'})
def test_noproxy_extra_keys_in_pattern(self):
lower_configs = {
1: {},
2: {}
}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(
KeyError, config_merger_callbacks.assign_noproxy,
None, None, lower_refs, 'noproxy',
default=['127.0.0.1', 'compass', '10.145.88.3'],
noproxy_pattern='%(hostname)s.%(clustername)s,%(ip)s',
clusterid=1, hostnames={1: 'host1', 2: 'host2'},
ips={1: '10.145.88.1', 2: '10.145.88.2'})
class TestOverrideIfEmpty(unittest2.TestCase):
"""test override if empty."""
def setUp(self):
super(TestOverrideIfEmpty, self).setUp()
logsetting.init()
def tearDown(self):
super(TestOverrideIfEmpty, self).tearDown()
def test_lower_config_none(self):
lower_config = None
lower_ref = config_reference.ConfigReference(lower_config)
override = config_merger_callbacks.override_if_empty(
None, None, lower_ref, 'override')
self.assertTrue(override)
def test_lower_config_empty(self):
lower_config = ''
lower_ref = config_reference.ConfigReference(lower_config)
override = config_merger_callbacks.override_if_empty(
None, None, lower_ref, 'override')
self.assertTrue(override)
lower_config = []
lower_ref = config_reference.ConfigReference(lower_config)
override = config_merger_callbacks.override_if_empty(
None, None, lower_ref, 'override')
self.assertTrue(override)
lower_config = {}
lower_ref = config_reference.ConfigReference(lower_config)
override = config_merger_callbacks.override_if_empty(
None, None, lower_ref, 'override')
self.assertTrue(override)
if __name__ == '__main__':
flags.init()

383
compass/tests/config_management/utils/test_config_reference.py
View File

@ -19,148 +19,335 @@
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import copy
import os
import unittest2
os.environ['COMPASS_IGNORE_SETTING'] = 'true'
from compass.utils import setting_wrapper as setting
reload(setting)
from compass.config_management.utils import config_reference
from compass.utils import flags
from compass.utils import logsetting
from compass.utils import util
class TestCleanConfig(unittest2.TestCase):
"""test clean_config function."""
def setUp(self):
super(TestCleanConfig, self).setUp()
logsetting.init()
def tearDown(self):
super(TestCleanConfig, self).tearDown()
def test_config_empty(self):
"""test config is empty."""
self.assertIsNone(config_reference.get_clean_config(None))
self.assertIsNone(config_reference.get_clean_config({}))
self.assertEqual([], config_reference.get_clean_config([]))
self.assertEqual('', config_reference.get_clean_config(''))
def test_recursive_empty_dict(self):
"""test config is recursively empty."""
self.assertIsNone(config_reference.get_clean_config({'test': {}}))
self.assertIsNone(config_reference.get_clean_config({'test': None}))
def test_nromal_dict(self):
"""test config is normal dict."""
config_list = [
{'abc': 'abc'},
{'abc': [1, 2, 3]},
{'abc': {'1': '123'}},
[1, 2, 3],
'abc',
]
for config in config_list:
self.assertEqual(config, config_reference.get_clean_config(config))
def test_partial_clean(self):
"""test config is partial cleaned."""
config_and_cleans = [
({'abc': 1, 'bcd': None}, {'abc': 1}),
({'abc': 1, 'bcd': {}}, {'abc': 1}),
({'abc': 1, 'bcd': {'m': {}}}, {'abc': 1}),
({'abc': 1, 'b': {'m': {}, 'n': 2}}, {'abc': 1, 'b': {'n': 2}}),
]
for config, expected_config in config_and_cleans:
self.assertEqual(
expected_config,
config_reference.get_clean_config(config))
class TestConfigReference(unittest2.TestCase):
"""test config reference class."""
def setUp(self):
super(TestConfigReference, self).setUp()
logsetting.init()
self.config_ = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8}
self.ref_ = config_reference.ConfigReference(self.config_)
def tearDown(self):
super(TestConfigReference, self).tearDown()
def test_init(self):
"""test init function."""
config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
config2 = {'5': {'6': 6}}
ref2 = config_reference.ConfigReference(config2['5'], ref, '5')
expected_config = copy.deepcopy(config)
util.merge_dict(expected_config, config2)
self.assertEqual(ref.config, expected_config)
self.assertEqual(id(ref.config['5']), id(ref2.config))
config3 = {'5': {'7': 7}}
ref3 = config_reference.ConfigReference(config3['5'], ref, '5')
self.assertEqual(id(ref.config['5']), id(ref3.config))
# create ConfigReference instance.
self.assertEqual(self.ref_.config, self.config_)
self.assertEqual(id(self.ref_.config), id(self.config_))
def test_ref(self):
"""test ref function."""
config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
self.assertRaises(KeyError, ref.ref, '')
self.assertRaises(KeyError, ref.ref, '/1/2/4')
self.assertEqual(ref.ref('.').config, config)
self.assertEqual(ref.ref('..').config, config)
self.assertEqual(ref.ref('/').config, config)
self.assertEqual(ref.ref('1').config, config['1'])
self.assertEqual(ref.ref('1/2').config, config['1']['2'])
self.assertEqual(ref.ref('1/2/.').config, config['1']['2'])
self.assertEqual(ref.ref('1/2/..').config, config['1'])
self.assertEqual(ref.ref('1/2//').config, config['1']['2'])
self.assertEqual(ref.ref('/1').config, config['1'])
self.assertEqual(ref.ref('/1/2').config, config['1']['2'])
subref = ref.ref('1')
self.assertEqual(subref.ref('2').config, config['1']['2'])
self.assertEqual(subref.ref('2/..').config, config['1'])
self.assertEqual(subref.ref('..').config, config)
self.assertEqual(subref.ref('../..').config, config)
self.assertEqual(subref.ref('/').config, config)
self.assertEqual(subref.ref('/4').config, config['4'])
def test_init_with_parent(self):
# create ConfigReference instance with parent param.
# it will add a key value pair in parent config if not exists.
config2 = {'5': {'6': 6}}
ref2 = config_reference.ConfigReference(config2['5'], self.ref_, '5')
expected_config = copy.deepcopy(self.config_)
util.merge_dict(expected_config, config2)
self.assertEqual(self.config_, expected_config)
self.assertEqual(id(self.config_['5']), id(ref2.config))
self.assertEqual(id(ref2.config), id(config2['5']))
def test_init_with_parent_update(self):
# create ConfigReference instance with parent param.
# it will update the key value pair in parent config if it exists.
config3 = {'5': {'7': 7}}
ref3 = config_reference.ConfigReference(config3['5'], self.ref_, '5')
expected_config = copy.deepcopy(self.config_)
util.merge_dict(expected_config, config3)
self.assertEqual(self.config_, expected_config)
self.assertEqual(id(self.config_['5']), id(ref3.config))
self.assertEqual(id(ref3.config), id(config3['5']))
def test_init_config_keys(self):
# config key should be string.
config_reference.ConfigReference(1)
config_reference.ConfigReference('1')
config_reference.ConfigReference([1, 2])
config_reference.ConfigReference({'1': 2})
config_reference.ConfigReference({u'1': 2})
self.assertRaises(
TypeError, config_reference.ConfigReference, {1: 2})
def test_init_parent_type(self):
# parent instance should be of ConfigReference.
self.assertRaises(
TypeError, config_reference.ConfigReference,
{'1': 2}, parent=object())
def test_init_parent_key_type(self):
# parent key should be string.
self.assertRaises(
TypeError, config_reference.ConfigReference,
{'1': 2}, parent=self.ref_, parent_key=6)
def test_ref_noexist_key(self):
# raise KeyError when accessing noexist key.
self.assertRaises(KeyError, self.ref_.ref, '')
self.assertRaises(KeyError, self.ref_.ref, '/1/2/4')
def test_ref_dot_key(self):
# . key returns the same reference.
self.assertEqual(id(self.ref_.ref('.')), id(self.ref_))
self.assertEqual(self.ref_.ref('.').config, self.config_)
def test_ref_double_dot_key(self):
# .. key returns the same reference if ref itself
# is the top level reference.
self.assertEqual(id(self.ref_.ref('..')), id(self.ref_))
self.assertEqual(self.ref_.ref('..').config, self.config_)
def test_ref_slash_key(self):
# / key returns the same reference if the ref itself is
# the top level reference.
self.assertEqual(id(self.ref_.ref('/')), id(self.ref_))
self.assertEqual(self.ref_.ref('/').config, self.config_)
def test_ref_key(self):
# ref(<key>) returns the reference of the <key>.
self.assertEqual(self.ref_.ref('1').config, self.config_['1'])
self.assertEqual(self.ref_.ref('1/2').config,
self.config_['1']['2'])
self.assertEqual(self.ref_.ref('1/2/.').config,
self.config_['1']['2'])
self.assertEqual(self.ref_.ref('1/2/..').config,
self.config_['1'])
self.assertEqual(self.ref_.ref('1/2//').config,
self.config_['1']['2'])
self.assertEqual(self.ref_.ref('/1').config,
self.config_['1'])
self.assertEqual(self.ref_.ref('/1/2').config,
self.config_['1']['2'])
def test_ref_key_in_parent(self):
# from sub ref, we can get the reference of it parent or root.
subref = self.ref_.ref('1')
self.assertEqual(id(subref.ref('..')), id(self.ref_))
self.assertEqual(subref.ref('..').config, self.config_)
self.assertEqual(id(subref.ref('../..')), id(self.ref_))
self.assertEqual(subref.ref('../..').config, self.config_)
self.assertEqual(id(subref.ref('/')), id(self.ref_))
self.assertEqual(subref.ref('/').config, self.config_)
self.assertEqual(subref.ref('2').config, self.config_['1']['2'])
self.assertEqual(subref.ref('2/..').config, self.config_['1'])
self.assertEqual(subref.ref('/4').config, self.config_['4'])
def test_ref_key_not_exist(self):
subref = self.ref_.ref('1')
self.assertRaises(KeyError, subref.ref, '/4/5')
self.assertRaises(KeyError, subref.ref, '/9')
subref2 = ref.ref('9', True)
self.assertEqual(ref.ref('9'), subref2)
def test_ref_key_not_exist_and_create(self):
# create sub reference if key does not exists and
# create_if_not_exist param is True.
subref2 = self.ref_.ref('9', True)
self.assertEqual(self.ref_.ref('9'), subref2)
def test_refs(self):
"""test refs function."""
config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8, '88': 88}
ref = config_reference.ConfigReference(config)
refkeys = ref.ref_keys('1')
# ref_keys will return all matching refs.
refkeys = self.ref_.ref_keys('1')
self.assertEqual(set(refkeys), set(['1']))
refkeys = ref.ref_keys('/1/*')
def test_refs_asterisks(self):
refkeys = self.ref_.ref_keys('/1/*')
self.assertEqual(set(refkeys), set(['/1/2', '/1/10']))
refkeys = ref.ref_keys('*')
self.assertEqual(set(refkeys), set(['1', '4', '8', '88']))
refkeys = ref.ref_keys('8*')
self.assertEqual(set(refkeys), set(['8', '88']))
self.assertRaises(KeyError, ref.ref_keys, '')
refkeys = self.ref_.ref_keys('*')
self.assertEqual(set(refkeys), set(['1', '4', '8']))
refkeys = self.ref_.ref_keys('8*')
self.assertEqual(set(refkeys), set(['8']))
def test_refs_empty_key(self):
# ref keys will raise KeyError if the param is empty.
self.assertRaises(KeyError, self.ref_.ref_keys, '')
def test_contains(self):
"""test contains function."""
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
self.assertIn('/1/2', ref)
self.assertIn('1/10/', ref)
self.assertIn('4/', ref)
self.assertIn('/1/2/..', ref)
self.assertNotIn('/1/3/7', ref)
self.assertNotIn('/1/2/3/..', ref)
self.assertIn('/1/2', self.ref_)
self.assertIn('1/10/', self.ref_)
self.assertIn('4/', self.ref_)
self.assertIn('/1/2/..', self.ref_)
self.assertNotIn('/1/3/7', self.ref_)
self.assertNotIn('/1/2/3/..', self.ref_)
def test_getitem(self):
"""test getitem function."""
self.assertEqual(self.ref_['1'], self.config_['1'])
self.assertEqual(self.ref_['1/2'], self.config_['1']['2'])
self.assertEqual(self.ref_['/1'], self.config_['1'])
self.assertEqual(self.ref_['1/2/../../4'], self.config_['4'])
def test_setitem(self):
"""test setitem function."""
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
ref['/1/2'] = '6'
self.assertEqual(config['1']['2'], '6')
self.assertEqual(ref['/1/2'], '6')
ref['1/10/5'] = 7
self.assertEqual(config['1']['10']['5'], 7)
self.assertEqual(ref['1/10/5'], 7)
ref['3/6/8'] = [1, 3, 5]
self.assertEqual(config['3']['6']['8'], [1, 3, 5])
self.assertEqual(ref['3/6/8'], [1, 3, 5])
self.ref_['/1/2'] = '6'
self.assertEqual(self.config_['1']['2'], '6')
self.assertEqual(self.ref_['/1/2'], '6')
self.ref_['1/10/5'] = 7
self.assertEqual(self.config_['1']['10']['5'], 7)
self.assertEqual(self.ref_['1/10/5'], 7)
self.ref_['3/6/8'] = [1, 3, 5]
self.assertEqual(self.config_['3']['6']['8'], [1, 3, 5])
self.assertEqual(self.ref_['3/6/8'], [1, 3, 5])
def test_del(self):
"""test del function."""
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
del ref['/8']
self.assertNotIn('8', config)
del ref['1/2']
self.assertNotIn('2', config['1'])
del ref['1']
self.assertNotIn('1', config)
self.assertRaises(KeyError, ref.__delitem__, '9')
del self.ref_['/8']
self.assertNotIn('8', self.config_)
del self.ref_['1/2']
self.assertNotIn('2', self.config_['1'])
del self.ref_['1']
self.assertNotIn('1', self.config_)
def test_del_noexist_key(self):
# del nonexist key will raise KeyError
self.assertRaises(KeyError, self.ref_.__delitem__, '9')
def test_len(self):
ref = config_reference.ConfigReference({})
self.assertEqual(len(ref), 0)
ref = config_reference.ConfigReference({'1': '2', '2': '4'})
self.assertEqual(len(ref), 2)
ref = config_reference.ConfigReference(
{'1': {'2': '3', '4': '5'}, '2': '4'})
self.assertEqual(len(ref), 4)
def test_bool(self):
ref = config_reference.ConfigReference({})
self.assertFalse(ref)
ref = config_reference.ConfigReference({'1': 1})
self.assertTrue(ref)
def test_get(self):
"""test get function."""
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
self.assertEqual(ref.get('1/2'), config['1']['2'])
self.assertIsNone(ref.get('1/3'))
self.assertEqual(ref.get('1/3', 3), 3)
self.assertNotIn('3', config['1'])
self.assertEqual(self.ref_.get('1/2'), self.config_['1']['2'])
self.assertIsNone(self.ref_.get('1/3'))
self.assertEqual(self.ref_.get('1/3', 3), 3)
self.assertNotIn('3', self.config_['1'])
def test_setdefault(self):
"""test setdefault function."""
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
self.assertEqual(ref.setdefault('1/2').config, config['1']['2'])
self.assertIsNone(ref.setdefault('1/3').config)
self.assertEqual(ref.setdefault('1/4', 4).config, 4)
self.assertEqual(4, config['1']['4'])
self.assertEqual(self.ref_.setdefault('1/2').config,
self.config_['1']['2'])
self.assertIsNone(self.ref_.setdefault('1/3').config)
self.assertEqual(self.ref_.setdefault('1/4', 4).config, 4)
self.assertEqual(4, self.config_['1']['4'])
def test_update(self):
"""test update function."""
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
expected_config = copy.deepcopy(config)
ref = config_reference.ConfigReference(config)
expected_config = copy.deepcopy(self.config_)
config2 = {'9': 9, '10': {'10': 10}}
util.merge_dict(expected_config, config2)
ref.update(config2)
self.assertEqual(ref.config, expected_config)
ref.update(10, False)
self.assertEqual(ref.config, expected_config)
ref.update(10)
self.assertEqual(ref.config, 10)
self.ref_.update(config2)
self.assertEqual(self.ref_.config, expected_config)
def test_update_nooverride(self):
# if override is False and ref config is not None, ignore the update.
expected_config = copy.deepcopy(self.config_)
self.ref_.update(10, False)
self.assertEqual(self.config_, expected_config)
def test_update_override(self):
# if override is True, it will force update the config.
self.ref_.update(10)
self.assertEqual(self.ref_.config, 10)
def test_iter(self):
"""test iter function."""
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
items = dict(self.ref_.items())
self.assertEqual({
'1': {'2': 3, '10': {}},
'1/2': 3,
'1/10': {},
'4': [5, 6, 7],
'8': 8}, items)
self.assertEqual(
set(self.ref_.keys()),
set(['1', '1/2', '1/10', '4', '8']))
def test_match(self):
config = {'1': {'2': 'abcdef'}, '3': ['efg', 'hij', 'k']}
ref = config_reference.ConfigReference(config)
keys = ref.keys()
self.assertEqual(set(keys), set(['1', '1/2', '1/10', '4', '8']))
self.assertTrue(ref.match({'1/2': 'abcdef'}))
self.assertFalse(ref.match({'1/2': 'abceef'}))
self.assertTrue(ref.match({'1/2': '[a-z]+'}))
self.assertFalse(ref.match({'1/2': '[0-9]+'}))
self.assertTrue(ref.match({'3': 'efg'}))
self.assertFalse(ref.match({'3': 'm'}))
self.assertTrue(ref.match({'3': 'hij'}))
def test_filter(self):
config = {'1': {'2': 'abcdef', '4': 4}, '3': ['efg', 'hij', 'k']}
ref = config_reference.ConfigReference(config)
self.assertEqual(ref.filter(['1/2', '1/4', '5']),
{'1/2': 'abcdef', '1/4': 4})
if __name__ == '__main__':
flags.init()
logsetting.init()
unittest2.main()

787
compass/tests/config_management/utils/test_config_translator.py
View File

@ -18,7 +18,6 @@
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import functools
import os
import unittest2
@ -30,12 +29,362 @@ from compass.utils import setting_wrapper as setting
reload(setting)
from compass.config_management.utils import config_reference
from compass.config_management.utils import config_translator
from compass.config_management.utils import config_translator_callbacks
from compass.utils import flags
from compass.utils import logsetting
class TestKeyTranslator(unittest2.TestCase):
"""test key translator class."""
def setUp(self):
super(TestKeyTranslator, self).setUp()
logsetting.init()
def tearDown(self):
super(TestKeyTranslator, self).tearDown()
def test_init(self):
# translated_keys should be callback or list of string or callback.
config_translator.KeyTranslator(
translated_keys=['/a/b/c', '/d/e'])
config_translator.KeyTranslator(
translated_keys=[u'/a/b/c', u'/d/e'])
config_translator.KeyTranslator(
translated_keys=(lambda sub_ref, ref_key: []))
config_translator.KeyTranslator(
translated_keys=[lambda sub_ref, ref_key: '/d/e'])
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys='/a/b/c')
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys={'/a/b/c': 'd/e'})
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=[5, 6, 7])
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=[('5', '6')])
def test_init_translated_keys_noasterrisks(self):
# the key in translated key should not contain '*'.
self.assertRaises(
KeyError, config_translator.KeyTranslator,
translated_keys=['/a/*/b'])
def test_init_from_keys(self):
# the from keys should be dict of string to string.
config_translator.KeyTranslator(
translated_keys=['/a/b/c', '/d/e'], from_keys={'m': '/m/n'})
config_translator.KeyTranslator(
translated_keys=['/a/b/c', '/d/e'], from_keys={u'm': u'/m/n'})
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_keys=['m'])
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_keys='m')
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_keys={5: 'm'})
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_keys={'m': 5})
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_keys={'m': ['/m/n']})
def test_init_from_keys_noasterisks(self):
# the value of the from_keys should not contain '*'.
self.assertRaises(
KeyError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_keys={'m': '/m/*/n'})
def test_init_from_values(self):
# from_values should be dict of string to string.
config_translator.KeyTranslator(
translated_keys=['/a/b/c'], from_values={'m': '/m'})
config_translator.KeyTranslator(
translated_keys=['/a/b/c'], from_values={u'm': u'/m'})
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_values=['m'])
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_values='m')
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_values={5: 'm'})
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_keys={'m': 5})
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_values={'m': ['/m/n']})
def test_init_from_values_noasterisks(self):
# the value of the from_values should not contain '*'.
self.assertRaises(
KeyError, config_translator.KeyTranslator,
translated_keys=['/a/b/c'], from_values={'m': '/m/*/n'})
def test_init_override_conditions(self):
# override_conditions should be dict of string to string
config_translator.KeyTranslator(
translated_keys=['1/2/3', '/4/5/6'],
override_conditions={'hello': 'hi'})
config_translator.KeyTranslator(
translated_keys=['1/2/3', '/4/5/6'],
override_conditions={u'hello': u'hi'})
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['1/2/3', '/4/5/6'],
override_conditions=['hello', 'hi'])
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['1/2/3', '/4/5/6'],
override_conditions='hello')
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['1/2/3', '/4/5/6'],
override_conditions={5: 'hi'})
self.assertRaises(
TypeError, config_translator.KeyTranslator,
translated_keys=['1/2/3', '/4/5/6'],
override_conditions={'hello': 5})
def test_init_override_conditions_noasterisks(self):
# the value in override_conditions should not contains '*'.
self.assertRaises(
KeyError, config_translator.KeyTranslator,
translated_keys=['1/2/3', '/4/5/6'],
override_conditions={'hello': 'hi*hi'})
def test_translate(self):
# test get translated keys.
# only keys in translated_keys is set in translted config.
config = {
'key1': 'abc',
'key2': 'bcd',
'key3': 'mnq'
}
ref = config_reference.ConfigReference(config)
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=['key2', 'key3'])
translator.translate(ref, 'key1', translated_ref)
self.assertEqual(translated_config, {'key2': 'abc', 'key3': 'abc'})
def test_translate_translated_keys_callback(self):
# translated_keys can be callback to dynamically
# get the translated_keys.
config = {
'key1': 'abc',
'key2': 'bcd',
'key3': 'mnq'
}
ref = config_reference.ConfigReference(config)
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=(
lambda sub_ref, ref_key: ['m%s' % ref_key, 'n%s' % ref_key]))
translator.translate(ref, 'key*', translated_ref)
self.assertEqual(
translated_config, {
'mkey1': 'abc', 'mkey2': 'bcd', 'mkey3': 'mnq',
'nkey1': 'abc', 'nkey2': 'bcd', 'nkey3': 'mnq',
}
)
def test_translate_translated_keys_each_key_callback(self):
# each translated key can be a callback to dynamically
# get the translated key.
config = {
'key1': 'abc',
'key2': 'bcd',
'key3': 'mnq'
}
ref = config_reference.ConfigReference(config)
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=['key1', (lambda sub_ref, ref_key: 'mkey2')])
translator.translate(ref, 'key1', translated_ref)
self.assertEqual(
translated_config, {'key1': 'abc', 'mkey2': 'abc'})
def test_translate_from_keys(self):
# generate translated_keys from some keys from config.
config = {
'key': 'abc',
'key_suffix': 'A',
'key_prefix': 'B'
}
def _generate_key(sub_ref, ref_key, prefix='', suffix=''):
return '%s%s%s' % (prefix, ref_key, suffix)
def _generate_keys(sub_ref, ref_key, prefix='', suffix=''):
return ['%s%s%s' % (prefix, ref_key, suffix)]
ref = config_reference.ConfigReference(config)
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=[_generate_key],
from_keys={'prefix': '/key_prefix', 'suffix': '/key_suffix'})
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'BkeyA': 'abc'})
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=_generate_keys,
from_keys={'prefix': '/key_prefix', 'suffix': '/key_suffix'})
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'BkeyA': 'abc'})
def test_translate_translated_value(self):
# translated_value can be set explictly.
config = {
'key': 'abc',
'key_suffix': 'A',
'key_prefix': 'B'
}
ref = config_reference.ConfigReference(config)
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=['mnq'],
translated_value='mnq')
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'mnq': 'mnq'})
def test_translated_value_by_callback_none(self):
# translated value can be generated from callback.
# the value will be ignored when generated translated_value is None.
config = {
'key': 'abc',
}
ref = config_reference.ConfigReference(config)
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
def _generate_none(
sub_ref, ref_key, translated_sub_ref, translated_key
):
return None
translator = config_translator.KeyTranslator(
translated_keys=['mnq'],
translated_value=_generate_none)
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'mnq': None})
def test_translate_translated_value_by_callback(self):
# translated_value can be set from some field of config.
config = {
'key': 'abc',
'key_suffix': 'A',
'key_prefix': 'B'
}
ref = config_reference.ConfigReference(config)
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
def _generate_value(
sub_ref, ref_key, translated_sub_ref, translated_key,
prefix='', suffix=''
):
return '%s%s%s' % (prefix, sub_ref.config, suffix)
translator = config_translator.KeyTranslator(
translated_keys=['mnq'],
translated_value=_generate_value,
from_values={'prefix': '/key_prefix', 'suffix': '/key_suffix'})
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'mnq': 'BabcA'})
def test_translate_nooverride(self):
# the translated key will be ignored when the key has already existed
# in translated config and override is False.
config = {
'key': 'abc',
}
ref = config_reference.ConfigReference(config)
translated_config = {'mnq': 'mnq'}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=['mnq'], override=False)
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'mnq': 'mnq'})
def test_translate_override(self):
# the translated config will be overrided if override param is True.
config = {
'key': 'abc',
}
ref = config_reference.ConfigReference(config)
translated_config = {'mnq': 'mnq'}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=['mnq'], override=True)
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'mnq': 'abc'})
def test_translate_override_by_callback(self):
# override param can be set from callback.
config = {
'key': 'abc',
}
ref = config_reference.ConfigReference(config)
translated_config = {'klm': 'klm', 'mnq': 'mnq'}
translated_ref = config_reference.ConfigReference(translated_config)
def _generate_override(
sub_ref, ref_key,
translated_sub_ref, translated_key
):
return translated_key == 'klm'
translator = config_translator.KeyTranslator(
translated_keys=['klm', 'mnq'], override=_generate_override)
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'klm': 'abc', 'mnq': 'mnq'})
def test_translate_override_conditions(self):
# override param can be set from some config fields.
config = {
'key': 'abc',
'key_suffix': 'A',
'key_prefix': 'B'
}
ref = config_reference.ConfigReference(config)
translated_config = {'BmA': 'BmA', 'mnq': 'mnq'}
translated_ref = config_reference.ConfigReference(translated_config)
def _generate_override2(
sub_ref, ref_key,
translated_sub_ref, translated_key, prefix='', suffix='',
):
return (
translated_key.startswith(prefix) and
translated_key.endswith(suffix))
translator = config_translator.KeyTranslator(
translated_keys=['BmA', 'mnq'],
override=_generate_override2,
override_conditions={
'prefix': '/key_prefix', 'suffix': '/key_suffix'
}
)
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'BmA': 'abc', 'mnq': 'mnq'})
class TestConfigTranslatorFunctions(unittest2.TestCase):
"""test config translator class."""
@ -46,327 +395,171 @@ class TestConfigTranslatorFunctions(unittest2.TestCase):
def tearDown(self):
super(TestConfigTranslatorFunctions, self).tearDown()
def test_translate_1(self):
"""config translate test."""
def test_init(self):
# mapping should be dict of string to list of KeyTranslator.
config_translator.ConfigTranslator(
mapping={
'key1': [config_translator.KeyTranslator(
translated_keys=['abc']
)]
}
)
config_translator.ConfigTranslator(
mapping={
u'key1': [config_translator.KeyTranslator(
translated_keys=['abc']
)]
}
)
self.assertRaises(
TypeError, config_translator.ConfigTranslator,
mapping=[config_translator.KeyTranslator(translated_keys=['abc'])]
)
self.assertRaises(
TypeError, config_translator.ConfigTranslator,
mapping=config_translator.KeyTranslator(translated_keys=['abc'])
)
self.assertRaises(
TypeError, config_translator.ConfigTranslator,
mapping={
'abc': config_translator.KeyTranslator(translated_keys=['abc'])
}
)
self.assertRaises(
TypeError, config_translator.ConfigTranslator,
mapping={
1: [config_translator.KeyTranslator(translated_keys=['abc'])]
}
)
self.assertRaises(
TypeError, config_translator.ConfigTranslator,
mapping={
'abc': [
{
'm': config_translator.KeyTranslator(
translated_keys=['abc'])
}
]
}
)
def test_translate(self):
"""test translate config."""
config = {
'networking': {
'interfaces': {
'management': {
'mac': '00:00:00:01:02:03',
'ip': '192.168.1.1',
'netmask': '255.255.255.0',
'promisc': 0,
'nic': 'eth0',
'gateway': '2.3.4.5',
'dns_alias': 'hello.ods.com',
},
'floating': {
'promisc': 1,
'nic': 'eth1',
},
'storage': {
'ip': '172.16.1.1',
'nic': 'eth2',
},
'tenant': {
'ip': '10.1.1.1',
'netmask': '255.255.0.0',
'nic': 'eth0',
},
},
'global': {
'name_servers': ['nameserver.ods.com'],
'search_path': 'ods.com',
'gateway': '10.0.0.1',
'proxy': 'http://1.2.3.4:3128',
'ntp_server': '1.2.3.4',
'ignore_proxy': '127.0.0.1',
},
},
}
expected_config = {
'name_servers_search': 'ods.com',
'gateway': '10.0.0.1',
'modify_interface': {
'dnsname-eth0': 'hello.ods.com',
'ipaddress-eth2': '172.16.1.1',
'static-eth2': True,
'static-eth1': True,
'static-eth0': True,
'netmask-eth0': '255.255.255.0',
'ipaddress-eth0': '192.168.1.1',
'macaddress-eth0': '00:00:00:01:02:03',
'management-eth2': False,
'management-eth0': True,
'management-eth1': False
},
'ksmeta': {
'promisc_nics': 'eth1',
'ntp_server': '1.2.3.4',
'proxy': 'http://1.2.3.4:3128',
'ignore_proxy': '127.0.0.1'
}
'key1': 'abc',
'key2': 'bcd'
}
translator = config_translator.ConfigTranslator(
mapping={
'/networking/global/gateway': [
'key1': [
config_translator.KeyTranslator(
translated_keys=['/gateway']
translated_keys=['mkey1']
)
],
'/networking/global/nameservers': [
'key2': [
config_translator.KeyTranslator(
translated_keys=['/name_servers']
)
],
'/networking/global/search_path': [
translated_keys=['mkey2'],
translated_value='mkey2'
),
config_translator.KeyTranslator(
translated_keys=['/name_servers_search']
translated_keys=['mkey2'],
translated_value='nkey2'
)
],
'/networking/global/proxy': [
config_translator.KeyTranslator(
translated_keys=['/ksmeta/proxy']
)
],
'/networking/global/ignore_proxy': [
config_translator.KeyTranslator(
translated_keys=['/ksmeta/ignore_proxy']
)
],
'/networking/global/ntp_server': [
config_translator.KeyTranslator(
translated_keys=['/ksmeta/ntp_server']
)
],
'/security/server_credentials/username': [
config_translator.KeyTranslator(
translated_keys=['/ksmeta/username']
)
],
'/security/server_credentials/password': [
config_translator.KeyTranslator(
translated_keys=['/ksmeta/password'],
translated_value=(
config_translator_callbacks.get_encrypted_value)
)
],
'/partition': [
config_translator.KeyTranslator(
translated_keys=['/ksmeta/partition']
)
],
'/networking/interfaces/*/mac': [
config_translator.KeyTranslator(
translated_keys=[functools.partial(
config_translator_callbacks.get_key_from_pattern,
to_pattern='/modify_interface/macaddress-%(nic)s'
)],
from_keys={'nic': '../nic'},
override=functools.partial(
config_translator_callbacks.override_path_has,
should_exist='management')
)
],
'/networking/interfaces/*/ip': [
config_translator.KeyTranslator(
translated_keys=[functools.partial(
config_translator_callbacks.get_key_from_pattern,
to_pattern='/modify_interface/ipaddress-%(nic)s')],
from_keys={'nic': '../nic'},
override=functools.partial(
config_translator_callbacks.override_path_has,
should_exist='management')
)
],
'/networking/interfaces/*/netmask': [
config_translator.KeyTranslator(
translated_keys=[functools.partial(
config_translator_callbacks.get_key_from_pattern,
to_pattern='/modify_interface/netmask-%(nic)s')],
from_keys={'nic': '../nic'},
override=functools.partial(
config_translator_callbacks.override_path_has,
should_exist='management')
)
],
'/networking/interfaces/*/dns_alias': [
config_translator.KeyTranslator(
translated_keys=[functools.partial(
config_translator_callbacks.get_key_from_pattern,
to_pattern='/modify_interface/dnsname-%(nic)s')],
from_keys={'nic': '../nic'},
override=functools.partial(
config_translator_callbacks.override_path_has,
should_exist='management')
)
],
'/networking/interfaces/*/nic': [
config_translator.KeyTranslator(
translated_keys=[functools.partial(
config_translator_callbacks.get_key_from_pattern,
to_pattern='/modify_interface/static-%(nic)s')],
from_keys={'nic': '../nic'},
translated_value=True,
override=functools.partial(
config_translator_callbacks.override_path_has,
should_exist='management'),
), config_translator.KeyTranslator(
translated_keys=[functools.partial(
config_translator_callbacks.get_key_from_pattern,
to_pattern='/modify_interface/management-%(nic)s'
)],
from_keys={'nic': '../nic'},
translated_value=functools.partial(
config_translator_callbacks.override_path_has,
should_exist='management'),
override=functools.partial(
config_translator_callbacks.override_path_has,
should_exist='management')
), config_translator.KeyTranslator(
translated_keys=['/ksmeta/promisc_nics'],
from_values={'condition': '../promisc'},
translated_value=config_translator_callbacks.add_value,
override=True,
)
],
]
}
)
translated_config = translator.translate(config)
self.assertEqual(translated_config, expected_config)
self.assertEqual(translated_config,
{'mkey1': 'abc', 'mkey2': 'mkey2'})
def test_translate_2(self):
"""config translate test."""
def test_translate_nooverride(self):
# the later KeyTranslator will be ignored if the former one
# has already set the value.
config = {
'key1': 'abc',
'key2': 'bcd'
}
translator = config_translator.ConfigTranslator(
mapping={
'/networking/interfaces/management/ip': [
'key1': [
config_translator.KeyTranslator(
translated_keys=[
'/db/mysql/bind_address',
'/mq/rabbitmg/bind_address',
'/mq/rabbitmq/bind_address',
'/endpoints/compute/metadata/host',
'/endpoints/compute/novnc/host',
'/endpoints/compute/service/host',
'/endpoints/compute/xvpvnc/host',
'/endpoints/ec2/admin/host',
'/endpoints/ec2/service/host',
'/endpoints/identity/admin/host',
'/endpoints/identity/service/host',
'/endpoints/image/registry/host',
'/endpoints/image/service/host',
'/endpoints/metering/service/host',
'/endpoints/network/service/host',
'/endpoints/volume/service/host',
],
translated_value=(
config_translator_callbacks.get_value_if),
from_values={'condition': '/has_dashboard_roles'},
override=config_translator_callbacks.override_if_any,
override_conditions={
'has_dashboard_roles': '/has_dashboard_roles'}
translated_keys=['mkey1']
)
],
'key2': [
config_translator.KeyTranslator(
translated_keys=['mkey2'],
translated_value='mkey2'
),
config_translator.KeyTranslator(
translated_keys=['mkey2'],
translated_value='nkey2',
override=False
)
]
}
)
config1 = {
'networking': {
'interfaces': {
'management': {
'ip': '1.2.3.4',
},
},
},
'has_dashboard_roles': True,
translated_config = translator.translate(config)
self.assertEqual(translated_config,
{'mkey1': 'abc', 'mkey2': 'mkey2'})
def test_translate_override(self):
# the later KeyTranslator will override the former one
# if override is set.
config = {
'key1': 'abc',
'key2': 'bcd'
}
expected_config1 = {
'db': {
'mysql': {
'bind_address': '1.2.3.4'
}
},
'endpoints': {
'compute': {
'novnc': {
'host': '1.2.3.4'
},
'xvpvnc': {
'host': '1.2.3.4'
},
'service': {
'host': '1.2.3.4'
},
'metadata': {
'host': '1.2.3.4'
}
},
'network': {
'service': {
'host': '1.2.3.4'
}
},
'image': {
'registry': {
'host': '1.2.3.4'
},
'service': {
'host': '1.2.3.4'
}
},
'metering': {
'service': {
'host': '1.2.3.4'
}
},
'volume': {
'service': {
'host': '1.2.3.4'
}
},
'ec2': {
'admin': {
'host': '1.2.3.4'
},
'service': {
'host': '1.2.3.4'
}
},
'identity': {
'admin': {
'host': '1.2.3.4'
},
'service': {
'host': '1.2.3.4'
}
}
},
'mq': {
'rabbitmg': {
'bind_address': '1.2.3.4'
},
'rabbitmq': {
'bind_address': '1.2.3.4'
}
translator = config_translator.ConfigTranslator(
mapping={
'key1': [
config_translator.KeyTranslator(
translated_keys=['mkey1']
)
],
'key2': [
config_translator.KeyTranslator(
translated_keys=['mkey2'],
translated_value='mkey2'
),
config_translator.KeyTranslator(
translated_keys=['mkey2'],
translated_value='nkey2',
override=True
)
]
}
}
translated_config1 = translator.translate(config1)
self.assertEqual(translated_config1, expected_config1)
)
translated_config = translator.translate(config)
self.assertEqual(translated_config,
{'mkey1': 'abc', 'mkey2': 'nkey2'})
config2 = {
'networking': {
'interfaces': {
'management': {
'ip': '1.2.3.4',
},
},
},
'has_dashboard_roles': False,
def test_translated_generated_value_none(self):
# When the generated value is None,
# the translated key will be ignored.
config = {
'key1': 'abc',
'key2': 'bcd'
}
expected_config2 = None
translated_config2 = translator.translate(config2)
self.assertEqual(translated_config2, expected_config2)
def _generate_none(
sub_ref, ref_key, translated_sub_ref, translated_key
):
return None
translator = config_translator.ConfigTranslator(
mapping={
'key1': [
config_translator.KeyTranslator(
translated_keys=['mkey1'],
translated_value=_generate_none
)
]
}
)
translated_config = translator.translate(config)
self.assertEqual(translated_config,
None)
if __name__ == '__main__':

64
compass/tests/db/test_database.py Normal file
View File

@ -0,0 +1,64 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""test util module.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import os
import unittest2
os.environ['COMPASS_IGNORE_SETTING'] = 'true'
from compass.utils import setting_wrapper as setting
reload(setting)
from compass.db import database
from compass.utils import flags
from compass.utils import logsetting
class TestDatabase(unittest2.TestCase):
"""Test database actions."""
def setUp(self):
super(TestDatabase, self).setUp()
logsetting.init()
database.init('sqlite://')
def tearDown(self):
super(TestDatabase, self).tearDown()
def test_init(self):
database.init('sqlite:///tmp/app.db')
self.assertEqual(str(database.ENGINE.url),
'sqlite:///tmp/app.db')
self.assertEqual(str(database.SCOPED_SESSION.bind.url),
'sqlite:///tmp/app.db')
def test_session(self):
with database.session() as session:
self.assertEqual(database.current_session(), session)
self.assertTrue(database.in_session())
self.assertFalse(database.in_session())
if __name__ == '__main__':
flags.init()
logsetting.init()
unittest2.main()