From a4bce397f0e3085f314e7231b37297747a8e3ea1 Mon Sep 17 00:00:00 2001 From: xiaodongwang Date: Tue, 25 Mar 2014 15:17:44 -0700 Subject: [PATCH] add haproxy support Change-Id: I63d9ab4b39591efa90c9bfc3be1595489d5a081d --- .../installers/plugins/chefhandler.py | 105 ++++++++++-------- .../providers/plugins/db_config_provider.py | 2 + .../config_management/utils/config_manager.py | 54 ++++++++- .../utils/config_merger_callbacks.py | 44 ++++++++ .../utils/config_translator_callbacks.py | 51 ++++++++- compass/tests/actions/deploy/data/test1 | 23 ++++ compass/tests/actions/deploy/data/test2 | 23 ++++ compass/tests/actions/deploy/data/test3 | 23 ++++ 8 files changed, 276 insertions(+), 49 deletions(-) diff --git a/compass/config_management/installers/plugins/chefhandler.py b/compass/config_management/installers/plugins/chefhandler.py index 4e7c2406..2376d428 100644 --- a/compass/config_management/installers/plugins/chefhandler.py +++ b/compass/config_management/installers/plugins/chefhandler.py @@ -17,6 +17,7 @@ .. moduleauthor:: Xiaodong Wang """ import fnmatch +import functools import logging from compass.config_management.installers import package_installer @@ -30,49 +31,12 @@ from compass.utils import util TO_CLUSTER_TRANSLATORS = { 'openstack': ConfigTranslator( mapping={ - '/security/console_credentials': [KeyTranslator( - translated_keys=['credential/identity/users/admin'], - )], - '/security/service_credentials': [KeyTranslator( - translated_keys=[ - '/credential/identity/users/compute', - '/credential/identity/users/image', - '/credential/identity/users/metering', - '/credential/identity/users/network', - '/credential/identity/users/object-store', - '/credential/identity/users/volume', - '/credential/mysql/compute', - '/credential/mysql/dashboard', - '/credential/mysql/identity', - '/credential/mysql/image', - '/credential/mysql/metering', - '/credential/mysql/network', - '/credential/mysql/volume', - ] - )], - '/security/service_credentials/password': [KeyTranslator( - translated_keys=[ - '/credential/mysql/super/password', - ] - )], - '/networking/interfaces/management/nic': [KeyTranslator( - translated_keys=['/networking/control/interface'], - )], - '/networking/global/ntp_server': [KeyTranslator( - translated_keys=['/ntp/ntpserver'] - )], - '/networking/interfaces/storage/nic': [KeyTranslator( - translated_keys=['/networking/storage/interface'] - )], - '/networking/interfaces/public/nic': [KeyTranslator( - translated_keys=['/networking/public/interface'] - )], - '/networking/interfaces/tenant/nic': [KeyTranslator( - translated_keys=[ - '/networking/tenant/interface', - '/networking/plugins/ovs/gre/local_ip_interface' - ] - )], + '/config_mapping': [KeyTranslator( + translated_keys=( + config_translator_callbacks.get_keys_from_config_mapping), + translated_value=( + config_translator_callbacks.get_value_from_config_mapping) + )] } ), } @@ -84,12 +48,18 @@ FROM_CLUSTER_TRANSLATORS = { '/role_assign_policy': [KeyTranslator( translated_keys=['/role_assign_policy'] )], - '/dashboard_roles': [KeyTranslator( - translated_keys=['/dashboard_roles'] + '/config_mapping': [KeyTranslator( + translated_keys=['/config_mapping'] )], '/role_mapping': [KeyTranslator( translated_keys=['/role_mapping'] )], + '/read_config_mapping': [KeyTranslator( + translated_keys=( + config_translator_callbacks.get_keys_from_config_mapping), + translated_value=( + config_translator_callbacks.get_value_from_config_mapping) + )], } ), } @@ -105,6 +75,45 @@ TO_HOST_TRANSLATORS = { translated_value=( config_translator_callbacks.get_value_from_role_mapping), from_values={'mapping': '/role_mapping'} + ), KeyTranslator( + translated_keys=[functools.partial( + config_translator_callbacks.get_key_from_pattern, + to_pattern='/node_mapping/%(node_name)s' + )], + from_keys={'node_name': '/node_name'} + )], + '/haproxy_roles': [KeyTranslator( + translated_keys=['/ha/status'], + translated_value='enable', + override=config_translator_callbacks.override_if_any, + override_conditions={'haproxy_roles': '/haproxy_roles'} + )], + '/haproxy/router_id': [KeyTranslator( + translated_keys=[functools.partial( + config_translator_callbacks.get_key_from_pattern, + to_pattern='/ha/keepalived/router_ids/%(node_name)s' + )], + from_keys={'node_name': '/node_name'} + )], + '/haproxy/priority': [KeyTranslator( + translated_keys=[functools.partial( + config_translator_callbacks.get_key_from_pattern, + to_pattern=( + '/ha/keepalived/instance_name/' + 'priorities/%(node_name)s' + ) + )], + from_keys={'node_name': '/node_name'} + )], + '/haproxy/state': [KeyTranslator( + translated_keys=[functools.partial( + config_translator_callbacks.get_key_from_pattern, + to_pattern=( + '/ha/keepalived/instance_name/' + 'states/%(node_name)s' + ) + )], + from_keys={'node_name': '/node_name'} )], } ), @@ -306,6 +315,12 @@ class Installer(package_installer.Installer): bag, clusterid, target_system) bag_item_dict = dict(bag_item) util.merge_dict(bag_item_dict, global_bag_item, False) + util.merge_dict(config, { + 'client_name': self._get_client_name( + config['fullname'], target_system), + 'node_name': self._get_node_name( + config['fullname'], target_system) + }) translated_config = TO_HOST_TRANSLATORS[target_system].translate( config) util.merge_dict(bag_item_dict, translated_config) diff --git a/compass/config_management/providers/plugins/db_config_provider.py b/compass/config_management/providers/plugins/db_config_provider.py index dd3f864c..7a2701dc 100644 --- a/compass/config_management/providers/plugins/db_config_provider.py +++ b/compass/config_management/providers/plugins/db_config_provider.py @@ -40,6 +40,8 @@ CLUSTER_DENIES = [] HOST_ALLOWS = [ '/roles', '/has_dashboard_roles', + '/dashboard_roles', + '/haproxy_roles', '/networking/interfaces/*/ip' ] HOST_DENIES = [] diff --git a/compass/config_management/utils/config_manager.py b/compass/config_management/utils/config_manager.py index 88d84e92..70c0620c 100644 --- a/compass/config_management/utils/config_manager.py +++ b/compass/config_management/utils/config_manager.py @@ -46,6 +46,12 @@ CLUSTER_HOST_MERGER = ConfigMerger( value=config_merger_callbacks.assign_roles_by_host_numbers, override=config_merger_callbacks.override_if_empty ), + ConfigMapping( + path_list=['/config_mapping'] + ), + ConfigMapping( + path_list=['/role_mapping'] + ), ConfigMapping( path_list=['/dashboard_roles'], from_lower_keys={'lower_values': '/roles'}, @@ -53,7 +59,16 @@ CLUSTER_HOST_MERGER = ConfigMerger( value=config_merger_callbacks.has_intersection ), ConfigMapping( - path_list=['/role_mapping'], + path_list=['/dashboard_roles'], + from_lower_keys={'lower_values': '/roles'}, + to_key='/dashboard_roles', + value=config_merger_callbacks.get_intersection + ), + ConfigMapping( + path_list=['/haproxy_roles'], + from_lower_keys={'lower_values': '/roles'}, + to_key='/haproxy_roles', + value=config_merger_callbacks.get_intersection ), ConfigMapping( path_list=[ @@ -61,6 +76,7 @@ CLUSTER_HOST_MERGER = ConfigMerger( '/networking/global/gateway', '/networking/global/proxy', '/networking/global/ntp_server', + '/networking/global/ha_vip', '/networking/interfaces/*/netmask', '/networking/interfaces/*/nic', '/networking/interfaces/*/promisc', @@ -98,7 +114,41 @@ CLUSTER_HOST_MERGER = ConfigMerger( to_key='search_path', value=functools.partial( config_merger_callbacks.assign_from_pattern, - upper_keys=['search_path', 'clusterid']) + upper_keys=['search_path', 'clusterid'] + ) + ), + ConfigMapping( + path_list=['/networking/global/ha_vip'], + to_key='/haproxy/router_id', + value=functools.partial( + config_merger_callbacks.assign_by_order, + orders=config_merger_callbacks.generate_order(0, -1) + ), + from_upper_keys={'prefix': '/haproxy/router_id_prefix'}, + from_lower_keys={'conditions': '/haproxy_roles'}, + ), + ConfigMapping( + path_list=['/networking/global/ha_vip'], + to_key='/haproxy/priority', + value=functools.partial( + config_merger_callbacks.assign_by_order, + orders=config_merger_callbacks.generate_order(0, -1) + ), + from_upper_keys={'prefix': '/haproxy/default_priority'}, + from_lower_keys={'conditions': '/haproxy_roles'}, + ), + ConfigMapping( + path_list=['/networking/global/ha_vip'], + to_key='/haproxy/state', + value=functools.partial( + config_merger_callbacks.assign_by_order, + prefix='' + ), + from_upper_keys={ + 'orders': '/haproxy/states_to_assign', + 'default_order': '/haproxy/default_state', + }, + from_lower_keys={'conditions': '/haproxy_roles'} )]) diff --git a/compass/config_management/utils/config_merger_callbacks.py b/compass/config_management/utils/config_merger_callbacks.py index 6168e158..cc3c2651 100644 --- a/compass/config_management/utils/config_merger_callbacks.py +++ b/compass/config_management/utils/config_merger_callbacks.py @@ -353,6 +353,23 @@ def has_intersection(upper_ref, from_key, _lower_refs, _to_key, return has +def get_intersection(upper_ref, from_key, _lower_refs, _to_key, + lower_values={}, **_kwargs): + """Get intersection of upper config and lower values.""" + intersections = {} + for lower_key, lower_value in lower_values.items(): + values = set(lower_value) + intersection = values.intersection(set(upper_ref.config)) + logging.debug( + 'lower_key %s values %s intersection' + 'with from_key %s value %s: %s', + lower_key, values, from_key, upper_ref.config, intersection) + if intersection: + intersections[lower_key] = list(intersection) + + return intersections + + def assign_ips(_upper_ref, _from_key, lower_refs, to_key, ip_start='192.168.0.1', ip_end='192.168.0.254', **_kwargs): @@ -401,6 +418,33 @@ def assign_ips(_upper_ref, _from_key, lower_refs, to_key, return host_ips +def generate_order(start=0, end=-1): + """generate order num.""" + while start < end or end < 0: + yield start + start += 1 + + +def assign_by_order(_upper_ref, _from_key, lower_refs, _to_key, + prefix='', + orders=[], default_order=0, + conditions={}, **kwargs): + """assign to_key by order.""" + host_values = {} + orders = iter(orders) + for lower_key, _ in lower_refs.items(): + if lower_key in conditions and conditions[lower_key]: + try: + order = orders.next() + except StopIteration: + order = default_order + + host_values[lower_key] = prefix + type(prefix)(order) + + logging.debug('assign orders: %s', host_values) + return host_values + + def assign_from_pattern(_upper_ref, _from_key, lower_refs, to_key, upper_keys=[], lower_keys=[], pattern='', **kwargs): """assign to_key by pattern.""" diff --git a/compass/config_management/utils/config_translator_callbacks.py b/compass/config_management/utils/config_translator_callbacks.py index 433b2250..195e6c5d 100644 --- a/compass/config_management/utils/config_translator_callbacks.py +++ b/compass/config_management/utils/config_translator_callbacks.py @@ -41,6 +41,15 @@ def get_key_from_pattern( return translated_key +def get_keys_from_config_mapping(ref, _path, **kwargs): + """get translated keys from config.""" + config = ref.config + translated_keys = config.keys() + logging.debug('got translated_keys %s from config mapping %s', + translated_keys, config) + return translated_keys + + def get_keys_from_role_mapping(ref, _path, mapping={}, **_kwargs): """get translated keys from roles.""" roles = ref.config @@ -56,6 +65,32 @@ def get_keys_from_role_mapping(ref, _path, mapping={}, **_kwargs): return translated_keys +def get_value_from_config_mapping( + ref, _path, _translated_ref, translated_path, **kwargs +): + """get translated_value from config and translated_path.""" + config = ref.config + if translated_path not in config: + return None + + value = config[translated_path] + if isinstance(value, basestring): + translated_value = ref.get(value) + logging.debug('got translated_value %s from %s', + translated_value, value) + else: + for value_in_list in value: + translated_value = ref.get(value_in_list) + logging.debug('got translated_value %s from %s', + translated_value, value_in_list) + if translated_value: + break + + logging.debug('got translated_value %s from translated_path %s', + translated_value, translated_path) + return translated_value + + def get_value_from_role_mapping( ref, _path, _translated_ref, translated_path, mapping={}, **_kwargs @@ -70,8 +105,20 @@ def get_value_from_role_mapping( continue value = mapping[role][translated_path] - translated_value = ref.get(value) - logging.debug('got translated_value %s from %s and roles %s', + if isinstance(value, basestring): + translated_value = ref.get(value) + logging.debug('got translated_value %s from %s', + translated_value, value) + else: + for value_in_list in value: + translated_value = ref.get(value_in_list) + logging.debug('got translated_value %s from %s', + translated_value, value_in_list) + if translated_value: + break + + logging.debug('got translated_value %s from roles %s ' + 'and translated_path %s', translated_value, roles, translated_path) return translated_value diff --git a/compass/tests/actions/deploy/data/test1 b/compass/tests/actions/deploy/data/test1 index 47cefd60..af6b2ce8 100644 --- a/compass/tests/actions/deploy/data/test1 +++ b/compass/tests/actions/deploy/data/test1 @@ -111,6 +111,29 @@ chef_MOCK = { 'os-network': 'openstack network node', 'os-compute-worker': 'openstack nova node' }, + 'config_mapping': { + '/credential/identity/users/admin': '/security/console_credentials', + '/credential/identity/users/compute': '/security/service_credentials', + '/credential/identity/users/image': '/security/service_credentials', + '/credential/identity/users/metering': '/security/service_credentials', + '/credential/identity/users/network': '/security/service_credentials', + '/credential/identity/users/object-store': '/security/service_credentials', + '/credential/identity/users/volume': '/security/service_credentials', + '/credential/mysql/compute': '/security/service_credentials', + '/credential/mysql/dashboard': '/security/service_credentials', + '/credential/mysql/identity': '/security/service_credentials', + '/credential/mysql/image': '/security/service_credentials', + '/credential/mysql/metering': '/security/service_credentials', + '/credential/mysql/network': '/security/service_credentials', + '/credential/mysql/volume': '/security/service_credentials', + '/credential/mysql/super/password': '/security/service_credentials/password', + '/networking/control/interface': '/networking/interfaces/management/nic', + '/ntp/ntpserver': '/networking/global/ntp_server', + '/networking/storage/interface': '/networking/interfaces/storage/nic', + '/networking/public/interface': '/networking/interfaces/public/nic', + '/networking/tenant/interface': '/networking/interfaces/tenant/nic', + '/networking/plugins/ovs/gre/local_ip_interface': '/networking/interfaces/tenant/nic', + }, 'role_mapping': { 'os-single-controller': { '/db/mysql/bind_address': '/networking/interfaces/management/ip', diff --git a/compass/tests/actions/deploy/data/test2 b/compass/tests/actions/deploy/data/test2 index 537c3d9b..a045b62e 100644 --- a/compass/tests/actions/deploy/data/test2 +++ b/compass/tests/actions/deploy/data/test2 @@ -128,6 +128,29 @@ chef_MOCK = { 'os-compute-worker': 'openstack nova node' }, 'dashboard_roles': ['os-single-controller'], + 'config_mapping': { + '/credential/identity/users/admin': '/security/console_credentials', + '/credential/identity/users/compute': '/security/service_credentials', + '/credential/identity/users/image': '/security/service_credentials', + '/credential/identity/users/metering': '/security/service_credentials', + '/credential/identity/users/network': '/security/service_credentials', + '/credential/identity/users/object-store': '/security/service_credentials', + '/credential/identity/users/volume': '/security/service_credentials', + '/credential/mysql/compute': '/security/service_credentials', + '/credential/mysql/dashboard': '/security/service_credentials', + '/credential/mysql/identity': '/security/service_credentials', + '/credential/mysql/image': '/security/service_credentials', + '/credential/mysql/metering': '/security/service_credentials', + '/credential/mysql/network': '/security/service_credentials', + '/credential/mysql/volume': '/security/service_credentials', + '/credential/mysql/super/password': '/security/service_credentials/password', + '/networking/control/interface': '/networking/interfaces/management/nic', + '/ntp/ntpserver': '/networking/global/ntp_server', + '/networking/storage/interface': '/networking/interfaces/storage/nic', + '/networking/public/interface': '/networking/interfaces/public/nic', + '/networking/tenant/interface': '/networking/interfaces/tenant/nic', + '/networking/plugins/ovs/gre/local_ip_interface': '/networking/interfaces/tenant/nic', + }, 'role_mapping': { 'os-single-controller': { '/db/mysql/bind_address': '/networking/interfaces/management/ip', diff --git a/compass/tests/actions/deploy/data/test3 b/compass/tests/actions/deploy/data/test3 index 71378e46..5c67c908 100644 --- a/compass/tests/actions/deploy/data/test3 +++ b/compass/tests/actions/deploy/data/test3 @@ -220,6 +220,29 @@ chef_MOCK = { 'os-compute-worker': 'openstack nova node' }, 'dashboard_roles': ['os-single-controller'], + 'config_mapping': { + '/credential/identity/users/admin': '/security/console_credentials', + '/credential/identity/users/compute': '/security/service_credentials', + '/credential/identity/users/image': '/security/service_credentials', + '/credential/identity/users/metering': '/security/service_credentials', + '/credential/identity/users/network': '/security/service_credentials', + '/credential/identity/users/object-store': '/security/service_credentials', + '/credential/identity/users/volume': '/security/service_credentials', + '/credential/mysql/compute': '/security/service_credentials', + '/credential/mysql/dashboard': '/security/service_credentials', + '/credential/mysql/identity': '/security/service_credentials', + '/credential/mysql/image': '/security/service_credentials', + '/credential/mysql/metering': '/security/service_credentials', + '/credential/mysql/network': '/security/service_credentials', + '/credential/mysql/volume': '/security/service_credentials', + '/credential/mysql/super/password': '/security/service_credentials/password', + '/networking/control/interface': '/networking/interfaces/management/nic', + '/ntp/ntpserver': '/networking/global/ntp_server', + '/networking/storage/interface': '/networking/interfaces/storage/nic', + '/networking/public/interface': '/networking/interfaces/public/nic', + '/networking/tenant/interface': '/networking/interfaces/tenant/nic', + '/networking/plugins/ovs/gre/local_ip_interface': '/networking/interfaces/tenant/nic', + }, 'role_mapping': { 'os-single-controller': { '/db/mysql/bind_address': '/networking/interfaces/management/ip',