Merge "add haproxy support"

2014-03-27 18:48:49 +00:00 · 2014-03-27 18:48:49 +00:00 · a9601ad07b
commit a9601ad07b
parent 01d233e888 a4bce397f0
8 changed files with 276 additions and 49 deletions
--- a/compass/config_management/installers/plugins/chefhandler.py
+++ b/compass/config_management/installers/plugins/chefhandler.py
@ -17,6 +17,7 @@
   .. moduleauthor:: Xiaodong Wang <xiaodongwang@gmail.com>
 """
 import fnmatch
+import functools
 import logging

 from compass.config_management.installers import package_installer
@ -30,49 +31,12 @@ from compass.utils import util
 TO_CLUSTER_TRANSLATORS = {
    'openstack': ConfigTranslator(
        mapping={
-            '/security/console_credentials': [KeyTranslator(
-                translated_keys=['credential/identity/users/admin'],
-            )],
-            '/security/service_credentials': [KeyTranslator(
-                translated_keys=[
-                    '/credential/identity/users/compute',
-                    '/credential/identity/users/image',
-                    '/credential/identity/users/metering',
-                    '/credential/identity/users/network',
-                    '/credential/identity/users/object-store',
-                    '/credential/identity/users/volume',
-                    '/credential/mysql/compute',
-                    '/credential/mysql/dashboard',
-                    '/credential/mysql/identity',
-                    '/credential/mysql/image',
-                    '/credential/mysql/metering',
-                    '/credential/mysql/network',
-                    '/credential/mysql/volume',
-                ]
-            )],
-            '/security/service_credentials/password': [KeyTranslator(
-                translated_keys=[
-                    '/credential/mysql/super/password',
-                ]
-            )],
-            '/networking/interfaces/management/nic': [KeyTranslator(
-                translated_keys=['/networking/control/interface'],
-            )],
-            '/networking/global/ntp_server': [KeyTranslator(
-                translated_keys=['/ntp/ntpserver']
-            )],
-            '/networking/interfaces/storage/nic': [KeyTranslator(
-                translated_keys=['/networking/storage/interface']
-            )],
-            '/networking/interfaces/public/nic': [KeyTranslator(
-                translated_keys=['/networking/public/interface']
-            )],
-            '/networking/interfaces/tenant/nic': [KeyTranslator(
-                translated_keys=[
-                    '/networking/tenant/interface',
-                    '/networking/plugins/ovs/gre/local_ip_interface'
-                ]
-            )],
+            '/config_mapping': [KeyTranslator(
+                translated_keys=(
+                    config_translator_callbacks.get_keys_from_config_mapping),
+                translated_value=(
+                    config_translator_callbacks.get_value_from_config_mapping)
+            )]
        }
    ),
 }
@ -84,12 +48,18 @@ FROM_CLUSTER_TRANSLATORS = {
            '/role_assign_policy': [KeyTranslator(
                translated_keys=['/role_assign_policy']
            )],
-            '/dashboard_roles': [KeyTranslator(
-                translated_keys=['/dashboard_roles']
+            '/config_mapping': [KeyTranslator(
+                translated_keys=['/config_mapping']
            )],
            '/role_mapping': [KeyTranslator(
                translated_keys=['/role_mapping']
            )],
+            '/read_config_mapping': [KeyTranslator(
+                translated_keys=(
+                    config_translator_callbacks.get_keys_from_config_mapping),
+                translated_value=(
+                    config_translator_callbacks.get_value_from_config_mapping)
+            )],
        }
    ),
 }
@ -105,6 +75,45 @@ TO_HOST_TRANSLATORS = {
                translated_value=(
                    config_translator_callbacks.get_value_from_role_mapping),
                from_values={'mapping': '/role_mapping'}
+            ), KeyTranslator(
+                translated_keys=[functools.partial(
+                    config_translator_callbacks.get_key_from_pattern,
+                    to_pattern='/node_mapping/%(node_name)s'
+                )],
+                from_keys={'node_name': '/node_name'}
+            )],
+            '/haproxy_roles': [KeyTranslator(
+                translated_keys=['/ha/status'],
+                translated_value='enable',
+                override=config_translator_callbacks.override_if_any,
+                override_conditions={'haproxy_roles': '/haproxy_roles'}
+            )],
+            '/haproxy/router_id': [KeyTranslator(
+                translated_keys=[functools.partial(
+                    config_translator_callbacks.get_key_from_pattern,
+                    to_pattern='/ha/keepalived/router_ids/%(node_name)s'
+                )],
+                from_keys={'node_name': '/node_name'}
+            )],
+            '/haproxy/priority': [KeyTranslator(
+                translated_keys=[functools.partial(
+                    config_translator_callbacks.get_key_from_pattern,
+                    to_pattern=(
+                        '/ha/keepalived/instance_name/'
+                        'priorities/%(node_name)s'
+                    )
+                )],
+                from_keys={'node_name': '/node_name'}
+            )],
+            '/haproxy/state': [KeyTranslator(
+                translated_keys=[functools.partial(
+                    config_translator_callbacks.get_key_from_pattern,
+                    to_pattern=(
+                        '/ha/keepalived/instance_name/'
+                        'states/%(node_name)s'
+                    )
+                )],
+                from_keys={'node_name': '/node_name'}
            )],
        }
    ),
@ -306,6 +315,12 @@ class Installer(package_installer.Installer):
            bag, clusterid, target_system)
        bag_item_dict = dict(bag_item)
        util.merge_dict(bag_item_dict, global_bag_item, False)
+        util.merge_dict(config, {
+            'client_name': self._get_client_name(
+                config['fullname'], target_system),
+            'node_name': self._get_node_name(
+                config['fullname'], target_system)
+        })
        translated_config = TO_HOST_TRANSLATORS[target_system].translate(
            config)
        util.merge_dict(bag_item_dict, translated_config)
--- a/compass/config_management/providers/plugins/db_config_provider.py
+++ b/compass/config_management/providers/plugins/db_config_provider.py
@ -40,6 +40,8 @@ CLUSTER_DENIES = []
 HOST_ALLOWS = [
    '/roles',
    '/has_dashboard_roles',
+    '/dashboard_roles',
+    '/haproxy_roles',
    '/networking/interfaces/*/ip'
 ]
 HOST_DENIES = []
--- a/compass/config_management/utils/config_manager.py
+++ b/compass/config_management/utils/config_manager.py
@ -46,6 +46,12 @@ CLUSTER_HOST_MERGER = ConfigMerger(
            value=config_merger_callbacks.assign_roles_by_host_numbers,
            override=config_merger_callbacks.override_if_empty
        ),
+        ConfigMapping(
+            path_list=['/config_mapping']
+        ),
+        ConfigMapping(
+            path_list=['/role_mapping']
+        ),
        ConfigMapping(
            path_list=['/dashboard_roles'],
            from_lower_keys={'lower_values': '/roles'},
@ -53,7 +59,16 @@ CLUSTER_HOST_MERGER = ConfigMerger(
            value=config_merger_callbacks.has_intersection
        ),
        ConfigMapping(
-            path_list=['/role_mapping'],
+            path_list=['/dashboard_roles'],
+            from_lower_keys={'lower_values': '/roles'},
+            to_key='/dashboard_roles',
+            value=config_merger_callbacks.get_intersection
+        ),
+        ConfigMapping(
+            path_list=['/haproxy_roles'],
+            from_lower_keys={'lower_values': '/roles'},
+            to_key='/haproxy_roles',
+            value=config_merger_callbacks.get_intersection
        ),
        ConfigMapping(
            path_list=[
@ -61,6 +76,7 @@ CLUSTER_HOST_MERGER = ConfigMerger(
                '/networking/global/gateway',
                '/networking/global/proxy',
                '/networking/global/ntp_server',
+                '/networking/global/ha_vip',
                '/networking/interfaces/*/netmask',
                '/networking/interfaces/*/nic',
                '/networking/interfaces/*/promisc',
@ -98,7 +114,41 @@ CLUSTER_HOST_MERGER = ConfigMerger(
            to_key='search_path',
            value=functools.partial(
                config_merger_callbacks.assign_from_pattern,
-                upper_keys=['search_path', 'clusterid'])
+                upper_keys=['search_path', 'clusterid']
+            )
+        ),
+        ConfigMapping(
+            path_list=['/networking/global/ha_vip'],
+            to_key='/haproxy/router_id',
+            value=functools.partial(
+                config_merger_callbacks.assign_by_order,
+                orders=config_merger_callbacks.generate_order(0, -1)
+            ),
+            from_upper_keys={'prefix': '/haproxy/router_id_prefix'},
+            from_lower_keys={'conditions': '/haproxy_roles'},
+        ),
+        ConfigMapping(
+            path_list=['/networking/global/ha_vip'],
+            to_key='/haproxy/priority',
+            value=functools.partial(
+                config_merger_callbacks.assign_by_order,
+                orders=config_merger_callbacks.generate_order(0, -1)
+            ),
+            from_upper_keys={'prefix': '/haproxy/default_priority'},
+            from_lower_keys={'conditions': '/haproxy_roles'},
+        ),
+        ConfigMapping(
+            path_list=['/networking/global/ha_vip'],
+            to_key='/haproxy/state',
+            value=functools.partial(
+                config_merger_callbacks.assign_by_order,
+                prefix=''
+            ),
+            from_upper_keys={
+                'orders': '/haproxy/states_to_assign',
+                'default_order': '/haproxy/default_state',
+            },
+            from_lower_keys={'conditions': '/haproxy_roles'}
        )])


--- a/compass/config_management/utils/config_merger_callbacks.py
+++ b/compass/config_management/utils/config_merger_callbacks.py
@ -353,6 +353,23 @@ def has_intersection(upper_ref, from_key, _lower_refs, _to_key,
    return has


+def get_intersection(upper_ref, from_key, _lower_refs, _to_key,
+                     lower_values={}, **_kwargs):
+    """Get intersection of upper config and  lower values."""
+    intersections = {}
+    for lower_key, lower_value in lower_values.items():
+        values = set(lower_value)
+        intersection = values.intersection(set(upper_ref.config))
+        logging.debug(
+            'lower_key %s values %s intersection'
+            'with from_key %s value %s: %s',
+            lower_key, values, from_key, upper_ref.config, intersection)
+        if intersection:
+            intersections[lower_key] = list(intersection)
+
+    return intersections
+
+
 def assign_ips(_upper_ref, _from_key, lower_refs, to_key,
               ip_start='192.168.0.1', ip_end='192.168.0.254',
               **_kwargs):
@ -401,6 +418,33 @@ def assign_ips(_upper_ref, _from_key, lower_refs, to_key,
    return host_ips


+def generate_order(start=0, end=-1):
+    """generate order num."""
+    while start < end or end < 0:
+        yield start
+        start += 1
+
+
+def assign_by_order(_upper_ref, _from_key, lower_refs, _to_key,
+                    prefix='',
+                    orders=[], default_order=0,
+                    conditions={}, **kwargs):
+    """assign to_key by order."""
+    host_values = {}
+    orders = iter(orders)
+    for lower_key, _ in lower_refs.items():
+        if lower_key in conditions and conditions[lower_key]:
+            try:
+                order = orders.next()
+            except StopIteration:
+                order = default_order
+
+            host_values[lower_key] = prefix + type(prefix)(order)
+
+    logging.debug('assign orders: %s', host_values)
+    return host_values
+
+
 def assign_from_pattern(_upper_ref, _from_key, lower_refs, to_key,
                        upper_keys=[], lower_keys=[], pattern='', **kwargs):
    """assign to_key by pattern."""
--- a/compass/config_management/utils/config_translator_callbacks.py
+++ b/compass/config_management/utils/config_translator_callbacks.py
@ -41,6 +41,15 @@ def get_key_from_pattern(
    return translated_key


+def get_keys_from_config_mapping(ref, _path, **kwargs):
+    """get translated keys from config."""
+    config = ref.config
+    translated_keys = config.keys()
+    logging.debug('got translated_keys %s from config mapping %s',
+                  translated_keys, config)
+    return translated_keys
+
+
 def get_keys_from_role_mapping(ref, _path, mapping={}, **_kwargs):
    """get translated keys from roles."""
    roles = ref.config
@ -56,6 +65,32 @@ def get_keys_from_role_mapping(ref, _path, mapping={}, **_kwargs):
    return translated_keys


+def get_value_from_config_mapping(
+    ref, _path, _translated_ref, translated_path, **kwargs
+):
+    """get translated_value from config and translated_path."""
+    config = ref.config
+    if translated_path not in config:
+        return None
+
+    value = config[translated_path]
+    if isinstance(value, basestring):
+        translated_value = ref.get(value)
+        logging.debug('got translated_value %s from %s',
+                      translated_value, value)
+    else:
+        for value_in_list in value:
+            translated_value = ref.get(value_in_list)
+            logging.debug('got translated_value %s from %s',
+                          translated_value, value_in_list)
+            if translated_value:
+                break
+
+    logging.debug('got translated_value %s from translated_path %s',
+                  translated_value, translated_path)
+    return translated_value
+
+
 def get_value_from_role_mapping(
    ref, _path, _translated_ref, translated_path,
    mapping={}, **_kwargs
@ -70,8 +105,20 @@ def get_value_from_role_mapping(
            continue

        value = mapping[role][translated_path]
-        translated_value = ref.get(value)
-        logging.debug('got translated_value %s from %s and roles %s',
+        if isinstance(value, basestring):
+            translated_value = ref.get(value)
+            logging.debug('got translated_value %s from %s',
+                          translated_value, value)
+        else:
+            for value_in_list in value:
+                translated_value = ref.get(value_in_list)
+                logging.debug('got translated_value %s from %s',
+                              translated_value, value_in_list)
+                if translated_value:
+                    break
+
+        logging.debug('got translated_value %s from roles %s '
+                      'and translated_path %s',
                      translated_value, roles, translated_path)
        return translated_value

--- a/compass/tests/actions/deploy/data/test1
+++ b/compass/tests/actions/deploy/data/test1
@ -111,6 +111,29 @@ chef_MOCK = {
               'os-network': 'openstack network node',
               'os-compute-worker': 'openstack nova node'
           },
+           'config_mapping': {
+	       '/credential/identity/users/admin': '/security/console_credentials',
+               '/credential/identity/users/compute': '/security/service_credentials',
+               '/credential/identity/users/image': '/security/service_credentials',
+               '/credential/identity/users/metering': '/security/service_credentials',
+               '/credential/identity/users/network': '/security/service_credentials',
+               '/credential/identity/users/object-store': '/security/service_credentials',
+               '/credential/identity/users/volume': '/security/service_credentials',
+               '/credential/mysql/compute': '/security/service_credentials',
+               '/credential/mysql/dashboard': '/security/service_credentials',
+               '/credential/mysql/identity': '/security/service_credentials',
+               '/credential/mysql/image': '/security/service_credentials',
+               '/credential/mysql/metering': '/security/service_credentials',
+               '/credential/mysql/network': '/security/service_credentials',
+               '/credential/mysql/volume': '/security/service_credentials',
+               '/credential/mysql/super/password': '/security/service_credentials/password',
+               '/networking/control/interface': '/networking/interfaces/management/nic',
+               '/ntp/ntpserver': '/networking/global/ntp_server',
+               '/networking/storage/interface': '/networking/interfaces/storage/nic',
+               '/networking/public/interface': '/networking/interfaces/public/nic',
+               '/networking/tenant/interface': '/networking/interfaces/tenant/nic',
+               '/networking/plugins/ovs/gre/local_ip_interface': '/networking/interfaces/tenant/nic',
+           },
           'role_mapping': {
               'os-single-controller': {
                   '/db/mysql/bind_address': '/networking/interfaces/management/ip',
--- a/compass/tests/actions/deploy/data/test2
+++ b/compass/tests/actions/deploy/data/test2
@ -128,6 +128,29 @@ chef_MOCK = {
               'os-compute-worker': 'openstack nova node'
           },
           'dashboard_roles': ['os-single-controller'],
+           'config_mapping': {
+	       '/credential/identity/users/admin': '/security/console_credentials',
+               '/credential/identity/users/compute': '/security/service_credentials',
+               '/credential/identity/users/image': '/security/service_credentials',
+               '/credential/identity/users/metering': '/security/service_credentials',
+               '/credential/identity/users/network': '/security/service_credentials',
+               '/credential/identity/users/object-store': '/security/service_credentials',
+               '/credential/identity/users/volume': '/security/service_credentials',
+               '/credential/mysql/compute': '/security/service_credentials',
+               '/credential/mysql/dashboard': '/security/service_credentials',
+               '/credential/mysql/identity': '/security/service_credentials',
+               '/credential/mysql/image': '/security/service_credentials',
+               '/credential/mysql/metering': '/security/service_credentials',
+               '/credential/mysql/network': '/security/service_credentials',
+               '/credential/mysql/volume': '/security/service_credentials',
+               '/credential/mysql/super/password': '/security/service_credentials/password',
+               '/networking/control/interface': '/networking/interfaces/management/nic',
+               '/ntp/ntpserver': '/networking/global/ntp_server',
+               '/networking/storage/interface': '/networking/interfaces/storage/nic',
+               '/networking/public/interface': '/networking/interfaces/public/nic',
+               '/networking/tenant/interface': '/networking/interfaces/tenant/nic',
+               '/networking/plugins/ovs/gre/local_ip_interface': '/networking/interfaces/tenant/nic',
+           },
           'role_mapping': {
               'os-single-controller': {
                   '/db/mysql/bind_address': '/networking/interfaces/management/ip',
--- a/compass/tests/actions/deploy/data/test3
+++ b/compass/tests/actions/deploy/data/test3
@ -220,6 +220,29 @@ chef_MOCK = {
               'os-compute-worker': 'openstack nova node'
           },
           'dashboard_roles': ['os-single-controller'],
+           'config_mapping': {
+	       '/credential/identity/users/admin': '/security/console_credentials',
+               '/credential/identity/users/compute': '/security/service_credentials',
+               '/credential/identity/users/image': '/security/service_credentials',
+               '/credential/identity/users/metering': '/security/service_credentials',
+               '/credential/identity/users/network': '/security/service_credentials',
+               '/credential/identity/users/object-store': '/security/service_credentials',
+               '/credential/identity/users/volume': '/security/service_credentials',
+               '/credential/mysql/compute': '/security/service_credentials',
+               '/credential/mysql/dashboard': '/security/service_credentials',
+               '/credential/mysql/identity': '/security/service_credentials',
+               '/credential/mysql/image': '/security/service_credentials',
+               '/credential/mysql/metering': '/security/service_credentials',
+               '/credential/mysql/network': '/security/service_credentials',
+               '/credential/mysql/volume': '/security/service_credentials',
+               '/credential/mysql/super/password': '/security/service_credentials/password',
+               '/networking/control/interface': '/networking/interfaces/management/nic',
+               '/ntp/ntpserver': '/networking/global/ntp_server',
+               '/networking/storage/interface': '/networking/interfaces/storage/nic',
+               '/networking/public/interface': '/networking/interfaces/public/nic',
+               '/networking/tenant/interface': '/networking/interfaces/tenant/nic',
+               '/networking/plugins/ovs/gre/local_ip_interface': '/networking/interfaces/tenant/nic',
+           },
           'role_mapping': {
               'os-single-controller': {
                   '/db/mysql/bind_address': '/networking/interfaces/management/ip',