diff --git a/compass/actions/update_progress.py b/compass/actions/update_progress.py index b320c6a0..cb7aa577 100644 --- a/compass/actions/update_progress.py +++ b/compass/actions/update_progress.py @@ -45,10 +45,11 @@ def update_progress(): After the progress got updated, these information will be stored back to the log_progressing_history for next time run. """ - with util.lock('log_progressing', blocking=False) as lock: + with util.lock('log_progressing', timeout=60, blocking=False) as lock: if not lock: logging.error( - 'failed to acquire lock to calculate installation progress') + 'failed to acquire lock to calculate installation progress' + ) return logging.info('update installing progress') diff --git a/compass/api/api.py b/compass/api/api.py index 026b7209..33fbb340 100644 --- a/compass/api/api.py +++ b/compass/api/api.py @@ -347,7 +347,7 @@ def list_user_permissions(user_id): @log_user_action @login_required def take_user_action(user_id): - """Update user permissions.""" + """Take user action.""" data = _get_request_data() update_permissions_func = _wrap_response( functools.partial( diff --git a/compass/db/api/metadata.py b/compass/db/api/metadata.py index 66f5e860..dfa0813f 100644 --- a/compass/db/api/metadata.py +++ b/compass/db/api/metadata.py @@ -219,6 +219,45 @@ def add_package_metadata_internal(session, exception_when_existing=True): return package_metadatas +def _filter_metadata(metadata, **kwargs): + if not isinstance(metadata, dict): + return metadata + filtered_metadata = {} + for key, value in metadata.items(): + if key == '_self': + default_value = value.get('default_value', None) + if default_value is None: + default_callback_params = value.get( + 'default_callback_params', {} + ) + callback_params = dict(kwargs) + if default_callback_params: + callback_params.update(default_callback_params) + default_callback = value.get('default_callback', None) + if default_callback: + default_value = default_callback(key, **callback_params) + options = value.get('options', None) + if options is None: + options_callback_params = value.get( + 'options_callback_params', {} + ) + callback_params = dict(kwargs) + if options_callback_params: + callback_params.update(options_callback_params) + + options_callback = value.get('options_callback', None) + if options_callback: + options = options_callback(key, **callback_params) + filtered_metadata[key] = value + if default_value is not None: + filtered_metadata[key]['default_value'] = default_value + if options is not None: + filtered_metadata[key]['options'] = options + else: + filtered_metadata[key] = _filter_metadata(value, **kwargs) + return filtered_metadata + + def get_package_metadatas_internal(session): metadata_mapping = {} adapters = utils.list_db_objects( @@ -227,7 +266,9 @@ def get_package_metadatas_internal(session): for adapter in adapters: if adapter.deployable: metadata_dict = adapter.metadata_dict() - metadata_mapping[adapter.id] = metadata_dict + metadata_mapping[adapter.id] = _filter_metadata( + metadata_dict, session=session + ) else: logging.info( 'ignore metadata since its adapter %s is not deployable', @@ -244,7 +285,9 @@ def get_os_metadatas_internal(session): for os in oses: if os.deployable: metadata_dict = os.metadata_dict() - metadata_mapping[os.id] = metadata_dict + metadata_mapping[os.id] = _filter_metadata( + metadata_dict, session=session + ) else: logging.info( 'ignore metadata since its os %s is not deployable', @@ -324,7 +367,7 @@ def _validate_config( continue if specified[key]['_self'].get('is_required', False): raise exception.InvalidParameter( - '%s/%s does not find is_required' % ( + '%s/%s does not find but it is required' % ( config_path, key ) ) @@ -335,7 +378,7 @@ def _validate_config( ) ): raise exception.InvalidParameter( - '%s/%s does not find required_in_whole_config' % ( + '%s/%s does not find but it is required in whole config' % ( config_path, key ) ) @@ -371,6 +414,9 @@ def _autofill_self_config( config_path, config, metadata, **kwargs ) return config + logging.debug( + 'autofill %s by metadata %s', config_path, metadata['_self'] + ) autofill_callback = metadata['_self'].get( 'autofill_callback', None ) diff --git a/compass/db/api/metadata_holder.py b/compass/db/api/metadata_holder.py index 8bbc94e6..231b4801 100644 --- a/compass/db/api/metadata_holder.py +++ b/compass/db/api/metadata_holder.py @@ -93,40 +93,15 @@ def _filter_metadata(metadata, **kwargs): filtered_metadata = {} for key, value in metadata.items(): if key == '_self': - filtered_metadata['_self'] = {} - default_value = value.get('default_value', None) - if default_value is None: - default_callback_params = value.get( - 'default_callback_params', {} - ) - callback_params = dict(kwargs) - if default_callback_params: - callback_params.update(default_callback_params) - default_callback = value.get('default_callback', None) - if default_callback: - default_value = default_callback(key, **callback_params) - options = value.get('options', None) - if options is None: - options_callback_params = value.get( - 'options_callback_params', {} - ) - callback_params = dict(kwargs) - if options_callback_params: - callback_params.update(options_callback_params) - - options_callback = value.get('options_callback', None) - if options_callback: - options = options_callback(key, **callback_params) filtered_metadata[key] = { 'name': value['name'], 'description': value.get('description', None), - 'default_value': default_value, - 'is_required': value.get( - 'is_required', False), + 'default_value': value.get('default_value', None), + 'is_required': value.get('is_required', False), 'required_in_whole_config': value.get( 'required_in_whole_config', False), 'js_validator': value.get('js_validator', None), - 'options': options, + 'options': value.get('options', None), 'required_in_options': value.get( 'required_in_options', False), 'field_type': value.get( @@ -135,7 +110,7 @@ def _filter_metadata(metadata, **kwargs): 'mapping_to': value.get('mapping_to', None) } else: - filtered_metadata[key] = _filter_metadata(value) + filtered_metadata[key] = _filter_metadata(value, **kwargs) return filtered_metadata @@ -223,8 +198,8 @@ def _autofill_config( ) metadatas = metadata_mapping[id] logging.debug( - 'auto fill %s config %s by metadata %s', - id_name, config, metadatas + 'auto fill %s config %s by params %s', + id_name, config, kwargs ) return metadata_api.autofill_config_internal( config, metadatas, **kwargs diff --git a/compass/db/callback.py b/compass/db/callback.py index 6ecc195e..35798bca 100644 --- a/compass/db/callback.py +++ b/compass/db/callback.py @@ -13,6 +13,7 @@ # limitations under the License. """Metadata Callback methods.""" +import logging import netaddr import random import re @@ -128,7 +129,39 @@ def autofill_callback_random_option(name, config, **kwargs): return config +def autofill_no_proxy(name, config, **kwargs): + logging.debug( + 'autofill %s config %s by params %s', + name, config, kwargs + ) + if 'cluster' in kwargs: + if config is None: + config = [] + if 'default_value' in kwargs: + for default_no_proxy in kwargs['default_value']: + if default_no_proxy and default_no_proxy not in config: + config.append(default_no_proxy) + cluster = kwargs['cluster'] + for clusterhost in cluster.clusterhosts: + host = clusterhost.host + hostname = host.name + if hostname not in config: + config.append(hostname) + for host_network in host.host_networks: + if host_network.is_mgmt: + ip = host_network.ip + if ip not in config: + config.append(ip) + if not config: + return config + return [no_proxy for no_proxy in config if no_proxy] + + def autofill_network_mapping(name, config, **kwargs): + logging.debug( + 'autofill %s config %s by params %s', + name, config, kwargs + ) if not config: return config if isinstance(config, basestring): diff --git a/compass/db/models.py b/compass/db/models.py index fbb8eb0a..69fc0da3 100644 --- a/compass/db/models.py +++ b/compass/db/models.py @@ -13,6 +13,7 @@ # limitations under the License. """Database model""" +import copy import datetime import logging import netaddr @@ -673,8 +674,12 @@ class ClusterHost(BASE, TimestampMixin, HelperMixin): @patched_package_config.setter def patched_package_config(self, value): - package_config = util.merge_dict(dict(self.package_config), value) - self.package_config = package_config + package_config = copy.deepcopy(self.package_config) + self.package_config = util.merge_dict(package_config, value) + logging.debug( + 'patch clusterhost %s package_config: %s', + self.clusterhost_id, value + ) self.config_validated = False @property @@ -683,9 +688,13 @@ class ClusterHost(BASE, TimestampMixin, HelperMixin): @put_package_config.setter def put_package_config(self, value): - package_config = dict(self.package_config) + package_config = copy.deepcopy(self.package_config) package_config.update(value) self.package_config = package_config + logging.debug( + 'put clusterhost %s package_config: %s', + self.clusterhost_id, value + ) self.config_validated = False @property @@ -949,8 +958,9 @@ class Host(BASE, TimestampMixin, HelperMixin): @patched_os_config.setter def patched_os_config(self, value): - self.os_config = util.merge_dict(dict(self.os_config), value) - logging.info('patch host os config in %s: %s', self.id, value) + os_config = copy.deepcopy(self.os_config) + self.os_config = util.merge_dict(os_config, value) + logging.debug('patch host os config in %s: %s', self.id, value) self.config_validated = False @property @@ -959,10 +969,10 @@ class Host(BASE, TimestampMixin, HelperMixin): @put_os_config.setter def put_os_config(self, value): - os_config = dict(self.os_config) + os_config = copy.deepcopy(self.os_config) os_config.update(value) self.os_config = os_config - logging.info('put host os config in %s: %s', self.id, value) + logging.debug('put host os config in %s: %s', self.id, value) self.config_validated = False def __init__(self, id, **kwargs): @@ -1289,8 +1299,9 @@ class Cluster(BASE, TimestampMixin, HelperMixin): @patched_os_config.setter def patched_os_config(self, value): - self.os_config = util.merge_dict(dict(self.os_config), value) - logging.info('patch cluster %s os config: %s', self.id, value) + os_config = copy.deepcopy(self.os_config) + self.os_config = util.merge_dict(os_config, value) + logging.debug('patch cluster %s os config: %s', self.id, value) self.config_validated = False @property @@ -1299,10 +1310,10 @@ class Cluster(BASE, TimestampMixin, HelperMixin): @put_os_config.setter def put_os_config(self, value): - os_config = dict(self.os_config) + os_config = copy.deepcopy(self.os_config) os_config.update(value) self.os_config = os_config - logging.info('put cluster %s os config: %s', self.id, value) + logging.debug('put cluster %s os config: %s', self.id, value) self.config_validated = False @property @@ -1311,9 +1322,9 @@ class Cluster(BASE, TimestampMixin, HelperMixin): @patched_package_config.setter def patched_package_config(self, value): - package_config = dict(self.package_config) + package_config = copy.deepcopy(self.package_config) self.package_config = util.merge_dict(package_config, value) - logging.info('patch cluster %s package config: %s', self.id, value) + logging.debug('patch cluster %s package config: %s', self.id, value) self.config_validated = False @property @@ -1325,7 +1336,7 @@ class Cluster(BASE, TimestampMixin, HelperMixin): package_config = dict(self.package_config) package_config.update(value) self.package_config = package_config - logging.info('put cluster %s package config: %s', self.id, value) + logging.debug('put cluster %s package config: %s', self.id, value) self.config_validated = False @property @@ -1715,9 +1726,10 @@ class Machine(BASE, HelperMixin, TimestampMixin): @patched_ipmi_credentials.setter def patched_ipmi_credentials(self, value): - self.ipmi_credentials = ( - util.merge_dict(dict(self.ipmi_credentials), value) - ) + if not value: + return + ipmi_credentials = copy.deepcopy(self.ipmi_credentials) + self.ipmi_credentials = util.merge_dict(ipmi_credentials, value) @property def patched_tag(self): @@ -1725,7 +1737,9 @@ class Machine(BASE, HelperMixin, TimestampMixin): @patched_tag.setter def patched_tag(self, value): - tag = dict(self.tag) + if not value: + return + tag = copy.deepcopy(self.tag) tag.update(value) self.tag = value @@ -1735,7 +1749,9 @@ class Machine(BASE, HelperMixin, TimestampMixin): @patched_location.setter def patched_location(self, value): - location = dict(self.location) + if not value: + return + location = copy.deepcopy(self.location) location.update(value) self.location = location @@ -1904,7 +1920,10 @@ class Switch(BASE, HelperMixin, TimestampMixin): @patched_credentials.setter def patched_credentials(self, value): - self.credentials = util.merge_dict(dict(self.credentials), value) + if not value: + return + credentials = copy.deepcopy(self.credentials) + self.credentials = util.merge_dict(credentials, value) @property def filters(self): diff --git a/compass/log_analyzor/file_matcher.py b/compass/log_analyzor/file_matcher.py index 7e57247d..f91b58f1 100644 --- a/compass/log_analyzor/file_matcher.py +++ b/compass/log_analyzor/file_matcher.py @@ -62,7 +62,7 @@ class FilterFileExist(FileFilter): """filter log file.""" file_exist = os.path.isfile(pathname) if not file_exist: - logging.error("%s is not exist", pathname) + logging.debug("%s is not exist", pathname) return file_exist @@ -150,7 +150,7 @@ class FileReaderFactory(object): pathname = os.path.join(self.logdir_, hostname, filename) logging.debug('get FileReader from %s', pathname) if not self.filefilter_.filter(pathname): - logging.error('%s is filtered', pathname) + logging.debug('%s is filtered', pathname) return None return FileReader(pathname, log_history) diff --git a/compass/log_analyzor/line_matcher.py b/compass/log_analyzor/line_matcher.py index 0bc25951..87a6f6f5 100644 --- a/compass/log_analyzor/line_matcher.py +++ b/compass/log_analyzor/line_matcher.py @@ -55,8 +55,8 @@ class ProgressCalculator(object): log_history['severity'] = severity logging.debug('update progress to %s', log_history) else: - logging.info('ignore update progress %s to %s', - progress_data, log_history) + logging.debug('ignore update progress %s to %s', + progress_data, log_history) def update(self, message, severity, log_history): """vritual method to update progress by message and severity. diff --git a/conf/os_metadata/general.conf b/conf/os_metadata/general.conf index 115ef442..8caf1571 100644 --- a/conf/os_metadata/general.conf +++ b/conf/os_metadata/general.conf @@ -48,6 +48,7 @@ METADATA = { 'field': 'general_list', 'default_callback': default_noproxy, 'options_callback': noproxy_options, + 'autofill_callback': autofill_no_proxy, 'mapping_to': 'no_proxy' } },