From d82d5457b66aacb46b1b1d7018cbb818a51d3aa2 Mon Sep 17 00:00:00 2001 From: Steve Lewis Date: Thu, 17 Nov 2016 15:47:37 -0800 Subject: [PATCH] Inventory generation: write hosts file refactor Refactor of the process for writing a hostnames and IPs file during inventory generation. The find_config_path is refactored as well since that was relied upon in the implementation of the above. To improve clarity of the file utility methods the pass_exception parameter has been renamed and logic has been inverted. Parent-Id: I0275eeeafeb2a4c89cd820d56dd0ad01d3124a80 Change-Id: I4c7b45f16f8ffecb91bf509daeca47b2c2450681 --- lib/filesystem.py | 79 ++++++++++++++++++++++++++++++++++++------- lib/generate.py | 86 +++++++++++++++++++++++------------------------ 2 files changed, 108 insertions(+), 57 deletions(-) diff --git a/lib/filesystem.py b/lib/filesystem.py index 8411255afb..ab454d1479 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -50,17 +50,17 @@ def _get_search_paths(preferred_path=None, suffix=None): return search_paths -def file_find(filename, preferred_path=None, pass_exception=False): - """Return the path to a file, or False if no file is found. +def file_find(filename, preferred_path=None, raise_if_missing=True): + """Return the path to an existing file, or False if no file is found. - If no file is found and pass_exception is True, the system will exit. + If no file is found and raise_if_missing is True, the system will exit. The file lookup will be done in the following directories: * ``preferred_path`` [Optional] * ``/etc/openstack_deploy/`` :param filename: ``str`` Name of the file to find :param preferred_path: ``str`` Additional directory to look in FIRST - :param pass_exception: ``bool`` Should a SystemExit be raised if the file + :param raise_if_missing: ``bool`` Should a SystemExit be raised if the file is not found """ @@ -69,11 +69,40 @@ def file_find(filename, preferred_path=None, pass_exception=False): for file_candidate in search_paths: if os.path.isfile(file_candidate): return file_candidate + + # The file was not found + if raise_if_missing: + raise SystemExit('No file found at: {}'.format(search_paths)) else: - if pass_exception is False: - raise SystemExit('No file found at: {}'.format(search_paths)) - else: - return False + return False + + +def dir_find(preferred_path=None, suffix=None, raise_if_missing=True): + """Return the path to the user configuration files. + + If no directory is found the system will exit. + + The lookup will be done in the following directories: + + * ``preferred_path`` [Optional] + * ``/etc/openstack_deploy/`` + + :param preferred_path: ``str`` Additional directory to look in FIRST + :param suffix: ``str`` Name of a subdirectory to find under standard paths + :param raise_if_missing: ``bool`` Should a SystemExit be raised if the + directory is not found. + """ + search_paths = _get_search_paths(preferred_path, suffix) + + for f in search_paths: + if os.path.isdir(f): + return f + + # The directory was not found + if raise_if_missing: + raise SystemExit('No directory found at:{}'.format(search_paths)) + else: + return False def _make_backup(backup_path, source_file_path): @@ -107,18 +136,42 @@ def _get_backup_name(basename): return '{}-{}.json'.format(basename, utctime) -def load_from_json(filename, preferred_path=None, pass_exception=False): +def write_hostnames(save_path, hostnames_ips): + """Write a list of all hosts and their given IP addresses + + NOTE: the file is saved in json format to a file with the name + ``openstack_hostnames_ips.yml`` + + :param save_path: path to save the file to, will use default location if + None or an invalid path is provided + :param hostnames_ips: the list of all hosts and their IP addresses + """ + + file_path = dir_find(save_path) + hostnames_ip_file = os.path.join(file_path, 'openstack_hostnames_ips.yml') + + with open(hostnames_ip_file, 'wb') as f: + f.write( + json.dumps( + hostnames_ips, + indent=4, + sort_keys=True + ) + ) + + +def load_from_json(filename, preferred_path=None, raise_if_missing=True): """Return a dictionary found in json format in a given file :param filename: ``str`` Name of the file to read from :param preferred_path: ``str`` Path to the json file to try FIRST - :param pass_exception: ``bool`` Should a SystemExit be raised if the file - is not found + :param raise_if_missing: ``bool`` Should a SystemExit be raised if the file + is not found :return ``(dict, str)`` Dictionary describing the JSON file contents or False, and the fully resolved file name loaded or None """ - target_file = file_find(filename, preferred_path, pass_exception) + target_file = file_find(filename, preferred_path, raise_if_missing) dictionary = False if target_file is not False: with open(target_file, 'rb') as f_handle: @@ -139,7 +192,7 @@ def load_inventory(preferred_path=None, default_inv=None): """ inventory, file_loaded = load_from_json(INVENTORY_FILENAME, preferred_path, - pass_exception=True) + raise_if_missing=False) if inventory is not False: logger.debug("Loaded existing inventory from {}".format(file_loaded)) _make_backup(preferred_path, file_loaded) diff --git a/lib/generate.py b/lib/generate.py index 34ffa849ea..7a032fb139 100755 --- a/lib/generate.py +++ b/lib/generate.py @@ -24,10 +24,8 @@ import uuid import warnings import yaml -from dictutils import append_if -from dictutils import merge_dict -from filesystem import load_inventory -from filesystem import save_inventory +import dictutils as du +import filesystem as filesys logger = logging.getLogger('osa-inventory') @@ -134,7 +132,8 @@ def _parse_belongs_to(key, belongs_to, inventory): """ for item in belongs_to: if key not in inventory[item]['children']: - appended = append_if(array=inventory[item]['children'], item=key) + appended = du.append_if(array=inventory[item]['children'], + item=key) if appended: logger.debug("Added %s to %s", key, item) @@ -168,7 +167,7 @@ def _build_container_hosts(container_affinity, container_hosts, type_and_name, for make_container in range(container_affinity): for i in container_hosts: if '{}-'.format(type_and_name) in i: - append_if(array=container_list, item=i) + du.append_if(array=container_list, item=i) existing_count = len(list(set(container_list))) if existing_count < container_affinity: @@ -188,7 +187,7 @@ def _build_container_hosts(container_affinity, container_hosts, type_and_name, "hosts": [], } - appended = append_if( + appended = du.append_if( array=inventory[container_host_type]["hosts"], item=container_host_name ) @@ -196,7 +195,7 @@ def _build_container_hosts(container_affinity, container_hosts, type_and_name, logger.debug("Added container %s to %s", container_host_name, container_host_type) - append_if(array=container_hosts, item=container_host_name) + du.append_if(array=container_hosts, item=container_host_name) else: if host_type not in hostvars: hostvars[host_type] = {} @@ -208,7 +207,7 @@ def _build_container_hosts(container_affinity, container_hosts, type_and_name, # Create a host types containers group and append it to inventory host_type_containers = '{}-host_containers'.format(host_type) - append_if(array=container_mapping, item=host_type_containers) + du.append_if(array=container_mapping, item=host_type_containers) hostvars_options.update({ 'properties': properties, @@ -273,25 +272,25 @@ def _append_to_host_groups(inventory, container_type, assignment, host_type, hdata['physical_host'] = host_type if container.startswith('{}-'.format(type_and_name)): - appended = append_if(array=iah, item=container) + appended = du.append_if(array=iah, item=container) if appended: logger.debug("Added host %s to %s hosts", container, assignment) elif is_metal is True: if component == assignment: - appended = append_if(array=iah, item=container) + appended = du.append_if(array=iah, item=container) if appended: logger.debug("Added is_metal host %s to %s hosts", container, assignment) if container.startswith('{}-'.format(type_and_name)): - appended = append_if(array=iph, item=container) + appended = du.append_if(array=iph, item=container) if appended: logger.debug("Added host %s to %s hosts", container, physical_group_type) elif is_metal is True: if container.startswith(host_type): - appended = append_if(array=iph, item=container) + appended = du.append_if(array=iph, item=container) if appended: logger.debug("Added is_metal host %s to %s hosts", container, physical_group_type) @@ -342,8 +341,8 @@ def _add_container_hosts(assignment, config, container_name, container_type, # If host_type is not in config do not append containers to it if host_type not in config[physical_host_type]: continue - appended = append_if(array=inventory['lxc_hosts']['hosts'], - item=host_type) + appended = du.append_if(array=inventory['lxc_hosts']['hosts'], + item=host_type) if appended: logger.debug("%s added to lxc_hosts group", host_type) @@ -445,7 +444,8 @@ def user_defined_setup(config, inventory): hvs[_key][_k] = _v ip.USED_IPS.add(_value['ip']) - appended = append_if(array=inventory[key]['hosts'], item=_key) + appended = du.append_if(array=inventory[key]['hosts'], + item=_key) if appended: logger.debug("Added host %s to group %s", _key, key) @@ -731,6 +731,8 @@ def container_skel_load(container_skel, inventory, config): def find_config_path(user_config_path=None): + # TODO(stevelle) REMOVE THIS FUNCTION: after other changes in chain + # prevents the clean removal right now """Return the path to the user configuration files. If no directory is found the system will exit. @@ -831,7 +833,7 @@ def _extra_config(user_defined_config, base_dir): for name in files: if name.endswith(('.yml', '.yaml')): with open(os.path.join(root_dir, name), 'rb') as f: - merge_dict( + du.merge_dict( user_defined_config, yaml.safe_load(f.read()) or {} ) @@ -973,6 +975,19 @@ def _check_all_conf_groups_present(config, environment): return retval +def _collect_hostnames(dynamic_inventory): + + # Generate a list of all hosts and their used IP addresses + hostnames_ips = {} + for _host, _vars in dynamic_inventory['_meta']['hostvars'].iteritems(): + host_hash = hostnames_ips[_host] = {} + for _key, _value in _vars.iteritems(): + if _key.endswith('address') or _key == 'ansible_host': + host_hash[_key] = _value + + return hostnames_ips + + def load_environment(config_path, environment): """Create an environment dictionary from config files @@ -981,9 +996,10 @@ def load_environment(config_path, environment): """ # Load all YAML files found in the env.d directory - env_plugins = os.path.join(config_path, 'env.d') + env_plugins = filesys.dir_find(config_path, 'env.d', + raise_if_missing=False) - if os.path.isdir(env_plugins): + if env_plugins is not False: _extra_config(user_defined_config=environment, base_dir=env_plugins) logger.debug("Loaded environment from %s", config_path) return environment @@ -1004,8 +1020,8 @@ def load_user_configuration(config_path): user_defined_config.update(yaml.safe_load(f.read()) or {}) # Load anything in a conf.d directory if found - base_dir = os.path.join(config_path, 'conf.d') - if os.path.isdir(base_dir): + base_dir = filesys.dir_find(config_path, 'conf.d', raise_if_missing=False) + if base_dir is not False: _extra_config(user_defined_config, base_dir) # Exit if no user_config was found and loaded @@ -1037,9 +1053,7 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs): logger.info("Beginning new inventory run") # Get the path to the user configuration files - config_path = find_config_path( - user_config_path=config - ) + config_path = filesys.dir_find(preferred_path=config) user_defined_config = load_user_configuration(config_path) base_env_dir = environment @@ -1047,7 +1061,7 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs): environment = load_environment(config_path, base_env) # Load existing inventory file if found - dynamic_inventory = load_inventory(config_path, INVENTORY_SKEL) + dynamic_inventory = filesys.load_inventory(config_path, INVENTORY_SKEL) # Save the users container cidr as a group variable cidr_networks = user_defined_config.get('cidr_networks') @@ -1109,31 +1123,15 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs): if _check_all_conf_groups_present(user_defined_config, environment): return 'Configuration ok!' - # Generate a list of all hosts and their used IP addresses - hostnames_ips = {} - for _host, _vars in dynamic_inventory['_meta']['hostvars'].iteritems(): - host_hash = hostnames_ips[_host] = {} - for _key, _value in _vars.iteritems(): - if _key.endswith('address') or _key == 'ansible_host': - host_hash[_key] = _value - # Save a list of all hosts and their given IP addresses - hostnames_ip_file = os.path.join( - config_path, 'openstack_hostnames_ips.yml') - with open(hostnames_ip_file, 'wb') as f: - f.write( - json.dumps( - hostnames_ips, - indent=4, - sort_keys=True - ) - ) + hostnames_ips = _collect_hostnames(dynamic_inventory) + filesys.write_hostnames(config, hostnames_ips) if logger.isEnabledFor(logging.DEBUG): num_hosts = len(dynamic_inventory['_meta']['hostvars']) logger.debug("%d hosts found." % num_hosts) # Save new dynamic inventory - save_inventory(dynamic_inventory_json, config_path) + filesys.save_inventory(dynamic_inventory_json, config_path) return dynamic_inventory_json