openstack/openstack-ansible
Code Issues Proposed changes

Inventory generation: write hosts file refactor

Browse Source 
Refactor of the process for writing a hostnames and IPs file
during inventory generation.

The find_config_path is refactored as well since that was relied
upon in the implementation of the above.

To improve clarity of the file utility methods the pass_exception
parameter has been renamed and logic has been inverted.

Parent-Id: I0275eeeafeb2a4c89cd820d56dd0ad01d3124a80
Change-Id: I4c7b45f16f8ffecb91bf509daeca47b2c2450681
This commit is contained in:
Steve Lewis 2016-11-17 15:47:37 -08:00
parent ad566bffd7
commit d82d5457b6
2 changed files with 108 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
lib/filesystem.py
View File

@ -50,17 +50,17 @@ def _get_search_paths(preferred_path=None, suffix=None):
return search_paths
def file_find(filename, preferred_path=None, pass_exception=False):
"""Return the path to a file, or False if no file is found.
def file_find(filename, preferred_path=None, raise_if_missing=True):
"""Return the path to an existing file, or False if no file is found.
If no file is found and pass_exception is True, the system will exit.
If no file is found and raise_if_missing is True, the system will exit.
The file lookup will be done in the following directories:
* ``preferred_path`` [Optional]
* ``/etc/openstack_deploy/``
:param filename: ``str`` Name of the file to find
:param preferred_path: ``str`` Additional directory to look in FIRST
:param pass_exception: ``bool`` Should a SystemExit be raised if the file
:param raise_if_missing: ``bool`` Should a SystemExit be raised if the file
is not found
"""
@ -69,11 +69,40 @@ def file_find(filename, preferred_path=None, pass_exception=False):
for file_candidate in search_paths:
if os.path.isfile(file_candidate):
return file_candidate
# The file was not found
if raise_if_missing:
raise SystemExit('No file found at: {}'.format(search_paths))
else:
if pass_exception is False:
raise SystemExit('No file found at: {}'.format(search_paths))
else:
return False
return False
def dir_find(preferred_path=None, suffix=None, raise_if_missing=True):
"""Return the path to the user configuration files.
If no directory is found the system will exit.
The lookup will be done in the following directories:
* ``preferred_path`` [Optional]
* ``/etc/openstack_deploy/``
:param preferred_path: ``str`` Additional directory to look in FIRST
:param suffix: ``str`` Name of a subdirectory to find under standard paths
:param raise_if_missing: ``bool`` Should a SystemExit be raised if the
directory is not found.
"""
search_paths = _get_search_paths(preferred_path, suffix)
for f in search_paths:
if os.path.isdir(f):
return f
# The directory was not found
if raise_if_missing:
raise SystemExit('No directory found at:{}'.format(search_paths))
else:
return False
def _make_backup(backup_path, source_file_path):
@ -107,18 +136,42 @@ def _get_backup_name(basename):
return '{}-{}.json'.format(basename, utctime)
def load_from_json(filename, preferred_path=None, pass_exception=False):
def write_hostnames(save_path, hostnames_ips):
"""Write a list of all hosts and their given IP addresses
NOTE: the file is saved in json format to a file with the name
``openstack_hostnames_ips.yml``
:param save_path: path to save the file to, will use default location if
None or an invalid path is provided
:param hostnames_ips: the list of all hosts and their IP addresses
"""
file_path = dir_find(save_path)
hostnames_ip_file = os.path.join(file_path, 'openstack_hostnames_ips.yml')
with open(hostnames_ip_file, 'wb') as f:
f.write(
json.dumps(
hostnames_ips,
indent=4,
sort_keys=True
)
)
def load_from_json(filename, preferred_path=None, raise_if_missing=True):
"""Return a dictionary found in json format in a given file
:param filename: ``str`` Name of the file to read from
:param preferred_path: ``str`` Path to the json file to try FIRST
:param pass_exception: ``bool`` Should a SystemExit be raised if the file
is not found
:param raise_if_missing: ``bool`` Should a SystemExit be raised if the file
is not found
:return ``(dict, str)`` Dictionary describing the JSON file contents or
False, and the fully resolved file name loaded or None
"""
target_file = file_find(filename, preferred_path, pass_exception)
target_file = file_find(filename, preferred_path, raise_if_missing)
dictionary = False
if target_file is not False:
with open(target_file, 'rb') as f_handle:
@ -139,7 +192,7 @@ def load_inventory(preferred_path=None, default_inv=None):
"""
inventory, file_loaded = load_from_json(INVENTORY_FILENAME, preferred_path,
pass_exception=True)
raise_if_missing=False)
if inventory is not False:
logger.debug("Loaded existing inventory from {}".format(file_loaded))
_make_backup(preferred_path, file_loaded)

86
lib/generate.py
View File

@ -24,10 +24,8 @@ import uuid
import warnings
import yaml
from dictutils import append_if
from dictutils import merge_dict
from filesystem import load_inventory
from filesystem import save_inventory
import dictutils as du
import filesystem as filesys
logger = logging.getLogger('osa-inventory')
@ -134,7 +132,8 @@ def _parse_belongs_to(key, belongs_to, inventory):
"""
for item in belongs_to:
if key not in inventory[item]['children']:
appended = append_if(array=inventory[item]['children'], item=key)
appended = du.append_if(array=inventory[item]['children'],
item=key)
if appended:
logger.debug("Added %s to %s", key, item)
@ -168,7 +167,7 @@ def _build_container_hosts(container_affinity, container_hosts, type_and_name,
for make_container in range(container_affinity):
for i in container_hosts:
if '{}-'.format(type_and_name) in i:
append_if(array=container_list, item=i)
du.append_if(array=container_list, item=i)
existing_count = len(list(set(container_list)))
if existing_count < container_affinity:
@ -188,7 +187,7 @@ def _build_container_hosts(container_affinity, container_hosts, type_and_name,
"hosts": [],
}
appended = append_if(
appended = du.append_if(
array=inventory[container_host_type]["hosts"],
item=container_host_name
)
@ -196,7 +195,7 @@ def _build_container_hosts(container_affinity, container_hosts, type_and_name,
logger.debug("Added container %s to %s",
container_host_name, container_host_type)
append_if(array=container_hosts, item=container_host_name)
du.append_if(array=container_hosts, item=container_host_name)
else:
if host_type not in hostvars:
hostvars[host_type] = {}
@ -208,7 +207,7 @@ def _build_container_hosts(container_affinity, container_hosts, type_and_name,
# Create a host types containers group and append it to inventory
host_type_containers = '{}-host_containers'.format(host_type)
append_if(array=container_mapping, item=host_type_containers)
du.append_if(array=container_mapping, item=host_type_containers)
hostvars_options.update({
'properties': properties,
@ -273,25 +272,25 @@ def _append_to_host_groups(inventory, container_type, assignment, host_type,
hdata['physical_host'] = host_type
if container.startswith('{}-'.format(type_and_name)):
appended = append_if(array=iah, item=container)
appended = du.append_if(array=iah, item=container)
if appended:
logger.debug("Added host %s to %s hosts",
container, assignment)
elif is_metal is True:
if component == assignment:
appended = append_if(array=iah, item=container)
appended = du.append_if(array=iah, item=container)
if appended:
logger.debug("Added is_metal host %s to %s hosts",
container, assignment)
if container.startswith('{}-'.format(type_and_name)):
appended = append_if(array=iph, item=container)
appended = du.append_if(array=iph, item=container)
if appended:
logger.debug("Added host %s to %s hosts",
container, physical_group_type)
elif is_metal is True:
if container.startswith(host_type):
appended = append_if(array=iph, item=container)
appended = du.append_if(array=iph, item=container)
if appended:
logger.debug("Added is_metal host %s to %s hosts",
container, physical_group_type)
@ -342,8 +341,8 @@ def _add_container_hosts(assignment, config, container_name, container_type,
# If host_type is not in config do not append containers to it
if host_type not in config[physical_host_type]:
continue
appended = append_if(array=inventory['lxc_hosts']['hosts'],
item=host_type)
appended = du.append_if(array=inventory['lxc_hosts']['hosts'],
item=host_type)
if appended:
logger.debug("%s added to lxc_hosts group", host_type)
@ -445,7 +444,8 @@ def user_defined_setup(config, inventory):
hvs[_key][_k] = _v
ip.USED_IPS.add(_value['ip'])
appended = append_if(array=inventory[key]['hosts'], item=_key)
appended = du.append_if(array=inventory[key]['hosts'],
item=_key)
if appended:
logger.debug("Added host %s to group %s",
_key, key)
@ -731,6 +731,8 @@ def container_skel_load(container_skel, inventory, config):
def find_config_path(user_config_path=None):
# TODO(stevelle) REMOVE THIS FUNCTION: after other changes in chain
# prevents the clean removal right now
"""Return the path to the user configuration files.
If no directory is found the system will exit.
@ -831,7 +833,7 @@ def _extra_config(user_defined_config, base_dir):
for name in files:
if name.endswith(('.yml', '.yaml')):
with open(os.path.join(root_dir, name), 'rb') as f:
merge_dict(
du.merge_dict(
user_defined_config,
yaml.safe_load(f.read()) or {}
)
@ -973,6 +975,19 @@ def _check_all_conf_groups_present(config, environment):
return retval
def _collect_hostnames(dynamic_inventory):
# Generate a list of all hosts and their used IP addresses
hostnames_ips = {}
for _host, _vars in dynamic_inventory['_meta']['hostvars'].iteritems():
host_hash = hostnames_ips[_host] = {}
for _key, _value in _vars.iteritems():
if _key.endswith('address') or _key == 'ansible_host':
host_hash[_key] = _value
return hostnames_ips
def load_environment(config_path, environment):
"""Create an environment dictionary from config files
@ -981,9 +996,10 @@ def load_environment(config_path, environment):
"""
# Load all YAML files found in the env.d directory
env_plugins = os.path.join(config_path, 'env.d')
env_plugins = filesys.dir_find(config_path, 'env.d',
raise_if_missing=False)
if os.path.isdir(env_plugins):
if env_plugins is not False:
_extra_config(user_defined_config=environment, base_dir=env_plugins)
logger.debug("Loaded environment from %s", config_path)
return environment
@ -1004,8 +1020,8 @@ def load_user_configuration(config_path):
user_defined_config.update(yaml.safe_load(f.read()) or {})
# Load anything in a conf.d directory if found
base_dir = os.path.join(config_path, 'conf.d')
if os.path.isdir(base_dir):
base_dir = filesys.dir_find(config_path, 'conf.d', raise_if_missing=False)
if base_dir is not False:
_extra_config(user_defined_config, base_dir)
# Exit if no user_config was found and loaded
@ -1037,9 +1053,7 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs):
logger.info("Beginning new inventory run")
# Get the path to the user configuration files
config_path = find_config_path(
user_config_path=config
)
config_path = filesys.dir_find(preferred_path=config)
user_defined_config = load_user_configuration(config_path)
base_env_dir = environment
@ -1047,7 +1061,7 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs):
environment = load_environment(config_path, base_env)
# Load existing inventory file if found
dynamic_inventory = load_inventory(config_path, INVENTORY_SKEL)
dynamic_inventory = filesys.load_inventory(config_path, INVENTORY_SKEL)
# Save the users container cidr as a group variable
cidr_networks = user_defined_config.get('cidr_networks')
@ -1109,31 +1123,15 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs):
if _check_all_conf_groups_present(user_defined_config, environment):
return 'Configuration ok!'
# Generate a list of all hosts and their used IP addresses
hostnames_ips = {}
for _host, _vars in dynamic_inventory['_meta']['hostvars'].iteritems():
host_hash = hostnames_ips[_host] = {}
for _key, _value in _vars.iteritems():
if _key.endswith('address') or _key == 'ansible_host':
host_hash[_key] = _value
# Save a list of all hosts and their given IP addresses
hostnames_ip_file = os.path.join(
config_path, 'openstack_hostnames_ips.yml')
with open(hostnames_ip_file, 'wb') as f:
f.write(
json.dumps(
hostnames_ips,
indent=4,
sort_keys=True
)
)
hostnames_ips = _collect_hostnames(dynamic_inventory)
filesys.write_hostnames(config, hostnames_ips)
if logger.isEnabledFor(logging.DEBUG):
num_hosts = len(dynamic_inventory['_meta']['hostvars'])
logger.debug("%d hosts found." % num_hosts)
# Save new dynamic inventory
save_inventory(dynamic_inventory_json, config_path)
filesys.save_inventory(dynamic_inventory_json, config_path)
return dynamic_inventory_json