Merge "Inventory filesystem refactor load funcs moved"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
12182c186d
@ -22,6 +22,9 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import tarfile
|
||||
import yaml
|
||||
|
||||
import dictutils as du
|
||||
|
||||
logger = logging.getLogger('osa-inventory')
|
||||
|
||||
@ -105,6 +108,23 @@ def dir_find(preferred_path=None, suffix=None, raise_if_missing=True):
|
||||
return False
|
||||
|
||||
|
||||
def _extra_config(user_defined_config, base_dir):
|
||||
"""Discover new items in any extra directories and add the new values.
|
||||
|
||||
:param user_defined_config: ``dict``
|
||||
:param base_dir: ``str``
|
||||
"""
|
||||
for root_dir, _, files in os.walk(base_dir):
|
||||
for name in files:
|
||||
if name.endswith(('.yml', '.yaml')):
|
||||
with open(os.path.join(root_dir, name), 'rb') as f:
|
||||
du.merge_dict(
|
||||
user_defined_config,
|
||||
yaml.safe_load(f.read()) or {}
|
||||
)
|
||||
logger.debug("Merged overrides from file {}".format(name))
|
||||
|
||||
|
||||
def _make_backup(backup_path, source_file_path):
|
||||
""" Create a backup of all previous inventory files as a tar archive
|
||||
|
||||
@ -217,3 +237,49 @@ def save_inventory(inventory_json, save_path):
|
||||
with open(inventory_file, 'wb') as f:
|
||||
f.write(inventory_json)
|
||||
logger.info("Inventory written")
|
||||
|
||||
|
||||
def load_environment(config_path, environment):
|
||||
"""Create an environment dictionary from config files
|
||||
|
||||
:param config_path: ``str`` path where the environment files are kept
|
||||
:param environment: ``dict`` dictionary to populate with environment data
|
||||
"""
|
||||
|
||||
# Load all YAML files found in the env.d directory
|
||||
env_plugins = dir_find(config_path, 'env.d', raise_if_missing=False)
|
||||
|
||||
if env_plugins is not False:
|
||||
_extra_config(user_defined_config=environment, base_dir=env_plugins)
|
||||
logger.debug("Loaded environment from {}".format(config_path))
|
||||
return environment
|
||||
|
||||
|
||||
def load_user_configuration(config_path):
|
||||
"""Create a user configuration dictionary from config files
|
||||
|
||||
:param config_path: ``str`` path where the configuration files are kept
|
||||
"""
|
||||
|
||||
user_defined_config = dict()
|
||||
|
||||
# Load the user defined configuration file
|
||||
user_config_file = os.path.join(config_path, 'openstack_user_config.yml')
|
||||
if os.path.isfile(user_config_file):
|
||||
with open(user_config_file, 'rb') as f:
|
||||
user_defined_config.update(yaml.safe_load(f.read()) or {})
|
||||
|
||||
# Load anything in a conf.d directory if found
|
||||
base_dir = dir_find(config_path, 'conf.d', raise_if_missing=False)
|
||||
if base_dir is not False:
|
||||
_extra_config(user_defined_config, base_dir)
|
||||
|
||||
# Exit if no user_config was found and loaded
|
||||
if not user_defined_config:
|
||||
raise SystemExit(
|
||||
'No user config loaded\n'
|
||||
'No openstack_user_config files are available in either \n{}'
|
||||
'\nor \n{}/conf.d directory'.format(config_path, config_path)
|
||||
)
|
||||
logger.debug("User configuration loaded from: {}".format(user_config_file))
|
||||
return user_defined_config
|
||||
|
||||
110
lib/generate.py
110
lib/generate.py
@ -19,10 +19,8 @@ import ip
|
||||
import json
|
||||
import logging
|
||||
import netaddr
|
||||
import os
|
||||
import uuid
|
||||
import warnings
|
||||
import yaml
|
||||
|
||||
import dictutils as du
|
||||
import filesystem as filesys
|
||||
@ -730,34 +728,6 @@ def container_skel_load(container_skel, inventory, config):
|
||||
)
|
||||
|
||||
|
||||
def find_config_path(user_config_path=None):
|
||||
# TODO(stevelle) REMOVE THIS FUNCTION: after other changes in chain
|
||||
# prevents the clean removal right now
|
||||
"""Return the path to the user configuration files.
|
||||
|
||||
If no directory is found the system will exit.
|
||||
|
||||
The lookup will be done in the following directories:
|
||||
|
||||
* user_config_path
|
||||
* ``/etc/openstack_deploy/``
|
||||
|
||||
:param user_config_path: ``str`` Location to look in FIRST for a file
|
||||
"""
|
||||
path_check = [
|
||||
os.path.join('/etc', 'openstack_deploy'),
|
||||
]
|
||||
|
||||
if user_config_path is not None:
|
||||
path_check.insert(0, os.path.expanduser(user_config_path))
|
||||
|
||||
for f in path_check:
|
||||
if os.path.isdir(f):
|
||||
return f
|
||||
else:
|
||||
raise SystemExit('No config found at: {}'.format(path_check))
|
||||
|
||||
|
||||
def _ensure_inventory_uptodate(inventory, container_skel):
|
||||
"""Update inventory if needed.
|
||||
|
||||
@ -823,23 +793,6 @@ def _parse_global_variables(user_cidr, inventory, user_defined_config):
|
||||
del inventory['all']['vars'][key]
|
||||
|
||||
|
||||
def _extra_config(user_defined_config, base_dir):
|
||||
"""Discover new items in any extra directories and add the new values.
|
||||
|
||||
:param user_defined_config: ``dict``
|
||||
:param base_dir: ``str``
|
||||
"""
|
||||
for root_dir, _, files in os.walk(base_dir):
|
||||
for name in files:
|
||||
if name.endswith(('.yml', '.yaml')):
|
||||
with open(os.path.join(root_dir, name), 'rb') as f:
|
||||
du.merge_dict(
|
||||
user_defined_config,
|
||||
yaml.safe_load(f.read()) or {}
|
||||
)
|
||||
logger.debug("Merged overrides from file %s", name)
|
||||
|
||||
|
||||
def _check_same_ip_to_multiple_host(config):
|
||||
"""Check for IPs assigned to multiple hosts
|
||||
|
||||
@ -988,51 +941,11 @@ def _collect_hostnames(dynamic_inventory):
|
||||
return hostnames_ips
|
||||
|
||||
|
||||
def load_environment(config_path, environment):
|
||||
"""Create an environment dictionary from config files
|
||||
|
||||
:param config_path: ``str`` path where the environment files are kept
|
||||
:param environment: ``dict`` dictionary to populate with environment data
|
||||
"""
|
||||
|
||||
# Load all YAML files found in the env.d directory
|
||||
env_plugins = filesys.dir_find(config_path, 'env.d',
|
||||
raise_if_missing=False)
|
||||
|
||||
if env_plugins is not False:
|
||||
_extra_config(user_defined_config=environment, base_dir=env_plugins)
|
||||
logger.debug("Loaded environment from %s", config_path)
|
||||
return environment
|
||||
|
||||
|
||||
def load_user_configuration(config_path):
|
||||
"""Create a user configuration dictionary from config files
|
||||
|
||||
:param config_path: ``str`` path where the configuration files are kept
|
||||
"""
|
||||
|
||||
user_defined_config = dict()
|
||||
|
||||
# Load the user defined configuration file
|
||||
user_config_file = os.path.join(config_path, 'openstack_user_config.yml')
|
||||
if os.path.isfile(user_config_file):
|
||||
with open(user_config_file, 'rb') as f:
|
||||
user_defined_config.update(yaml.safe_load(f.read()) or {})
|
||||
|
||||
# Load anything in a conf.d directory if found
|
||||
base_dir = filesys.dir_find(config_path, 'conf.d', raise_if_missing=False)
|
||||
if base_dir is not False:
|
||||
_extra_config(user_defined_config, base_dir)
|
||||
|
||||
# Exit if no user_config was found and loaded
|
||||
if not user_defined_config:
|
||||
raise SystemExit(
|
||||
'No user config loaded\n'
|
||||
'No openstack_user_config files are available in either \n{}'
|
||||
'\nor \n{}/conf.d directory'.format(config_path, config_path)
|
||||
)
|
||||
logger.debug("User configuration loaded from: %s", user_config_file)
|
||||
return user_defined_config
|
||||
def _prepare_debug_logger():
|
||||
log_fmt = "%(lineno)d - %(funcName)s: %(message)s"
|
||||
logging.basicConfig(format=log_fmt, filename='inventory.log')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.info("Beginning new inventory run")
|
||||
|
||||
|
||||
def main(config=None, check=False, debug=False, environment=None, **kwargs):
|
||||
@ -1047,18 +960,15 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs):
|
||||
:param environment: ``str`` Directory containing the base env.d
|
||||
"""
|
||||
if debug:
|
||||
log_fmt = "%(lineno)d - %(funcName)s: %(message)s"
|
||||
logging.basicConfig(format=log_fmt, filename='inventory.log')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.info("Beginning new inventory run")
|
||||
_prepare_debug_logger()
|
||||
|
||||
# Get the path to the user configuration files
|
||||
config_path = filesys.dir_find(preferred_path=config)
|
||||
|
||||
user_defined_config = load_user_configuration(config_path)
|
||||
user_defined_config = filesys.load_user_configuration(config_path)
|
||||
base_env_dir = environment
|
||||
base_env = load_environment(base_env_dir, {})
|
||||
environment = load_environment(config_path, base_env)
|
||||
base_env = filesys.load_environment(base_env_dir, {})
|
||||
environment = filesys.load_environment(config_path, base_env)
|
||||
|
||||
# Load existing inventory file if found
|
||||
dynamic_inventory = filesys.load_inventory(config_path, INVENTORY_SKEL)
|
||||
@ -1129,7 +1039,7 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs):
|
||||
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
num_hosts = len(dynamic_inventory['_meta']['hostvars'])
|
||||
logger.debug("%d hosts found." % num_hosts)
|
||||
logger.debug("%d hosts found.", num_hosts)
|
||||
|
||||
# Save new dynamic inventory
|
||||
filesys.save_inventory(dynamic_inventory_json, config_path)
|
||||
|
||||
@ -426,7 +426,7 @@ class TestAnsibleInventoryFormatConstraints(unittest.TestCase):
|
||||
class TestUserConfiguration(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.longMessage = True
|
||||
self.loaded_user_configuration = di.load_user_configuration(TARGET_DIR)
|
||||
self.loaded_user_configuration = fs.load_user_configuration(TARGET_DIR)
|
||||
|
||||
def test_loading_user_configuration(self):
|
||||
"""Test that the user configuration can be loaded"""
|
||||
@ -436,7 +436,7 @@ class TestUserConfiguration(unittest.TestCase):
|
||||
class TestEnvironments(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.longMessage = True
|
||||
self.loaded_environment = di.load_environment(BASE_ENV_DIR, {})
|
||||
self.loaded_environment = fs.load_environment(BASE_ENV_DIR, {})
|
||||
|
||||
def test_loading_environment(self):
|
||||
"""Test that the environment can be loaded"""
|
||||
@ -458,14 +458,14 @@ class TestIps(unittest.TestCase):
|
||||
# Allow custom assertion errors.
|
||||
self.longMessage = True
|
||||
|
||||
@mock.patch('generate.load_environment')
|
||||
@mock.patch('generate.load_user_configuration')
|
||||
@mock.patch('filesystem.load_environment')
|
||||
@mock.patch('filesystem.load_user_configuration')
|
||||
def test_duplicates(self, mock_load_config, mock_load_env):
|
||||
"""Test that no duplicate IPs are made on any network."""
|
||||
|
||||
# Grab our values read from the file system just once.
|
||||
mock_load_config.return_value = get_config()
|
||||
mock_load_env.return_value = di.load_environment(BASE_ENV_DIR, {})
|
||||
mock_load_env.return_value = fs.load_environment(BASE_ENV_DIR, {})
|
||||
|
||||
mock_open = mock.mock_open()
|
||||
|
||||
@ -843,7 +843,7 @@ class TestGlobalOverridesConfigDeletion(TestConfigCheckBase):
|
||||
|
||||
class TestEnsureInventoryUptoDate(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.env = di.load_environment(BASE_ENV_DIR, {})
|
||||
self.env = fs.load_environment(BASE_ENV_DIR, {})
|
||||
# Copy because we manipulate the structure in each test;
|
||||
# not copying would modify the global var in the target code
|
||||
self.inv = copy.deepcopy(di.INVENTORY_SKEL)
|
||||
@ -900,7 +900,7 @@ class TestEnsureInventoryUptoDate(unittest.TestCase):
|
||||
|
||||
class OverridingEnvBase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.base_env = di.load_environment(BASE_ENV_DIR, {})
|
||||
self.base_env = fs.load_environment(BASE_ENV_DIR, {})
|
||||
|
||||
# Use the cinder configuration as our sample for override testing
|
||||
with open(path.join(BASE_ENV_DIR, 'env.d', 'cinder.yml'), 'r') as f:
|
||||
@ -926,7 +926,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
|
||||
|
||||
self.write_override_env()
|
||||
|
||||
di.load_environment(TARGET_DIR, self.base_env)
|
||||
fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
test_vol = self.base_env['container_skel']['cinder_volumes_container']
|
||||
self.assertFalse(test_vol['properties']['is_metal'])
|
||||
@ -942,7 +942,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
|
||||
|
||||
self.write_override_env()
|
||||
|
||||
di.load_environment(TARGET_DIR, self.base_env)
|
||||
fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
test_vol = self.base_env['container_skel']['cinder_volumes_container']
|
||||
|
||||
@ -954,7 +954,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
|
||||
|
||||
self.write_override_env()
|
||||
|
||||
di.load_environment(TARGET_DIR, self.base_env)
|
||||
fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
test_vol = self.base_env['container_skel']['cinder_volumes_container']
|
||||
|
||||
@ -966,7 +966,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
|
||||
|
||||
self.write_override_env()
|
||||
|
||||
di.load_environment(TARGET_DIR, self.base_env)
|
||||
fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
test_vol = self.base_env['container_skel']['cinder_volumes_container']
|
||||
|
||||
@ -978,7 +978,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
|
||||
|
||||
self.write_override_env()
|
||||
|
||||
di.load_environment(TARGET_DIR, self.base_env)
|
||||
fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
test_vol = self.base_env['container_skel']['cinder_volumes_container']
|
||||
|
||||
@ -994,7 +994,7 @@ class TestOverridingEnvIntegration(OverridingEnvBase):
|
||||
self.inv = fs.load_inventory(TARGET_DIR, di.INVENTORY_SKEL)
|
||||
|
||||
def skel_setup(self):
|
||||
self.environment = di.load_environment(TARGET_DIR, self.base_env)
|
||||
self.environment = fs.load_environment(TARGET_DIR, self.base_env)
|
||||
|
||||
di.skel_setup(self.environment, self.inv)
|
||||
|
||||
@ -1173,7 +1173,7 @@ class TestLxcHosts(TestConfigCheckBase):
|
||||
|
||||
class TestConfigMatchesEnvironment(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.env = di.load_environment(BASE_ENV_DIR, {})
|
||||
self.env = fs.load_environment(BASE_ENV_DIR, {})
|
||||
|
||||
def test_matching_keys(self):
|
||||
config = get_config()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user