Merge "Inventory filesystem refactor load funcs moved"

lib/filesystem.py

@@ -22,6 +22,9 @@ import json
 import logging
 import os
 import tarfile
+import yaml
+
+import dictutils as du
 
 logger = logging.getLogger('osa-inventory')
 
@@ -105,6 +108,23 @@ def dir_find(preferred_path=None, suffix=None, raise_if_missing=True):
         return False
 
 
+def _extra_config(user_defined_config, base_dir):
+    """Discover new items in any extra directories and add the new values.
+
+    :param user_defined_config: ``dict``
+    :param base_dir: ``str``
+    """
+    for root_dir, _, files in os.walk(base_dir):
+        for name in files:
+            if name.endswith(('.yml', '.yaml')):
+                with open(os.path.join(root_dir, name), 'rb') as f:
+                    du.merge_dict(
+                        user_defined_config,
+                        yaml.safe_load(f.read()) or {}
+                    )
+                    logger.debug("Merged overrides from file {}".format(name))
+
+
 def _make_backup(backup_path, source_file_path):
     """ Create a backup of all previous inventory files as a tar archive
 
@@ -217,3 +237,49 @@ def save_inventory(inventory_json, save_path):
     with open(inventory_file, 'wb') as f:
         f.write(inventory_json)
         logger.info("Inventory written")
+
+
+def load_environment(config_path, environment):
+    """Create an environment dictionary from config files
+
+    :param config_path: ``str`` path where the environment files are kept
+    :param environment: ``dict`` dictionary to populate with environment data
+    """
+
+    # Load all YAML files found in the env.d directory
+    env_plugins = dir_find(config_path, 'env.d', raise_if_missing=False)
+
+    if env_plugins is not False:
+        _extra_config(user_defined_config=environment, base_dir=env_plugins)
+    logger.debug("Loaded environment from {}".format(config_path))
+    return environment
+
+
+def load_user_configuration(config_path):
+    """Create a user configuration dictionary from config files
+
+    :param config_path: ``str`` path where the configuration files are kept
+    """
+
+    user_defined_config = dict()
+
+    # Load the user defined configuration file
+    user_config_file = os.path.join(config_path, 'openstack_user_config.yml')
+    if os.path.isfile(user_config_file):
+        with open(user_config_file, 'rb') as f:
+            user_defined_config.update(yaml.safe_load(f.read()) or {})
+
+    # Load anything in a conf.d directory if found
+    base_dir = dir_find(config_path, 'conf.d', raise_if_missing=False)
+    if base_dir is not False:
+        _extra_config(user_defined_config, base_dir)
+
+    # Exit if no user_config was found and loaded
+    if not user_defined_config:
+        raise SystemExit(
+            'No user config loaded\n'
+            'No openstack_user_config files are available in either \n{}'
+            '\nor \n{}/conf.d directory'.format(config_path, config_path)
+        )
+    logger.debug("User configuration loaded from: {}".format(user_config_file))
+    return user_defined_config

lib/generate.py

@@ -19,10 +19,8 @@ import ip
 import json
 import logging
 import netaddr
-import os
 import uuid
 import warnings
-import yaml
 
 import dictutils as du
 import filesystem as filesys
@@ -730,34 +728,6 @@ def container_skel_load(container_skel, inventory, config):
                 )
 
 
-def find_config_path(user_config_path=None):
-    # TODO(stevelle) REMOVE THIS FUNCTION: after other changes in chain
-    # prevents the clean removal right now
-    """Return the path to the user configuration files.
-
-    If no directory is found the system will exit.
-
-    The lookup will be done in the following directories:
-
-      * user_config_path
-      * ``/etc/openstack_deploy/``
-
-    :param user_config_path: ``str`` Location to look in FIRST for a file
-    """
-    path_check = [
-        os.path.join('/etc', 'openstack_deploy'),
-    ]
-
-    if user_config_path is not None:
-        path_check.insert(0, os.path.expanduser(user_config_path))
-
-    for f in path_check:
-        if os.path.isdir(f):
-            return f
-    else:
-        raise SystemExit('No config found at: {}'.format(path_check))
-
-
 def _ensure_inventory_uptodate(inventory, container_skel):
     """Update inventory if needed.
 
@@ -823,23 +793,6 @@ def _parse_global_variables(user_cidr, inventory, user_defined_config):
                     del inventory['all']['vars'][key]
 
 
-def _extra_config(user_defined_config, base_dir):
-    """Discover new items in any extra directories and add the new values.
-
-    :param user_defined_config: ``dict``
-    :param base_dir: ``str``
-    """
-    for root_dir, _, files in os.walk(base_dir):
-        for name in files:
-            if name.endswith(('.yml', '.yaml')):
-                with open(os.path.join(root_dir, name), 'rb') as f:
-                    du.merge_dict(
-                        user_defined_config,
-                        yaml.safe_load(f.read()) or {}
-                    )
-                    logger.debug("Merged overrides from file %s", name)
-
-
 def _check_same_ip_to_multiple_host(config):
     """Check for IPs assigned to multiple hosts
 
@@ -988,51 +941,11 @@ def _collect_hostnames(dynamic_inventory):
     return hostnames_ips
 
 
-def load_environment(config_path, environment):
+def _prepare_debug_logger():
-    """Create an environment dictionary from config files
+    log_fmt = "%(lineno)d - %(funcName)s: %(message)s"
-
+    logging.basicConfig(format=log_fmt, filename='inventory.log')
-    :param config_path: ``str`` path where the environment files are kept
+    logger.setLevel(logging.DEBUG)
-    :param environment: ``dict`` dictionary to populate with environment data
+    logger.info("Beginning new inventory run")
-    """
-
-    # Load all YAML files found in the env.d directory
-    env_plugins = filesys.dir_find(config_path, 'env.d',
-                                   raise_if_missing=False)
-
-    if env_plugins is not False:
-        _extra_config(user_defined_config=environment, base_dir=env_plugins)
-    logger.debug("Loaded environment from %s", config_path)
-    return environment
-
-
-def load_user_configuration(config_path):
-    """Create a user configuration dictionary from config files
-
-    :param config_path: ``str`` path where the configuration files are kept
-    """
-
-    user_defined_config = dict()
-
-    # Load the user defined configuration file
-    user_config_file = os.path.join(config_path, 'openstack_user_config.yml')
-    if os.path.isfile(user_config_file):
-        with open(user_config_file, 'rb') as f:
-            user_defined_config.update(yaml.safe_load(f.read()) or {})
-
-    # Load anything in a conf.d directory if found
-    base_dir = filesys.dir_find(config_path, 'conf.d', raise_if_missing=False)
-    if base_dir is not False:
-        _extra_config(user_defined_config, base_dir)
-
-    # Exit if no user_config was found and loaded
-    if not user_defined_config:
-        raise SystemExit(
-            'No user config loaded\n'
-            'No openstack_user_config files are available in either \n{}'
-            '\nor \n{}/conf.d directory'.format(config_path, config_path)
-        )
-    logger.debug("User configuration loaded from: %s", user_config_file)
-    return user_defined_config
 
 
 def main(config=None, check=False, debug=False, environment=None, **kwargs):
@@ -1047,18 +960,15 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs):
     :param environment: ``str`` Directory containing the base env.d
     """
     if debug:
-        log_fmt = "%(lineno)d - %(funcName)s: %(message)s"
+        _prepare_debug_logger()
-        logging.basicConfig(format=log_fmt, filename='inventory.log')
-        logger.setLevel(logging.DEBUG)
-        logger.info("Beginning new inventory run")
 
     # Get the path to the user configuration files
     config_path = filesys.dir_find(preferred_path=config)
 
-    user_defined_config = load_user_configuration(config_path)
+    user_defined_config = filesys.load_user_configuration(config_path)
     base_env_dir = environment
-    base_env = load_environment(base_env_dir, {})
+    base_env = filesys.load_environment(base_env_dir, {})
-    environment = load_environment(config_path, base_env)
+    environment = filesys.load_environment(config_path, base_env)
 
     # Load existing inventory file if found
     dynamic_inventory = filesys.load_inventory(config_path, INVENTORY_SKEL)
@@ -1129,7 +1039,7 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs):
 
     if logger.isEnabledFor(logging.DEBUG):
         num_hosts = len(dynamic_inventory['_meta']['hostvars'])
-        logger.debug("%d hosts found." % num_hosts)
+        logger.debug("%d hosts found.", num_hosts)
 
     # Save new dynamic inventory
     filesys.save_inventory(dynamic_inventory_json, config_path)

tests/test_inventory.py

@@ -426,7 +426,7 @@ class TestAnsibleInventoryFormatConstraints(unittest.TestCase):
 class TestUserConfiguration(unittest.TestCase):
     def setUp(self):
         self.longMessage = True
-        self.loaded_user_configuration = di.load_user_configuration(TARGET_DIR)
+        self.loaded_user_configuration = fs.load_user_configuration(TARGET_DIR)
 
     def test_loading_user_configuration(self):
         """Test that the user configuration can be loaded"""
@@ -436,7 +436,7 @@ class TestUserConfiguration(unittest.TestCase):
 class TestEnvironments(unittest.TestCase):
     def setUp(self):
         self.longMessage = True
-        self.loaded_environment = di.load_environment(BASE_ENV_DIR, {})
+        self.loaded_environment = fs.load_environment(BASE_ENV_DIR, {})
 
     def test_loading_environment(self):
         """Test that the environment can be loaded"""
@@ -458,14 +458,14 @@ class TestIps(unittest.TestCase):
         # Allow custom assertion errors.
         self.longMessage = True
 
-    @mock.patch('generate.load_environment')
+    @mock.patch('filesystem.load_environment')
-    @mock.patch('generate.load_user_configuration')
+    @mock.patch('filesystem.load_user_configuration')
     def test_duplicates(self, mock_load_config, mock_load_env):
         """Test that no duplicate IPs are made on any network."""
 
         # Grab our values read from the file system just once.
         mock_load_config.return_value = get_config()
-        mock_load_env.return_value = di.load_environment(BASE_ENV_DIR, {})
+        mock_load_env.return_value = fs.load_environment(BASE_ENV_DIR, {})
 
         mock_open = mock.mock_open()
 
@@ -843,7 +843,7 @@ class TestGlobalOverridesConfigDeletion(TestConfigCheckBase):
 
 class TestEnsureInventoryUptoDate(unittest.TestCase):
     def setUp(self):
-        self.env = di.load_environment(BASE_ENV_DIR, {})
+        self.env = fs.load_environment(BASE_ENV_DIR, {})
         # Copy because we manipulate the structure in each test;
         # not copying would modify the global var in the target code
         self.inv = copy.deepcopy(di.INVENTORY_SKEL)
@@ -900,7 +900,7 @@ class TestEnsureInventoryUptoDate(unittest.TestCase):
 
 class OverridingEnvBase(unittest.TestCase):
     def setUp(self):
-        self.base_env = di.load_environment(BASE_ENV_DIR, {})
+        self.base_env = fs.load_environment(BASE_ENV_DIR, {})
 
         # Use the cinder configuration as our sample for override testing
         with open(path.join(BASE_ENV_DIR, 'env.d', 'cinder.yml'), 'r') as f:
@@ -926,7 +926,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
 
         self.write_override_env()
 
-        di.load_environment(TARGET_DIR, self.base_env)
+        fs.load_environment(TARGET_DIR, self.base_env)
 
         test_vol = self.base_env['container_skel']['cinder_volumes_container']
         self.assertFalse(test_vol['properties']['is_metal'])
@@ -942,7 +942,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
 
         self.write_override_env()
 
-        di.load_environment(TARGET_DIR, self.base_env)
+        fs.load_environment(TARGET_DIR, self.base_env)
 
         test_vol = self.base_env['container_skel']['cinder_volumes_container']
 
@@ -954,7 +954,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
 
         self.write_override_env()
 
-        di.load_environment(TARGET_DIR, self.base_env)
+        fs.load_environment(TARGET_DIR, self.base_env)
 
         test_vol = self.base_env['container_skel']['cinder_volumes_container']
 
@@ -966,7 +966,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
 
         self.write_override_env()
 
-        di.load_environment(TARGET_DIR, self.base_env)
+        fs.load_environment(TARGET_DIR, self.base_env)
 
         test_vol = self.base_env['container_skel']['cinder_volumes_container']
 
@@ -978,7 +978,7 @@ class TestOverridingEnvVars(OverridingEnvBase):
 
         self.write_override_env()
 
-        di.load_environment(TARGET_DIR, self.base_env)
+        fs.load_environment(TARGET_DIR, self.base_env)
 
         test_vol = self.base_env['container_skel']['cinder_volumes_container']
 
@@ -994,7 +994,7 @@ class TestOverridingEnvIntegration(OverridingEnvBase):
         self.inv = fs.load_inventory(TARGET_DIR, di.INVENTORY_SKEL)
 
     def skel_setup(self):
-        self.environment = di.load_environment(TARGET_DIR, self.base_env)
+        self.environment = fs.load_environment(TARGET_DIR, self.base_env)
 
         di.skel_setup(self.environment, self.inv)
 
@@ -1173,7 +1173,7 @@ class TestLxcHosts(TestConfigCheckBase):
 
 class TestConfigMatchesEnvironment(unittest.TestCase):
     def setUp(self):
-        self.env = di.load_environment(BASE_ENV_DIR, {})
+        self.env = fs.load_environment(BASE_ENV_DIR, {})
 
     def test_matching_keys(self):
         config = get_config()