diff --git a/lib/filesystem.py b/lib/filesystem.py index a6af89ade5..3c76d37fbe 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -16,8 +16,16 @@ # (c) 2015, Major Hayden # +import copy +import datetime import json +import logging import os +import tarfile + +logger = logging.getLogger('osa-inventory') + +INVENTORY_FILENAME = 'openstack_inventory.json' def _get_search_paths(preferred_path=None, suffix=None): @@ -49,6 +57,7 @@ def file_find(filename, preferred_path=None, pass_exception=False): :param pass_exception: ``bool`` Should a SystemExit be raised if the file is not found """ + search_paths = _get_search_paths(preferred_path, suffix=filename) for file_candidate in search_paths: @@ -61,25 +70,79 @@ def file_find(filename, preferred_path=None, pass_exception=False): return False -def save_to_json(filename, dictionary): - """Write out the given dictionary - - :param filename: ``str`` Name of the file to write to - :param dictionary: ``dict`` A dictionary to write - """ - target_file = file_find(filename) - with open(target_file, 'wb') as f_handle: - inventory_json = json.dumps(dictionary, indent=2) - f_handle.write(inventory_json) +def make_backup(config_path, inventory_file_path): + # Create a backup of all previous inventory files as a tar archive + inventory_backup_file = os.path.join( + config_path, + 'backup_openstack_inventory.tar' + ) + with tarfile.open(inventory_backup_file, 'a') as tar: + basename = os.path.basename(inventory_file_path) + backup_name = get_backup_name(basename) + tar.add(inventory_file_path, arcname=backup_name) + logger.debug("Backup written to {}".format(inventory_backup_file)) -def load_from_json(filename): +def get_backup_name(basename): + utctime = datetime.datetime.utcnow() + utctime = utctime.strftime("%Y%m%d_%H%M%S") + return '{}-{}.json'.format(basename, utctime) + + +def load_from_json(filename, preferred_path=None, pass_exception=False): """Return a dictionary found in a given file :param filename: ``str`` Name of the file to read from + :param preferred_path: ``str`` Path to the json file to try FIRST + :param pass_exception: ``bool`` Should a SystemExit be raised if the file + is not found + :return ``(dict, str)`` Dictionary describing the JSON file contents or + False, and the fully resolved file name loaded or None """ - target_file = file_find(filename) - with open(target_file, 'rb') as f_handle: - dictionary = json.loads(f_handle.read()) - return dictionary + target_file = file_find(filename, preferred_path, pass_exception) + dictionary = False + if target_file is not False: + with open(target_file, 'rb') as f_handle: + dictionary = json.loads(f_handle.read()) + + return dictionary, target_file + + +def load_inventory(preferred_path=None, default_inv=None): + """Create an inventory dictionary from the given source file or a default + inventory. If an inventory is found then a backup tarball is created + as well. + + :param preferred_path: ``str`` Path to the inventory directory to try FIRST + :param default_inv: ``dict`` Default inventory skeleton + + :return: ``dict`` A dictionary found or ``default_inv`` + """ + + inventory, file_loaded = load_from_json(INVENTORY_FILENAME, preferred_path, + pass_exception=True) + if inventory is not False: + logger.debug("Loaded existing inventory from {}".format(file_loaded)) + make_backup(preferred_path, file_loaded) + else: + logger.debug("No existing inventory, created fresh skeleton.") + inventory = copy.deepcopy(default_inv) + + return inventory + + +def save_inventory(inventory_json, save_path): + """Save an inventory dictionary + + :param inventory_json: ``str`` String of JSON formatted inventory to store + :param save_path: ``str`` Path of the directory to save to + """ + + if INVENTORY_FILENAME == save_path: + inventory_file = file_find(save_path) + else: + inventory_file = os.path.join(save_path, INVENTORY_FILENAME) + with open(inventory_file, 'wb') as f: + f.write(inventory_json) + logger.info("Inventory written") diff --git a/lib/generate.py b/lib/generate.py index 8d33881372..ffbeb01edd 100755 --- a/lib/generate.py +++ b/lib/generate.py @@ -15,21 +15,20 @@ # # (c) 2014, Kevin Carter -import copy -import datetime import json import logging import netaddr import os import Queue import random -import tarfile import uuid import warnings import yaml from dictutils import append_if from dictutils import merge_dict +from filesystem import load_inventory +from filesystem import save_inventory logger = logging.getLogger('osa-inventory') @@ -1102,40 +1101,6 @@ def load_user_configuration(config_path): return user_defined_config -def make_backup(config_path, inventory_file_path): - # Create a backup of all previous inventory files as a tar archive - inventory_backup_file = os.path.join( - config_path, - 'backup_openstack_inventory.tar' - ) - with tarfile.open(inventory_backup_file, 'a') as tar: - basename = os.path.basename(inventory_file_path) - backup_name = get_backup_name(basename) - tar.add(inventory_file_path, arcname=backup_name) - logger.debug("Backup written to %s", inventory_backup_file) - - -def get_backup_name(basename): - utctime = datetime.datetime.utcnow() - utctime = utctime.strftime("%Y%m%d_%H%M%S") - return '{}-{}.json'.format(basename, utctime) - - -def get_inventory(config_path, inventory_file_path): - if os.path.isfile(inventory_file_path): - with open(inventory_file_path, 'rb') as f: - dynamic_inventory = json.loads(f.read()) - logger.debug("Loaded existing inventory from %s", - inventory_file_path) - - make_backup(config_path, inventory_file_path) - else: - dynamic_inventory = copy.deepcopy(INVENTORY_SKEL) - logger.debug("No existing inventory, created fresh skeleton.") - - return dynamic_inventory - - def main(config=None, check=False, debug=False, environment=None, **kwargs): """Run the main application. @@ -1164,11 +1129,7 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs): environment = load_environment(config_path, base_env) # Load existing inventory file if found - dynamic_inventory_file = os.path.join( - config_path, 'openstack_inventory.json' - ) - - dynamic_inventory = get_inventory(config_path, dynamic_inventory_file) + dynamic_inventory = load_inventory(config_path, INVENTORY_SKEL) # Save the users container cidr as a group variable cidr_networks = user_defined_config.get('cidr_networks') @@ -1255,8 +1216,6 @@ def main(config=None, check=False, debug=False, environment=None, **kwargs): logger.debug("%d hosts found." % num_hosts) # Save new dynamic inventory - with open(dynamic_inventory_file, 'wb') as f: - f.write(dynamic_inventory_json) - logger.info("Inventory written") + save_inventory(dynamic_inventory_json, config_path) return dynamic_inventory_json diff --git a/lib/manage.py b/lib/manage.py index 99261d30b3..2f3021a488 100644 --- a/lib/manage.py +++ b/lib/manage.py @@ -24,7 +24,7 @@ import prettytable from dictutils import recursive_dict_removal from filesystem import load_from_json -from filesystem import save_to_json +from filesystem import save_inventory def args(): @@ -307,7 +307,7 @@ def main(): user_args = args() # Get the contents of the system inventory - inventory = load_from_json(user_args['file']) + inventory, filename = load_from_json(user_args['file']) # Make a table with hosts in the left column and details about each in the # columns to the right @@ -325,11 +325,13 @@ def main(): print(json.dumps(export_host_info(inventory), indent=2)) elif user_args['clear_ips'] is True: remove_ip_addresses(inventory) - save_to_json(user_args['file'], inventory) + inventory_json = json.dumps(inventory, indent=2) + save_inventory(inventory_json, filename) print('Success. . .') else: recursive_dict_removal(inventory, user_args['remove_item']) - save_to_json(user_args['file'], inventory) + inventory_json = json.dumps(inventory, indent=2) + save_inventory(inventory_json, filename) print('Success. . .') if __name__ == "__main__": diff --git a/tests/test_inventory.py b/tests/test_inventory.py index b98b9205dd..d41b7f59e5 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -20,6 +20,7 @@ sys.path.append(path.join(os.getcwd(), LIB_DIR)) sys.path.append(path.join(os.getcwd(), INV_DIR)) import dynamic_inventory +import filesystem as fs import generate as di TARGET_DIR = path.join(os.getcwd(), 'tests', 'inventory') @@ -843,18 +844,18 @@ class TestMultipleRuns(unittest.TestCase): def test_creating_backup_file(self): inventory_file_path = os.path.join(TARGET_DIR, 'openstack_inventory.json') - get_backup_name_path = 'generate.get_backup_name' + get_backup_name_path = 'filesystem.get_backup_name' backup_name = 'openstack_inventory.json-20160531_171804.json' tar_file = mock.MagicMock() tar_file.__enter__.return_value = tar_file # run make backup with faked tarfiles and date - with mock.patch('generate.tarfile.open') as tar_open: + with mock.patch('filesystem.tarfile.open') as tar_open: tar_open.return_value = tar_file with mock.patch(get_backup_name_path) as backup_mock: backup_mock.return_value = backup_name - di.make_backup(TARGET_DIR, inventory_file_path) + fs.make_backup(TARGET_DIR, inventory_file_path) backup_path = path.join(TARGET_DIR, 'backup_openstack_inventory.tar') @@ -881,9 +882,7 @@ class TestMultipleRuns(unittest.TestCase): # Generate the initial inventory files get_inventory(clean=False) - inventory_file_path = os.path.join(TARGET_DIR, - 'openstack_inventory.json') - inv = di.get_inventory(TARGET_DIR, inventory_file_path) + inv = fs.load_inventory(TARGET_DIR) self.assertIsInstance(inv, dict) self.assertIn('_meta', inv) # This test is basically just making sure we get more than @@ -1046,7 +1045,7 @@ class TestOverridingEnvIntegration(OverridingEnvBase): self.user_defined_config = get_config() # Inventory is necessary since keys are assumed present - self.inv = di.get_inventory(TARGET_DIR, '') + self.inv = fs.load_inventory(TARGET_DIR, di.INVENTORY_SKEL) def skel_setup(self): self.environment = di.load_environment(TARGET_DIR, self.base_env) @@ -1198,7 +1197,7 @@ class TestDebugLogging(unittest.TestCase): self.assertFalse(mock_logging.basicConfig.called) # Even though logging is disabled, we still call these # all over the place; they just choose not to do anything. - self.assertTrue(mock_logger.info.called) + # NOTE: No info messages are published when debug is False self.assertTrue(mock_logger.debug.called)