diff --git a/.gitignore b/.gitignore index 59253dd1..29d54a59 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ ChangeLog etc/tuskar/tuskar.conf *.sqlite tuskar/api/templates/tripleo-heat-templates/ +tags diff --git a/tuskar/manager/plan.py b/tuskar/manager/plan.py index 7f229127..fef1d954 100644 --- a/tuskar/manager/plan.py +++ b/tuskar/manager/plan.py @@ -22,9 +22,11 @@ from tuskar.storage.stores import DeploymentPlanStore from tuskar.storage.stores import EnvironmentFileStore from tuskar.storage.stores import MasterSeedStore from tuskar.storage.stores import MasterTemplateStore +from tuskar.storage.stores import ResourceRegistryMappingStore from tuskar.storage.stores import ResourceRegistryStore from tuskar.storage.stores import TemplateStore from tuskar.templates import composer +from tuskar.templates.heat import RegistryEntry from tuskar.templates import namespace as ns_utils from tuskar.templates import parser from tuskar.templates import plan @@ -42,6 +44,7 @@ class PlansManager(object): self.plan_store = DeploymentPlanStore() self.seed_store = MasterSeedStore() self.registry_store = ResourceRegistryStore() + self.registry_mapping_store = ResourceRegistryMappingStore() self.template_store = TemplateStore() self.master_template_store = MasterTemplateStore() self.environment_store = EnvironmentFileStore() @@ -189,6 +192,18 @@ class PlansManager(object): seed_role, role_namespace) + # Update environment file to add top level mappings, which is made + # up of all non-role files present in the resource registry + reg_mapping = self.registry_mapping_store.list() + + environment = deployment_plan.environment + for entry in parsed_registry_env.registry_entries: + # check if registry_mapping is in database, if so add to + # environment (later will become environment.yaml) + if any(x.name == entry.filename for x in reg_mapping): + additem = RegistryEntry(entry.alias, entry.filename) + environment.add_registry_entry(additem, unique=True) + # Save the updated plan. updated = self._save_updated_plan(plan_uuid, deployment_plan) @@ -359,6 +374,11 @@ class PlansManager(object): role.version) files_dict[filename] = contents + # in addition to provider roles above, return non-role template files + reg_mapping = self.registry_mapping_store.list() + for entry in reg_mapping: + files_dict[entry.name] = entry.contents + return files_dict def _find_roles(self, environment): @@ -386,7 +406,9 @@ class PlansManager(object): role.description, role) return tuskar_role - roles = [load_role(e) for e in environment.registry_entries] + reg_mapping = self.registry_mapping_store.list() + roles = [load_role(e) for e in environment.registry_entries + if not any(x.name == e.filename for x in reg_mapping)] return roles @staticmethod diff --git a/tuskar/storage/load_roles.py b/tuskar/storage/load_roles.py index bded5752..af84fd51 100644 --- a/tuskar/storage/load_roles.py +++ b/tuskar/storage/load_roles.py @@ -21,8 +21,10 @@ from os import path from tuskar.storage.exceptions import UnknownName from tuskar.storage.stores import MasterSeedStore +from tuskar.storage.stores import ResourceRegistryMappingStore from tuskar.storage.stores import ResourceRegistryStore from tuskar.storage.stores import TemplateStore +from tuskar.templates import parser MASTER_SEED_NAME = '_master_seed' @@ -124,4 +126,24 @@ def load_roles(roles, seed_file=None, resource_registry_path=None, else: updated.append(RESOURCE_REGISTRY_NAME) + parsed_env = parser.parse_environment(contents) + + mapping_store = ResourceRegistryMappingStore() + dirname = path.dirname(resource_registry_path) + role_paths = [r[1] for r in roles] + for entry in parsed_env.registry_entries: + complete_path = path.join(dirname, entry.filename) + # skip adding if entry is not a filename (is another alias) or + # if template has already been stored as a role + if (not entry.is_filename() or complete_path in role_paths): + continue + + mapping_created, _ = _create_or_update(entry.filename, + _load_file(complete_path), + store=mapping_store) + if mapping_created: + created.append(entry.filename) + else: + updated.append(entry.filename) + return all_roles, created, updated diff --git a/tuskar/storage/stores.py b/tuskar/storage/stores.py index fe422e66..dd6e8658 100644 --- a/tuskar/storage/stores.py +++ b/tuskar/storage/stores.py @@ -225,6 +225,10 @@ class ResourceRegistryStore(_NamedStore): object_type = "registry" +class ResourceRegistryMappingStore(_NamedStore): + object_type = "registry_mapping" + + class EnvironmentFileStore(_BaseStore): """Environment File for Heat environment files. diff --git a/tuskar/templates/heat.py b/tuskar/templates/heat.py index cee0e7ee..737e398f 100644 --- a/tuskar/templates/heat.py +++ b/tuskar/templates/heat.py @@ -17,6 +17,8 @@ These objects were created against the HOT specification found at: http://docs.openstack.org/developer/heat/template_guide/hot_spec.html """ +from os import path + from tuskar.templates import namespace as ns_utils @@ -460,12 +462,19 @@ class Environment(object): return True return False - def add_registry_entry(self, entry): + def add_registry_entry(self, entry, unique=False): """Adds a registry entry to the environment. :type entry: tuskar.templates.heat.RegistryEntry + :param unique: toggles if registry is treated as a set + :type unique: boolean """ - self._registry_entries.append(entry) + if unique: + setentries = set(self._registry_entries) + setentries.add(entry) + self._registry_entries = list(setentries) + else: + self._registry_entries.append(entry) def remove_registry_entry(self, entry): """Removes a registry entry from the environment. @@ -506,6 +515,7 @@ class RegistryEntry(object): super(RegistryEntry, self).__init__() self.alias = alias self.filename = filename + # TODO(jpeeler) rename self.filename to mapping def __str__(self): msg = 'RegistryEntry: alias=%(alias)s, filename=%(f)s' @@ -515,6 +525,12 @@ class RegistryEntry(object): } return msg % data + def is_filename(self): + if ('::' in self.filename or + path.splitext(self.filename)[1] not in ('.yaml', '.yml')): + return False + return True + def _safe_strip(value): """Strips the value if it is not None. diff --git a/tuskar/tests/manager/test_plan.py b/tuskar/tests/manager/test_plan.py index 1b98dbf6..537bd395 100644 --- a/tuskar/tests/manager/test_plan.py +++ b/tuskar/tests/manager/test_plan.py @@ -25,6 +25,7 @@ from tuskar.storage.stores import DeploymentPlanStore from tuskar.storage.stores import EnvironmentFileStore from tuskar.storage.stores import MasterSeedStore from tuskar.storage.stores import MasterTemplateStore +from tuskar.storage.stores import ResourceRegistryMappingStore from tuskar.storage.stores import ResourceRegistryStore from tuskar.storage.stores import TemplateStore from tuskar.templates import namespace as ns_utils @@ -109,6 +110,7 @@ resources: RESOURCE_REGISTRY = """ resource_registry: OS::TripleO::Role: r1.yaml + OS::TripleO::Another: required_file.yaml """ RESOURCE_REGISTRY_WRONG_TYPE = """ @@ -127,6 +129,7 @@ class PlansManagerTestCase(TestCase): self.template_store = TemplateStore() self.seed_store = MasterSeedStore() self.registry_store = ResourceRegistryStore() + self.registry_mapping_store = ResourceRegistryMappingStore() def test_create_plan(self): # Tests @@ -182,6 +185,9 @@ class PlansManagerTestCase(TestCase): # Setup self.seed_store.create(MASTER_SEED_NAME, TEST_SEED) self.registry_store.create(RESOURCE_REGISTRY_NAME, RESOURCE_REGISTRY) + # more setup (this is normally called in load_roles) + self.registry_mapping_store.create('required_file.yaml', + 'some fake template data') test_role = self._add_test_role() test_plan = self.plans_manager.create_plan('p1', 'd1') @@ -206,6 +212,12 @@ class PlansManagerTestCase(TestCase): {'ip_addresses': {'get_attr': ['r1-1-servers', 'foo_ip']}}) + # verify both entries are present from RESOURCE_REGISTRY + parsed_env = parser.parse_environment( + db_plan.environment_file.contents + ) + self.assertEqual(2, len(parsed_env.registry_entries)) + def test_add_unknown_role_to_seeded_plan(self): # Setup self.seed_store.create(MASTER_SEED_NAME, TEST_SEED) @@ -388,5 +400,59 @@ class PlansManagerTestCase(TestCase): parsed_role = parser.parse_template(templates[role_filename]) self.assertEqual(parsed_role.description, 'Test provider resource foo') + def test_package_templates_seeded_plan(self): + # Setup + self.seed_store.create(MASTER_SEED_NAME, TEST_SEED) + self.registry_store.create(RESOURCE_REGISTRY_NAME, RESOURCE_REGISTRY) + # more setup (this is normally called in load_roles) + self.registry_mapping_store.create('required_file.yaml', + 'some fake template data') + + test_role = self._add_test_role() + test_plan = self.plans_manager.create_plan('p1', 'd1') + self.plans_manager.add_role_to_plan(test_plan.uuid, test_role.uuid) + + # Test + templates = self.plans_manager.package_templates(test_plan.uuid) + + # Verify + self.assertTrue(isinstance(templates, dict)) + self.assertEqual(4, len(templates)) + + self.assertTrue('plan.yaml' in templates) + parsed_plan = parser.parse_template(templates['plan.yaml']) + self.assertEqual(parsed_plan.description, 'd1') + + self.assertTrue('environment.yaml' in templates) + self.assertTrue('required_file.yaml' in templates) + parsed_env = parser.parse_environment(templates['environment.yaml']) + self.assertEqual(2, len(parsed_env.registry_entries)) + + role_filename = name_utils.role_template_filename('r1', '1') + self.assertTrue(role_filename in templates) + parsed_role = parser.parse_template(templates[role_filename]) + self.assertEqual(parsed_role.description, 'Test provider resource foo') + + def test_find_roles(self): + # Setup + self.seed_store.create(MASTER_SEED_NAME, TEST_SEED) + self.registry_store.create(RESOURCE_REGISTRY_NAME, RESOURCE_REGISTRY) + # more setup (this is normally called in load_roles) + self.registry_mapping_store.create('required_file.yaml', + 'some fake template data') + test_role = self._add_test_role() + test_plan = self.plans_manager.create_plan('p1', 'd1') + + # Test + self.plans_manager.add_role_to_plan(test_plan.uuid, test_role.uuid) + + # Verify only one role is found + db_plan = self.plan_store.retrieve(test_plan.uuid) + parsed_env = parser.parse_environment( + db_plan.environment_file.contents + ) + roles = self.plans_manager._find_roles(parsed_env) + self.assertEqual(1, len(roles)) + def _add_test_role(self): return self.template_store.create('r1', TEST_TEMPLATE) diff --git a/tuskar/tests/storage/test_load_roles.py b/tuskar/tests/storage/test_load_roles.py index 3c575286..ce34e7f7 100644 --- a/tuskar/tests/storage/test_load_roles.py +++ b/tuskar/tests/storage/test_load_roles.py @@ -33,19 +33,22 @@ class LoadRoleTests(TestCase): self.roles = [path.join(self.directory, role) for role in roles_name] for role in self.roles: - self._create_role(role) + self._create_file(role) def tearDown(self): super(LoadRoleTests, self).tearDown() rmtree(self.directory) - def _create_role(self, role): - """Create a mock role file which simple contains it's own name as - the file contents. + def _create_file(self, file, data=None): + """Create a mock file which contains its own name as the file + contents when the data argument is empty. """ - with open(role, 'w') as f: - f.write("CONTENTS FOR {0}".format(role)) + if data is None: + data = "CONTENTS FOR {0}".format(file) + + with open(file, 'w') as f: + f.write(data) def test_dry_run(self): @@ -83,7 +86,7 @@ class LoadRoleTests(TestCase): def test_import_with_seed(self): # Setup - self._create_role(path.join(self.directory, 'seed')) + self._create_file(path.join(self.directory, 'seed')) # Test seed_file = path.join(self.directory, 'seed') @@ -94,3 +97,30 @@ class LoadRoleTests(TestCase): self.assertEqual(['_master_seed', 'role1', 'role2'], sorted(total)) self.assertEqual(['_master_seed', 'role1', 'role2'], sorted(created)) self.assertEqual([], updated) + + def test_import_seed_and_registry(self): + env_data = """ +resource_registry: + OS::TripleO::Role: role1.yaml + OS::TripleO::Another: required_file.yaml + """ + + # Setup + self._create_file(path.join(self.directory, 'seed')) + self._create_file(path.join(self.directory, 'environment'), env_data) + self._create_file(path.join(self.directory, 'required_file.yaml')) + + # Test + seed_file = path.join(self.directory, 'seed') + env_file = path.join(self.directory, 'environment') + all_roles, created, updated = load_roles.load_roles( + self.roles, + seed_file=seed_file, + resource_registry_path=env_file) + + # Verify + self.assertEqual(['_master_seed', '_registry', + 'role1', 'role2'], sorted(all_roles)) + self.assertEqual(['_master_seed', '_registry', 'required_file.yaml', + 'role1', 'role2'], sorted(created)) + self.assertEqual([], updated) diff --git a/tuskar/tests/templates/test_heat.py b/tuskar/tests/templates/test_heat.py index 63e0852c..e5afb852 100644 --- a/tuskar/tests/templates/test_heat.py +++ b/tuskar/tests/templates/test_heat.py @@ -433,6 +433,11 @@ class EnvironmentTests(unittest.TestCase): e.remove_registry_entry(re) self.assertEqual(0, len(e.registry_entries)) + # Test unique add + e.add_registry_entry(re, unique=True) + e.add_registry_entry(re, unique=True) + self.assertEqual(1, len(e.registry_entries)) + def test_remove_registry_entry_not_found(self): e = heat.Environment() self.assertRaises(ValueError, e.remove_registry_entry, @@ -495,6 +500,17 @@ class EnvironmentParameterTests(unittest.TestCase): self.assertEqual('test-value', p.value) +class RegistryEntryTest(unittest.TestCase): + + def test_is_filename(self): + re = heat.RegistryEntry('Tuskar::compute-1', 'provider-compute-1.yaml') + self.assertTrue(re.is_filename()) + + re = heat.RegistryEntry('OS::TripleO::StructuredDeployment', + 'OS::Heat::StructuredDeployment') + self.assertFalse(re.is_filename()) + + class ModuleMethodTests(unittest.TestCase): def test_safe_strip(self):