Support non-role resources in resource registry

This change allows for resources utilized in the master seed that are
specified in the resource registry, but are not a provider resource to
be stored and saved to disk. There is no additional information required
to pass as all the data is stored while loading roles.

Previously tuskar always assumed that there was a 1:1 correspondence
between the final resource registry and the number of roles, but this is
no longer the case. The API for /v2/plans/templates has changed to
include non-role resources specified in the resource registry (CLI
operation tuskar plan-templates).

(Also, ignores tags file in git)

Change-Id: Ia15c0fdad0c036564b81f664e49fca280bc7ce7e

.gitignore

@@ -32,3 +32,4 @@ ChangeLog
 etc/tuskar/tuskar.conf
 *.sqlite
 tuskar/api/templates/tripleo-heat-templates/
+tags

tuskar/manager/plan.py

@@ -22,9 +22,11 @@ from tuskar.storage.stores import DeploymentPlanStore
 from tuskar.storage.stores import EnvironmentFileStore
 from tuskar.storage.stores import MasterSeedStore
 from tuskar.storage.stores import MasterTemplateStore
+from tuskar.storage.stores import ResourceRegistryMappingStore
 from tuskar.storage.stores import ResourceRegistryStore
 from tuskar.storage.stores import TemplateStore
 from tuskar.templates import composer
+from tuskar.templates.heat import RegistryEntry
 from tuskar.templates import namespace as ns_utils
 from tuskar.templates import parser
 from tuskar.templates import plan
@@ -42,6 +44,7 @@ class PlansManager(object):
         self.plan_store = DeploymentPlanStore()
         self.seed_store = MasterSeedStore()
         self.registry_store = ResourceRegistryStore()
+        self.registry_mapping_store = ResourceRegistryMappingStore()
         self.template_store = TemplateStore()
         self.master_template_store = MasterTemplateStore()
         self.environment_store = EnvironmentFileStore()
@@ -189,6 +192,18 @@ class PlansManager(object):
                 seed_role,
                 role_namespace)
 
+            # Update environment file to add top level mappings, which is made
+            # up of all non-role files present in the resource registry
+            reg_mapping = self.registry_mapping_store.list()
+
+            environment = deployment_plan.environment
+            for entry in parsed_registry_env.registry_entries:
+                # check if registry_mapping is in database, if so add to
+                # environment (later will become environment.yaml)
+                if any(x.name == entry.filename for x in reg_mapping):
+                    additem = RegistryEntry(entry.alias, entry.filename)
+                    environment.add_registry_entry(additem, unique=True)
+
         # Save the updated plan.
         updated = self._save_updated_plan(plan_uuid, deployment_plan)
 
@@ -359,6 +374,11 @@ class PlansManager(object):
                                                          role.version)
             files_dict[filename] = contents
 
+        # in addition to provider roles above, return non-role template files
+        reg_mapping = self.registry_mapping_store.list()
+        for entry in reg_mapping:
+            files_dict[entry.name] = entry.contents
+
         return files_dict
 
     def _find_roles(self, environment):
@@ -386,7 +406,9 @@ class PlansManager(object):
                                       role.description, role)
             return tuskar_role
 
-        roles = [load_role(e) for e in environment.registry_entries]
+        reg_mapping = self.registry_mapping_store.list()
+        roles = [load_role(e) for e in environment.registry_entries
+                 if not any(x.name == e.filename for x in reg_mapping)]
         return roles
 
     @staticmethod

tuskar/storage/load_roles.py

@@ -21,8 +21,10 @@ from os import path
 
 from tuskar.storage.exceptions import UnknownName
 from tuskar.storage.stores import MasterSeedStore
+from tuskar.storage.stores import ResourceRegistryMappingStore
 from tuskar.storage.stores import ResourceRegistryStore
 from tuskar.storage.stores import TemplateStore
+from tuskar.templates import parser
 
 
 MASTER_SEED_NAME = '_master_seed'
@@ -124,4 +126,24 @@ def load_roles(roles, seed_file=None, resource_registry_path=None,
         else:
             updated.append(RESOURCE_REGISTRY_NAME)
 
+        parsed_env = parser.parse_environment(contents)
+
+        mapping_store = ResourceRegistryMappingStore()
+        dirname = path.dirname(resource_registry_path)
+        role_paths = [r[1] for r in roles]
+        for entry in parsed_env.registry_entries:
+            complete_path = path.join(dirname, entry.filename)
+            # skip adding if entry is not a filename (is another alias) or
+            # if template has already been stored as a role
+            if (not entry.is_filename() or complete_path in role_paths):
+                continue
+
+            mapping_created, _ = _create_or_update(entry.filename,
+                                                   _load_file(complete_path),
+                                                   store=mapping_store)
+            if mapping_created:
+                created.append(entry.filename)
+            else:
+                updated.append(entry.filename)
+
     return all_roles, created, updated

tuskar/storage/stores.py

@@ -225,6 +225,10 @@ class ResourceRegistryStore(_NamedStore):
     object_type = "registry"
 
 
+class ResourceRegistryMappingStore(_NamedStore):
+    object_type = "registry_mapping"
+
+
 class EnvironmentFileStore(_BaseStore):
     """Environment File for Heat environment files.
 

tuskar/templates/heat.py

@@ -17,6 +17,8 @@ These objects were created against the HOT specification found at:
 http://docs.openstack.org/developer/heat/template_guide/hot_spec.html
 """
 
+from os import path
+
 from tuskar.templates import namespace as ns_utils
 
 
@@ -460,12 +462,19 @@ class Environment(object):
                 return True
         return False
 
-    def add_registry_entry(self, entry):
+    def add_registry_entry(self, entry, unique=False):
         """Adds a registry entry to the environment.
 
         :type entry: tuskar.templates.heat.RegistryEntry
+        :param unique: toggles if registry is treated as a set
+        :type unique: boolean
         """
-        self._registry_entries.append(entry)
+        if unique:
+            setentries = set(self._registry_entries)
+            setentries.add(entry)
+            self._registry_entries = list(setentries)
+        else:
+            self._registry_entries.append(entry)
 
     def remove_registry_entry(self, entry):
         """Removes a registry entry from the environment.
@@ -506,6 +515,7 @@ class RegistryEntry(object):
         super(RegistryEntry, self).__init__()
         self.alias = alias
         self.filename = filename
+        # TODO(jpeeler) rename self.filename to mapping
 
     def __str__(self):
         msg = 'RegistryEntry: alias=%(alias)s, filename=%(f)s'
@@ -515,6 +525,12 @@ class RegistryEntry(object):
         }
         return msg % data
 
+    def is_filename(self):
+        if ('::' in self.filename or
+                path.splitext(self.filename)[1] not in ('.yaml', '.yml')):
+            return False
+        return True
+
 
 def _safe_strip(value):
     """Strips the value if it is not None.

tuskar/tests/manager/test_plan.py

@@ -25,6 +25,7 @@ from tuskar.storage.stores import DeploymentPlanStore
 from tuskar.storage.stores import EnvironmentFileStore
 from tuskar.storage.stores import MasterSeedStore
 from tuskar.storage.stores import MasterTemplateStore
+from tuskar.storage.stores import ResourceRegistryMappingStore
 from tuskar.storage.stores import ResourceRegistryStore
 from tuskar.storage.stores import TemplateStore
 from tuskar.templates import namespace as ns_utils
@@ -109,6 +110,7 @@ resources:
 RESOURCE_REGISTRY = """
 resource_registry:
   OS::TripleO::Role: r1.yaml
+  OS::TripleO::Another: required_file.yaml
 """
 
 RESOURCE_REGISTRY_WRONG_TYPE = """
@@ -127,6 +129,7 @@ class PlansManagerTestCase(TestCase):
         self.template_store = TemplateStore()
         self.seed_store = MasterSeedStore()
         self.registry_store = ResourceRegistryStore()
+        self.registry_mapping_store = ResourceRegistryMappingStore()
 
     def test_create_plan(self):
         # Tests
@@ -182,6 +185,9 @@ class PlansManagerTestCase(TestCase):
         # Setup
         self.seed_store.create(MASTER_SEED_NAME, TEST_SEED)
         self.registry_store.create(RESOURCE_REGISTRY_NAME, RESOURCE_REGISTRY)
+        # more setup (this is normally called in load_roles)
+        self.registry_mapping_store.create('required_file.yaml',
+                                           'some fake template data')
         test_role = self._add_test_role()
         test_plan = self.plans_manager.create_plan('p1', 'd1')
 
@@ -206,6 +212,12 @@ class PlansManagerTestCase(TestCase):
                          {'ip_addresses':
                           {'get_attr': ['r1-1-servers', 'foo_ip']}})
 
+        # verify both entries are present from RESOURCE_REGISTRY
+        parsed_env = parser.parse_environment(
+            db_plan.environment_file.contents
+        )
+        self.assertEqual(2, len(parsed_env.registry_entries))
+
     def test_add_unknown_role_to_seeded_plan(self):
         # Setup
         self.seed_store.create(MASTER_SEED_NAME, TEST_SEED)
@@ -388,5 +400,59 @@ class PlansManagerTestCase(TestCase):
         parsed_role = parser.parse_template(templates[role_filename])
         self.assertEqual(parsed_role.description, 'Test provider resource foo')
 
+    def test_package_templates_seeded_plan(self):
+        # Setup
+        self.seed_store.create(MASTER_SEED_NAME, TEST_SEED)
+        self.registry_store.create(RESOURCE_REGISTRY_NAME, RESOURCE_REGISTRY)
+        # more setup (this is normally called in load_roles)
+        self.registry_mapping_store.create('required_file.yaml',
+                                           'some fake template data')
+
+        test_role = self._add_test_role()
+        test_plan = self.plans_manager.create_plan('p1', 'd1')
+        self.plans_manager.add_role_to_plan(test_plan.uuid, test_role.uuid)
+
+        # Test
+        templates = self.plans_manager.package_templates(test_plan.uuid)
+
+        # Verify
+        self.assertTrue(isinstance(templates, dict))
+        self.assertEqual(4, len(templates))
+
+        self.assertTrue('plan.yaml' in templates)
+        parsed_plan = parser.parse_template(templates['plan.yaml'])
+        self.assertEqual(parsed_plan.description, 'd1')
+
+        self.assertTrue('environment.yaml' in templates)
+        self.assertTrue('required_file.yaml' in templates)
+        parsed_env = parser.parse_environment(templates['environment.yaml'])
+        self.assertEqual(2, len(parsed_env.registry_entries))
+
+        role_filename = name_utils.role_template_filename('r1', '1')
+        self.assertTrue(role_filename in templates)
+        parsed_role = parser.parse_template(templates[role_filename])
+        self.assertEqual(parsed_role.description, 'Test provider resource foo')
+
+    def test_find_roles(self):
+        # Setup
+        self.seed_store.create(MASTER_SEED_NAME, TEST_SEED)
+        self.registry_store.create(RESOURCE_REGISTRY_NAME, RESOURCE_REGISTRY)
+        # more setup (this is normally called in load_roles)
+        self.registry_mapping_store.create('required_file.yaml',
+                                           'some fake template data')
+        test_role = self._add_test_role()
+        test_plan = self.plans_manager.create_plan('p1', 'd1')
+
+        # Test
+        self.plans_manager.add_role_to_plan(test_plan.uuid, test_role.uuid)
+
+        # Verify only one role is found
+        db_plan = self.plan_store.retrieve(test_plan.uuid)
+        parsed_env = parser.parse_environment(
+            db_plan.environment_file.contents
+        )
+        roles = self.plans_manager._find_roles(parsed_env)
+        self.assertEqual(1, len(roles))
+
     def _add_test_role(self):
         return self.template_store.create('r1', TEST_TEMPLATE)

tuskar/tests/storage/test_load_roles.py

@@ -33,19 +33,22 @@ class LoadRoleTests(TestCase):
         self.roles = [path.join(self.directory, role) for role in roles_name]
 
         for role in self.roles:
-            self._create_role(role)
+            self._create_file(role)
 
     def tearDown(self):
         super(LoadRoleTests, self).tearDown()
         rmtree(self.directory)
 
-    def _create_role(self, role):
-        """Create a mock role file which simple contains it's own name as
-        the file contents.
+    def _create_file(self, file, data=None):
+        """Create a mock file which contains its own name as the file
+        contents when the data argument is empty.
         """
 
-        with open(role, 'w') as f:
-            f.write("CONTENTS FOR {0}".format(role))
+        if data is None:
+            data = "CONTENTS FOR {0}".format(file)
+
+        with open(file, 'w') as f:
+            f.write(data)
 
     def test_dry_run(self):
 
@@ -83,7 +86,7 @@ class LoadRoleTests(TestCase):
 
     def test_import_with_seed(self):
         # Setup
-        self._create_role(path.join(self.directory, 'seed'))
+        self._create_file(path.join(self.directory, 'seed'))
 
         # Test
         seed_file = path.join(self.directory, 'seed')
@@ -94,3 +97,30 @@ class LoadRoleTests(TestCase):
         self.assertEqual(['_master_seed', 'role1', 'role2'], sorted(total))
         self.assertEqual(['_master_seed', 'role1', 'role2'], sorted(created))
         self.assertEqual([], updated)
+
+    def test_import_seed_and_registry(self):
+        env_data = """
+resource_registry:
+  OS::TripleO::Role: role1.yaml
+  OS::TripleO::Another: required_file.yaml
+        """
+
+        # Setup
+        self._create_file(path.join(self.directory, 'seed'))
+        self._create_file(path.join(self.directory, 'environment'), env_data)
+        self._create_file(path.join(self.directory, 'required_file.yaml'))
+
+        # Test
+        seed_file = path.join(self.directory, 'seed')
+        env_file = path.join(self.directory, 'environment')
+        all_roles, created, updated = load_roles.load_roles(
+            self.roles,
+            seed_file=seed_file,
+            resource_registry_path=env_file)
+
+        # Verify
+        self.assertEqual(['_master_seed', '_registry',
+                          'role1', 'role2'], sorted(all_roles))
+        self.assertEqual(['_master_seed', '_registry', 'required_file.yaml',
+                          'role1', 'role2'], sorted(created))
+        self.assertEqual([], updated)

tuskar/tests/templates/test_heat.py

@@ -433,6 +433,11 @@ class EnvironmentTests(unittest.TestCase):
         e.remove_registry_entry(re)
         self.assertEqual(0, len(e.registry_entries))
 
+        # Test unique add
+        e.add_registry_entry(re, unique=True)
+        e.add_registry_entry(re, unique=True)
+        self.assertEqual(1, len(e.registry_entries))
+
     def test_remove_registry_entry_not_found(self):
         e = heat.Environment()
         self.assertRaises(ValueError, e.remove_registry_entry,
@@ -495,6 +500,17 @@ class EnvironmentParameterTests(unittest.TestCase):
         self.assertEqual('test-value', p.value)
 
 
+class RegistryEntryTest(unittest.TestCase):
+
+    def test_is_filename(self):
+        re = heat.RegistryEntry('Tuskar::compute-1', 'provider-compute-1.yaml')
+        self.assertTrue(re.is_filename())
+
+        re = heat.RegistryEntry('OS::TripleO::StructuredDeployment',
+                                'OS::Heat::StructuredDeployment')
+        self.assertFalse(re.is_filename())
+
+
 class ModuleMethodTests(unittest.TestCase):
 
     def test_safe_strip(self):