Fix bug with subfolder copying

Save path in manifest object in relative way, not in absolute

metadata_service/config.py

@@ -21,16 +21,7 @@ server_opts = [
     cfg.IntOpt('port', default='5000'),
 ]
 
-# type_dirs_opts = [cfg.StrOpt(x) for x in DATA_TYPES]
+type_dirs_opts = [cfg.StrOpt(x) for x in DATA_TYPES]
-
-type_dirs_opts = [
-    cfg.StrOpt(MANIFEST),
-    cfg.StrOpt(UI),
-    cfg.StrOpt(WORKFLOW),
-    cfg.StrOpt(HEAT),
-    cfg.StrOpt(AGENT),
-    cfg.StrOpt(SCRIPTS)
-]
 
 cfg.set_defaults(log.log_opts,
                  default_log_levels=['qpid.messaging=INFO',
@@ -39,7 +30,7 @@ cfg.set_defaults(log.log_opts,
 CONF = cfg.CONF
 CONF.register_cli_opts(server_opts)
 CONF.register_opts(type_dirs_opts)
-
+CONF.register_opts(type_dirs_opts, group='output')
 
 ARGV = []
 

metadata_service/etc/murano-repository.conf

@@ -12,17 +12,17 @@ manifests = /home/fervent/Projects/my_repo/Services
 
 # Parameter name corresponds to section in manifest file
 # Parameter value corresponds to relative path to data type
-
 ui = ui_forms
 workflows = workflows
 heat = heat_templates
 agent = agent_templates
 scripts = scripts
 
-# Configure how to archive data types
+# Configure archive structure
-# [output]
+# data_type = desired folder
-# ui = ui_forms: ui_forms
+[output]
-# workflows = workflows: workflows
+ui = service_forms
-# heat = heat_templates: heat_templates
+workflows = workflows
-# agent = agent_templates: agent_templates
+heat = templates/cf
-# scripts = scripts: scripts
+agent = templates/agent
+scripts = templates/agent/script

metadata_service/utils/archiver.py

@@ -16,33 +16,13 @@ import tarfile
 import tempfile
 import shutil
 import logging as log
-
+from oslo.config import cfg
 from metadata_service.consts import DATA_TYPES
+OUTPUT_CONF = cfg.CONF.output
+CONF = cfg.CONF
 
 
 class Archiver(object):
-    def __init__(self,
-                 ui_forms_target="service_forms",
-                 workflows_target="workflows",
-                 heat_templates_target="templates/cf",
-                 agent_templates_target="templates/agent",
-                 scripts_target="templates/agent/scripts"):
-        """
-        ui_forms_target -- relative path for a ui_forms location in
-                                                            resulted archive
-        workflows_target -- relative path for a desired workflow location in
-                                                            resulted archive
-        heat_templates_target -- relative path for a desired heat templates
-                                                location in resulted archive
-        agent_templates_target -- relative path for a heat templates location
-        scripts_target -- relative path for a agent script location
-        """
-        self.archive_structure = {"ui_forms": ui_forms_target,
-                                     "workflows": workflows_target,
-                                     "heat_templates": heat_templates_target,
-                                     "agent_templates": agent_templates_target,
-                                     "scripts": scripts_target
-                                  }
 
     def create(self, manifests, *types):
         """
@@ -60,34 +40,41 @@ class Archiver(object):
                     continue
                 if hasattr(manifest, data_type):
                     dst_directory = os.path.join(temp_dir,
-                                                 self.archive_structure[
+                                                 getattr(OUTPUT_CONF,
-                                                     data_type])
+                                                         data_type))
+                    scr_directory = os.path.join(CONF.manifests,
+                                                 getattr(CONF, data_type))
+
                     if not os.path.exists(dst_directory):
                         os.makedirs(dst_directory)
 
-                    for file_path in getattr(manifest, data_type):
+                    for path in getattr(manifest, data_type):
-                        basedir, filename = os.path.split(file_path)
+                        source = os.path.join(scr_directory, path)
-                        destination = os.path.join(dst_directory,
+                        destination = os.path.join(dst_directory, path)
-                                                   filename)
+                        base_dir = os.path.dirname(destination)
+
+                        if (base_dir != dst_directory) \
+                           and (not os.path.exists(base_dir)):
+                            os.makedirs(os.path.dirname(destination))
                         try:
-                            shutil.copyfile(file_path, destination)
+                            shutil.copyfile(source, destination)
                         except IOError:
                             log.error("Unable to copy file "
                                       "{0}".format(file))
                 else:
                     log.info(
                         "Manifest for {0} service has no file definitions for "
-                        "{1}").format(manifest.service_display_name, data_type)
+                        "{1}".format(manifest.service_display_name, data_type))
 
-        target_archeve = "service_metadata.tar"
+        target_archive = "service_metadata.tar"
-        with tarfile.open(target_archeve, "w") as tar:
+        with tarfile.open(target_archive, "w") as tar:
             for item in os.listdir(temp_dir):
                 tar.add(os.path.join(temp_dir, item), item)
         try:
             shutil.rmtree(temp_dir, ignore_errors=True)
         except Exception as e:
             log.error("Unable to delete temp directory: {0}".format(e))
-        return os.path.abspath(target_archeve)
+        return os.path.abspath(target_archive)
 
 
 

metadata_service/utils/parser.py

@@ -17,7 +17,7 @@ import yaml
 from oslo.config import cfg
 import logging as log
 from metadata_service.manifest import Manifest
-from metadata_service.consts import DATA_TYPES
+from metadata_service.consts import DATA_TYPES, MANIFEST
 CONF = cfg.CONF
 
 
@@ -45,22 +45,26 @@ class ManifestParser(object):
                                                             exc))
                         continue
 
+                #check for existence files specified in manifests
                 for key, value in service_manifest_data.iteritems():
                     valid_file_info = True
                     if key in DATA_TYPES:
-                        root_directory = getattr(CONF, key)
+                        if key != MANIFEST:
+                            root_directory = os.path.join(CONF.manifests,
+                                                          getattr(CONF, key))
+                        else:
+                            root_directory = CONF.manifests
+
                         if not isinstance(value, list):
                             log.error("{0} section should represent a file"
                                       " listing in manifest {1}"
                                       "".format(root_directory, file))
                             valid_file_info = False
                             continue
-                        for i, filename in enumerate(value):
+                        for filename in value:
                             absolute_path = os.path.join(root_directory,
                                                          filename)
 
-                            service_manifest_data[key][i] = absolute_path
-
                             if not os.path.exists(absolute_path):
                                 valid_file_info = False
                                 log.warning(