Merge "Add new api call: add new service in one archive"

muranorepository/api/v1.py

@@ -14,6 +14,7 @@
 
 import os
 import re
+import tarfile
 import tempfile
 from flask import Blueprint, send_file
 from flask import jsonify, request, abort
@@ -56,7 +57,7 @@ def _update_cache(data_type):
     archive_manager.create(cache_dir, manifests, CLIENTS_DICT[client])
 
 
-def _get_archive(manifests, client, hash_sum):
+def _get_archive(client, hash_sum):
     types = CLIENTS_DICT.get(client)
     archive_manager = Archiver()
     cache_dir = os.path.join(CACHE_DIR, client)
@@ -70,6 +71,7 @@ def _get_archive(manifests, client, hash_sum):
 
     if archive_manager.hashes_match(cache_dir, existing_hash, hash_sum):
         return None
+    manifests = ManifestParser().parse()
     return archive_manager.create(cache_dir, manifests, types)
 
 
@@ -124,17 +126,15 @@ def _check_data_type(data_type):
 
 
 def _get_manifest_files(manifest):
-    return dict((k, v)
-                for k, v in manifest.__dict__.iteritems() if k in DATA_TYPES)
+    return dict((k, v) for k, v in manifest.__dict__.iteritems()
+                if k in DATA_TYPES)
 
 
 @v1_api.route('/client/<path:client_type>')
 def get_archive_data(client_type):
-    manifests = ManifestParser().parse()
     if client_type not in CLIENTS_DICT.keys():
         abort(404)
-    path_to_archive = _get_archive(manifests,
-                                   client_type,
+    path_to_archive = _get_archive(client_type,
                                    request.args.get('hash'))
     if path_to_archive:
         return send_file(path_to_archive, mimetype='application/octet-stream')
@@ -193,6 +193,7 @@ def _get_locations_in_nested_path_or_get_file(data_type, path):
 @v1_api.route('/admin/<data_type>/<path:path>', methods=['POST'])
 def upload_file_in_nested_path(data_type, path):
     _check_data_type(data_type)
+
     if data_type == MANIFEST:
         make_response('It is forbidden to upload manifests to subfolders', 403)
     return _save_file(request, data_type, path)
@@ -262,3 +263,27 @@ def get_files_for_service(service_name):
     if not data:
         abort(404)
     return jsonify(service_files=data)
+
+
+@v1_api.route('/admin/services/<service_name>', methods=['POST'])
+def upload_new_service(service_name):
+    if not re.match(r'^\w+(\.\w+)*\w+$', service_name):
+        abort(404)
+    file_to_upload = request.files.get('file')
+
+    if file_to_upload:
+        filename = secure_filename(file_to_upload.filename)
+    else:
+        return make_response('There is no file to upload', 403)
+    path_to_archive = os.path.join(CACHE_DIR, filename)
+    file_to_upload.save(path_to_archive)
+    if not tarfile.is_tarfile(path_to_archive):
+        return make_response('Uploading file should be a tar archive', 403)
+
+    archive_manager = Archiver()
+    result = archive_manager.extract(path_to_archive)
+    if result:
+        return jsonify(result='success')
+    else:
+        #ToDo: Pass error msg there
+        return make_response('Uploading file failed.', 400)

muranorepository/utils/archiver.py

@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 import os
+import glob
 import tarfile
 import tempfile
 import shutil
@@ -82,7 +83,7 @@ class Archiver(object):
             return hsum
         else:
             log.info(
-                'Archive {0} does not exist, no hash to calculate'.format(
+                "Archive '{0}' doesn't exist, no hash to calculate".format(
                     archive_path))
             return None
 
@@ -209,3 +210,57 @@ class Archiver(object):
         path = os.path.join(cache_dir, hash)
         log.info('Deleting archive package from {0}.'.format(path))
         shutil.rmtree(path, ignore_errors=True)
+
+    def extract(self, path_to_archive):
+        """
+        path_to_archive - path to archive to extract from
+        ---
+        return value - True if succeeded , False otherwise
+        """
+        try:
+            path_to_extract = tempfile.mkdtemp()
+            with tarfile.open(path_to_archive) as archive:
+                archive.extractall(path_to_extract)
+            # assert manifest file
+            manifests = glob.glob(os.path.join(path_to_extract,
+                                               '*-manifest.yaml'))
+            if len(manifests) != 1:
+                raise AssertionError('There should be one '
+                                     'manifest file in archive')
+
+            shutil.copy(manifests[0], CONF.manifests)
+            #Todo: Check manifest is valid
+            for item in os.listdir(path_to_extract):
+                item_path = os.path.join(path_to_extract, item)
+                if os.path.isdir(item_path):
+                    if item in DATA_TYPES:
+                        file_list = []
+                        for path, subdirs, files in os.walk(item_path):
+                            # ToDo: Extract to a separate
+                            # function and use in v1.py also
+                            nested = False
+                            if path != item_path:
+                                base, subfolder = path.rsplit(item_path, 2)
+                                nested = True
+                            for name in files:
+                                if nested:
+                                    name = os.path.join(subfolder[1:], name)
+                                file_list.append(name)
+                        self._copy_data(file_list,
+                                        item_path,
+                                        os.path.join(
+                                            CONF.manifests,
+                                            self.src_directories[item]),
+                                        overwrite=False)
+                    else:
+                        log.warning(
+                            'Uploading archive contents folder {0} that does '
+                            'not correspond to supported data types: {1}. '
+                            'It will be ignored'.format(item, DATA_TYPES))
+            return True
+        except Exception as e:
+            log.error('Unable to extract archive due to {0}'.format(e.message))
+            return False
+        finally:
+            os.remove(path_to_archive)
+            shutil.rmtree(path_to_extract, ignore_errors=True)