murano-repository/muranorepository/api/utils.py

import os
import shutil
import re
import tempfile
import datetime

from flask import jsonify, abort
from flask import make_response
from werkzeug import secure_filename

from muranorepository.utils.parser import ManifestParser
from muranorepository.utils.parser import serialize
from muranorepository.utils.archiver import Archiver
from muranorepository.utils import utils
from muranorepository.consts import DATA_TYPES, MANIFEST
from muranorepository.consts import CLIENTS_DICT
from muranorepository.consts import ARCHIVE_PKG_NAME
from muranorepository.config import cfg
from muranorepository.openstack.common.gettextutils import _  # noqa
import logging as log
CONF = cfg.CONF


def reset_cache():
    try:
        cache_dir = utils.get_cache_folder()
        if os.path.exists(cache_dir):
            shutil.rmtree(cache_dir, ignore_errors=True)
            os.mkdir(cache_dir)
    except:
        log.exception(_('Error while cleaning cache'))
        return make_response(_('Unable to reset cache'), 500)


def compose_path(data_type, path=None):
    tenant_dir = utils.get_tenant_folder()
    utils.check_tenant_dir_existence(tenant_dir)
    return os.path.join(tenant_dir,
                        getattr(CONF, data_type),
                        path or '')


def get_archive(client, hash_sum):
    types = CLIENTS_DICT.get(client)
    archive_manager = Archiver()
    cache_dir = os.path.join(utils.get_cache_folder(), client)
    if not os.path.exists(cache_dir):
        os.makedirs(cache_dir)
        existing_hash = None
    else:
        existing_hash = archive_manager.get_existing_hash(cache_dir)

    if existing_hash and hash_sum is None:
        log.debug(_('Transferring existing archive'))
        return os.path.join(cache_dir, existing_hash, ARCHIVE_PKG_NAME)

    if archive_manager.hashes_match(cache_dir, existing_hash, hash_sum):
        return None

    manifests = ManifestParser().parse()
    return archive_manager.create(cache_dir, manifests, types)


def get_locations(data_type, result_path):
    locations = []
    if data_type == MANIFEST:
        for item in os.listdir(result_path):
            if '-manifest' in item and \
                    os.path.isfile(os.path.join(result_path, item)):
                locations.append(item)
    else:
        for path, subdirs, files in os.walk(result_path):
            for name in files:
                if path != result_path:
                    # need to add directory names for nested files
                    # add keep path relative to result_path
                    base, diff = path.rsplit(result_path, 2)
                    # split base path and remove slash
                    if diff.startswith('/'):
                        diff = diff[1:]
                    name = os.path.join(diff, name)
                locations.append(name)
    return jsonify({data_type: locations})


def save_file(request, data_type, path=None, filename=None):
    if path:
        path_to_folder = compose_path(data_type, path)
        #subfolder should already exists
        if not os.path.exists(path_to_folder):
            abort(404)
    else:
        path_to_folder = compose_path(data_type)

    if request.content_type == 'application/octet-stream':
        data = request.environ['wsgi.input'].read()
        if not data:
            return make_response(_('No file to upload'), 400)
        if not filename:
            return make_response(_("'filename' should be in "
                                   "request arguments"), 400)

        with tempfile.NamedTemporaryFile(delete=False) as uploaded_file:
            uploaded_file.write(data)
        path_to_file = os.path.join(path_to_folder, filename)
        if os.path.exists(path_to_file):
            abort(403)
        shutil.move(uploaded_file.name, path_to_file)
    else:
        file_to_upload = request.files.get('file')
        if file_to_upload:
            filename = secure_filename(file_to_upload.filename)
            path_to_file = os.path.join(path_to_folder, filename)
            if os.path.exists(path_to_file):
                abort(403)
            file_to_upload.save(path_to_file)
        else:
            return make_response(_('No file to upload'), 400)
    reset_cache()
    return jsonify(result='success')


def check_data_type(data_type):
    if data_type not in DATA_TYPES:
        abort(404)


def get_manifest_files(manifest):
    return dict((k, v) for k, v in manifest.__dict__.iteritems()
                if k in DATA_TYPES)


def get_manifest_info(manifest):
    return dict((k, v) for k, v in manifest.__dict__.iteritems()
                if k not in DATA_TYPES)


def exclude_common_files(files, manifests):
    all_manifest_files = [get_manifest_files(manifest)
                          for manifest in manifests]
    for data_type in files.keys():
        files[data_type] = set(files[data_type])
        for manifest_files in all_manifest_files:
            if manifest_files.get(data_type):
                files[data_type] -= set(manifest_files[data_type])
    return files


def check_service_name(service_name):
    if not re.match(r'^\w+(\.\w+)*\w+$', service_name):
        abort(404)


def perform_deletion(files_for_deletion, manifest_for_deletion):
    def backup_data():
        backup_dir = os.path.join(
            utils.get_cache_folder(),
            'Backup_{0}'.format(datetime.datetime.utcnow()))
        log.debug(_('Creating service data backup to {0}'.format(backup_dir)))
        shutil.copytree(utils.get_tenant_folder(), backup_dir)
        return backup_dir

    def release_backup(backup):
        try:
            shutil.rmtree(backup, ignore_errors=True)
        except OSError:
            log.exception(_('Release Backup: '
                            'Backup {0} deletion failed'.format(backup)))

    def restore_backup(backup):
        log.debug(_('Restore service data after unsuccessful deletion'))
        shutil.rmtree(utils.get_tenant_folder(), ignore_errors=True)
        os.rename(backup, utils.get_tenant_folder())

    backup_dir = backup_data()
    service_name = manifest_for_deletion.full_service_name
    path_to_manifest = os.path.join(utils.get_tenant_folder(),
                                    '{0}-manifest.yaml'.format(service_name))
    try:
        if os.path.exists(path_to_manifest):
            log.debug(_('Deleting manifest file {0}'.format(path_to_manifest)))
            os.remove(path_to_manifest)

        for data_type, files in files_for_deletion.iteritems():
            data_type_dir = os.path.join(utils.get_tenant_folder(),
                                         getattr(CONF, data_type))
            for file in files:
                path_to_delete = os.path.join(data_type_dir, file)
                if os.path.exists(path_to_delete):
                    log.debug(_('Delete {0}: Removing {1} file'.format(
                        service_name, path_to_delete)))
                    os.remove(path_to_delete)
    except:
        log.exception(_('Deleting operation failed'))
        restore_backup(backup_dir)
        abort(500)
    else:
        release_backup(backup_dir)
        reset_cache()
        return jsonify(result='success')


def save_archive(request):
    err_resp = make_response(_('There is no data to upload'), 400)
    if request.content_type == 'application/octet-stream':
        data = request.environ['wsgi.input'].read()
        if not data:
            return err_resp
        with tempfile.NamedTemporaryFile(delete=False) as uploaded_file:
                uploaded_file.write(data)
        path_to_archive = uploaded_file.name
    else:
        file_to_upload = request.files.get('file')
        if not file_to_upload:
            return err_resp
        path_to_archive = tempfile.NamedTemporaryFile(delete=False).name
        file_to_upload.save(path_to_archive)
    return path_to_archive


def create_or_update_service(service_id, data):
    manifest_directory = utils.get_tenant_folder()
    utils.check_tenant_dir_existence(manifest_directory)

    required = ['service_display_name']
    optional = {'enabled': True,
                'version': 0.1,
                'description': '',
                'author': '',
                'service_version': 1}

    for parameter in required:
        if not data.get(parameter):
            return make_response(_('There is no {parameter} in json'.format(
                parameter=parameter)), 400)
    for parameter in optional.keys():
        if not data.get(parameter):
            data[parameter] = optional[parameter]

    path_to_manifest = os.path.join(manifest_directory,
                                    service_id + '-manifest.yaml')

    backup_done = False
    with tempfile.NamedTemporaryFile() as backup:
        # make a backup
        if os.path.exists(path_to_manifest):
            backup_done = True
            shutil.copy(path_to_manifest, backup.name)
    try:
        with open(path_to_manifest, 'w') as service_manifest:
            service_manifest.write(serialize(data))
    except:
        log.exception(_('Unable to write to service '
                        'manifest file {0}'.format(path_to_manifest)))
        if backup_done:
            shutil.move(backup.name, path_to_manifest)
        elif os.path.exists(path_to_manifest):
            os.remove(path_to_manifest)
        return make_response(_('Error during service manifest creation'), 500)
    return jsonify(result='success')