From 9946642d018bc130244d31131cf5b40e4bb32b32 Mon Sep 17 00:00:00 2001 From: Ekaterina Fedorova Date: Tue, 24 Dec 2013 17:02:36 +0400 Subject: [PATCH] Add per-tenant isolation * Fix some exceptions * use tmpfile to store uploaded archive Implements blueprint per-tenant-isolation Change-Id: I9a302823fb208cc73808e573cf4a9812a0c9cf79 --- etc/murano-repository.conf.sample | 2 +- muranorepository/api/utils.py | 52 ++++++++++++++++-------------- muranorepository/cmd/run.py | 13 ++++---- muranorepository/config.py | 2 +- muranorepository/consts.py | 3 ++ muranorepository/utils/archiver.py | 27 +++++++++------- muranorepository/utils/parser.py | 52 ++++++++++++++++-------------- muranorepository/utils/utils.py | 40 +++++++++++++++++++++++ 8 files changed, 123 insertions(+), 68 deletions(-) create mode 100644 muranorepository/utils/utils.py diff --git a/etc/murano-repository.conf.sample b/etc/murano-repository.conf.sample index b274486..3bef245 100644 --- a/etc/murano-repository.conf.sample +++ b/etc/murano-repository.conf.sample @@ -5,7 +5,7 @@ host = 0.0.0.0 port = 8084 # Directory for cache, OS temp directory is used by default -#cache_dir = +# data_dir = /tmp/muranorepository-data # Show more verbose log output (sets INFO log level output) verbose = True diff --git a/muranorepository/api/utils.py b/muranorepository/api/utils.py index 6ec02b9..519bb5a 100644 --- a/muranorepository/api/utils.py +++ b/muranorepository/api/utils.py @@ -12,6 +12,7 @@ from werkzeug import secure_filename from muranorepository.utils.parser import ManifestParser from muranorepository.utils.parser import serialize from muranorepository.utils.archiver import Archiver +from muranorepository.utils import utils from muranorepository.consts import DATA_TYPES, MANIFEST from muranorepository.consts import CLIENTS_DICT from muranorepository.consts import ARCHIVE_PKG_NAME @@ -23,16 +24,26 @@ CONF = cfg.CONF def reset_cache(): try: - shutil.rmtree(CONF.cache_dir, ignore_errors=True) - os.mkdir(CONF.cache_dir) + cache_dir = utils.get_cache_folder() + shutil.rmtree(cache_dir, ignore_errors=True) + os.mkdir(cache_dir) except: + log.exception(_('Error while cleaning cache')) return make_response('Unable to reset cache', 500) +def compose_path(data_type, path=None): + tenant_dir = utils.get_tenant_folder() + utils.check_tenant_dir_existence(tenant_dir) + return os.path.join(tenant_dir, + getattr(CONF, data_type), + path or '') + + def get_archive(client, hash_sum): types = CLIENTS_DICT.get(client) archive_manager = Archiver() - cache_dir = os.path.join(CONF.cache_dir, client) + cache_dir = os.path.join(utils.get_cache_folder(), client) if not os.path.exists(cache_dir): os.makedirs(cache_dir) existing_hash = None @@ -45,6 +56,7 @@ def get_archive(client, hash_sum): if archive_manager.hashes_match(cache_dir, existing_hash, hash_sum): return None + manifests = ManifestParser().parse() return archive_manager.create(cache_dir, manifests, types) @@ -106,13 +118,6 @@ def save_file(request, data_type, path=None, filename=None): return jsonify(result='success') -def compose_path(data_type, path=None): - if path: - return os.path.join(CONF.manifests, getattr(CONF, data_type), path) - else: - return os.path.join(CONF.manifests, getattr(CONF, data_type)) - - def check_data_type(data_type): if data_type not in DATA_TYPES: abort(404) @@ -147,11 +152,10 @@ def check_service_name(service_name): def perform_deletion(files_for_deletion, manifest_for_deletion): def backup_data(): backup_dir = os.path.join( - os.path.dirname(CONF.manifests), - 'Backup_{0}'.format(datetime.datetime.utcnow()) - ) - log.debug(_('Creating service data backup to {0}'.format(backup_dir))) - shutil.copytree(CONF.manifests, backup_dir) + utils.get_cache_folder(), + 'Backup_{0}'.format(datetime.datetime.utcnow())) + log.debug('Creating service data backup to {0}'.format(backup_dir)) + shutil.copytree(utils.get_tenant_folder(), backup_dir) return backup_dir def release_backup(backup): @@ -163,12 +167,12 @@ def perform_deletion(files_for_deletion, manifest_for_deletion): def restore_backup(backup): log.debug('Restore service data after unsuccessful deletion') - shutil.rmtree(CONF.manifests, ignore_errors=True) - os.rename(backup, CONF.manifests) + shutil.rmtree(utils.get_tenant_folder(), ignore_errors=True) + os.rename(backup, utils.get_tenant_folder()) backup_dir = backup_data() service_name = manifest_for_deletion.full_service_name - path_to_manifest = os.path.join(CONF.manifests, + path_to_manifest = os.path.join(utils.get_tenant_folder(), '{0}-manifest.yaml'.format(service_name)) try: if os.path.exists(path_to_manifest): @@ -176,8 +180,8 @@ def perform_deletion(files_for_deletion, manifest_for_deletion): os.remove(path_to_manifest) for data_type, files in files_for_deletion.iteritems(): - data_type_dir = os.path.join(CONF.manifests, getattr(CONF, - data_type)) + data_type_dir = os.path.join(utils.get_tenant_folder(), + getattr(CONF, data_type)) for file in files: path_to_delete = os.path.join(data_type_dir, file) if os.path.exists(path_to_delete): @@ -205,11 +209,9 @@ def save_archive(request): path_to_archive = uploaded_file.name else: file_to_upload = request.files.get('file') - if file_to_upload: - filename = secure_filename(file_to_upload.filename) - else: + if not file_to_upload: return err_resp - path_to_archive = os.path.join(CONF.cache_dir, filename) + path_to_archive = tempfile.NamedTemporaryFile(delete=False).name file_to_upload.save(path_to_archive) return path_to_archive @@ -230,7 +232,7 @@ def create_or_update_service(service_id, data): if not data.get(parameter): data[parameter] = optional[parameter] - path_to_manifest = os.path.join(CONF.manifests, + path_to_manifest = os.path.join(utils.get_tenant_folder(), service_id + '-manifest.yaml') backup_done = False diff --git a/muranorepository/cmd/run.py b/muranorepository/cmd/run.py index cab1a30..b8b5e89 100644 --- a/muranorepository/cmd/run.py +++ b/muranorepository/cmd/run.py @@ -53,13 +53,14 @@ def main(): log.setup('muranorepository') #configuring and initializing cache directory - if cfg.CONF.cache_dir is None: - cfg.CONF.cache_dir = os.path.join( - tempfile.gettempdir(), 'muranorepository-cache' + if cfg.CONF.data_dir is None: + cfg.CONF.data_dir = os.path.join( + tempfile.gettempdir(), 'muranorepository-data' ) - if not os.path.exists(cfg.CONF.cache_dir): - os.mkdir(cfg.CONF.cache_dir) - LOG.info('Cache is located at: {0}'.format(cfg.CONF.cache_dir)) + + if not os.path.exists(cfg.CONF.data_dir): + os.mkdir(cfg.CONF.data_dir) + LOG.info('Cache is located at: {0}'.format(cfg.CONF.data_dir)) app = server.make_app({ 'auth_host': cfg.CONF.keystone.auth_host, diff --git a/muranorepository/config.py b/muranorepository/config.py index b0162b6..94d4c12 100644 --- a/muranorepository/config.py +++ b/muranorepository/config.py @@ -21,7 +21,7 @@ server_opts = [ cfg.IntOpt('port', default=5000) ] -cache_opt = cfg.StrOpt('cache_dir') +cache_opt = cfg.StrOpt('data_dir') keystone_opts = [ cfg.StrOpt('auth_host', default='localhost'), diff --git a/muranorepository/consts.py b/muranorepository/consts.py index ef366d0..6628790 100644 --- a/muranorepository/consts.py +++ b/muranorepository/consts.py @@ -30,3 +30,6 @@ ARCHIVE_PKG_NAME = 'data.tar.gz' UI_FIELDS_IN_MANIFEST = {'description': 'description', 'type': 'full_service_name', 'name': 'service_display_name'} + +SERVICE_DEFINITIONS_FOLDER_NAME = 'service_definitions' +CACHE_FOLDER_NAME = 'cache' diff --git a/muranorepository/utils/archiver.py b/muranorepository/utils/archiver.py index 94fc468..8d8cfb2 100644 --- a/muranorepository/utils/archiver.py +++ b/muranorepository/utils/archiver.py @@ -23,6 +23,7 @@ from oslo.config import cfg from .parser import serialize from muranorepository.consts import DATA_TYPES, ARCHIVE_PKG_NAME from muranorepository.consts import UI, UI_FIELDS_IN_MANIFEST +from muranorepository.utils import utils from muranorepository.openstack.common.gettextutils import _ # noqa CONF = cfg.CONF @@ -206,16 +207,17 @@ class Archiver(object): if hasattr(manifest, data_type): file_list = getattr(manifest, data_type) - scr_directory = os.path.join( - CONF.manifests, self.src_directories[data_type]) + src_directory = os.path.join( + utils.get_tenant_folder(), + self.src_directories[data_type]) dst_directory = os.path.join( temp_dir, self.dst_directories[data_type]) if data_type == UI: self._compose_ui_forms(manifest, file_list, - scr_directory, dst_directory) + src_directory, dst_directory) else: self._copy_data(file_list, - scr_directory, + src_directory, dst_directory) else: log.info( @@ -234,23 +236,25 @@ class Archiver(object): for data_type in DATA_TYPES: if hasattr(manifest, data_type): file_list = getattr(manifest, data_type) - scr_directory = os.path.join( - CONF.manifests, self.src_directories[data_type]) + src_directory = os.path.join( + utils.get_tenant_folder(), self.src_directories[data_type]) dst_directory = os.path.join( temp_dir, self.dst_directories[data_type]) - self._copy_data(file_list, scr_directory, dst_directory) + self._copy_data(file_list, src_directory, dst_directory) else: log.info( _('{0} manifest has no file definitions for ' '{1}'.format(manifest.service_display_name, data_type))) #Add manifest file into archive manifest_filename = manifest.full_service_name + '-manifest.yaml' - self._copy_data([manifest_filename], CONF.manifests, temp_dir) + self._copy_data([manifest_filename], + utils.get_tenant_folder(), + temp_dir) return self._compose_archive(file_name, temp_dir) def remove_existing_hash(self, cache_dir, hash): path = os.path.join(cache_dir, hash) - log.info(_('Deleting archive package from {0}.'.format(path))) + log.info('Deleting archive package from {0}.'.format(path)) shutil.rmtree(path, ignore_errors=True) def extract(self, path_to_archive): @@ -260,6 +264,7 @@ class Archiver(object): return value - True if succeeded , False otherwise """ try: + root_folder = utils.get_tenant_folder() path_to_extract = tempfile.mkdtemp() archive = tarfile.open(path_to_archive) try: @@ -277,7 +282,7 @@ class Archiver(object): 'file in archive')) return False - shutil.copy(manifests[0], CONF.manifests) + shutil.copy(manifests[0], root_folder) #Todo: Check manifest is valid for item in os.listdir(path_to_extract): item_path = os.path.join(path_to_extract, item) @@ -298,7 +303,7 @@ class Archiver(object): self._copy_data(file_list, item_path, os.path.join( - CONF.manifests, + root_folder, self.src_directories[item]), overwrite=False) else: diff --git a/muranorepository/utils/parser.py b/muranorepository/utils/parser.py index 4c8800d..0af918c 100644 --- a/muranorepository/utils/parser.py +++ b/muranorepository/utils/parser.py @@ -16,9 +16,11 @@ import os import yaml from oslo.config import cfg import logging as log + from muranorepository.manifest import Manifest from muranorepository.consts import DATA_TYPES, MANIFEST from muranorepository.openstack.common.gettextutils import _ # noqa +from muranorepository.utils import utils CONF = cfg.CONF @@ -48,8 +50,9 @@ def serialize(data): class ManifestParser(object): def __init__(self, manifest_directory=None): if not manifest_directory: - manifest_directory = CONF.manifests + manifest_directory = utils.get_tenant_folder() self.manifest_directory = manifest_directory + utils.check_tenant_dir_existence(self.manifest_directory) def _validate_manifest(self, file, service_manifest_data): service_id = service_manifest_data.get('full_service_name') @@ -70,9 +73,9 @@ class ManifestParser(object): root_directory = self.manifest_directory if not isinstance(value, list): - log.error(_("'{0}' section should represent a file" - " listing in manifest {1}".format( - root_directory, file))) + log.error(_("'{0}' section should represent a file listing" + " in manifest {1}".format(root_directory, + file))) valid_file_info = False continue for filename in value: @@ -90,28 +93,29 @@ class ManifestParser(object): def parse(self): manifests = [] - for file in os.listdir(self.manifest_directory): - manifest_file = os.path.join(self.manifest_directory, file) - if os.path.isfile(manifest_file): - if not file.endswith(".yaml"): - log.warning(_("Extension of '{0}' file is not yaml. " - "Only yaml file supported for " - "service manifest files.".format(file))) - continue + if os.path.exists(self.manifest_directory): + for file in os.listdir(self.manifest_directory): + manifest_file = os.path.join(self.manifest_directory, file) + if os.path.isfile(manifest_file): + if not file.endswith(".yaml"): + log.warning(_("Extension of {0} file is not yaml. " + "Only yaml file supported for " + "service manifest files.".format(file))) + continue - try: - with open(manifest_file) as stream: - manifest_data = yaml.load(stream) - except yaml.YAMLError: - log.exception(_("Failed to load manifest " - "file '{0}'".format(manifest_file))) - continue + try: + with open(manifest_file) as stream: + manifest_data = yaml.load(stream) + except yaml.YAMLError: + log.exception(_("Failed to load manifest file " + " '{0}'".format(manifest_file))) + continue - manifest_is_valid, use_manifest = self._validate_manifest( - file, manifest_data) - if use_manifest: - manifest_data["valid"] = manifest_is_valid - manifests.append(Manifest(manifest_data)) + manifest_is_valid, use_manifest = self._validate_manifest( + file, manifest_data) + if use_manifest: + manifest_data["valid"] = manifest_is_valid + manifests.append(Manifest(manifest_data)) return manifests diff --git a/muranorepository/utils/utils.py b/muranorepository/utils/utils.py new file mode 100644 index 0000000..d9dfbba --- /dev/null +++ b/muranorepository/utils/utils.py @@ -0,0 +1,40 @@ +# Copyright (c) 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import os +import shutil +from oslo.config import cfg +from flask import request +from muranorepository.consts import SERVICE_DEFINITIONS_FOLDER_NAME +from muranorepository.consts import CACHE_FOLDER_NAME +CONF = cfg.CONF + + +def get_tenant_id(): + return request.environ['keystone.' + 'token_info']['access']['token']['tenant']['id'] + + +def get_tenant_folder(): + return os.path.join(CONF.data_dir, + SERVICE_DEFINITIONS_FOLDER_NAME, + get_tenant_id()) + + +def get_cache_folder(): + return os.path.join(CONF.data_dir, CACHE_FOLDER_NAME, get_tenant_id()) + + +def check_tenant_dir_existence(path): + if not os.path.exists(path): + shutil.copytree(CONF.manifests, path)