From 926fe339190d741807d18d7bc830a4793425c32b Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Tue, 2 Jun 2015 09:13:58 +0200 Subject: [PATCH] Cached file system and Connections flush This is to improve performance of resource/connections creation --- config.yaml | 6 +- example.py | 6 +- solar/solar/core/signals.py | 10 +- solar/solar/interfaces/db/__init__.py | 5 +- .../interfaces/db/cached_file_system_db.py | 106 ++++++++++++++++++ solar/solar/interfaces/db/file_system_db.py | 5 - 6 files changed, 125 insertions(+), 13 deletions(-) create mode 100644 solar/solar/interfaces/db/cached_file_system_db.py diff --git a/config.yaml b/config.yaml index 3af564ca..569a5df4 100644 --- a/config.yaml +++ b/config.yaml @@ -1,13 +1,13 @@ -clients-data-file: /vagrant/tmp/connections.yaml +clients-data-file: /tmp/connections.yaml -tmp: /vagrant/tmp +tmp: /tmp examples-dir: /vagrant/examples extensions-dir: /vagrant/solar/solar/extensions file-system-db: - storage-path: /vagrant/tmp/storage + storage-path: /tmp/storage template-dir: /vagrant/templates diff --git a/example.py b/example.py index 18c6f05c..05a57583 100644 --- a/example.py +++ b/example.py @@ -12,7 +12,6 @@ from solar.core import validation from solar.interfaces.db import get_db - @click.group() def main(): pass @@ -160,6 +159,11 @@ def deploy(): signals.connect(glance_keystone_user, glance_registry_container, {'admin_token': 'keystone_admin_token'}) signals.connect(haproxy_config, glance_registry_container, {'ip': 'keystone_host'}) + if hasattr(db, 'flush'): + db.flush() + + signals.Connections.flush() + has_errors = False for r in resource.Resource.__subclasses__(): diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index 5afbb72a..ddc08fb2 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -30,7 +30,7 @@ class Connections(object): if [receiver.name, dst] not in CLIENTS[emitter.name][src]: CLIENTS[emitter.name][src].append([receiver.name, dst]) - utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS) + #utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS) @staticmethod def remove(emitter, src, receiver, dst): @@ -39,7 +39,7 @@ class Connections(object): if destination != [receiver.name, dst] ] - utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS) + #utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS) @staticmethod def reconnect_all(): @@ -65,6 +65,10 @@ class Connections(object): if os.path.exists(path): os.remove(path) + @staticmethod + def flush(): + utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS) + def guess_mapping(emitter, receiver): """Guess connection mapping between emitter and receiver. @@ -136,7 +140,7 @@ def disconnect_by_src(emitter_name, src, receiver): if destination[0] != receiver.name ] - utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS) + #utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS) def notify(source, key, value): diff --git a/solar/solar/interfaces/db/__init__.py b/solar/solar/interfaces/db/__init__.py index e633458d..5a02f78f 100644 --- a/solar/solar/interfaces/db/__init__.py +++ b/solar/solar/interfaces/db/__init__.py @@ -1,9 +1,12 @@ from solar.interfaces.db.file_system_db import FileSystemDB +from solar.interfaces.db.cached_file_system_db import CachedFileSystemDB mapping = { + 'cached_file_system': CachedFileSystemDB, 'file_system': FileSystemDB } def get_db(): # Should be retrieved from config - return mapping['file_system']() + #return mapping['file_system']() + return mapping['cached_file_system']() diff --git a/solar/solar/interfaces/db/cached_file_system_db.py b/solar/solar/interfaces/db/cached_file_system_db.py new file mode 100644 index 00000000..14a77bec --- /dev/null +++ b/solar/solar/interfaces/db/cached_file_system_db.py @@ -0,0 +1,106 @@ +from solar.third_party.dir_dbm import DirDBM + +import os +import types +import yaml + +from solar import utils +from solar import errors + + +class CachedFileSystemDB(DirDBM): + STORAGE_PATH = utils.read_config()['file-system-db']['storage-path'] + RESOURCE_COLLECTION_NAME = 'resource' + + _CACHE = {} + + def __init__(self): + utils.create_dir(self.STORAGE_PATH) + super(CachedFileSystemDB, self).__init__(self.STORAGE_PATH) + self.entities = {} + + def __setitem__(self, k, v): + """ + C{dirdbm[k] = v} + Create or modify a textfile in this directory + @type k: strings @param k: key to setitem + @type v: strings @param v: value to associate with C{k} + """ + assert type(k) == types.StringType, "DirDBM key must be a string" + # NOTE: Can be not a string if _writeFile in the child is redefined + # assert type(v) == types.StringType, "DirDBM value must be a string" + k = self._encode(k) + + # we create a new file with extension .new, write the data to it, and + # if the write succeeds delete the old file and rename the new one. + old = os.path.join(self.dname, k) + if os.path.exists(old): + new = old + ".rpl" # replacement entry + else: + new = old + ".new" # new entry + try: + self._writeFile(old, v) + except: + raise + + def get_resource(self, uid): + return self[self._make_key(self.RESOURCE_COLLECTION_NAME, uid)] + + def get_obj_resource(self, uid): + from solar.core.resource import wrap_resource + raw_resource = self[self._make_key(self.RESOURCE_COLLECTION_NAME, uid)] + + return wrap_resource(raw_resource) + + def add_resource(self, uid, resource): + self[self._make_key(self.RESOURCE_COLLECTION_NAME, uid)] = resource + + def store(self, collection, obj): + if 'id' in obj: + self[self._make_key(collection, obj['id'])] = obj + else: + raise errors.CannotFindID('Cannot find id for object {0}'.format(obj)) + + def store_list(self, collection, objs): + for obj in objs: + self.store(collection, obj) + + def get_list(self, collection): + collection_keys = filter( + lambda k: k.startswith('{0}-'.format(collection)), + self.keys()) + + return map(lambda k: self[k], collection_keys) + + def get_record(self, collection, _id): + key = self._make_key(collection, _id) + if key not in self: + return None + + return self[key] + + def _make_key(self, collection, _id): + return '{0}-{1}'.format(collection, _id) + + def _readFile(self, path): + if path not in self._CACHE: + data = yaml.load(super(CachedFileSystemDB, self)._readFile(path)) + self._CACHE[path] = data + return data + + return self._CACHE[path] + + def _writeFile(self, path, data): + self._CACHE[path] = data + + def _encode(self, key): + """Override method of the parent not to use base64 as a key for encoding""" + return key + + def _decode(self, key): + """Override method of the parent not to use base64 as a key for encoding""" + return key + + def flush(self): + for path, data in self._CACHE.items(): + super(CachedFileSystemDB, self)._writeFile(path, yaml.dump(data)) diff --git a/solar/solar/interfaces/db/file_system_db.py b/solar/solar/interfaces/db/file_system_db.py index 950c0e46..724cbdca 100644 --- a/solar/solar/interfaces/db/file_system_db.py +++ b/solar/solar/interfaces/db/file_system_db.py @@ -1,10 +1,5 @@ from solar.third_party.dir_dbm import DirDBM - -import os -from fnmatch import fnmatch -from copy import deepcopy - import yaml from solar import utils