Cached file system and Connections flush

This is to improve performance of resource/connections creation

config.yaml

@@ -1,13 +1,13 @@
-clients-data-file: /vagrant/tmp/connections.yaml
+clients-data-file: /tmp/connections.yaml
 
-tmp: /vagrant/tmp
+tmp: /tmp
 
 examples-dir: /vagrant/examples
 
 extensions-dir: /vagrant/solar/solar/extensions
 
 file-system-db:
-  storage-path: /vagrant/tmp/storage
+  storage-path: /tmp/storage
 
 template-dir: /vagrant/templates
 

example.py

@@ -12,7 +12,6 @@ from solar.core import validation
 from solar.interfaces.db import get_db
 
 
-
 @click.group()
 def main():
     pass
@@ -160,6 +159,11 @@ def deploy():
     signals.connect(glance_keystone_user, glance_registry_container, {'admin_token': 'keystone_admin_token'})
     signals.connect(haproxy_config, glance_registry_container, {'ip': 'keystone_host'})
 
+    if hasattr(db, 'flush'):
+        db.flush()
+
+    signals.Connections.flush()
+
 
     has_errors = False
     for r in resource.Resource.__subclasses__():

solar/solar/core/signals.py

@@ -30,7 +30,7 @@ class Connections(object):
         if [receiver.name, dst] not in CLIENTS[emitter.name][src]:
             CLIENTS[emitter.name][src].append([receiver.name, dst])
 
-        utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS)
+        #utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS)
 
     @staticmethod
     def remove(emitter, src, receiver, dst):
@@ -39,7 +39,7 @@ class Connections(object):
             if destination != [receiver.name, dst]
         ]
 
-        utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS)
+        #utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS)
 
     @staticmethod
     def reconnect_all():
@@ -65,6 +65,10 @@ class Connections(object):
         if os.path.exists(path):
             os.remove(path)
 
+    @staticmethod
+    def flush():
+        utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS)
+
 
 def guess_mapping(emitter, receiver):
     """Guess connection mapping between emitter and receiver.
@@ -136,7 +140,7 @@ def disconnect_by_src(emitter_name, src, receiver):
             if destination[0] != receiver.name
         ]
 
-    utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS)
+    #utils.save_to_config_file(CLIENTS_CONFIG_KEY, CLIENTS)
 
 
 def notify(source, key, value):

solar/solar/interfaces/db/__init__.py

@@ -1,9 +1,12 @@
 from solar.interfaces.db.file_system_db import FileSystemDB
+from solar.interfaces.db.cached_file_system_db import CachedFileSystemDB
 
 mapping = {
+    'cached_file_system': CachedFileSystemDB,
     'file_system': FileSystemDB
 }
 
 def get_db():
     # Should be retrieved from config
-    return mapping['file_system']()
+    #return mapping['file_system']()
+    return mapping['cached_file_system']()

solar/solar/interfaces/db/cached_file_system_db.py

@@ -0,0 +1,106 @@
+from solar.third_party.dir_dbm import DirDBM
+
+import os
+import types
+import yaml
+
+from solar import utils
+from solar import errors
+
+
+class CachedFileSystemDB(DirDBM):
+    STORAGE_PATH = utils.read_config()['file-system-db']['storage-path']
+    RESOURCE_COLLECTION_NAME = 'resource'
+
+    _CACHE = {}
+
+    def __init__(self):
+        utils.create_dir(self.STORAGE_PATH)
+        super(CachedFileSystemDB, self).__init__(self.STORAGE_PATH)
+        self.entities = {}
+
+    def __setitem__(self, k, v):
+        """
+        C{dirdbm[k] = v}
+        Create or modify a textfile in this directory
+        @type k: strings        @param k: key to setitem
+        @type v: strings        @param v: value to associate with C{k}
+        """
+        assert type(k) == types.StringType, "DirDBM key must be a string"
+        # NOTE: Can be not a string if _writeFile in the child is redefined
+        # assert type(v) == types.StringType, "DirDBM value must be a string"
+        k = self._encode(k)
+
+        # we create a new file with extension .new, write the data to it, and
+        # if the write succeeds delete the old file and rename the new one.
+        old = os.path.join(self.dname, k)
+        if os.path.exists(old):
+            new = old + ".rpl" # replacement entry
+        else:
+            new = old + ".new" # new entry
+        try:
+            self._writeFile(old, v)
+        except:
+            raise
+
+    def get_resource(self, uid):
+        return self[self._make_key(self.RESOURCE_COLLECTION_NAME, uid)]
+
+    def get_obj_resource(self, uid):
+        from solar.core.resource import wrap_resource
+        raw_resource = self[self._make_key(self.RESOURCE_COLLECTION_NAME, uid)]
+
+        return wrap_resource(raw_resource)
+
+    def add_resource(self, uid, resource):
+        self[self._make_key(self.RESOURCE_COLLECTION_NAME, uid)] = resource
+
+    def store(self, collection, obj):
+        if 'id' in obj:
+            self[self._make_key(collection, obj['id'])] = obj
+        else:
+            raise errors.CannotFindID('Cannot find id for object {0}'.format(obj))
+
+    def store_list(self, collection, objs):
+        for obj in objs:
+            self.store(collection, obj)
+
+    def get_list(self, collection):
+        collection_keys = filter(
+            lambda k: k.startswith('{0}-'.format(collection)),
+            self.keys())
+
+        return map(lambda k: self[k], collection_keys)
+
+    def get_record(self, collection, _id):
+        key = self._make_key(collection, _id)
+        if key not in self:
+            return None
+
+        return self[key]
+
+    def _make_key(self, collection, _id):
+        return '{0}-{1}'.format(collection, _id)
+
+    def _readFile(self, path):
+        if path not in self._CACHE:
+            data = yaml.load(super(CachedFileSystemDB, self)._readFile(path))
+            self._CACHE[path] = data
+            return data
+
+        return self._CACHE[path]
+
+    def _writeFile(self, path, data):
+        self._CACHE[path] = data
+
+    def _encode(self, key):
+        """Override method of the parent not to use base64 as a key for encoding"""
+        return key
+
+    def _decode(self, key):
+        """Override method of the parent not to use base64 as a key for encoding"""
+        return key
+
+    def flush(self):
+        for path, data in self._CACHE.items():
+            super(CachedFileSystemDB, self)._writeFile(path, yaml.dump(data))

solar/solar/interfaces/db/file_system_db.py

@@ -1,10 +1,5 @@
 from solar.third_party.dir_dbm import DirDBM
 
-
-import os
-from fnmatch import fnmatch
-from copy import deepcopy
-
 import yaml
 
 from solar import utils