From 05c2100eef308cc842f096b463ca62e3b1a5bae7 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 19 Oct 2015 10:24:22 +0200 Subject: [PATCH 001/191] New db layer core idea. --- solar/solar/dblayer/__init__.py | 0 solar/solar/dblayer/model.py | 405 +++++++++++++++++++++++++ solar/solar/dblayer/solar_models.py | 277 +++++++++++++++++ solar/solar/dblayer/sql.py | 386 +++++++++++++++++++++++ solar/solar/dblayer/test/conftest.py | 53 ++++ solar/solar/dblayer/test/test_basic.py | 76 +++++ solar/solar/dblayer/test/test_real.py | 193 ++++++++++++ 7 files changed, 1390 insertions(+) create mode 100644 solar/solar/dblayer/__init__.py create mode 100644 solar/solar/dblayer/model.py create mode 100644 solar/solar/dblayer/solar_models.py create mode 100644 solar/solar/dblayer/sql.py create mode 100644 solar/solar/dblayer/test/conftest.py create mode 100644 solar/solar/dblayer/test/test_basic.py create mode 100644 solar/solar/dblayer/test/test_real.py diff --git a/solar/solar/dblayer/__init__.py b/solar/solar/dblayer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py new file mode 100644 index 00000000..a6ed1d6d --- /dev/null +++ b/solar/solar/dblayer/model.py @@ -0,0 +1,405 @@ +from threading import local, current_thread +from random import getrandbits +import uuid +from functools import wraps +from operator import itemgetter + + +LOCAL = local() + +class DBLayerException(Exception): + pass + + +class DBLayerNotFound(DBLayerException): + pass + + +class DBLayerNoRiakObj(DBLayerException): + pass + + +class NONE: + """A None like type""" + pass + +def get_cache(instance, _): + th = current_thread() + l = LOCAL + try: + cache_id = l.cache_id + except AttributeError: + cache_id = uuid.UUID(int=getrandbits(128), version=4).hex + setattr(th, 'cache_id', cache_id) + if getattr(th, 'cache_id', None) == cache_id: + setattr(l, 'obj_cache', {}) + setattr(l, 'db_ch_state', {}) + l.db_ch_state.setdefault('index', set()) + return l + + +def clear_cache(): + th = current_thread() + cache_id = uuid.UUID(int=getrandbits(128), version=4).hex + setattr(th, 'cache_id', cache_id) + + +def get_bucket(_, owner, mcs): + name = owner.get_bucket_name() + bucket = mcs.riak_client.bucket(name) + return bucket + + +def changes_state_for(_type): + def _inner1(f): + @wraps(f) + def _inner2(obj, *args, **kwargs): + obj._c.db_ch_state['index'].add(obj.key) + return f(obj, *args, **kwargs) + return _inner2 + return _inner1 + + +def clears_state_for(_type): + def _inner1(f): + @wraps(f) + def _inner2(obj, *args, **kwargs): + try: + obj._c.db_ch_state[_type].remove(obj.key) + except KeyError: + pass + return f(obj, *args, **kwargs) + return _inner2 + return _inner1 + + +def requires_clean_state(_type): + def _inner1(f): + @wraps(f) + def _inner2(obj, *args, **kwargs): + check_state_for(_type, obj) + return f(obj, *args, **kwargs) + return _inner2 + return _inner1 + + +def check_state_for(_type, obj): + state = obj._c.db_ch_state.get(_type) + if state: + raise Exception("Dirty state, save all objects first") + + +class Replacer(object): + + def __init__(self, name, fget, *args): + self.name = name + self.fget = fget + self.args = args + + def __get__(self, instance, owner): + val = self.fget(instance, owner, *self.args) + if instance is not None: + setattr(instance, self.name, val) + else: + setattr(owner, self.name, val) + return val + + +class FieldBase(object): + def __init__(self, fname, default): + self._fname = fname + self._default = default + + @property + def fname(self): + return self._fname + + @fname.setter + def fname(self, value): + if self._fname is None: + self._fname = value + else: + raise Exception("fname already set") + + @property + def default(self): + if self._default is NONE: + return self._default + if callable(self._default): + return self._default() + return self._default + + +class Field(FieldBase): + + def __init__(self, _type, default=NONE): + self._type = _type + super(Field, self).__init__(fname=None, default=default) + + def __get__(self, instance, owner): + if instance is None: + return self + return instance._riak_object.data[self.fname] + + def __set__(self, instance, value): + if not isinstance(value, self._type): + raise Exception("Invalid type %r for %r" % (type(value), self.fname)) + instance._modified_fields.add(self.fname) + instance._riak_object.data[self.fname] = value + + def __str__(self): + return "<%s:%r>" % (self.__class__.__name__, self.fname) + + __repr__ = __str__ + + +class IndexFieldWrp(object): + + def __init__(self, field_obj, instance): + self._field_obj = field_obj + self._instance = instance + self._c = self._instance._c + + @property + def fname(self): + return self._field_obj.fname + + def __str__(self): + return "<%s for field %s>" % (self.__class__.__name__, self._field_obj) + + def _get_field_val(self, name): + return self._instance._riak_object.data[self.fname][name] + + def __getitem__(self, name): + return self._get_field_val(name) + + def __setitem__(self, name, value): + inst = self._instance + inst._riak_object.set_index('%s_bin' % self.fname, '{}|{}'.format(name, value)) + + def __delitem__(self, name): + inst = self._instance + del inst._riak_object.data[self.fname][name] + indexes = inst._riak_object.indexes + + # TODO: move this to backend layer + for ind_name, ind_value in indexes: + if ind_name == ('%s_bin' % self.fname): + if ind_value.startswith('{}|'.format(name)): + inst._remove_index(ind_name, ind_value) + break + + +class IndexField(FieldBase): + + _wrp_class = IndexFieldWrp + + def __init__(self, fname=None, default=NONE): + super(IndexField, self).__init__(fname, default) + + def _on_no_inst(self, instance, owner): + return self + + def __get__(self, instance, owner): + if instance is None: + return self._on_no_inst(instance, owner) + cached = getattr(instance, '_real_obj_%s' % self.fname, None) + if cached: + return cached + obj = self._wrp_class(self, instance) + setattr(instance, '_real_obj_%s' % self.fname, obj) + return obj + + def __set__(self, instance, value): + wrp = getattr(instance, self.fname) + instance._riak_object.data[self.fname] = self.default + for f_name, f_value in value.iteritems(): + wrp[f_name] = f_value + + + def _parse_key(self, k): + if '=' in k: + val, subval = k.split('=', 1) + if subval is None: + subval = '' + if not isinstance(subval, basestring): + subval = str(subval) + return '{}|{}'.format(val, subval) + + def filter(self, startkey, endkey=None, **kwargs): + startkey = self._parse_key(startkey) + if endkey is None: + if startkey.endswith('*'): + startkey = startkey[:-1] + endkey = startkey + '~' + else: + endkey = startkey + ' ' + kwargs.setdefault('max_results', 1000000) + kwargs['return_terms'] = False + res = self._declared_in._get_index('{}_bin'.format(self.fname), + startkey=startkey, + endkey=endkey, + **kwargs).results + return set(res) + + + + +class ModelMeta(type): + + def __new__(mcs, name, bases, attrs): + cls = super(ModelMeta, mcs).__new__(mcs, name, bases, attrs) + model_fields = set((name for (name, attr) in attrs.items() + if isinstance(attr, FieldBase))) + for f in model_fields: + field = getattr(cls, f) + if hasattr(field, 'fname') and field.fname is None: + setattr(field, 'fname', f) + # need to set declared_in because `with_tag` + # no need to wrap descriptor with another object then + setattr(field, '_declared_in', cls) + + for base in bases: + try: + model_fields_base = base._model_fields + except AttributeError: + continue + else: + for given in base._model_fields: + model_fields.add(given) + + # set the fields just in case + cls._model_fields = model_fields + cls.bucket = Replacer('bucket', get_bucket, mcs) + return cls + + + @classmethod + def setup(mcs, riak_client): + mcs.riak_client = riak_client + + +class Model(object): + + __metaclass__ = ModelMeta + + _c = Replacer('_c', get_cache) + + _key = None + _new = None + _real_riak_object = None + + _changed = False + + def __init__(self, key=None): + self._modified_fields = set() + self.key = key + + @property + def key(self): + return self._key + + @key.setter + def key(self, value): + if self._key is None: + self._key = value + else: + raise Exception("Can't set key again") + + @property + def _riak_object(self): + if self._real_riak_object is None: + raise DBLayerNoRiakObj("You cannot access _riak_object now") + return self._real_riak_object + + @_riak_object.setter + def _riak_object(self, value): + if self._real_riak_object is not None: + raise DBLayerException("Already have _riak_object") + self._real_riak_object = value + + @changes_state_for('index') + def _set_index(self, *args, **kwargs): + return self._riak_object.set_index(*args, **kwargs) + + @changes_state_for('index') + def _add_index(self, *args, **kwargs): + return self._riak_object.add_index(*args, **kwargs) + + @changes_state_for('index') + def _add_index(self, *args, **kwargs): + return self._riak_object.add_index(*args, **kwargs) + + @changes_state_for('index') + def _remove_index(self, *args, **kwargs): + return self._riak_object.remove_index(*args, **kwargs) + + @classmethod + def _get_index(cls, *args, **kwargs): + return cls.bucket.get_index(*args, **kwargs) + + @property + def _bucket(self): + return self._riak_object.bucket + + @classmethod + def get_bucket_name(cls): + # XXX: should be changed and more smart + return cls.__name__ + + def changed(self): + return True if self._modified_fields else False + + def __str__(self): + if self._riak_object is None: + return "<%s not initialized>" % (self.__class__.__name__) + return "<%s %s:%s>" % (self.__class__.__name__, self._riak_object.bucket.name, self.key) + + + @classmethod + def new(cls, key, data): + return cls.from_dict(key, data) + + @classmethod + def from_riakobj(cls, riak_obj): + obj = cls(riak_obj.key) + obj._riak_object = riak_obj + if obj._new is None: + obj._new = False + cls._c.obj_cache[riak_obj.key] = obj + return obj + + @classmethod + def from_dict(cls, key, data): + riak_obj = cls.bucket.new(key, data={}) + obj = cls.from_riakobj(riak_obj) + obj._new = True + for field in cls._model_fields: + val = data.get(field, NONE) + default = getattr(cls, field).default + # if val is NONE and default is NONE: + # continue + if val is NONE and default is not NONE: + setattr(obj, field, default) + elif val is not NONE: + setattr(obj, field, val) + return obj + + @classmethod + def get(cls, key): + try: + return cls._c.obj_cache[key] + except KeyError: + riak_object = cls.bucket.get(key) + if not riak_object.exists: + raise DBLayerNotFound(key) + else: + obj = cls.from_riakobj(riak_object) + return obj + + @clears_state_for('index') + def save(self, force=False): + if self.changed() or force or self._new: + return self._riak_object.store() + else: + raise Exception("No changes") diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py new file mode 100644 index 00000000..e07315a0 --- /dev/null +++ b/solar/solar/dblayer/solar_models.py @@ -0,0 +1,277 @@ +from solar.dblayer.model import (Model, Field, IndexField, + IndexFieldWrp, + DBLayerException, + requires_clean_state, check_state_for) + +from operator import itemgetter + +class DBLayerSolarException(DBLayerException): + pass + + +class InputsFieldWrp(IndexFieldWrp): + + def __init__(self, *args, **kwargs): + super(InputsFieldWrp, self).__init__(*args, **kwargs) + # TODO: add cache for lookup + self._cache = {} + + def connect(self, my_inp_name, other_resource, other_inp_name): + # TODO: for now connections are attached to target resource + # in future we might change it to separate object + my_resource = self._instance + + + other_ind_name = '{}_emit_bin'.format(self.fname) + other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, + other_inp_name, + my_resource.key, + my_inp_name) + + my_ind_name = '{}_recv_bin'.format(self.fname) + my_ind_val = '{}|{}|{}|{}'.format(my_resource.key, + my_inp_name, + other_resource.key, + other_inp_name) + + # ensure no conflicting connections are done + # TODO: move this to backend layer + indexes = my_resource._riak_object.indexes + to_del = [] + for ind_name, ind_value in indexes: + if ind_name == my_ind_name: + mr, mn = ind_value.split('|')[:2] + if mr == my_resource.key and mn == my_inp_name: + to_del.append((ind_name, ind_value)) + elif ind_name == other_ind_name: + mr, mn = ind_value.rsplit('|')[2:] + if mr == my_resource.key and mn == my_inp_name: + to_del.append((ind_name, ind_value)) + + for ind_name, ind_value in to_del: + my_resource._remove_index(ind_name, value=ind_value) + + # add new + my_resource._add_index(my_ind_name, + my_ind_val) + my_resource._add_index(other_ind_name, + other_ind_val) + try: + del self._cache[my_inp_name] + except KeyError: + pass + return True + + + def _has_own_input(self, name): + try: + return self._cache[name] + except KeyError: + pass + my_name = self._instance.key + bucket = self._instance._bucket + _input = bucket.get_index('%s_bin' % self.fname, + startkey='{}|{}'.format(my_name, name), + endkey='{}|{}~'.format(my_name, name), + max_results=1, + return_terms=True).results + if not _input : + raise DBLayerSolarException('No input {} for {}'.format(name, my_name)) + if not _input[0][0].startswith('{}|{}'.format(my_name, name)): + # double check for ranges + raise DBLayerSolarException('No input {} for {}'.format(name, my_name)) + return True + + def _get_field_val(self, name): + # maybe it should be tco + check_state_for('index', self._instance) + try: + return self._cache[name] + except KeyError: + pass + fname = self.fname + my_name = self._instance.key + bucket = self._instance._bucket + # TODO: _has_own_input is slow, check if its really needed + self._has_own_input(name) + ind_name = '{}_recv_bin'.format(fname) + # XXX: possible optimization + # get all values for resource and cache it (use dirty to check) + recvs = bucket.get_index(ind_name, + startkey='{}|{}|'.format(my_name, name), + endkey='{}|{}|~'.format(my_name, name), + max_results=1, + return_terms=True).results + if not recvs: + _res = self._get_raw_field_val(name) + self._cache[name] = _res + return _res + recvs = recvs[0] + index_val, obj_key = recvs + _, inp, emitter_key, emitter_inp = index_val.split('|', 4) + res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + self._cache[name] = res + return res + + def _get_raw_field_val(self, name): + return self._instance._riak_object.data[self.fname][name] + + def __getitem__(self, name): + return self._get_field_val(name) + + def __delitem__(self, name): + self._has_own_input(name) + try: + del self._cache[name] + except KeyError: + pass + inst = self._instance + inst._riak_object.remove_index('%s_bin' % self.fname, '{}|{}'.format(self._instance.key, name)) + del inst._riak_object.data[self.fname][name] + + def __setitem__(self, name, value): + return self._set_field_value(name, value) + + def _set_field_value(self, name, value): + fname = self.fname + my_name = self._instance.key + ind_name = '{}_recv_bin'.format(fname) + bucket = self._instance._bucket + recvs = bucket.get_index(ind_name, + startkey='{}|{}|'.format(my_name, name), + endkey='{}|{}|~'.format(my_name,name), + max_results=1, + return_terms=True).results + if recvs: + recvs = recvs[0] + inp, emitter_name, emitter_inp = recvs[0].split('|', 3) + raise Exception("I'm connected with resource %r input %r" % (emitter_name, emitter_inp)) + # inst = self._instance + robj = self._instance._riak_object + self._instance._add_index('%s_bin' % self.fname, '{}|{}'.format(my_name, name)) + try: + robj.data[self.fname][name] = value + except KeyError: + robj.data[self.fname] = {name: value} + self._cache[name] = value + return True + + +class InputsField(IndexField): + _wrp_class = InputsFieldWrp + + def __set__(self, instance, value): + wrp = getattr(instance, self.fname) + instance._riak_object.data[self.fname] = self.default + for inp_name, inp_value in value.iteritems(): + wrp[inp_name] = inp_value + + +class TagsFieldWrp(IndexFieldWrp): + + def __getitem__(self, name): + raise TypeError('You cannot get tags like this') + + def __setitem__(self, name, value): + raise TypeError('You cannot set tags like this') + + def __delitem__(self, name, value): + raise TypeError('You cannot set tags like this') + + def __iter__(self): + return iter(self._instance._riak_object.data[self.fname]) + + def set(self, name, value=None): + if '=' in name and value is None: + name, value = name.split('=', 1) + if value is None: + value = '' + inst = self._instance + indexes = inst._riak_object.indexes.copy() # copy it + + inst._add_index('{}_bin'.format(self.fname), '{}~{}'.format(name, value)) + try: + fld = inst._riak_object.data[self.fname] + except IndexError: + fld = inst._riak_object.data[self.fname] = [] + full_value = '{}={}'.format(name, value) + try: + fld.append(full_value) + except KeyError: + fld = [full_value] + return True + + def has_tag(self, name, subval=None): + fld = self._instance._riak_object.data[self.fname] + if not name in fld: + return False + if subval is not None: + subvals = fld[name] + return subval in subvals + return True + + def remove(self, name, value=None): + if '=' in name and value is None: + name, value = name.split('=', 1) + if value is None: + value = '' + inst = self._instance + fld = inst._riak_object.data[self.fname] + full_value = '{}={}'.format(name, value) + try: + vals = fld.remove(full_value) + except ValueError: + pass + else: + inst._remove_index('{}_bin'.format(self.fname), '{}~{}'.format(name, value)) + return True + + + + +class TagsField(IndexField): + _wrp_class = TagsFieldWrp + + def __set__(self, instance, value): + wrp = getattr(instance, self.fname) + instance._riak_object.data[self.fname] = self.default + for val in value: + wrp.set(val) + + def filter(self, name, subval=None): + check_state_for('index', self._declared_in) + if '=' in name and subval is None: + name, subval = name.split('=', 1) + if subval is None: + subval = '' + if not isinstance(subval, basestring): + subval = str(subval) + # maxresults because of riak bug with small number of results + # https://github.com/basho/riak/issues/608 + if not subval.endswith('*'): + res = self._declared_in._get_index('{}_bin'.format(self.fname), + startkey='{}~{}'.format(name, subval), + endkey='{}~{} '.format(name, subval), # space required + max_results=100000, + return_terms=True).results + else: + subval = subval.replace('*', '') + res = self._declared_in._get_index('{}_bin'.format(self.fname), + startkey='{}~{}'.format(name, subval), + endkey='{}~{}~'.format(name, subval), # space required + max_results=100000, + return_terms=True).results + return set(map(itemgetter(1), res)) + + +class Resource(Model): + + name = Field(str) + inputs = InputsField(default=dict) + tags = TagsField(default=list) + + def connect(self, other, mappings): + my_inputs = self.inputs + other_inputs = other.inputs + for my_name, other_name in mappings.iteritems(): + other_inputs.connect(other_name, self, my_name) diff --git a/solar/solar/dblayer/sql.py b/solar/solar/dblayer/sql.py new file mode 100644 index 00000000..e35f0c48 --- /dev/null +++ b/solar/solar/dblayer/sql.py @@ -0,0 +1,386 @@ +# -*- coding: utf-8 -*- +from collections import deque +import inspect +import os +import uuid +import json +import sys +from peewee import CharField, BlobField, IntegerField, \ + ForeignKeyField, Model, BooleanField, TextField, Field, Database + +from threading import RLock + + +encoder = json.dumps +decoder = json.loads + + + +class _DataField(BlobField): + + def db_value(self, value): + return super(_DataField, self).db_value(encoder(value)) + + def python_value(self, value): + return decoder(super(_DataField, self).python_value(value)) + + +class _LinksField(_DataField): + + def db_value(self, value): + return super(_LinksField, self).db_value(list(value)) + + def python_value(self, value): + ret = super(_LinksField, self).python_value(value) + return [tuple(e) for e in ret] + + +class _SqlBucket(Model): + + def __init__(self, *args, **kwargs): + self._new = kwargs.pop('_new', False) + ed = kwargs.pop('encoded_data', None) + if ed: + self.encoded_data = ed + if 'data' not in kwargs: + kwargs['data'] = {} + super(_SqlBucket, self).__init__(*args, **kwargs) + + key = CharField(primary_key=True, null=False) + data = _DataField(null=False) + vclock = CharField(max_length=32, null=False) + links = _LinksField(null=False, default=list) + + @property + def encoded_data(self): + return self.data.get('_encoded_data') + + @encoded_data.setter + def encoded_data(self, value): + self.data['_encoded_data'] = value + + def save(self, force_insert=False, only=None): + if self._new: + force_insert = True + self._new = False + ret = super(_SqlBucket, self).save(force_insert, only) + return ret + + @property + def sql_session(self): + return self.bucket.sql_session + + +class FieldWrp(object): + + def __init__(self, name): + self.name = name + + def __get__(self, instance, owner): + return getattr(instance._sql_bucket_obj, self.name) + + def __set__(self, instance, value): + setattr(instance._sql_bucket_obj, self.name, value) + + +class _SqlIdx(Model): + name = CharField(null=False, index=True) + value = CharField(null=False, index=True) + + +class RiakObj(object): + key = FieldWrp('key') + data = FieldWrp('data') + vclock = FieldWrp('vclock') + links = FieldWrp('links') + encoded_data = FieldWrp('encoded_data') + + def __init__(self, sql_bucket_obj, new=False): + self._sql_bucket_obj = sql_bucket_obj + self.new = sql_bucket_obj._new + self.fetch_indexes() + + @property + def sql_session(self): + return self._sql_bucket_obj.sql_session + + @property + def bucket(self): + return self._sql_bucket_obj.bucket + + @property + def indexes(self): + self.fetch_indexes() + return self._indexes + + def fetch_indexes(self): + if not hasattr(self, '_indexes'): + idxes = self.bucket._sql_idx.select().where( + self.bucket._sql_idx.key == self.key) + self._indexes = set((idx.name, idx.value) for idx in idxes) + + @indexes.setter + def indexes(self, value): + assert isinstance(value, set) + self._indexes = value + + def _save_indexes(self): + # TODO: possible optimization + # update only what's needed + # don't delete all at first + q = self.bucket._sql_idx.delete().where( + self.bucket._sql_idx.key == self.key) + q.execute() + + for iname, ival in self.indexes: + idx = self.bucket._sql_idx(key=self.key, name=iname, value=ival) + idx.save() + + def add_index(self, field, value): + self.indexes.add((field, value)) + return self + + def set_index(self, field, value): + to_rem = set((x for x in self.indexes if x[0] == field)) + self.indexes.difference_update(to_rem) + return self.add_index(field, value) + + def remove_index(self, field=None, value=None): + if field is None and value is None: + # q = self.bucket._sql_idx.delete().where( + # self.bucket._sql_idx.key == self.key) + # q.execute() + self.indexes.clear() + elif field is not None and value is None: + # q = self.bucket._sql_idx.delete().where( + # (self.bucket._sql_idx.key == self.key) & + # (self.bucket._sql_idx.name == field)) + # q.execute() + to_rem = set((x for x in self.indexes if x[0] == field)) + self.indexes.difference_update(to_rem) + elif field is not None and value is not None: + # q = self.bucket._sql_idx.delete().where( + # (self.bucket._sql_idx.key == self.key) & + # (self.bucket._sql_idx.name == field) & + # (self.bucket._sql_idx.value == value)) + # q.execute() + to_rem = set((x for x in self.indexes if x[0] == field and x[1] == value)) + self.indexes.difference_update(to_rem) + return self + + def store(self, return_body=True): + self.vclock = uuid.uuid4().hex + assert self._sql_bucket_obj is not None + self._sql_bucket_obj.save() + self._save_indexes() + return self + + def delete(self): + self._sql_bucket_obj.delete() + return self + + @property + def exists(self): + return not self.new + + def get_link(self, tag): + return next(x[1] for x in self.links if x[2] == tag) + + def set_link(self, obj, tag=None): + if isinstance(obj, tuple): + newlink = obj + else: + newlink = (obj.bucket.name, obj.key, tag) + + multi = [x for x in self.links if x[0:1] == newlink[0:1]] + for item in multi: + self.links.remove(item) + + self.links.append(newlink) + return self + + def del_link(self, obj=None, tag=None): + assert obj is not None or tag is not None + if tag is not None: + links = [x for x in self.links if x[2] != tag] + else: + links = self.links + if obj is not None: + if not isinstance(obj, tuple): + obj = (obj.bucket.name, obj.key, tag) + links = [x for x in links if x[0:1] == obj[0:1]] + self.links = links + return self + + +class IndexPage(object): + + def __init__(self, index, results, return_terms, max_results, continuation): + self.max_results = max_results + self.index = index + if not return_terms: + self.results = tuple(x[0] for x in results) + else: + self.results = tuple(results) + + if not max_results or not self.results: + self.continuation = None + else: + self.continuation = str(continuation + len(self.results)) + self.return_terms = return_terms + + def __len__(self): + return len(self.results) + + def __getitem__(self, item): + return self.results[item] + + +class Bucket(object): + + def __init__(self, name, client): + self.client = client + table_name = "bucket_%s" % name.lower() + self.name = table_name + idx_table_name = 'idx_%s' % name.lower() + + class ModelMeta: + db_table = table_name + database = self.client.sql_session + + self._sql_model = type(table_name, (_SqlBucket,), + {'Meta': ModelMeta, + 'bucket': self}) + _idx_key = ForeignKeyField(self._sql_model, null=False, index=True) + + class IdxMeta: + db_table = idx_table_name + database = self.client.sql_session + + self._sql_idx = type(idx_table_name, (_SqlIdx,), + {'Meta': IdxMeta, + 'bucket': self, + 'key': _idx_key}) + + def search(self, q, rows=10, start=0, sort=''): + raise NotImplementedError() + + def create_search(self, index): + raise NotImplementedError() + + def set_property(self, name, value): + return + + def get_properties(self): + return {'search_index': False} + + def get(self, key): + try: + ret = self._sql_model.get(self._sql_model.key == key) + except self._sql_model.DoesNotExist: + ret = None + new = ret is None + if new: + ret = self._sql_model(key=key, _new=new) + return RiakObj(ret, new) + + def new(self, key, data=None, encoded_data=None, **kwargs): + if key is not None: + try: + ret = self._sql_model.get(self._sql_model.key == key) + except self._sql_model.DoesNotExist: + ret = None + new = ret is None + else: + key = uuid.uuid4().hex + new = True + if new: + ret = self._sql_model(key=key, _new=new) + ret.key = key + ret.data = data if data is not None else {} + if encoded_data: + ret.encoded_data = encoded_data + ret.links = [] + ret.vclock = "new" + return RiakObj(ret, new) + + def get_index(self, index, startkey, endkey, return_terms=None, + max_results=None, continuation=None, timeout=None, fmt=None, + term_regex=None): + if startkey > endkey: + startkey, endkey = endkey, startkey + + if index == '$key': + if return_terms: + q = self._sql_model.select( + self._sql_model.value, self._sql_model.key) + else: + q = self._sql_model.select(self._sql_model.key) + q = q.where( + self._sql_model.key >= startkey, self._sql_model.key <= endkey + ).order_by(self._sql_model.key) + else: + if return_terms: + q = self._sql_idx.select( + self._sql_idx.value, self._sql_idx.key) + else: + q = self._sql_idx.select(self._sql_idx.key) + q = q.where( + self._sql_idx.name == index, + self._sql_idx.value >= startkey, self._sql_idx.value <= endkey + ).order_by(self._sql_idx.value) + + max_results = int(max_results or 0) + continuation = int(continuation or 0) + if max_results: + q = q.limit(max_results) + if continuation: + q = q.offset(continuation) + + q = q.tuples() + + return IndexPage(index, q, return_terms, max_results, continuation) + + def multiget(self, keys): + if not keys: + return [] + else: + q = self._sql_model.select().where(self._sql_model.key << list(keys)) + print q + return map(RiakObj, list(q)) + + @property + def sql_session(self): + return self.client.sql_session + + +class SqlClient(object): + block = RLock() + + search_dir = None + + def __init__(self, *args, **kwargs): + db_class_str = kwargs.pop("db_class", 'SqliteDatabase') + try: + mod, fromlist = db_class_str.split('.') + except ValueError: + mod = 'peewee' + fromlist = db_class_str + __import__(mod, fromlist=[fromlist]) + db_class = getattr(sys.modules[mod], fromlist) + session = db_class(*args, **kwargs) + self._sql_session = session + self.buckets = {} + + def bucket(self, name): + with self.block: + if name not in self.buckets: + b = Bucket(name, self) + b._sql_model.create_table(fail_silently=True) + b._sql_idx.create_table(fail_silently=True) + self.buckets[name] = b + return self.buckets[name] + + @property + def sql_session(self): + return self._sql_session diff --git a/solar/solar/dblayer/test/conftest.py b/solar/solar/dblayer/test/conftest.py new file mode 100644 index 00000000..2545fe81 --- /dev/null +++ b/solar/solar/dblayer/test/conftest.py @@ -0,0 +1,53 @@ +from riak import RiakClient +from solar.dblayer.model import * +import pytest +import time +import string +import random + +def patched_get_bucket_name(cls): + return cls.__name__ + str(time.time()) + +class RndObj(object): + + def __init__(self, name): + self.rnd = name + ''.join((random.choice(string.ascii_lowercase) for x in xrange(8))) + self.calls = 0 + + def next(self): + num = self.calls + self.calls += 1 + return (self.rnd + str(num)) + + def __iter__(self): + return self + +@pytest.fixture(scope='function') +def rk(request): + + name = request.module.__name__ + request.function.__name__ + + obj = RndObj(name) + + return obj + +@pytest.fixture(scope='function') +def rt(request): + + name = request.module.__name__ + request.function.__name__ + + obj = RndObj(name) + + return obj + +Model.get_bucket_name = classmethod(patched_get_bucket_name) + +from solar.dblayer.sql import SqlClient +client = SqlClient(':memory:', threadlocals=False, autocommit=False) +# client = SqlClient('blah.db', threadlocals=True, +# autocommit=False, pragmas=(('journal_mode', 'WAL'), +# ('synchronous', 'NORMAL'))) +# client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) +# client = RiakClient(protocol='http', host='10.0.0.3', http_port=18098) + +ModelMeta.setup(client) diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py new file mode 100644 index 00000000..8b671574 --- /dev/null +++ b/solar/solar/dblayer/test/test_basic.py @@ -0,0 +1,76 @@ +import pytest +from solar.dblayer.model import (Field, IndexField, + clear_cache, Model, + DBLayerNotFound, DBLayerNoRiakObj) + + +class M1(Model): + + f1 = Field(str) + f2 = Field(int) + ind = IndexField(default=dict) + + +def test_from_dict(rk): + key = next(rk) + m1 = M1.from_dict(key, {'f1': 'blah', 'f2': 150}) + + m1.save() + m11 = M1.get(key) + assert m1.key == key + assert m1 is m11 + + +def test_not_exists(rk): + key = next(rk) + with pytest.raises(DBLayerNotFound): + M1.get(key) + + m1 = M1.from_dict(key, {'f1': 'blah', 'f2': 150}) + m1.save() + M1.get(key) + + +def test_update(rk): + k = next(rk) + m1 = M1.from_dict(k, {'f1': 'blah', 'f2': 150}) + m1.save() + m1.f1 = 'blub' + assert m1.f1 == 'blub' + m1.save() + assert m1.f1 == 'blub' + m11 = M1.get(k) + assert m11.f1 == 'blub' + + clear_cache() + m12 = M1.get(k) + assert m12.f1 == 'blub' + + +def test_lazy(rk): + k = next(rk) + m1 = M1.from_dict(k, {'f1': 'blah', 'f2': 150}) + m1.save() + clear_cache() + + m1 = M1(k) + with pytest.raises(DBLayerNoRiakObj): + assert m1.f1 == 'blah' + + +def test_normal_index(rk): + key = next(rk) + key2 = next(rk) + + m1 = M1.from_dict(key, {'f1': 'blah', 'f2': 150, + 'ind': {'blah': 'something'}}) + m1.save() + + m2 = M1.from_dict(key2, {'f1': 'blah', 'f2': 150, + 'ind': {'blah': 'something2'}}) + m2.save() + assert M1.ind.filter('blah=somethi*') == set([key, key2]) + assert M1.ind.filter('blah=something') == set([key]) + assert M1.ind.filter('blah=something2') == set([key2]) + + diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py new file mode 100644 index 00000000..5e33b487 --- /dev/null +++ b/solar/solar/dblayer/test/test_real.py @@ -0,0 +1,193 @@ +import pytest +import random + +from solar.dblayer.model import Model, Field, IndexField, clear_cache, check_state_for +from solar.dblayer.solar_models import Resource, DBLayerSolarException + + + +def test_changes_state(rk): + key = next(rk) + r = Resource.from_dict(key, {'name': 'a name'}) + r.inputs['a'] = 1 + with pytest.raises(Exception): + # raise exception when something is changed + val = r.inputs['a'] + r.save() + check_state_for('index', r) + + +def test_basic_input(rk): + key = next(rk) + r = Resource.from_dict(key, {'name': 'a name'}) + r.inputs['a'] = 1 + r.save() + assert r.inputs['a'] == 1 + assert len(r._riak_object.indexes) == 1 + del r.inputs['a'] + r.save() + with pytest.raises(DBLayerSolarException): + assert r.inputs['a'] == 1 + assert len(r._riak_object.indexes) == 0 + + +def test_input_in_dict(rk): + key = next(rk) + r = Resource.from_dict(key, {'name': 'a name', + 'inputs': {'input1': 15, + 'input2': None}}) + r.save() + assert r._riak_object.data['inputs']['input1'] == 15 + assert r.inputs['input1'] == 15 + + assert r._riak_object.data['inputs']['input2'] == None + assert r.inputs['input2'] == None + + +def test_basic_connect(rk): + k1 = next(rk) + k2 = next(rk) + + r1 = Resource.from_dict(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + r2 = Resource.from_dict(k2, {'name': 'second', + 'inputs': {'input1': None, + 'input2': None}}) + + r1.connect(r2, {'input1': 'input1', 'input2': 'input2'}) + r1.save() + r2.save() + + assert r1._riak_object.data['inputs']['input1'] == 10 + assert r1.inputs['input1'] == 10 + + assert r2._riak_object.data['inputs']['input1'] == None + assert r2.inputs['input1'] == 10 + + assert r1._riak_object.data['inputs']['input2'] == 15 + assert r1.inputs['input2'] == 15 + + assert r2._riak_object.data['inputs']['input2'] == None + assert r2.inputs['input2'] == 15 + + +@pytest.mark.parametrize('depth', (3, 4, 5, 10)) +def test_adv_connect(rk, depth): + k1 = next(rk) + k2 = next(rk) + + r1 = Resource.from_dict(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + prev = Resource.from_dict(k2, {'name': 'second', + 'inputs': {'input1': None, + 'input2': None, + 'input3': 0}}) + conn = {'input1': 'input1', 'input2': 'input2'} + r1.save() + r1.connect(prev, conn) + prev.save() + created = [prev] + + for x in xrange(depth - 1): + k = next(rk) + res = Resource.from_dict(k, {'name': 'next %d' % (x + 1), + 'inputs': {'input1': None, + 'input2': None, + 'input3': x + 1}}) + created.append(res) + prev.connect(res, conn) + res.save() + prev = res + + for i, c in enumerate(created): + assert c.inputs['input1'] == 10 + assert c.inputs['input2'] == 15 + assert c.inputs['input3'] == i + + +@pytest.mark.parametrize('depth', (1, 3, 5, 10, 50, 100)) +def test_perf_inputs(rk, depth): + k1 = next(rk) + r1 = Resource.from_dict(k1, {'name': 'first', + 'inputs': {'input1': 'target'}}) + + r1.save() + prev = r1 + for x in xrange(depth): + k = next(rk) + res = Resource.from_dict(k, {'name': 'next %d' % (x + 1), + 'inputs': {'input1': None}}) + prev.connect(res, {'input1': 'input1'}) + res.save() + prev = res + + import time + st = time.time() + assert res.inputs['input1'] == 'target' + end = time.time() + print end - st + + +def test_change_connect(rk): + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + + r1 = Resource.from_dict(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + r2 = Resource.from_dict(k2, {'name': 'second', + 'inputs': {'input1': None, + 'input2': None, + 'input3': 0}}) + r3 = Resource.from_dict(k3, {'name': 'first', + 'inputs': {'input1': 30, + 'input2': 35}}) + + r1.connect(r2, {'input1': 'input1', 'input2': 'input2'}) + r3.connect(r2, {'input1': 'input1'}) + + r1.save() + r2.save() + r3.save() + + assert r2.inputs['input1'] == 30 + assert r2.inputs['input2'] == 15 + + + +def test_simple_tag(rk, rt): + k1 = next(rk) + tag = next(rt) + + r1 = Resource.from_dict(k1, {'name': 'first', + 'tags': ['%s' % tag, '%s=10' % tag]}) + + r1.save() + assert list(r1.tags) == ['%s=' % tag, '%s=10' % tag] + + +def test_list_by_tag(rk, rt): + k1 = next(rk) + k2 = next(rk) + tag1 = next(rt) + tag2 = next(rt) + r1 = Resource.from_dict(k1, {'name': 'first', + 'tags': [tag1, '%s=10' % tag1]}) + r1.save() + + r2 = Resource.from_dict(k2, {'name': 'first', + 'tags': [tag1, '%s=10' % tag2]}) + r2.save() + + assert len(Resource.tags.filter(tag1)) == 2 + assert Resource.tags.filter(tag1) == set([k1, k2]) + assert len(Resource.tags.filter('other_tag')) == 0 + + assert len(Resource.tags.filter(tag2)) == 0 + assert len(Resource.tags.filter(tag2, 10)) == 1 + assert Resource.tags.filter(tag2, 10) == set([k2]) + + assert len(Resource.tags.filter(tag2, '*')) == 1 From 4a65504cf6ddf2f9500310619cbf16b4f2202ae6 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 19 Oct 2015 14:34:55 +0200 Subject: [PATCH 002/191] Better defaults and from_dict behaviour --- solar/solar/dblayer/model.py | 35 ++++++++++++++++++-------- solar/solar/dblayer/test/test_basic.py | 13 ++++++++-- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index a6ed1d6d..d9df93d7 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -132,9 +132,9 @@ class FieldBase(object): class Field(FieldBase): - def __init__(self, _type, default=NONE): + def __init__(self, _type, fname=None, default=NONE): self._type = _type - super(Field, self).__init__(fname=None, default=default) + super(Field, self).__init__(fname=fname, default=default) def __get__(self, instance, owner): if instance is None: @@ -250,11 +250,12 @@ class ModelMeta(type): def __new__(mcs, name, bases, attrs): cls = super(ModelMeta, mcs).__new__(mcs, name, bases, attrs) model_fields = set((name for (name, attr) in attrs.items() - if isinstance(attr, FieldBase))) + if isinstance(attr, FieldBase) and not name.startswith('_'))) for f in model_fields: field = getattr(cls, f) if hasattr(field, 'fname') and field.fname is None: setattr(field, 'fname', f) + setattr(field, 'gname', f) # need to set declared_in because `with_tag` # no need to wrap descriptor with another object then setattr(field, '_declared_in', cls) @@ -269,7 +270,8 @@ class ModelMeta(type): model_fields.add(given) # set the fields just in case - cls._model_fields = model_fields + cls._model_fields = [getattr(cls, x) for x in model_fields] + cls.bucket = Replacer('bucket', get_bucket, mcs) return cls @@ -370,19 +372,30 @@ class Model(object): return obj @classmethod - def from_dict(cls, key, data): + def from_dict(cls, key, data=None): + if isinstance(key, dict) and data is None: + data = key + try: + key = data['key'] + except KeyError: + raise DBLayerException("No key specified") + if key and 'key' in data and data['key'] != key: + raise DBLayerException("Different key values detected") + data['key'] = key riak_obj = cls.bucket.new(key, data={}) obj = cls.from_riakobj(riak_obj) obj._new = True for field in cls._model_fields: - val = data.get(field, NONE) - default = getattr(cls, field).default - # if val is NONE and default is NONE: - # continue + # if field is cls._pkey_field: + # continue # pkey already set + fname = field.fname + gname = field.gname + val = data.get(fname, NONE) + default = field.default if val is NONE and default is not NONE: - setattr(obj, field, default) + setattr(obj, gname, default) elif val is not NONE: - setattr(obj, field, val) + setattr(obj, gname, val) return obj @classmethod diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index 8b671574..b3482fc7 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -1,23 +1,32 @@ import pytest from solar.dblayer.model import (Field, IndexField, clear_cache, Model, - DBLayerNotFound, DBLayerNoRiakObj) + DBLayerNotFound, + DBLayerNoRiakObj, + DBLayerException) class M1(Model): f1 = Field(str) f2 = Field(int) + f3 = Field(int, fname='some_field') + ind = IndexField(default=dict) def test_from_dict(rk): key = next(rk) - m1 = M1.from_dict(key, {'f1': 'blah', 'f2': 150}) + + with pytest.raises(DBLayerException): + M1.from_dict({'f1': 'blah', 'f2': 150, 'some_field': 250}) + + m1 = M1.from_dict({'key': key, 'f1': 'blah', 'f2': 150, 'some_field': 250}) m1.save() m11 = M1.get(key) assert m1.key == key + assert m1.f3 == 250 assert m1 is m11 From 1effb84f22bd10cf438480179c412a90c676a8fc Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 19 Oct 2015 15:35:23 +0200 Subject: [PATCH 003/191] Update tests --- solar/solar/dblayer/model.py | 10 ++++++++-- solar/solar/dblayer/test/test_basic.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index d9df93d7..952129a5 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -410,9 +410,15 @@ class Model(object): obj = cls.from_riakobj(riak_object) return obj + def _reset_state(self): + self._new = False + self._modified_fields.clear() + @clears_state_for('index') def save(self, force=False): if self.changed() or force or self._new: - return self._riak_object.store() + res = self._riak_object.store() + self._reset_state() + return res else: - raise Exception("No changes") + raise DBLayerException("No changes") diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index b3482fc7..4ac46ae0 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -83,3 +83,25 @@ def test_normal_index(rk): assert M1.ind.filter('blah=something2') == set([key2]) +def test_update(rk): + key = next(rk) + + m1 = M1.from_dict(key, {'f1': 'blah', 'f2': 150}) + assert m1.changed() is True + m1.save() + + assert m1.changed() is False + with pytest.raises(DBLayerException): + m1.save() + + m1.f1 = 'updated' + assert m1.changed() is True + + m1.save() + + assert m1.f1 == 'updated' + + clear_cache() + m11 = M1.get(key) + assert m11.f1 == 'updated' + From 5f7d191a2d1501bbb310d458b37b2638594898f6 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 19 Oct 2015 19:32:42 +0200 Subject: [PATCH 004/191] Introduced nested model --- solar/solar/dblayer/model.py | 68 +++++++++++++++++++++++--- solar/solar/dblayer/test/test_basic.py | 2 - 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 952129a5..61d4cd1a 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -139,13 +139,13 @@ class Field(FieldBase): def __get__(self, instance, owner): if instance is None: return self - return instance._riak_object.data[self.fname] + return instance._data_container[self.fname] def __set__(self, instance, value): if not isinstance(value, self._type): raise Exception("Invalid type %r for %r" % (type(value), self.fname)) - instance._modified_fields.add(self.fname) - instance._riak_object.data[self.fname] = value + instance._field_changed(self) + instance._data_container[self.fname] = value def __str__(self): return "<%s:%r>" % (self.__class__.__name__, self.fname) @@ -168,7 +168,7 @@ class IndexFieldWrp(object): return "<%s for field %s>" % (self.__class__.__name__, self._field_obj) def _get_field_val(self, name): - return self._instance._riak_object.data[self.fname][name] + return self._instance._data_container[self.fname][name] def __getitem__(self, name): return self._get_field_val(name) @@ -179,7 +179,7 @@ class IndexFieldWrp(object): def __delitem__(self, name): inst = self._instance - del inst._riak_object.data[self.fname][name] + del inst._data_container[self.fname][name] indexes = inst._riak_object.indexes # TODO: move this to backend layer @@ -212,7 +212,7 @@ class IndexField(FieldBase): def __set__(self, instance, value): wrp = getattr(instance, self.fname) - instance._riak_object.data[self.fname] = self.default + instance._data_container[self.fname] = self.default for f_name, f_value in value.iteritems(): wrp[f_name] = f_value @@ -281,6 +281,55 @@ class ModelMeta(type): mcs.riak_client = riak_client +class NestedField(FieldBase): + + def __init__(self, _class, fname=None, default=NONE): + self._class = _class + super(NestedField, self).__init__(fname=fname, default=default) + + def __get__(self, instance, owner): + if instance is None: + return self + cached = getattr(instance, '_real_obj_%s' % self.fname, None) + if cached: + return cached + obj = self._class(self, instance) + setattr(instance, '_real_obj_%s' % self.fname, obj) + return obj + + def __set__(self, instance, value): + obj = getattr(instance, self.fname) + obj.from_dict(value) + + +class NestedModel(object): + + __metaclass__ = ModelMeta + + def __init__(self, field, parent): + self._field = field + self._parent = parent + + def from_dict(self, data): + for field in self._model_fields: + fname = field.fname + gname = field.gname + val = data.get(fname, NONE) + default = field.default + if val is NONE and default is not NONE: + setattr(self, gname, default) + elif val is not NONE: + setattr(self, gname, val) + return + + @property + def _data_container(self): + return self._parent._data_container + + def _field_changed(self, field): + return self._parent._modified_fields + + class Model(object): __metaclass__ = ModelMeta @@ -320,6 +369,10 @@ class Model(object): raise DBLayerException("Already have _riak_object") self._real_riak_object = value + @property + def _data_container(self): + return self._riak_object.data + @changes_state_for('index') def _set_index(self, *args, **kwargs): return self._riak_object.set_index(*args, **kwargs) @@ -349,6 +402,9 @@ class Model(object): # XXX: should be changed and more smart return cls.__name__ + def _field_changed(self, field): + self._modified_fields.add(field.fname) + def changed(self): return True if self._modified_fields else False diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index 4ac46ae0..bf001b61 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -5,7 +5,6 @@ from solar.dblayer.model import (Field, IndexField, DBLayerNoRiakObj, DBLayerException) - class M1(Model): f1 = Field(str) @@ -104,4 +103,3 @@ def test_update(rk): clear_cache() m11 = M1.get(key) assert m11.f1 == 'updated' - From 8dd4da19ab991790c7886d3b4de9612159052a05 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 19 Oct 2015 21:30:27 +0200 Subject: [PATCH 005/191] Nested improvements (required for inputs) --- solar/solar/dblayer/model.py | 71 +++++++++++++++++++++++++++-- solar/solar/dblayer/solar_models.py | 40 ++++++++++++---- 2 files changed, 97 insertions(+), 14 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 61d4cd1a..230d9137 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -283,8 +283,9 @@ class ModelMeta(type): class NestedField(FieldBase): - def __init__(self, _class, fname=None, default=NONE): + def __init__(self, _class, fname=None, default=NONE, hash_key=None): self._class = _class + self._hash_key = hash_key super(NestedField, self).__init__(fname=fname, default=default) def __get__(self, instance, owner): @@ -293,7 +294,10 @@ class NestedField(FieldBase): cached = getattr(instance, '_real_obj_%s' % self.fname, None) if cached: return cached - obj = self._class(self, instance) + if self._hash_key is not None: + obj = NestedModelHash(self, instance, self._class, self._hash_key) + else: + obj = self._class(self, instance) setattr(instance, '_real_obj_%s' % self.fname, obj) return obj @@ -301,11 +305,17 @@ class NestedField(FieldBase): obj = getattr(instance, self.fname) obj.from_dict(value) + def __delete__(self, instance): + obj = getattr(instance, self.fname) + obj.delete() + class NestedModel(object): __metaclass__ = ModelMeta + _nested_value = None + def __init__(self, field, parent): self._field = field self._parent = parent @@ -324,10 +334,60 @@ class NestedModel(object): @property def _data_container(self): - return self._parent._data_container + pdc = self._parent._data_container + try: + ppdc = pdc[self._field.fname] + except KeyError: + ppdc = pdc[self._field.fname] = {} + if self._field._hash_key is None: + return ppdc + else: + try: + ret = ppdc[self._nested_value] + except KeyError: + ret = ppdc[self._nested_value] = {} + return ret def _field_changed(self, field): - return self._parent._modified_fields + return self._parent._modified_fields.add(self._field.fname) + + def delete(self): + if self._field._hash_key is None: + del self._parent._data_container[self._field.fname] + + + +class NestedModelHash(object): + + def __init__(self, field, parent, _class, hash_key): + self._field = field + self._parent = parent + self._class = _class + self._hash_key = hash_key + self._cache = {} + + def __getitem__(self, name): + try: + return self._cache[name] + except KeyError: + obj = self._class(self._field, self._parent) + obj._nested_value = name + self._cache[name] = obj + return obj + + def __setitem__(self, name, value): + obj = self[name] + return obj.from_dict(value) + + def __delitem__(self, name): + obj = self[name] + obj.delete() + del self._cache[name] + + def from_dict(self, data): + hk = data[self._hash_key] + self[hk] = data + class Model(object): @@ -478,3 +538,6 @@ class Model(object): return res else: raise DBLayerException("No changes") + + def delete(self): + raise NotImplementedError() diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index e07315a0..fd1a6c15 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -114,22 +114,24 @@ class InputsFieldWrp(IndexFieldWrp): return res def _get_raw_field_val(self, name): - return self._instance._riak_object.data[self.fname][name] + return self._instance._data_container[self.fname][name] def __getitem__(self, name): return self._get_field_val(name) def __delitem__(self, name): self._has_own_input(name) + self._instance._field_changed(self) try: del self._cache[name] except KeyError: pass inst = self._instance inst._riak_object.remove_index('%s_bin' % self.fname, '{}|{}'.format(self._instance.key, name)) - del inst._riak_object.data[self.fname][name] + del inst._data_container[self.fname][name] def __setitem__(self, name, value): + self._instance._field_changed(self) return self._set_field_value(name, value) def _set_field_value(self, name, value): @@ -162,7 +164,7 @@ class InputsField(IndexField): def __set__(self, instance, value): wrp = getattr(instance, self.fname) - instance._riak_object.data[self.fname] = self.default + instance._data_container[self.fname] = self.default for inp_name, inp_value in value.iteritems(): wrp[inp_name] = inp_value @@ -179,7 +181,7 @@ class TagsFieldWrp(IndexFieldWrp): raise TypeError('You cannot set tags like this') def __iter__(self): - return iter(self._instance._riak_object.data[self.fname]) + return iter(self._instance._data_container[self.fname]) def set(self, name, value=None): if '=' in name and value is None: @@ -191,9 +193,9 @@ class TagsFieldWrp(IndexFieldWrp): inst._add_index('{}_bin'.format(self.fname), '{}~{}'.format(name, value)) try: - fld = inst._riak_object.data[self.fname] + fld = inst._data_container[self.fname] except IndexError: - fld = inst._riak_object.data[self.fname] = [] + fld = inst._data_container[self.fname] = [] full_value = '{}={}'.format(name, value) try: fld.append(full_value) @@ -202,7 +204,7 @@ class TagsFieldWrp(IndexFieldWrp): return True def has_tag(self, name, subval=None): - fld = self._instance._riak_object.data[self.fname] + fld = self._instance._data_container[self.fname] if not name in fld: return False if subval is not None: @@ -216,7 +218,7 @@ class TagsFieldWrp(IndexFieldWrp): if value is None: value = '' inst = self._instance - fld = inst._riak_object.data[self.fname] + fld = inst._data_container[self.fname] full_value = '{}={}'.format(name, value) try: vals = fld.remove(full_value) @@ -228,13 +230,12 @@ class TagsFieldWrp(IndexFieldWrp): - class TagsField(IndexField): _wrp_class = TagsFieldWrp def __set__(self, instance, value): wrp = getattr(instance, self.fname) - instance._riak_object.data[self.fname] = self.default + instance._data_container[self.fname] = self.default for val in value: wrp.set(val) @@ -264,9 +265,28 @@ class TagsField(IndexField): return set(map(itemgetter(1), res)) +# class MetaInput(NestedModel): + +# name = Field(str) +# schema = Field(str) +# value = None # TODO: implement it +# is_list = Field(bool) +# is_hash = Field(bool) + + class Resource(Model): name = Field(str) + + version = Field(str) + base_name = Field(str) + base_path = Field(str) + actions_path = Field(str) + handler = Field(str) + puppet_module = Field(str) # remove + meta_inputs = Field(dict, default=dict) + state = Field(str) # on_set/on_get would be useful + inputs = InputsField(default=dict) tags = TagsField(default=list) From a9d2d471407a1348ca71990af11a23fcf64e0e65 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 21 Oct 2015 14:52:15 +0200 Subject: [PATCH 006/191] Added session_start/session_end + added it to conftest --- solar/solar/dblayer/model.py | 2 ++ solar/solar/dblayer/riak_client.py | 13 +++++++++++++ solar/solar/dblayer/{sql.py => sql_client.py} | 19 ++++++++++++++++++- solar/solar/dblayer/test/conftest.py | 19 +++++++++++++++---- 4 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 solar/solar/dblayer/riak_client.py rename solar/solar/dblayer/{sql.py => sql_client.py} (96%) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 230d9137..ddea60e2 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -279,6 +279,8 @@ class ModelMeta(type): @classmethod def setup(mcs, riak_client): mcs.riak_client = riak_client + mcs.session_start = riak_client.session_start + mcs.session_end = riak_client.session_end class NestedField(FieldBase): diff --git a/solar/solar/dblayer/riak_client.py b/solar/solar/dblayer/riak_client.py new file mode 100644 index 00000000..6d3f4231 --- /dev/null +++ b/solar/solar/dblayer/riak_client.py @@ -0,0 +1,13 @@ +from riak import RiakClient as OrigRiakClient + +from solar.dblayer.model import clear_cache + + +class RiakClient(OrigRiakClient): + + def session_start(self): + clear_cache() + + def session_end(self, result=True): + # ignore result + clear_cache() diff --git a/solar/solar/dblayer/sql.py b/solar/solar/dblayer/sql_client.py similarity index 96% rename from solar/solar/dblayer/sql.py rename to solar/solar/dblayer/sql_client.py index e35f0c48..f5230f83 100644 --- a/solar/solar/dblayer/sql.py +++ b/solar/solar/dblayer/sql_client.py @@ -3,14 +3,18 @@ from collections import deque import inspect import os import uuid -import json import sys + from peewee import CharField, BlobField, IntegerField, \ ForeignKeyField, Model, BooleanField, TextField, Field, Database +from solar.dblayer.model import clear_cache from threading import RLock +# msgpack is way faster but less readable +# using json for easier debug +import json encoder = json.dumps decoder = json.loads @@ -384,3 +388,16 @@ class SqlClient(object): @property def sql_session(self): return self._sql_session + + def session_start(self): + clear_cache() + sess = self._sql_session + sess.begin() + + def session_end(self, result=True): + sess = self._sql_session + if result: + sess.commit() + else: + sess.rollback() + clear_cache() diff --git a/solar/solar/dblayer/test/conftest.py b/solar/solar/dblayer/test/conftest.py index 2545fe81..1c03e089 100644 --- a/solar/solar/dblayer/test/conftest.py +++ b/solar/solar/dblayer/test/conftest.py @@ -1,5 +1,4 @@ -from riak import RiakClient -from solar.dblayer.model import * +from solar.dblayer.model import Model, ModelMeta import pytest import time import string @@ -40,13 +39,25 @@ def rt(request): return obj + + +def pytest_runtest_teardown(item, nextitem): + ModelMeta.session_end(result=True) + return nextitem + +def pytest_runtest_call(item): + ModelMeta.session_start() + + Model.get_bucket_name = classmethod(patched_get_bucket_name) -from solar.dblayer.sql import SqlClient +from solar.dblayer.sql_client import SqlClient client = SqlClient(':memory:', threadlocals=False, autocommit=False) -# client = SqlClient('blah.db', threadlocals=True, +# client = SqlClient('/tmp/blah.db', threadlocals=True, # autocommit=False, pragmas=(('journal_mode', 'WAL'), # ('synchronous', 'NORMAL'))) + +# from solar.dblayer.riak_client import RiakClient # client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) # client = RiakClient(protocol='http', host='10.0.0.3', http_port=18098) From 8fa54a7dfbd98ef727cf4fe54315d717e8b0cb3c Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 26 Oct 2015 17:12:02 +0100 Subject: [PATCH 007/191] Fix typo in IndexField --- solar/solar/dblayer/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index ddea60e2..c163eb5b 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -175,7 +175,7 @@ class IndexFieldWrp(object): def __setitem__(self, name, value): inst = self._instance - inst._riak_object.set_index('%s_bin' % self.fname, '{}|{}'.format(name, value)) + inst._add_index('%s_bin' % self.fname, '{}|{}'.format(name, value)) def __delitem__(self, name): inst = self._instance From 0a06d06a1ab17fde9ae04d4713fa7f148afbe229 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 26 Oct 2015 18:44:42 +0100 Subject: [PATCH 008/191] Added IndexedField --- solar/solar/dblayer/model.py | 146 ++++++++++++++++++++++++++++++++++- 1 file changed, 143 insertions(+), 3 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index c163eb5b..456b6845 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -1,9 +1,9 @@ from threading import local, current_thread from random import getrandbits import uuid -from functools import wraps +from functools import wraps, total_ordering from operator import itemgetter - +import time LOCAL = local() @@ -89,6 +89,112 @@ def check_state_for(_type, obj): raise Exception("Dirty state, save all objects first") +@total_ordering +class StrInt(object): + + precision = 3 + positive_char = 'p' + negative_char = 'n' + format_size = 10 + precision + + + def __init__(self, val=None): + self._val = self._make_val(val) + + def __str__(self): + return self._val.__str__() + + def __repr__(self): + return "<%s: %r>" % (self.__class__.__name__, self._val) + + @classmethod + def p_max(cls): + return cls(int('9' * cls.format_size)) + + @classmethod + def p_min(cls): + return cls(1) + + @classmethod + def n_max(cls): + return -cls.p_max() + + @classmethod + def n_min(cls): + return -cls.p_min() + + def __neg__(self): + time_ = self.int_val() + ret = self.__class__(-time_) + return ret + + @classmethod + def to_hex(cls, value): + char = cls.positive_char + if value < 0: + value = int('9' * cls.format_size) + value + char = cls.negative_char + f = '%s%%.%dx' % (char, cls.format_size - 2) + value = f % value + if value[-1] == 'L': + value = value[:-1] + return value + + @classmethod + def from_hex(cls, value): + v = int(value[1:], 16) + if value[0] == cls.negative_char: + v -= int('9' * self.format_size) + return v + + def int_val(self): + return self.from_hex(self._val) + + @classmethod + def from_simple(cls, value): + return cls(value) + + @classmethod + def to_simple(cls, value): + return value._val + + @classmethod + def _make_val(cls, val): + if val is None: + val = time.time() + if isinstance(val, (long, int, float)): + if isinstance(val, float): + val = int(val * (10 ** cls.precision)) + val = cls.to_hex(val) + elif isinstance(val, cls): + val = val._val + return val + + def __eq__(self, other): + if not isinstance(other, self.__class__): + raise Exception("Cannot compare %r with %r" % (self, other)) + so = other._val[0] + ss = self._val[0] + son = so == other.negative_char + ssn = so == self.negative_char + if son != ssn: + return False + return self._val[1:] == other._val[1:] + + def __gt__(self, other): + if not isinstance(other, self.__class__): + raise Exception("Cannot compare %r with %r" % (self, other)) + so = other._val[0] + ss = self._val[0] + if ss == self.positive_char and so == other.negative_char: + return -1 + elif ss == self.negative_char and so == other.positive_char: + return 1 + else: + return other._val[1:] < self._val[1:] + + + class Replacer(object): def __init__(self, name, fget, *args): @@ -132,6 +238,8 @@ class FieldBase(object): class Field(FieldBase): + _simple_types = {int, float, long, str, unicode, basestring, list, tuple, dict} + def __init__(self, _type, fname=None, default=NONE): self._type = _type super(Field, self).__init__(fname=fname, default=default) @@ -139,13 +247,20 @@ class Field(FieldBase): def __get__(self, instance, owner): if instance is None: return self - return instance._data_container[self.fname] + val = instance._data_container[self.fname] + if self._type in self._simple_types: + return val + else: + return self._type.from_simple(val) def __set__(self, instance, value): if not isinstance(value, self._type): raise Exception("Invalid type %r for %r" % (type(value), self.fname)) + if self._type not in self._simple_types: + value = self._type.to_simple(value) instance._field_changed(self) instance._data_container[self.fname] = value + return value def __str__(self): return "<%s:%r>" % (self.__class__.__name__, self.fname) @@ -153,6 +268,31 @@ class Field(FieldBase): __repr__ = __str__ +class IndexedField(Field): + + def __set__(self, instance, value): + value = super(IndexedField, self).__set__(instance, value) + instance._set_index('{}_bin'.format(self.fname), value) + return value + + def _filter(self, startkey, endkey=None, **kwargs): + if isinstance(startkey, self._type) and self._type not in self._simple_types: + startkey = self._type.to_simple(startkey) + if isinstance(endkey, self._type) and self._type not in self._simple_types: + endkey = self._type.to_simple(endkey) + kwargs.setdefault('max_results', 1000000) + res = self._declared_in._get_index('{}_bin'.format(self.fname), + startkey=startkey, + endkey=endkey, + **kwargs).results + return res + + def filter(self, *args, **kwargs): + kwargs['return_terms'] = False + res = self._filter(*args, **kwargs) + return set(res) + + class IndexFieldWrp(object): def __init__(self, field_obj, instance): From 1e289032e291aee878c34acfa03a9af932bc2590 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 26 Oct 2015 18:47:26 +0100 Subject: [PATCH 009/191] Added updated field to Resource (using IndexedField) --- solar/solar/dblayer/solar_models.py | 11 ++++++++++- solar/solar/dblayer/test/test_real.py | 21 +++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index fd1a6c15..ab3f7819 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -1,7 +1,9 @@ from solar.dblayer.model import (Model, Field, IndexField, IndexFieldWrp, DBLayerException, - requires_clean_state, check_state_for) + requires_clean_state, check_state_for, + StrInt, + IndexedField) from operator import itemgetter @@ -290,8 +292,15 @@ class Resource(Model): inputs = InputsField(default=dict) tags = TagsField(default=list) + updated = IndexedField(StrInt) + def connect(self, other, mappings): my_inputs = self.inputs other_inputs = other.inputs for my_name, other_name in mappings.iteritems(): other_inputs.connect(other_name, self, my_name) + + def save(self, *args, **kwargs): + if self.changed(): + self.updated = StrInt() + return super(Resource, self).save(*args, **kwargs) diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 5e33b487..b73db052 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -1,11 +1,12 @@ import pytest import random -from solar.dblayer.model import Model, Field, IndexField, clear_cache, check_state_for +from solar.dblayer.model import Model, Field, IndexField, clear_cache, check_state_for, StrInt from solar.dblayer.solar_models import Resource, DBLayerSolarException + def test_changes_state(rk): key = next(rk) r = Resource.from_dict(key, {'name': 'a name'}) @@ -23,12 +24,12 @@ def test_basic_input(rk): r.inputs['a'] = 1 r.save() assert r.inputs['a'] == 1 - assert len(r._riak_object.indexes) == 1 + assert len(r._riak_object.indexes) == 2 del r.inputs['a'] r.save() with pytest.raises(DBLayerSolarException): assert r.inputs['a'] == 1 - assert len(r._riak_object.indexes) == 0 + assert len(r._riak_object.indexes) == 1 def test_input_in_dict(rk): @@ -72,7 +73,7 @@ def test_basic_connect(rk): assert r2.inputs['input2'] == 15 -@pytest.mark.parametrize('depth', (3, 4, 5, 10)) +@pytest.mark.parametrize('depth', (3, 4, 5, 10, 25, 50)) def test_adv_connect(rk, depth): k1 = next(rk) k2 = next(rk) @@ -191,3 +192,15 @@ def test_list_by_tag(rk, rt): assert Resource.tags.filter(tag2, 10) == set([k2]) assert len(Resource.tags.filter(tag2, '*')) == 1 + + +def test_updated_behaviour(rk): + k1 = next(rk) + + _cmp = StrInt() + r1 = Resource.from_dict(k1, {'name': 'blah'}) + r1.save() + assert isinstance(r1._riak_object.data['updated'], basestring) + assert not isinstance(r1.updated, basestring) + assert r1.updated >= _cmp + assert k1 in Resource.updated.filter(StrInt.p_min(), StrInt.p_max()) From 459529984192dfe47ed695151c9fa95efb517273 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 26 Oct 2015 18:50:30 +0100 Subject: [PATCH 010/191] Added some optimization comments --- solar/solar/dblayer/solar_models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index ab3f7819..38f53bbe 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -72,6 +72,8 @@ class InputsFieldWrp(IndexFieldWrp): pass my_name = self._instance.key bucket = self._instance._bucket + # XXX: possible optmization + # could fetch all my inputs and cache it _input = bucket.get_index('%s_bin' % self.fname, startkey='{}|{}'.format(my_name, name), endkey='{}|{}~'.format(my_name, name), From 4f60b6e0b3f8455432ba2a6c4f2f97414f128773 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 27 Oct 2015 09:42:03 +0100 Subject: [PATCH 011/191] Use instance._get_index everywhere in solar_models --- solar/solar/dblayer/solar_models.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 38f53bbe..38b41334 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -71,14 +71,13 @@ class InputsFieldWrp(IndexFieldWrp): except KeyError: pass my_name = self._instance.key - bucket = self._instance._bucket # XXX: possible optmization # could fetch all my inputs and cache it - _input = bucket.get_index('%s_bin' % self.fname, - startkey='{}|{}'.format(my_name, name), - endkey='{}|{}~'.format(my_name, name), - max_results=1, - return_terms=True).results + _input = self._instance._get_index('%s_bin' % self.fname, + startkey='{}|{}'.format(my_name, name), + endkey='{}|{}~'.format(my_name, name), + max_results=1, + return_terms=True).results if not _input : raise DBLayerSolarException('No input {} for {}'.format(name, my_name)) if not _input[0][0].startswith('{}|{}'.format(my_name, name)): @@ -95,17 +94,16 @@ class InputsFieldWrp(IndexFieldWrp): pass fname = self.fname my_name = self._instance.key - bucket = self._instance._bucket # TODO: _has_own_input is slow, check if its really needed self._has_own_input(name) ind_name = '{}_recv_bin'.format(fname) # XXX: possible optimization # get all values for resource and cache it (use dirty to check) - recvs = bucket.get_index(ind_name, - startkey='{}|{}|'.format(my_name, name), - endkey='{}|{}|~'.format(my_name, name), - max_results=1, - return_terms=True).results + recvs = self._instance._get_index(ind_name, + startkey='{}|{}|'.format(my_name, name), + endkey='{}|{}|~'.format(my_name, name), + max_results=1, + return_terms=True).results if not recvs: _res = self._get_raw_field_val(name) self._cache[name] = _res @@ -142,8 +140,7 @@ class InputsFieldWrp(IndexFieldWrp): fname = self.fname my_name = self._instance.key ind_name = '{}_recv_bin'.format(fname) - bucket = self._instance._bucket - recvs = bucket.get_index(ind_name, + recvs = self._instance._get_index(ind_name, startkey='{}|{}|'.format(my_name, name), endkey='{}|{}|~'.format(my_name,name), max_results=1, From 65d2af1aaa10ae7fe740a624539d58e85125d826 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 27 Oct 2015 13:03:52 +0100 Subject: [PATCH 012/191] Added test_nested file --- solar/solar/dblayer/test/test_nested.py | 66 +++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 solar/solar/dblayer/test/test_nested.py diff --git a/solar/solar/dblayer/test/test_nested.py b/solar/solar/dblayer/test/test_nested.py new file mode 100644 index 00000000..95ae3528 --- /dev/null +++ b/solar/solar/dblayer/test/test_nested.py @@ -0,0 +1,66 @@ +import pytest +from solar.dblayer.model import (Field, IndexField, + clear_cache, Model, + NestedField, + NestedModel, + DBLayerNotFound, + DBLayerNoRiakObj, + DBLayerException) + + +class N1(NestedModel): + + f_nested1 = Field(str) + f_nested2 = Field(int, default=150) + + + +class M1(Model): + + f1 = Field(str) + f2 = NestedField(N1) + f3 = NestedField(N1, hash_key='f_nested1') + + + +def test_nested_simple(rk): + + key = next(rk) + + m1 = M1.from_dict(key, {'f1': 'blah', + 'f2': {'f_nested1': 'foo'}}) + + assert m1.f2.f_nested1 == 'foo' + assert m1.f2.f_nested2 == 150 + assert m1._modified_fields == set(['f1', 'f2']) + assert m1._data_container == {'f1': 'blah', 'f2': {'f_nested1': 'foo', 'f_nested2': 150}} + del m1.f2 + assert m1._data_container == {'f1': 'blah'} + + +def test_nested(rk): + key = next(rk) + + m1 = M1.from_dict(key, {'f1': 'blah', + 'f2': {'f_nested1': 'foo'}, + 'f3': {'f_nested1': 'foo', 'f_nested2': 150}}) + + assert m1.f2.f_nested1 == 'foo' + assert m1.f2.f_nested2 == 150 + assert m1.f3['foo'].f_nested2 == 150 + + m1.f3['blah'].f_nested2 = 250 + + assert m1.f3['foo'].f_nested2 == 150 + assert m1.f3['blah'].f_nested2 == 250 + assert m1._modified_fields == set(['f1', 'f2', 'f3']) + + exp = {'f1': 'blah', 'f2': {'f_nested1': 'foo', 'f_nested2': 150}, 'f3': {'blah': {'f_nested2': 250}, 'foo': {'f_nested1': 'foo', 'f_nested2': 150}}} + assert m1._data_container == exp + + del m1.f2 + exp.pop('f2') + assert m1._data_container == exp + + assert m1._modified_fields == set(['f1', 'f2', 'f3']) + From 472cc38500437040bdc29818bd44fa97c6726389 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 27 Oct 2015 14:36:28 +0100 Subject: [PATCH 013/191] has_own_input speedup --- solar/solar/dblayer/solar_models.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 38b41334..df0d4d9e 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -71,19 +71,12 @@ class InputsFieldWrp(IndexFieldWrp): except KeyError: pass my_name = self._instance.key - # XXX: possible optmization - # could fetch all my inputs and cache it - _input = self._instance._get_index('%s_bin' % self.fname, - startkey='{}|{}'.format(my_name, name), - endkey='{}|{}~'.format(my_name, name), - max_results=1, - return_terms=True).results - if not _input : + try: + self._get_raw_field_val(name) + except KeyError: raise DBLayerSolarException('No input {} for {}'.format(name, my_name)) - if not _input[0][0].startswith('{}|{}'.format(my_name, name)): - # double check for ranges - raise DBLayerSolarException('No input {} for {}'.format(name, my_name)) - return True + else: + return True def _get_field_val(self, name): # maybe it should be tco @@ -94,7 +87,6 @@ class InputsFieldWrp(IndexFieldWrp): pass fname = self.fname my_name = self._instance.key - # TODO: _has_own_input is slow, check if its really needed self._has_own_input(name) ind_name = '{}_recv_bin'.format(fname) # XXX: possible optimization From 9280f2c4c8e3fd412042d03afccb7588bd7db9d3 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 27 Oct 2015 14:37:57 +0100 Subject: [PATCH 014/191] Added actions to resource model --- solar/solar/dblayer/solar_models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index df0d4d9e..52af28e6 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -275,6 +275,7 @@ class Resource(Model): base_name = Field(str) base_path = Field(str) actions_path = Field(str) + actions = Field(dict) handler = Field(str) puppet_module = Field(str) # remove meta_inputs = Field(dict, default=dict) From b55568986b8747bc95c329418b2983f2e8cd6c5b Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 27 Oct 2015 16:08:20 +0100 Subject: [PATCH 015/191] Added to_dict to model --- solar/solar/dblayer/model.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 456b6845..b1850ebc 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -571,6 +571,7 @@ class Model(object): raise DBLayerException("Already have _riak_object") self._real_riak_object = value + @property def _data_container(self): return self._riak_object.data @@ -610,6 +611,9 @@ class Model(object): def changed(self): return True if self._modified_fields else False + def to_dict(self): + return dict(self._riak_object.data) + def __str__(self): if self._riak_object is None: return "<%s not initialized>" % (self.__class__.__name__) From 83a5a3d1d728055c82337c38479af04a2f5fdd87 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 27 Oct 2015 17:48:42 +0100 Subject: [PATCH 016/191] resource.py updates --- solar/solar/core/resource/resource.py | 68 +++++++++++++-------------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 0ebdd8d8..52ffbd10 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -30,6 +30,8 @@ from uuid import uuid4 from hashlib import md5 +from solar.dblayer.solar_models import Resource + def read_meta(base_path): base_meta_file = os.path.join(base_path, 'meta.yaml') @@ -74,26 +76,25 @@ class Resource(object): self.auto_extend_inputs(inputs) - self.db_obj = orm.DBResource(**{ - 'id': name, - 'name': name, - 'actions_path': metadata.get('actions_path', ''), - 'actions': metadata.get('actions', ''), - 'base_name': metadata.get('base_name', ''), - 'base_path': metadata.get('base_path', ''), - 'handler': metadata.get('handler', ''), - 'puppet_module': metadata.get('puppet_module', ''), - 'version': metadata.get('version', ''), - 'meta_inputs': inputs, - 'tags': tags - + self.db_obj = Resource.from_dict(name, + 'id': name, + 'name': name, + 'actions_path': metadata.get('actions_path', ''), + 'actions': metadata.get('actions', ''), + 'base_name': metadata.get('base_name', ''), + 'base_path': metadata.get('base_path', ''), + 'handler': metadata.get('handler', ''), + 'puppet_module': metadata.get('puppet_module', ''), + 'version': metadata.get('version', ''), + 'meta_inputs': inputs, + 'tags': tags, + 'stae': RESOURCE_STATE.created.name }) - self.db_obj.state = RESOURCE_STATE.created.name - self.db_obj.tags = tags or [] - self.db_obj.save() self.create_inputs(args) + self.db_obj.save() + # Load @dispatch(orm.DBResource) def __init__(self, resource_db): @@ -147,15 +148,15 @@ class Resource(object): args = args or {} for name, v in self.db_obj.meta_inputs.items(): value = args.get(name, v.get('value')) - - self.db_obj.add_input(name, v['schema'], value) + self.db_obj.inputs[name] = value @property def args(self): - ret = {} - for i in self.resource_inputs().values(): - ret[i.name] = i.backtrack_value() - return ret + return self.db_obj.inputs + # ret = {} + # for i in self.resource_inputs().values(): + # ret[i.name] = i.backtrack_value() + # return ret def update(self, args): # TODO: disconnect input when it is updated and end_node @@ -224,18 +225,10 @@ class Resource(object): return rst def resource_inputs(self): - return { - i.name: i for i in self.db_obj.inputs.as_set() - } + return self.db_obj.inputs def to_dict(self): ret = self.db_obj.to_dict() - ret['input'] = {} - for k, v in self.args.items(): - ret['input'][k] = { - 'value': v, - } - return ret def color_repr(self): @@ -258,11 +251,14 @@ class Resource(object): def connect_with_events(self, receiver, mapping=None, events=None, use_defaults=False): - signals.connect(self, receiver, mapping=mapping) - if use_defaults: - api.add_default_events(self, receiver) - if events: - api.add_events(self.name, events) + self.db_obj.connect(receiver, mappings=mappings) + # signals.connect(self, receiver, mapping=mapping) + # TODO: implement events + return + # if use_defaults: + # api.add_default_events(self, receiver) + # if events: + # api.add_events(self.name, events) def connect(self, receiver, mapping=None, events=None): return self.connect_with_events( From ab49e199ad618677db45e7a32af5384052cfb121 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 27 Oct 2015 17:59:11 +0100 Subject: [PATCH 017/191] load + corrected imports --- solar/solar/core/resource/resource.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 52ffbd10..e2f4d604 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -30,7 +30,7 @@ from uuid import uuid4 from hashlib import md5 -from solar.dblayer.solar_models import Resource +from solar.dblayer.solar_models import Resource as DBResource def read_meta(base_path): @@ -76,7 +76,7 @@ class Resource(object): self.auto_extend_inputs(inputs) - self.db_obj = Resource.from_dict(name, + self.db_obj = DBResource.from_dict(name, 'id': name, 'name': name, 'actions_path': metadata.get('actions_path', ''), @@ -96,7 +96,7 @@ class Resource(object): self.db_obj.save() # Load - @dispatch(orm.DBResource) + @dispatch(DBResource) def __init__(self, resource_db): self.db_obj = resource_db self.name = resource_db.name @@ -267,7 +267,7 @@ class Resource(object): def load(name): - r = orm.DBResource.load(name) + r = DBResource.get(name) if not r: raise Exception('Resource {} does not exist in DB'.format(name)) From 60a4a088a25a423862a4ac11809963c5baf8f6f3 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 00:45:28 +0100 Subject: [PATCH 018/191] Wrapped json.dumps in sqlclient --- solar/solar/dblayer/sql_client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/solar/solar/dblayer/sql_client.py b/solar/solar/dblayer/sql_client.py index f5230f83..5a95a5ce 100644 --- a/solar/solar/dblayer/sql_client.py +++ b/solar/solar/dblayer/sql_client.py @@ -16,7 +16,13 @@ from threading import RLock # using json for easier debug import json encoder = json.dumps -decoder = json.loads + +def wrapped_loads(data, *args, **kwargs): + if not isinstance(data, basestring): + data = str(data) + return json.loads(data, *args, **kwargs) + +decoder = wrapped_loads From 37c28ec36e1c43f74bf8abe518154a44add304e6 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 00:45:28 +0100 Subject: [PATCH 019/191] Wrapped json.dumps in sqlclient --- solar/solar/dblayer/sql_client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/solar/solar/dblayer/sql_client.py b/solar/solar/dblayer/sql_client.py index f5230f83..5a95a5ce 100644 --- a/solar/solar/dblayer/sql_client.py +++ b/solar/solar/dblayer/sql_client.py @@ -16,7 +16,13 @@ from threading import RLock # using json for easier debug import json encoder = json.dumps -decoder = json.loads + +def wrapped_loads(data, *args, **kwargs): + if not isinstance(data, basestring): + data = str(data) + return json.loads(data, *args, **kwargs) + +decoder = wrapped_loads From 59cbb3988399bfe07991563bde7c2699e37accab Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 00:46:19 +0100 Subject: [PATCH 020/191] Added save_lazy + changed how cache works --- solar/solar/dblayer/model.py | 92 +++++++++++++++++++++++++++++------- 1 file changed, 75 insertions(+), 17 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 456b6845..37499b4d 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -23,19 +23,44 @@ class NONE: """A None like type""" pass -def get_cache(instance, _): - th = current_thread() - l = LOCAL - try: - cache_id = l.cache_id - except AttributeError: - cache_id = uuid.UUID(int=getrandbits(128), version=4).hex - setattr(th, 'cache_id', cache_id) - if getattr(th, 'cache_id', None) == cache_id: - setattr(l, 'obj_cache', {}) - setattr(l, 'db_ch_state', {}) - l.db_ch_state.setdefault('index', set()) - return l + +class SingleClassCache(object): + + __slots__ = ['obj_cache', 'db_ch_state', 'lazy_save', 'origin_class'] + + def __init__(self, origin_class): + self.obj_cache = {} + self.db_ch_state = {'index': set()} + self.lazy_save = set() + self.origin_class = origin_class + + +class ClassCache(object): + + def __get__(self, _, owner): + th = current_thread() + l = LOCAL + # better don't duplicate class names + cache_name = owner.__name__ + try: + cache_id = l.cache_id + except AttributeError: + cache_id = uuid.UUID(int=getrandbits(128), version=4).hex + setattr(l, 'cache_id', cache_id) + if getattr(th, 'cache_id', None) != cache_id: + # new cache + setattr(th, 'cache_id', cache_id) + c = SingleClassCache(owner) + setattr(l, '_model_caches', {}) + l._model_caches[cache_name] = c + try: + # already had this owner in cache + return l._model_caches[cache_name] + except KeyError: + # old cache but first time this owner + c = SingleClassCache(owner) + l._model_caches[cache_name] = c + return c def clear_cache(): @@ -387,6 +412,8 @@ class IndexField(FieldBase): class ModelMeta(type): + _defined_models = set() + def __new__(mcs, name, bases, attrs): cls = super(ModelMeta, mcs).__new__(mcs, name, bases, attrs) model_fields = set((name for (name, attr) in attrs.items() @@ -409,18 +436,38 @@ class ModelMeta(type): for given in base._model_fields: model_fields.add(given) - # set the fields just in case + cls._model_fields = [getattr(cls, x) for x in model_fields] + if bases == (object, ): + return cls + cls.bucket = Replacer('bucket', get_bucket, mcs) + mcs._defined_models.add(cls) return cls @classmethod def setup(mcs, riak_client): mcs.riak_client = riak_client - mcs.session_start = riak_client.session_start - mcs.session_end = riak_client.session_end + + @classmethod + def session_end(mcs, result=True): + for cls in mcs._defined_models: + for to_save in cls._c.lazy_save: + try: + to_save.save() + except DBLayerException: + continue + else: + print 'saved', to_save + cls._c.lazy_save.clear() + mcs.riak_client.session_end(result) + + @classmethod + def session_start(mcs): + clear_cache() + mcs.riak_client.session_start() class NestedField(FieldBase): @@ -536,7 +583,7 @@ class Model(object): __metaclass__ = ModelMeta - _c = Replacer('_c', get_cache) + _c = ClassCache() _key = None _new = None @@ -656,6 +703,9 @@ class Model(object): setattr(obj, gname, val) return obj + def __hash__(self): + return hash(self.key) + @classmethod def get(cls, key): try: @@ -681,5 +731,13 @@ class Model(object): else: raise DBLayerException("No changes") + def save_lazy(self): + self._c.lazy_save.add(self) + def delete(self): + ls = self._c.lazy_save + try: + ls.remove(self.key) + except KeyError: + pass raise NotImplementedError() From 4914b2f40bf438ba2b75be824b8d13afcd99146e Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 00:46:34 +0100 Subject: [PATCH 021/191] Improved tests --- solar/solar/dblayer/test/test_basic.py | 71 ++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index bf001b61..f256887f 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -14,6 +14,19 @@ class M1(Model): ind = IndexField(default=dict) +class M2(Model): + f1 = Field(str) + + ind = IndexField(default=dict) + + +class M3(Model): + f1 = Field(str) + + ind = IndexField(default=dict) + + + def test_from_dict(rk): key = next(rk) @@ -103,3 +116,61 @@ def test_update(rk): clear_cache() m11 = M1.get(key) assert m11.f1 == 'updated' + + +def test_different_models(rk): + key = next(rk) + + m2 = M2.from_dict(key, {'f1': 'm2', 'ind': {'blah': 'blub'}}) + m3 = M3.from_dict(key, {'f1': 'm3', 'ind': {'blah': 'blub'}}) + + m2.save() + m3.save() + + assert M2.get(key).f1 == 'm2' + assert M3.get(key).f1 == 'm3' + + +def test_cache_behaviour(rk): + key1 = next(rk) + + m1 = M1.from_dict(key1, {'f1': 'm1'}) + + m11 = M1.get(key1) + assert m1 is m11 + m1.save() + assert m1 is m11 + + m12 = M1.get(key1) + assert m1 is m12 + + clear_cache() + m13 = M1.get(key1) + assert m1 is not m13 + + +def test_save_lazy(rk): + key1 = next(rk) + key2 = next(rk) + + m1 = M1.from_dict(key1, {'f1': 'm1'}) + m2 = M1.from_dict(key2, {'f1': 'm2'}) + m1.save_lazy() + m2.save_lazy() + + m1g = M1.get(key1) + m2g = M1.get(key2) + + assert m1 is m1g + assert m2 is m2g + + assert M1._c.lazy_save == {m1, m2} + M1.session_end() + assert M1._c.lazy_save == set() + + clear_cache() + m1g2 = M1.get(key1) + m2g2 = M1.get(key2) + + assert m1g is not m1g2 + assert m2g is not m2g2 From c8f96d4eccccc9a541bf6a8dc754b49e42ec6487 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 00:46:19 +0100 Subject: [PATCH 022/191] Added save_lazy + changed how cache works --- solar/solar/dblayer/model.py | 92 +++++++++++++++++++++++++++++------- 1 file changed, 75 insertions(+), 17 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index b1850ebc..8eae42c0 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -23,19 +23,44 @@ class NONE: """A None like type""" pass -def get_cache(instance, _): - th = current_thread() - l = LOCAL - try: - cache_id = l.cache_id - except AttributeError: - cache_id = uuid.UUID(int=getrandbits(128), version=4).hex - setattr(th, 'cache_id', cache_id) - if getattr(th, 'cache_id', None) == cache_id: - setattr(l, 'obj_cache', {}) - setattr(l, 'db_ch_state', {}) - l.db_ch_state.setdefault('index', set()) - return l + +class SingleClassCache(object): + + __slots__ = ['obj_cache', 'db_ch_state', 'lazy_save', 'origin_class'] + + def __init__(self, origin_class): + self.obj_cache = {} + self.db_ch_state = {'index': set()} + self.lazy_save = set() + self.origin_class = origin_class + + +class ClassCache(object): + + def __get__(self, _, owner): + th = current_thread() + l = LOCAL + # better don't duplicate class names + cache_name = owner.__name__ + try: + cache_id = l.cache_id + except AttributeError: + cache_id = uuid.UUID(int=getrandbits(128), version=4).hex + setattr(l, 'cache_id', cache_id) + if getattr(th, 'cache_id', None) != cache_id: + # new cache + setattr(th, 'cache_id', cache_id) + c = SingleClassCache(owner) + setattr(l, '_model_caches', {}) + l._model_caches[cache_name] = c + try: + # already had this owner in cache + return l._model_caches[cache_name] + except KeyError: + # old cache but first time this owner + c = SingleClassCache(owner) + l._model_caches[cache_name] = c + return c def clear_cache(): @@ -387,6 +412,8 @@ class IndexField(FieldBase): class ModelMeta(type): + _defined_models = set() + def __new__(mcs, name, bases, attrs): cls = super(ModelMeta, mcs).__new__(mcs, name, bases, attrs) model_fields = set((name for (name, attr) in attrs.items() @@ -409,18 +436,38 @@ class ModelMeta(type): for given in base._model_fields: model_fields.add(given) - # set the fields just in case + cls._model_fields = [getattr(cls, x) for x in model_fields] + if bases == (object, ): + return cls + cls.bucket = Replacer('bucket', get_bucket, mcs) + mcs._defined_models.add(cls) return cls @classmethod def setup(mcs, riak_client): mcs.riak_client = riak_client - mcs.session_start = riak_client.session_start - mcs.session_end = riak_client.session_end + + @classmethod + def session_end(mcs, result=True): + for cls in mcs._defined_models: + for to_save in cls._c.lazy_save: + try: + to_save.save() + except DBLayerException: + continue + else: + print 'saved', to_save + cls._c.lazy_save.clear() + mcs.riak_client.session_end(result) + + @classmethod + def session_start(mcs): + clear_cache() + mcs.riak_client.session_start() class NestedField(FieldBase): @@ -536,7 +583,7 @@ class Model(object): __metaclass__ = ModelMeta - _c = Replacer('_c', get_cache) + _c = ClassCache() _key = None _new = None @@ -660,6 +707,9 @@ class Model(object): setattr(obj, gname, val) return obj + def __hash__(self): + return hash(self.key) + @classmethod def get(cls, key): try: @@ -685,5 +735,13 @@ class Model(object): else: raise DBLayerException("No changes") + def save_lazy(self): + self._c.lazy_save.add(self) + def delete(self): + ls = self._c.lazy_save + try: + ls.remove(self.key) + except KeyError: + pass raise NotImplementedError() From 7602b3b181599f1bc002248d121da99474fd0967 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 00:46:34 +0100 Subject: [PATCH 023/191] Improved tests --- solar/solar/dblayer/test/test_basic.py | 71 ++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index bf001b61..f256887f 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -14,6 +14,19 @@ class M1(Model): ind = IndexField(default=dict) +class M2(Model): + f1 = Field(str) + + ind = IndexField(default=dict) + + +class M3(Model): + f1 = Field(str) + + ind = IndexField(default=dict) + + + def test_from_dict(rk): key = next(rk) @@ -103,3 +116,61 @@ def test_update(rk): clear_cache() m11 = M1.get(key) assert m11.f1 == 'updated' + + +def test_different_models(rk): + key = next(rk) + + m2 = M2.from_dict(key, {'f1': 'm2', 'ind': {'blah': 'blub'}}) + m3 = M3.from_dict(key, {'f1': 'm3', 'ind': {'blah': 'blub'}}) + + m2.save() + m3.save() + + assert M2.get(key).f1 == 'm2' + assert M3.get(key).f1 == 'm3' + + +def test_cache_behaviour(rk): + key1 = next(rk) + + m1 = M1.from_dict(key1, {'f1': 'm1'}) + + m11 = M1.get(key1) + assert m1 is m11 + m1.save() + assert m1 is m11 + + m12 = M1.get(key1) + assert m1 is m12 + + clear_cache() + m13 = M1.get(key1) + assert m1 is not m13 + + +def test_save_lazy(rk): + key1 = next(rk) + key2 = next(rk) + + m1 = M1.from_dict(key1, {'f1': 'm1'}) + m2 = M1.from_dict(key2, {'f1': 'm2'}) + m1.save_lazy() + m2.save_lazy() + + m1g = M1.get(key1) + m2g = M1.get(key2) + + assert m1 is m1g + assert m2 is m2g + + assert M1._c.lazy_save == {m1, m2} + M1.session_end() + assert M1._c.lazy_save == set() + + clear_cache() + m1g2 = M1.get(key1) + m2g2 = M1.get(key2) + + assert m1g is not m1g2 + assert m2g is not m2g2 From 514f206527ddd7d907ab38d4a8725404f76e5746 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 10:48:12 +0100 Subject: [PATCH 024/191] Naive tracking of index changes --- solar/solar/dblayer/model.py | 18 +++++++++++------- solar/solar/dblayer/test/test_basic.py | 11 +++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 37499b4d..ec1094f8 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -593,6 +593,8 @@ class Model(object): def __init__(self, key=None): self._modified_fields = set() + # TODO: that _indexes_changed should be smarter + self._indexes_changed = False self.key = key @property @@ -623,19 +625,18 @@ class Model(object): return self._riak_object.data @changes_state_for('index') - def _set_index(self, *args, **kwargs): - return self._riak_object.set_index(*args, **kwargs) - - @changes_state_for('index') - def _add_index(self, *args, **kwargs): - return self._riak_object.add_index(*args, **kwargs) + def _set_index(self, name, value): + self._indexes_changed = True + return self._riak_object.set_index(name, value) @changes_state_for('index') def _add_index(self, *args, **kwargs): + self._indexes_changed = True return self._riak_object.add_index(*args, **kwargs) @changes_state_for('index') def _remove_index(self, *args, **kwargs): + self._indexes_changed = True return self._riak_object.remove_index(*args, **kwargs) @classmethod @@ -655,7 +656,9 @@ class Model(object): self._modified_fields.add(field.fname) def changed(self): - return True if self._modified_fields else False + if self._modified_fields: + return True + return self._indexes_changed def __str__(self): if self._riak_object is None: @@ -721,6 +724,7 @@ class Model(object): def _reset_state(self): self._new = False self._modified_fields.clear() + self._indexes_hash = None @clears_state_for('index') def save(self, force=False): diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index f256887f..b1c03097 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -174,3 +174,14 @@ def test_save_lazy(rk): assert m1g is not m1g2 assert m2g is not m2g2 + + +def test_changed_index(rk): + key1 = next(rk) + + m1 = M1.from_dict(key1, {'f1': 'm1'}) + + m1.save() + # don't use _add_index directly + m1._add_index('test_bin', 'blah') + m1.save() From 72198f78105f4b8f240de88a5d1f31fc674e37bf Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 10:51:58 +0100 Subject: [PATCH 025/191] removed debug print --- solar/solar/dblayer/model.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index ec1094f8..6a08bb10 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -459,8 +459,6 @@ class ModelMeta(type): to_save.save() except DBLayerException: continue - else: - print 'saved', to_save cls._c.lazy_save.clear() mcs.riak_client.session_end(result) From f5b98323b64bd9ec6ee66a2e78d7fb8d474b0116 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 12:15:01 +0100 Subject: [PATCH 026/191] Added multi_get --- solar/solar/dblayer/model.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 6a08bb10..93be78fb 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -719,6 +719,12 @@ class Model(object): obj = cls.from_riakobj(riak_object) return obj + @classmethod + def multi_get(cls, keys): + # TODO: parallel execution + ret = map(cls.get, keys) + return ret + def _reset_state(self): self._new = False self._modified_fields.clear() From aee0fcb1fa5063d9449dbdabda56fe687753b2d0 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 18:20:01 +0100 Subject: [PATCH 027/191] Redesigned inputs flow --- solar/solar/dblayer/solar_models.py | 44 +++++++++++++++++++++++++-- solar/solar/dblayer/test/test_real.py | 23 ++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 52af28e6..7dd8b5b2 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -18,11 +18,25 @@ class InputsFieldWrp(IndexFieldWrp): # TODO: add cache for lookup self._cache = {} - def connect(self, my_inp_name, other_resource, other_inp_name): + + def _connect_dict_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): + if ':' in other_inp_name: + raise NotImplementedError("Not supported `:` in this direction") + if ':' in my_inp_name: + my_inp_name, nested_key = my_inp_name.split(':', 1) + else: + nested_key = "" + + if '|' in nested_key: + nested_key, nested_tag = nested_key.split('|', 1) + else: + nested_tag = my_resource.name + raise NotImplementedError() + + + def _connect_simple_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): # TODO: for now connections are attached to target resource # in future we might change it to separate object - my_resource = self._instance - other_ind_name = '{}_emit_bin'.format(self.fname) other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, @@ -65,6 +79,30 @@ class InputsFieldWrp(IndexFieldWrp): return True + def connect(self, my_inp_name, other_resource, other_inp_name): + my_resource = self._instance + if ':' in my_inp_name: + tmp_name = my_inp_name.split(':', 1)[0] + my_input = self._get_raw_field_val(tmp_name) + else: + my_input = self._get_raw_field_val(my_inp_name) + other_input = other_resource.inputs._get_raw_field_val(other_inp_name) + if isinstance(my_input, dict): + my_side = 'dict' + else: + my_side = 'simple' + if isinstance(other_input, dict): + other_side = 'dict' + else: + other_side = 'simple' + method = '_connect_{}_{}'.format(my_side, other_side) + try: + meth = getattr(self, method) + except AttributeError: + raise Exception("Unknown connection %r %r" % (my_side, other_side)) + else: + return meth(my_resource, my_inp_name, other_resource, other_inp_name) + def _has_own_input(self, name): try: return self._cache[name] diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index b73db052..2f322de6 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -204,3 +204,26 @@ def test_updated_behaviour(rk): assert not isinstance(r1.updated, basestring) assert r1.updated >= _cmp assert k1 in Resource.updated.filter(StrInt.p_min(), StrInt.p_max()) + + + +def test_advanced_inputs(rk): + k1 = next(rk) + k2 = next(rk) + + r1 = Resource.from_dict(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + r2 = Resource.from_dict(k2, {'name': 'second', + 'inputs': {'input': {'input1': None, + 'input2': None}}}) + + + r1.connect(r2, {'input1': 'input:input1', + 'input2': 'input:input2'}) + + r1.save() + r2.save() + + assert r2.inputs['input']['input1'] == 10 + assert r2.inputs['input']['input2'] == 15 From 17760146c670014d6f870d5434acd38ccb2d6ae9 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 18:20:47 +0100 Subject: [PATCH 028/191] Some Resource integration with new backend --- solar/solar/core/resource/resource.py | 36 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index e2f4d604..41e4208d 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -76,20 +76,22 @@ class Resource(object): self.auto_extend_inputs(inputs) - self.db_obj = DBResource.from_dict(name, - 'id': name, - 'name': name, - 'actions_path': metadata.get('actions_path', ''), - 'actions': metadata.get('actions', ''), - 'base_name': metadata.get('base_name', ''), - 'base_path': metadata.get('base_path', ''), - 'handler': metadata.get('handler', ''), - 'puppet_module': metadata.get('puppet_module', ''), - 'version': metadata.get('version', ''), - 'meta_inputs': inputs, - 'tags': tags, - 'stae': RESOURCE_STATE.created.name - }) + self.db_obj = DBResource.from_dict( + name, + { + 'id': name, + 'name': name, + 'actions_path': metadata.get('actions_path', ''), + 'actions': metadata.get('actions', ''), + 'base_name': metadata.get('base_name', ''), + 'base_path': metadata.get('base_path', ''), + 'handler': metadata.get('handler', ''), + 'puppet_module': metadata.get('puppet_module', ''), + 'version': metadata.get('version', ''), + 'meta_inputs': inputs, + 'tags': tags, + 'stae': RESOURCE_STATE.created.name + }) self.create_inputs(args) @@ -224,6 +226,12 @@ class Resource(object): receiver.resource.name, receiver_input]) return rst + def graph(self): + mdg = networkx.MultiDiGraph() + for input_name, input_value in self.inputs: + mdg.add_edges_from(input.edges()) + return mdg + def resource_inputs(self): return self.db_obj.inputs From 8afca13319a0635629acb3a4fce7d45502307391 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 28 Oct 2015 22:48:03 +0100 Subject: [PATCH 029/191] More inputs behaviour refactoring --- solar/solar/dblayer/solar_models.py | 62 ++++++++++++++++++++--- solar/solar/dblayer/test/test_real.py | 73 +++++++++++++++++++-------- 2 files changed, 107 insertions(+), 28 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 7dd8b5b2..6decc698 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -4,8 +4,12 @@ from solar.dblayer.model import (Model, Field, IndexField, requires_clean_state, check_state_for, StrInt, IndexedField) - +from types import NoneType from operator import itemgetter +from enum import Enum + +InputTypes = Enum('InputTypes', + 'simple list hash list_hash') class DBLayerSolarException(DBLayerException): pass @@ -13,6 +17,8 @@ class DBLayerSolarException(DBLayerException): class InputsFieldWrp(IndexFieldWrp): + _simple_types = (NoneType, int, float, basestring, str, unicode) + def __init__(self, *args, **kwargs): super(InputsFieldWrp, self).__init__(*args, **kwargs) # TODO: add cache for lookup @@ -30,10 +36,34 @@ class InputsFieldWrp(IndexFieldWrp): if '|' in nested_key: nested_key, nested_tag = nested_key.split('|', 1) else: - nested_tag = my_resource.name + nested_tag = other_resource.key raise NotImplementedError() + def _connect_list_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): + other_ind_name = '{}_emit_bin'.format(self.fname) + other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, + other_inp_name, + my_resource.key, + my_inp_name) + + my_ind_name = '{}_recv_bin'.format(self.fname) + my_ind_val = '{}|{}|{}|{}'.format(my_resource.key, + my_inp_name, + other_resource.key, + other_inp_name) + + my_resource._add_index(my_ind_name, + my_ind_val) + my_resource._add_index(other_ind_name, + other_ind_val) + try: + del self._cache[my_inp_name] + except KeyError: + pass + return True + + def _connect_simple_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): # TODO: for now connections are attached to target resource # in future we might change it to separate object @@ -79,22 +109,39 @@ class InputsFieldWrp(IndexFieldWrp): return True + def _input_type(self, resource, name): + # XXX: it could be worth to precalculate it + schema = resource.meta_inputs[name]['schema'] + if isinstance(schema, self._simple_types): + return InputTypes.simple + if isinstance(schema, list): + if len(schema) > 0 and isinstance(schema[0], dict): + return InputTypes.list_hash + return InputTypes.list + if isinstance(schema, dict): + return InputTypes.hash + raise Exception("Unknown type") + + def connect(self, my_inp_name, other_resource, other_inp_name): my_resource = self._instance + other_type = self._input_type(other_resource, other_inp_name) + my_type = self._input_type(my_resource, my_inp_name) + raise Exception() if ':' in my_inp_name: tmp_name = my_inp_name.split(':', 1)[0] my_input = self._get_raw_field_val(tmp_name) else: my_input = self._get_raw_field_val(my_inp_name) other_input = other_resource.inputs._get_raw_field_val(other_inp_name) - if isinstance(my_input, dict): - my_side = 'dict' - else: + if isinstance(my_input, self._simple_types): my_side = 'simple' - if isinstance(other_input, dict): - other_side = 'dict' else: + my_side = type(my_input).__name__ + if isinstance(other_input, self._simple_types): other_side = 'simple' + else: + other_side = type(other_input).__name__ method = '_connect_{}_{}'.format(my_side, other_side) try: meth = getattr(self, method) @@ -132,7 +179,6 @@ class InputsFieldWrp(IndexFieldWrp): recvs = self._instance._get_index(ind_name, startkey='{}|{}|'.format(my_name, name), endkey='{}|{}|~'.format(my_name, name), - max_results=1, return_terms=True).results if not recvs: _res = self._get_raw_field_val(name) diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 2f322de6..6a6b5f9a 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -5,11 +5,23 @@ from solar.dblayer.model import Model, Field, IndexField, clear_cache, check_sta from solar.dblayer.solar_models import Resource, DBLayerSolarException +def create_resource(key, data): + mi = data.get('meta_inputs', {}) + for inp_name, inp_value in data.get('inputs', {}).items(): + if isinstance(inp_value, list): + schema = ['str!'] + elif isinstance(inp_value, dict): + schema = {} + else: + schema = '%s!' % type(inp_value).__name__ + mi.setdefault(inp_name, {"schema": schema}) + data['meta_inputs'] = mi + return Resource.from_dict(key, data) def test_changes_state(rk): key = next(rk) - r = Resource.from_dict(key, {'name': 'a name'}) + r = create_resource(key, {'name': 'a name'}) r.inputs['a'] = 1 with pytest.raises(Exception): # raise exception when something is changed @@ -20,7 +32,7 @@ def test_changes_state(rk): def test_basic_input(rk): key = next(rk) - r = Resource.from_dict(key, {'name': 'a name'}) + r = create_resource(key, {'name': 'a name'}) r.inputs['a'] = 1 r.save() assert r.inputs['a'] == 1 @@ -34,7 +46,7 @@ def test_basic_input(rk): def test_input_in_dict(rk): key = next(rk) - r = Resource.from_dict(key, {'name': 'a name', + r = create_resource(key, {'name': 'a name', 'inputs': {'input1': 15, 'input2': None}}) r.save() @@ -49,10 +61,10 @@ def test_basic_connect(rk): k1 = next(rk) k2 = next(rk) - r1 = Resource.from_dict(k1, {'name': 'first', + r1 = create_resource(k1, {'name': 'first', 'inputs': {'input1': 10, 'input2': 15}}) - r2 = Resource.from_dict(k2, {'name': 'second', + r2 = create_resource(k2, {'name': 'second', 'inputs': {'input1': None, 'input2': None}}) @@ -78,10 +90,10 @@ def test_adv_connect(rk, depth): k1 = next(rk) k2 = next(rk) - r1 = Resource.from_dict(k1, {'name': 'first', + r1 = create_resource(k1, {'name': 'first', 'inputs': {'input1': 10, 'input2': 15}}) - prev = Resource.from_dict(k2, {'name': 'second', + prev = create_resource(k2, {'name': 'second', 'inputs': {'input1': None, 'input2': None, 'input3': 0}}) @@ -93,7 +105,7 @@ def test_adv_connect(rk, depth): for x in xrange(depth - 1): k = next(rk) - res = Resource.from_dict(k, {'name': 'next %d' % (x + 1), + res = create_resource(k, {'name': 'next %d' % (x + 1), 'inputs': {'input1': None, 'input2': None, 'input3': x + 1}}) @@ -111,14 +123,14 @@ def test_adv_connect(rk, depth): @pytest.mark.parametrize('depth', (1, 3, 5, 10, 50, 100)) def test_perf_inputs(rk, depth): k1 = next(rk) - r1 = Resource.from_dict(k1, {'name': 'first', + r1 = create_resource(k1, {'name': 'first', 'inputs': {'input1': 'target'}}) r1.save() prev = r1 for x in xrange(depth): k = next(rk) - res = Resource.from_dict(k, {'name': 'next %d' % (x + 1), + res = create_resource(k, {'name': 'next %d' % (x + 1), 'inputs': {'input1': None}}) prev.connect(res, {'input1': 'input1'}) res.save() @@ -136,14 +148,14 @@ def test_change_connect(rk): k2 = next(rk) k3 = next(rk) - r1 = Resource.from_dict(k1, {'name': 'first', + r1 = create_resource(k1, {'name': 'first', 'inputs': {'input1': 10, 'input2': 15}}) - r2 = Resource.from_dict(k2, {'name': 'second', + r2 = create_resource(k2, {'name': 'second', 'inputs': {'input1': None, 'input2': None, 'input3': 0}}) - r3 = Resource.from_dict(k3, {'name': 'first', + r3 = create_resource(k3, {'name': 'first', 'inputs': {'input1': 30, 'input2': 35}}) @@ -163,7 +175,7 @@ def test_simple_tag(rk, rt): k1 = next(rk) tag = next(rt) - r1 = Resource.from_dict(k1, {'name': 'first', + r1 = create_resource(k1, {'name': 'first', 'tags': ['%s' % tag, '%s=10' % tag]}) r1.save() @@ -175,11 +187,11 @@ def test_list_by_tag(rk, rt): k2 = next(rk) tag1 = next(rt) tag2 = next(rt) - r1 = Resource.from_dict(k1, {'name': 'first', + r1 = create_resource(k1, {'name': 'first', 'tags': [tag1, '%s=10' % tag1]}) r1.save() - r2 = Resource.from_dict(k2, {'name': 'first', + r2 = create_resource(k2, {'name': 'first', 'tags': [tag1, '%s=10' % tag2]}) r2.save() @@ -198,7 +210,7 @@ def test_updated_behaviour(rk): k1 = next(rk) _cmp = StrInt() - r1 = Resource.from_dict(k1, {'name': 'blah'}) + r1 = create_resource(k1, {'name': 'blah'}) r1.save() assert isinstance(r1._riak_object.data['updated'], basestring) assert not isinstance(r1.updated, basestring) @@ -207,14 +219,35 @@ def test_updated_behaviour(rk): -def test_advanced_inputs(rk): +def test_list_inputs(rk): k1 = next(rk) k2 = next(rk) - r1 = Resource.from_dict(k1, {'name': 'first', + r1 = create_resource(k1, {'name': 'first', 'inputs': {'input1': 10, 'input2': 15}}) - r2 = Resource.from_dict(k2, {'name': 'second', + r2 = create_resource(k2, {'name': 'second', + 'inputs': {'input': []}}) + + r1.connect(r2, {'input1': 'input'}) + # r1.connect(r2, {'input2': 'input'}) + + r1.save() + r2.save() + + print r2._riak_object.indexes + + assert r2.inputs['input'] == [10, 15] + + +def test_dict_inputs(rk): + k1 = next(rk) + k2 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + r2 = create_resource(k2, {'name': 'second', 'inputs': {'input': {'input1': None, 'input2': None}}}) From e7849af1a25837ecc064c5e8020c3818e772a75a Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 14:07:20 +0100 Subject: [PATCH 030/191] Allow comparsion StrInt with hex value of it --- solar/solar/dblayer/model.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 28b33205..0574d751 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -196,6 +196,12 @@ class StrInt(object): return val def __eq__(self, other): + if isinstance(other, basestring): + first_ch = other[0] + if first_ch not in (self.positive_char, self.negative_char): + raise Exception("Cannot compare %r with %r" % (self, other)) + else: + other = self.from_simple(other) if not isinstance(other, self.__class__): raise Exception("Cannot compare %r with %r" % (self, other)) so = other._val[0] @@ -207,6 +213,12 @@ class StrInt(object): return self._val[1:] == other._val[1:] def __gt__(self, other): + if isinstance(other, basestring): + first_ch = other[0] + if first_ch not in (self.positive_char, self.negative_char): + raise Exception("Cannot compare %r with %r" % (self, other)) + else: + other = self.from_simple(other) if not isinstance(other, self.__class__): raise Exception("Cannot compare %r with %r" % (self, other)) so = other._val[0] From 6a40f5203021770ff523cba08736ee5799c083a9 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 14:07:20 +0100 Subject: [PATCH 031/191] Allow comparsion StrInt with hex value of it --- solar/solar/dblayer/model.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 93be78fb..df473053 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -196,6 +196,12 @@ class StrInt(object): return val def __eq__(self, other): + if isinstance(other, basestring): + first_ch = other[0] + if first_ch not in (self.positive_char, self.negative_char): + raise Exception("Cannot compare %r with %r" % (self, other)) + else: + other = self.from_simple(other) if not isinstance(other, self.__class__): raise Exception("Cannot compare %r with %r" % (self, other)) so = other._val[0] @@ -207,6 +213,12 @@ class StrInt(object): return self._val[1:] == other._val[1:] def __gt__(self, other): + if isinstance(other, basestring): + first_ch = other[0] + if first_ch not in (self.positive_char, self.negative_char): + raise Exception("Cannot compare %r with %r" % (self, other)) + else: + other = self.from_simple(other) if not isinstance(other, self.__class__): raise Exception("Cannot compare %r with %r" % (self, other)) so = other._val[0] From 4a69cd635f467a99918132ebd494be37ffb31e7f Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 16:05:30 +0100 Subject: [PATCH 032/191] Redesigned inputs again, this approach should be clean enough --- solar/solar/dblayer/solar_models.py | 219 +++++++++++++++---------- solar/solar/dblayer/test/test_basic.py | 10 ++ 2 files changed, 138 insertions(+), 91 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 6decc698..dcd1558c 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -25,88 +25,88 @@ class InputsFieldWrp(IndexFieldWrp): self._cache = {} - def _connect_dict_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): - if ':' in other_inp_name: - raise NotImplementedError("Not supported `:` in this direction") - if ':' in my_inp_name: - my_inp_name, nested_key = my_inp_name.split(':', 1) - else: - nested_key = "" + # def _connect_dict_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): + # if ':' in other_inp_name: + # raise NotImplementedError("Not supported `:` in this direction") + # if ':' in my_inp_name: + # my_inp_name, nested_key = my_inp_name.split(':', 1) + # else: + # nested_key = "" - if '|' in nested_key: - nested_key, nested_tag = nested_key.split('|', 1) - else: - nested_tag = other_resource.key - raise NotImplementedError() + # if '|' in nested_key: + # nested_key, nested_tag = nested_key.split('|', 1) + # else: + # nested_tag = other_resource.key + # raise NotImplementedError() - def _connect_list_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): - other_ind_name = '{}_emit_bin'.format(self.fname) - other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, - other_inp_name, - my_resource.key, - my_inp_name) + # def _connect_list_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): + # other_ind_name = '{}_emit_bin'.format(self.fname) + # other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, + # other_inp_name, + # my_resource.key, + # my_inp_name) - my_ind_name = '{}_recv_bin'.format(self.fname) - my_ind_val = '{}|{}|{}|{}'.format(my_resource.key, - my_inp_name, - other_resource.key, - other_inp_name) + # my_ind_name = '{}_recv_bin'.format(self.fname) + # my_ind_val = '{}|{}|{}|{}'.format(my_resource.key, + # my_inp_name, + # other_resource.key, + # other_inp_name) - my_resource._add_index(my_ind_name, - my_ind_val) - my_resource._add_index(other_ind_name, - other_ind_val) - try: - del self._cache[my_inp_name] - except KeyError: - pass - return True + # my_resource._add_index(my_ind_name, + # my_ind_val) + # my_resource._add_index(other_ind_name, + # other_ind_val) + # try: + # del self._cache[my_inp_name] + # except KeyError: + # pass + # return True - def _connect_simple_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): - # TODO: for now connections are attached to target resource - # in future we might change it to separate object + # def _connect_simple_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): + # # TODO: for now connections are attached to target resource + # # in future we might change it to separate object - other_ind_name = '{}_emit_bin'.format(self.fname) - other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, - other_inp_name, - my_resource.key, - my_inp_name) + # other_ind_name = '{}_emit_bin'.format(self.fname) + # other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, + # other_inp_name, + # my_resource.key, + # my_inp_name) - my_ind_name = '{}_recv_bin'.format(self.fname) - my_ind_val = '{}|{}|{}|{}'.format(my_resource.key, - my_inp_name, - other_resource.key, - other_inp_name) + # my_ind_name = '{}_recv_bin'.format(self.fname) + # my_ind_val = '{}|{}|{}|{}'.format(my_resource.key, + # my_inp_name, + # other_resource.key, + # other_inp_name) - # ensure no conflicting connections are done - # TODO: move this to backend layer - indexes = my_resource._riak_object.indexes - to_del = [] - for ind_name, ind_value in indexes: - if ind_name == my_ind_name: - mr, mn = ind_value.split('|')[:2] - if mr == my_resource.key and mn == my_inp_name: - to_del.append((ind_name, ind_value)) - elif ind_name == other_ind_name: - mr, mn = ind_value.rsplit('|')[2:] - if mr == my_resource.key and mn == my_inp_name: - to_del.append((ind_name, ind_value)) + # # ensure no conflicting connections are done + # # TODO: move this to backend layer + # indexes = my_resource._riak_object.indexes + # to_del = [] + # for ind_name, ind_value in indexes: + # if ind_name == my_ind_name: + # mr, mn = ind_value.split('|')[:2] + # if mr == my_resource.key and mn == my_inp_name: + # to_del.append((ind_name, ind_value)) + # elif ind_name == other_ind_name: + # mr, mn = ind_value.rsplit('|')[2:] + # if mr == my_resource.key and mn == my_inp_name: + # to_del.append((ind_name, ind_value)) - for ind_name, ind_value in to_del: - my_resource._remove_index(ind_name, value=ind_value) + # for ind_name, ind_value in to_del: + # my_resource._remove_index(ind_name, value=ind_value) - # add new - my_resource._add_index(my_ind_name, - my_ind_val) - my_resource._add_index(other_ind_name, - other_ind_val) - try: - del self._cache[my_inp_name] - except KeyError: - pass - return True + # # add new + # my_resource._add_index(my_ind_name, + # my_ind_val) + # my_resource._add_index(other_ind_name, + # other_ind_val) + # try: + # del self._cache[my_inp_name] + # except KeyError: + # pass + # return True def _input_type(self, resource, name): @@ -123,32 +123,69 @@ class InputsFieldWrp(IndexFieldWrp): raise Exception("Unknown type") + def _connect_my_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): + my_ind_name = '{}_recv_bin'.format(self.fname) + my_ind_val = '{}|{}|{}|{}'.format(my_resource.key, + my_inp_name, + other_resource.key, + other_inp_name) + + + my_resource._add_index(my_ind_name, my_ind_val) + return True + + + def _connect_other_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): + other_ind_name = '{}_emit_bin'.format(self.fname) + other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, + other_inp_name, + my_resource.key, + my_inp_name) + my_resource._add_index(other_ind_name, + other_ind_val) + return True + + def connect(self, my_inp_name, other_resource, other_inp_name): my_resource = self._instance other_type = self._input_type(other_resource, other_inp_name) my_type = self._input_type(my_resource, my_inp_name) - raise Exception() - if ':' in my_inp_name: - tmp_name = my_inp_name.split(':', 1)[0] - my_input = self._get_raw_field_val(tmp_name) - else: - my_input = self._get_raw_field_val(my_inp_name) - other_input = other_resource.inputs._get_raw_field_val(other_inp_name) - if isinstance(my_input, self._simple_types): - my_side = 'simple' - else: - my_side = type(my_input).__name__ - if isinstance(other_input, self._simple_types): - other_side = 'simple' - else: - other_side = type(other_input).__name__ - method = '_connect_{}_{}'.format(my_side, other_side) + + # set my side + my_meth = getattr(self, '_connect_my_{}'.format(my_type.name)) + my_meth(my_resource, my_inp_name, other_resource, other_inp_name) + + # set other side + other_meth = getattr(self, '_connect_other_{}'.format(other_type.name)) + other_meth(my_resource, my_inp_name, other_resource, other_inp_name) + try: - meth = getattr(self, method) - except AttributeError: - raise Exception("Unknown connection %r %r" % (my_side, other_side)) - else: - return meth(my_resource, my_inp_name, other_resource, other_inp_name) + del self._cache[my_inp_name] + except KeyError: + pass + return True + # raise Exception() + # if ':' in my_inp_name: + # tmp_name = my_inp_name.split(':', 1)[0] + # my_input = self._get_raw_field_val(tmp_name) + # else: + # my_input = self._get_raw_field_val(my_inp_name) + # other_input = other_resource.inputs._get_raw_field_val(other_inp_name) + # if isinstance(my_input, self._simple_types): + # my_side = 'simple' + # else: + # my_side = type(my_input).__name__ + # if isinstance(other_input, self._simple_types): + # other_side = 'simple' + # else: + # other_side = type(other_input).__name__ + # method = '_connect_{}_{}'.format(my_side, other_side) + # try: + # meth = getattr(self, method) + # except AttributeError: + # raise Exception("Unknown connection %r %r" % (my_side, other_side)) + # else: + # return meth(my_resource, my_inp_name, other_resource, other_inp_name) def _has_own_input(self, name): try: diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index b1c03097..cd7c03ec 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -1,6 +1,7 @@ import pytest from solar.dblayer.model import (Field, IndexField, clear_cache, Model, + StrInt, DBLayerNotFound, DBLayerNoRiakObj, DBLayerException) @@ -185,3 +186,12 @@ def test_changed_index(rk): # don't use _add_index directly m1._add_index('test_bin', 'blah') m1.save() + + +def test_strint_comparsions(): + a = StrInt(-1) + b = StrInt(-2) + c = StrInt.to_simple(b) + assert isinstance(c, basestring) + assert a > b + assert a > c From 2d186ea21306b8e6e237a100763a3e082cc39b7c Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 16:31:28 +0100 Subject: [PATCH 033/191] dict_to_dict input works --- solar/solar/dblayer/solar_models.py | 5 ++++- solar/solar/dblayer/test/test_real.py | 26 +++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index dcd1558c..d0440fef 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -134,7 +134,6 @@ class InputsFieldWrp(IndexFieldWrp): my_resource._add_index(my_ind_name, my_ind_val) return True - def _connect_other_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): other_ind_name = '{}_emit_bin'.format(self.fname) other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, @@ -151,6 +150,10 @@ class InputsFieldWrp(IndexFieldWrp): other_type = self._input_type(other_resource, other_inp_name) my_type = self._input_type(my_resource, my_inp_name) + if my_type == other_type: + my_type = InputTypes.simple + other_type = InputTypes.simple + # set my side my_meth = getattr(self, '_connect_my_{}'.format(my_type.name)) my_meth(my_resource, my_inp_name, other_resource, other_inp_name) diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 6a6b5f9a..d02bae40 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -230,7 +230,7 @@ def test_list_inputs(rk): 'inputs': {'input': []}}) r1.connect(r2, {'input1': 'input'}) - # r1.connect(r2, {'input2': 'input'}) + r1.connect(r2, {'input2': 'input'}) r1.save() r2.save() @@ -240,6 +240,30 @@ def test_list_inputs(rk): assert r2.inputs['input'] == [10, 15] +def test_dict_to_dict_inputs(rk): + k1 = next(rk) + k2 = next(rk) + + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'input': {'input1': 10, + 'input2': 15} + }}) + r2 = create_resource(k2, {'name': 'second', + 'inputs': {'input': {'input1': None, + 'input2': None, + 'input3': None}}}) + + r1.connect(r2, {'input': 'input'}) + r1.save() + r2.save() + + assert r2.inputs['input']['input1'] == 10 + assert r2.inputs['input']['input2'] == 15 + assert 'input3' not in r2.inputs['input'] + + + def test_dict_inputs(rk): k1 = next(rk) k2 = next(rk) From d6a664efbdb765ba0387d8417834659fa31dc591 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 16:44:15 +0100 Subject: [PATCH 034/191] simple to list inputs are working --- solar/solar/dblayer/solar_models.py | 32 +++++++++++++++++++++++---- solar/solar/dblayer/test/test_real.py | 2 -- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index d0440fef..9c35ab14 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -122,7 +122,6 @@ class InputsFieldWrp(IndexFieldWrp): return InputTypes.hash raise Exception("Unknown type") - def _connect_my_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): my_ind_name = '{}_recv_bin'.format(self.fname) my_ind_val = '{}|{}|{}|{}'.format(my_resource.key, @@ -145,12 +144,17 @@ class InputsFieldWrp(IndexFieldWrp): return True + def _connect_my_list(self, my_resource, my_inp_name, other_resource, other_inp_name): + ret = self._connect_my_simple(my_resource, my_inp_name, other_resource, other_inp_name) + return ret + def connect(self, my_inp_name, other_resource, other_inp_name): my_resource = self._instance other_type = self._input_type(other_resource, other_inp_name) my_type = self._input_type(my_resource, my_inp_name) if my_type == other_type: + # if the type is the same map 1:1 my_type = InputTypes.simple other_type = InputTypes.simple @@ -216,14 +220,24 @@ class InputsFieldWrp(IndexFieldWrp): ind_name = '{}_recv_bin'.format(fname) # XXX: possible optimization # get all values for resource and cache it (use dirty to check) + kwargs = dict(startkey='{}|{}|'.format(my_name, name), + endkey='{}|{}|~'.format(my_name, name), + return_terms=True) + my_type = self._input_type(self._instance, name) + if my_type == InputTypes.simple: + kwargs['max_results'] = 1 + else: + kwargs['max_results'] = 99999 recvs = self._instance._get_index(ind_name, - startkey='{}|{}|'.format(my_name, name), - endkey='{}|{}|~'.format(my_name, name), - return_terms=True).results + **kwargs).results if not recvs: _res = self._get_raw_field_val(name) self._cache[name] = _res return _res + my_meth = getattr(self, '_map_field_val_{}'.format(my_type.name)) + return my_meth(recvs, my_name) + + def _map_field_val_simple(self, recvs, name): recvs = recvs[0] index_val, obj_key = recvs _, inp, emitter_key, emitter_inp = index_val.split('|', 4) @@ -231,6 +245,16 @@ class InputsFieldWrp(IndexFieldWrp): self._cache[name] = res return res + def _map_field_val_list(self, recvs, name): + res = [] + for recv in recvs: + index_val, obj_key = recv + _, inp, emitter_key, emitter_inp = index_val.split('|', 4) + cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + res.append(cres) + self._cache[name] = res + return res + def _get_raw_field_val(self, name): return self._instance._data_container[self.fname][name] diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index d02bae40..07957af7 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -235,8 +235,6 @@ def test_list_inputs(rk): r1.save() r2.save() - print r2._riak_object.indexes - assert r2.inputs['input'] == [10, 15] From dedde4712fee472da7e388897c10e31c98897b75 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 17:55:49 +0100 Subject: [PATCH 035/191] list to list + dict to dict inputs --- solar/solar/dblayer/solar_models.py | 41 +++++++++++++++++++++------ solar/solar/dblayer/test/test_real.py | 17 +++++++++++ 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 9c35ab14..c22297fe 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -122,12 +122,14 @@ class InputsFieldWrp(IndexFieldWrp): return InputTypes.hash raise Exception("Unknown type") - def _connect_my_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): + def _connect_my_simple(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): + types_mapping = '|{}_{}'.format(my_type.value, other_type.value) my_ind_name = '{}_recv_bin'.format(self.fname) my_ind_val = '{}|{}|{}|{}'.format(my_resource.key, my_inp_name, other_resource.key, other_inp_name) + my_ind_val += types_mapping my_resource._add_index(my_ind_name, my_ind_val) @@ -144,8 +146,8 @@ class InputsFieldWrp(IndexFieldWrp): return True - def _connect_my_list(self, my_resource, my_inp_name, other_resource, other_inp_name): - ret = self._connect_my_simple(my_resource, my_inp_name, other_resource, other_inp_name) + def _connect_my_list(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): + ret = self._connect_my_simple(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) return ret def connect(self, my_inp_name, other_resource, other_inp_name): @@ -160,7 +162,7 @@ class InputsFieldWrp(IndexFieldWrp): # set my side my_meth = getattr(self, '_connect_my_{}'.format(my_type.name)) - my_meth(my_resource, my_inp_name, other_resource, other_inp_name) + my_meth(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) # set other side other_meth = getattr(self, '_connect_other_{}'.format(other_type.name)) @@ -246,12 +248,33 @@ class InputsFieldWrp(IndexFieldWrp): return res def _map_field_val_list(self, recvs, name): - res = [] - for recv in recvs: + if len(recvs) == 1: + recv = recvs[0] index_val, obj_key = recv - _, inp, emitter_key, emitter_inp = index_val.split('|', 4) - cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) - res.append(cres) + _, inp, emitter_key, emitter_inp, mapping_type = index_val.split('|', 4) + res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + if mapping_type != "{}_{}".format(InputTypes.simple.value, InputTypes.simple.value): + res = [res] + else: + res = [] + for recv in recvs: + index_val, obj_key = recv + _, inp, emitter_key, emitter_inp, mapping_type = index_val.split('|', 4) + cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + res.append(cres) + self._cache[name] = res + return res + + def _map_field_val_hash(self, recvs, name): + if len(recvs) == 1: + recv = recvs[0] + index_val, obj_key = recv + _, inp, emitter_key, emitter_inp, mapping_type = index_val.split('|', 4) + res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + if mapping_type != "{}_{}".format(InputTypes.simple.value, InputTypes.simple.value): + raise NotImplementedError() + else: + raise NotImplementedError() self._cache[name] = res return res diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 07957af7..07974002 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -261,6 +261,23 @@ def test_dict_to_dict_inputs(rk): assert 'input3' not in r2.inputs['input'] +def test_list_to_list_inputs(rk): + k1 = next(rk) + k2 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'input': [10, 15]}}) + r2 = create_resource(k2, {'name': 'second', + 'inputs': {'input': []}}) + + r1.connect(r2, {'input': 'input'}) + + r1.save() + r2.save() + + assert r2.inputs['input'] == [10, 15] + + def test_dict_inputs(rk): k1 = next(rk) From fc396d427ea7ffc044165352539093283d0a5432 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 21:05:58 +0100 Subject: [PATCH 036/191] simple to dict input works --- solar/solar/dblayer/solar_models.py | 47 +++++++++++++++++++++++---- solar/solar/dblayer/test/test_real.py | 3 +- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index c22297fe..2b24622d 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -7,6 +7,7 @@ from solar.dblayer.model import (Model, Field, IndexField, from types import NoneType from operator import itemgetter from enum import Enum +from itertools import groupby InputTypes = Enum('InputTypes', 'simple list hash list_hash') @@ -111,6 +112,8 @@ class InputsFieldWrp(IndexFieldWrp): def _input_type(self, resource, name): # XXX: it could be worth to precalculate it + if ':' in name: + name = name.split(":", 1)[0] schema = resource.meta_inputs[name]['schema'] if isinstance(schema, self._simple_types): return InputTypes.simple @@ -133,7 +136,7 @@ class InputsFieldWrp(IndexFieldWrp): my_resource._add_index(my_ind_name, my_ind_val) - return True + return my_inp_name def _connect_other_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): other_ind_name = '{}_emit_bin'.format(self.fname) @@ -143,13 +146,33 @@ class InputsFieldWrp(IndexFieldWrp): my_inp_name) my_resource._add_index(other_ind_name, other_ind_val) - return True + return other_inp_name def _connect_my_list(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): ret = self._connect_my_simple(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) return ret + def _connect_my_hash(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): + my_key, my_val = my_inp_name.split(':', 1) + if '|' in my_val: + my_val, my_tag = my_val.split('|', 1) + else: + my_tag = other_resource.name + types_mapping = '|{}_{}'.format(my_type.value, other_type.value) + my_ind_name = '{}_recv_bin'.format(self.fname) + my_ind_val = '{}|{}|{}|{}|{}|{}'.format(my_resource.key, + my_key, + other_resource.key, + other_inp_name, + my_tag, + my_val + ) + my_ind_val += types_mapping + + my_resource._add_index(my_ind_name, my_ind_val) + return my_key + def connect(self, my_inp_name, other_resource, other_inp_name): my_resource = self._instance other_type = self._input_type(other_resource, other_inp_name) @@ -162,14 +185,14 @@ class InputsFieldWrp(IndexFieldWrp): # set my side my_meth = getattr(self, '_connect_my_{}'.format(my_type.name)) - my_meth(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) + my_affected = my_meth(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) # set other side other_meth = getattr(self, '_connect_other_{}'.format(other_type.name)) other_meth(my_resource, my_inp_name, other_resource, other_inp_name) try: - del self._cache[my_inp_name] + del self._cache[my_affected] except KeyError: pass return True @@ -259,7 +282,7 @@ class InputsFieldWrp(IndexFieldWrp): res = [] for recv in recvs: index_val, obj_key = recv - _, inp, emitter_key, emitter_inp, mapping_type = index_val.split('|', 4) + _, _, emitter_key, emitter_inp, mapping_type = index_val.split('|', 4) cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) res.append(cres) self._cache[name] = res @@ -274,7 +297,19 @@ class InputsFieldWrp(IndexFieldWrp): if mapping_type != "{}_{}".format(InputTypes.simple.value, InputTypes.simple.value): raise NotImplementedError() else: - raise NotImplementedError() + items = [] + tags = set() + for recv in recvs: + index_val, obj_key = recv + _, _, emitter_key, emitter_inp, my_tag, my_val, mapping_type = index_val.split('|', 6) + cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + items.append((my_tag, my_val, cres)) + tags.add(my_tag) + if len(tags) != 1: + raise Exception("Detected dict with different tags") + res = {} + for _, my_val, value in items: + res[my_val] = value self._cache[name] = res return res diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 07974002..95cb6e7b 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -278,8 +278,7 @@ def test_list_to_list_inputs(rk): assert r2.inputs['input'] == [10, 15] - -def test_dict_inputs(rk): +def test_simple_to_dict_inputs(rk): k1 = next(rk) k2 = next(rk) From c80781be64c8e7e23dfcf639935b8c5a128b556d Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 21:07:21 +0100 Subject: [PATCH 037/191] Invalid unpacking fixed --- solar/solar/dblayer/solar_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 2b24622d..c5021982 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -265,7 +265,7 @@ class InputsFieldWrp(IndexFieldWrp): def _map_field_val_simple(self, recvs, name): recvs = recvs[0] index_val, obj_key = recvs - _, inp, emitter_key, emitter_inp = index_val.split('|', 4) + _, inp, emitter_key, emitter_inp, _mapping_type = index_val.split('|', 4) res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) self._cache[name] = res return res From 25023278c6bf510a5853c1c889ce6cd46202fd47 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 21:18:29 +0100 Subject: [PATCH 038/191] dict input with tags --- solar/solar/dblayer/test/test_real.py | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 95cb6e7b..6911c5b6 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -298,3 +298,31 @@ def test_simple_to_dict_inputs(rk): assert r2.inputs['input']['input1'] == 10 assert r2.inputs['input']['input2'] == 15 + + +def test_simple_do_dict_inputs_with_tag(rk): + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + r3 = create_resource(k3, {'name': 'first', + 'inputs': {'input1': 110, + 'input2': 115}}) + r2 = create_resource(k2, {'name': 'second', + 'inputs': {'input': {'input1': None, + 'input2': None}}}) + + + r1.connect(r2, {'input1': 'input:input1|tag'}) + r3.connect(r2, {'input2': 'input:input2|tag'}) + + r1.save() + r2.save() + r3.save() + + assert r2.inputs['input']['input1'] == 10 + assert r2.inputs['input']['input2'] == 115 + From 7e7ababacd229d8b8f0d60885bdba3cafa127dcd Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 21:18:56 +0100 Subject: [PATCH 039/191] Removed old code --- solar/solar/dblayer/solar_models.py | 107 ---------------------------- 1 file changed, 107 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index c5021982..e87db73d 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -25,91 +25,6 @@ class InputsFieldWrp(IndexFieldWrp): # TODO: add cache for lookup self._cache = {} - - # def _connect_dict_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): - # if ':' in other_inp_name: - # raise NotImplementedError("Not supported `:` in this direction") - # if ':' in my_inp_name: - # my_inp_name, nested_key = my_inp_name.split(':', 1) - # else: - # nested_key = "" - - # if '|' in nested_key: - # nested_key, nested_tag = nested_key.split('|', 1) - # else: - # nested_tag = other_resource.key - # raise NotImplementedError() - - - # def _connect_list_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): - # other_ind_name = '{}_emit_bin'.format(self.fname) - # other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, - # other_inp_name, - # my_resource.key, - # my_inp_name) - - # my_ind_name = '{}_recv_bin'.format(self.fname) - # my_ind_val = '{}|{}|{}|{}'.format(my_resource.key, - # my_inp_name, - # other_resource.key, - # other_inp_name) - - # my_resource._add_index(my_ind_name, - # my_ind_val) - # my_resource._add_index(other_ind_name, - # other_ind_val) - # try: - # del self._cache[my_inp_name] - # except KeyError: - # pass - # return True - - - # def _connect_simple_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): - # # TODO: for now connections are attached to target resource - # # in future we might change it to separate object - - # other_ind_name = '{}_emit_bin'.format(self.fname) - # other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, - # other_inp_name, - # my_resource.key, - # my_inp_name) - - # my_ind_name = '{}_recv_bin'.format(self.fname) - # my_ind_val = '{}|{}|{}|{}'.format(my_resource.key, - # my_inp_name, - # other_resource.key, - # other_inp_name) - - # # ensure no conflicting connections are done - # # TODO: move this to backend layer - # indexes = my_resource._riak_object.indexes - # to_del = [] - # for ind_name, ind_value in indexes: - # if ind_name == my_ind_name: - # mr, mn = ind_value.split('|')[:2] - # if mr == my_resource.key and mn == my_inp_name: - # to_del.append((ind_name, ind_value)) - # elif ind_name == other_ind_name: - # mr, mn = ind_value.rsplit('|')[2:] - # if mr == my_resource.key and mn == my_inp_name: - # to_del.append((ind_name, ind_value)) - - # for ind_name, ind_value in to_del: - # my_resource._remove_index(ind_name, value=ind_value) - - # # add new - # my_resource._add_index(my_ind_name, - # my_ind_val) - # my_resource._add_index(other_ind_name, - # other_ind_val) - # try: - # del self._cache[my_inp_name] - # except KeyError: - # pass - # return True - - def _input_type(self, resource, name): # XXX: it could be worth to precalculate it if ':' in name: @@ -196,28 +111,6 @@ class InputsFieldWrp(IndexFieldWrp): except KeyError: pass return True - # raise Exception() - # if ':' in my_inp_name: - # tmp_name = my_inp_name.split(':', 1)[0] - # my_input = self._get_raw_field_val(tmp_name) - # else: - # my_input = self._get_raw_field_val(my_inp_name) - # other_input = other_resource.inputs._get_raw_field_val(other_inp_name) - # if isinstance(my_input, self._simple_types): - # my_side = 'simple' - # else: - # my_side = type(my_input).__name__ - # if isinstance(other_input, self._simple_types): - # other_side = 'simple' - # else: - # other_side = type(other_input).__name__ - # method = '_connect_{}_{}'.format(my_side, other_side) - # try: - # meth = getattr(self, method) - # except AttributeError: - # raise Exception("Unknown connection %r %r" % (my_side, other_side)) - # else: - # return meth(my_resource, my_inp_name, other_resource, other_inp_name) def _has_own_input(self, name): try: From 7d07d5b44810ca28a8b2a3bce890b989255a4ad6 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 21:36:16 +0100 Subject: [PATCH 040/191] Fixed typo --- solar/solar/dblayer/test/test_real.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 6911c5b6..4a056472 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -300,7 +300,7 @@ def test_simple_to_dict_inputs(rk): assert r2.inputs['input']['input2'] == 15 -def test_simple_do_dict_inputs_with_tag(rk): +def test_simple_to_dict_inputs_with_tag(rk): k1 = next(rk) k2 = next(rk) k3 = next(rk) From 225f70d97e9946f9a428a72352168f1c25754045 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 22:26:21 +0100 Subject: [PATCH 041/191] dict list inputs are working --- solar/solar/dblayer/solar_models.py | 21 ++++++++++++++ solar/solar/dblayer/test/test_real.py | 42 ++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index e87db73d..4001b049 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -88,6 +88,9 @@ class InputsFieldWrp(IndexFieldWrp): my_resource._add_index(my_ind_name, my_ind_val) return my_key + def _connect_my_list_hash(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): + return self._connect_my_hash(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) + def connect(self, my_inp_name, other_resource, other_inp_name): my_resource = self._instance other_type = self._input_type(other_resource, other_inp_name) @@ -206,6 +209,24 @@ class InputsFieldWrp(IndexFieldWrp): self._cache[name] = res return res + def _map_field_val_list_hash(self, recvs, name): + items = [] + tags = set() + for recv in recvs: + index_val, obj_key = recv + _, _, emitter_key, emitter_inp, my_tag, my_val, mapping_type = index_val.split('|', 6) + cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + items.append((my_tag, my_val, cres)) + tmp_res = {} + for my_tag, my_val, value in items: + try: + tmp_res[my_tag][my_val] = value + except KeyError: + tmp_res[my_tag] = {my_val: value} + res = tmp_res.values() + self._cache[name] = res + return res + def _get_raw_field_val(self, name): return self._instance._data_container[self.fname][name] diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 4a056472..364aba8e 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -9,7 +9,10 @@ def create_resource(key, data): mi = data.get('meta_inputs', {}) for inp_name, inp_value in data.get('inputs', {}).items(): if isinstance(inp_value, list): - schema = ['str!'] + if len(inp_value) == 1 and isinstance(inp_value[0], dict): + schema = [{}] + else: + schema = ['str!'] elif isinstance(inp_value, dict): schema = {} else: @@ -326,3 +329,40 @@ def test_simple_to_dict_inputs_with_tag(rk): assert r2.inputs['input']['input1'] == 10 assert r2.inputs['input']['input2'] == 115 + +def test_simple_to_listdict_inputs(rk): + + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + k4 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + r3 = create_resource(k3, {'name': 'first', + 'inputs': {'input1': 110, + 'input2': 115}}) + r4 = create_resource(k4, {'name': 'first', + 'inputs': {'input1': 1110, + 'input2': 1115}}) + r2 = create_resource(k2, {'name': 'second', + 'inputs': {'input': [{'input1': None, + 'input2': None}]}}) + + + r1.connect(r2, {'input1': 'input:input1', + 'input2': 'input:input2'}) + r3.connect(r2, {'input2': 'input:input2|tag2', + 'input1': 'input:input1|tag1'}) + r4.connect(r2, {'input2': 'input:input2|tag1', + 'input1': 'input:input1|tag2'}) + + r1.save() + r2.save() + r3.save() + r4.save() + + assert r2.inputs['input'] == [{u'input2': 1115, u'input1': 110}, + {u'input2': 115, u'input1': 1110}, + {u'input2': 15, u'input1': 10}] From 67405b6625a2fc4ab00e1a6e4621518f20bd86fa Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 29 Oct 2015 23:00:35 +0100 Subject: [PATCH 042/191] Simple input connect update works --- solar/solar/dblayer/solar_models.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 4001b049..e8dd0bcb 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -49,6 +49,12 @@ class InputsFieldWrp(IndexFieldWrp): other_inp_name) my_ind_val += types_mapping + for ind_name, ind_value in my_resource._riak_object.indexes: + if ind_name == my_ind_name: + mr, mn, _ = ind_value.split('|', 2) + if mr == my_resource.key and mn == my_inp_name: + my_resource._remove_index(ind_name, ind_value) + break my_resource._add_index(my_ind_name, my_ind_val) return my_inp_name @@ -59,6 +65,14 @@ class InputsFieldWrp(IndexFieldWrp): other_inp_name, my_resource.key, my_inp_name) + + for ind_name, ind_value in my_resource._riak_object.indexes: + if ind_name == other_ind_name: + mr, mn = ind_value.rsplit('|')[2:] + if mr == my_resource.key and mn == my_inp_name: + my_resource._remove_index(ind_name, ind_value) + break + my_resource._add_index(other_ind_name, other_ind_val) return other_inp_name @@ -202,6 +216,7 @@ class InputsFieldWrp(IndexFieldWrp): items.append((my_tag, my_val, cres)) tags.add(my_tag) if len(tags) != 1: + # TODO: add it also for during connecting raise Exception("Detected dict with different tags") res = {} for _, my_val, value in items: From 7c42888523e2b843964531adb679477da69e785e Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 10:38:28 +0100 Subject: [PATCH 043/191] When field is str make it basestring --- solar/solar/dblayer/model.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 0574d751..8cb7b763 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -278,6 +278,8 @@ class Field(FieldBase): _simple_types = {int, float, long, str, unicode, basestring, list, tuple, dict} def __init__(self, _type, fname=None, default=NONE): + if _type == str: + _type = basestring self._type = _type super(Field, self).__init__(fname=fname, default=default) @@ -292,7 +294,7 @@ class Field(FieldBase): def __set__(self, instance, value): if not isinstance(value, self._type): - raise Exception("Invalid type %r for %r" % (type(value), self.fname)) + raise Exception("Invalid type %r for %r, expected %r" % (type(value), self.fname, self._type)) if self._type not in self._simple_types: value = self._type.to_simple(value) instance._field_changed(self) From 7c961a93bcb36e79873da9561017d0e61f25708a Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 11:11:26 +0100 Subject: [PATCH 044/191] Added logic for nullable fields (transforms None => NONE) --- solar/solar/dblayer/model.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 8cb7b763..bd6d5c73 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -275,6 +275,8 @@ class FieldBase(object): class Field(FieldBase): + # in from_dict, when you set value to None, then types that are *not* there are set to NONE + _not_nullable_types = {int, float, long, str, unicode, basestring} _simple_types = {int, float, long, str, unicode, basestring, list, tuple, dict} def __init__(self, _type, fname=None, default=NONE): @@ -674,7 +676,9 @@ class Model(object): return self._indexes_changed def to_dict(self): - return dict(self._riak_object.data) + d = dict(self._riak_object.data) + d['key'] = self.key + return d def __str__(self): if self._riak_object is None: @@ -716,6 +720,8 @@ class Model(object): gname = field.gname val = data.get(fname, NONE) default = field.default + if val is None and field._type not in field._not_nullable_types: + val = NONE if val is NONE and default is not NONE: setattr(obj, gname, default) elif val is not NONE: From ac17e839af27a4dd9a9ed49e4bb1c115a28a1627 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 11:12:16 +0100 Subject: [PATCH 045/191] Resource creation works --- solar/solar/core/resource/resource.py | 17 +++++++++-------- solar/solar/dblayer/solar_models.py | 4 ++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 41e4208d..7878c87a 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -82,7 +82,7 @@ class Resource(object): 'id': name, 'name': name, 'actions_path': metadata.get('actions_path', ''), - 'actions': metadata.get('actions', ''), + 'actions': metadata.get('actions', {}), 'base_name': metadata.get('base_name', ''), 'base_path': metadata.get('base_path', ''), 'handler': metadata.get('handler', ''), @@ -90,7 +90,7 @@ class Resource(object): 'version': metadata.get('version', ''), 'meta_inputs': inputs, 'tags': tags, - 'stae': RESOURCE_STATE.created.name + 'state': RESOURCE_STATE.created.name }) self.create_inputs(args) @@ -167,9 +167,10 @@ class Resource(object): resource_inputs = self.resource_inputs() for k, v in args.items(): - i = resource_inputs[k] - i.value = v - i.save() + self.db_obj.inputs[k] = v + # i = resource_inputs[k] + # i.value = v + # i.save() def delete(self): return self.db_obj.delete() @@ -244,8 +245,8 @@ class Resource(object): arg_color = 'yellow' - return ("{resource_s}({name_s}='{id}', {base_path_s}={base_path} " - "{args_s}={input}, {tags_s}={tags})").format( + return ("{resource_s}({name_s}='{key}', {base_path_s}={base_path} " + "{args_s}={inputs}, {tags_s}={tags})").format( resource_s=click.style('Resource', fg='white', bold=True), name_s=click.style('name', fg=arg_color, bold=True), base_path_s=click.style('base_path', fg=arg_color, bold=True), @@ -259,7 +260,7 @@ class Resource(object): def connect_with_events(self, receiver, mapping=None, events=None, use_defaults=False): - self.db_obj.connect(receiver, mappings=mappings) + self.db_obj.connect(receiver.db_obj, mapping=mapping) # signals.connect(self, receiver, mapping=mapping) # TODO: implement events return diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index e8dd0bcb..59926a19 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -421,10 +421,10 @@ class Resource(Model): updated = IndexedField(StrInt) - def connect(self, other, mappings): + def connect(self, other, mapping): my_inputs = self.inputs other_inputs = other.inputs - for my_name, other_name in mappings.iteritems(): + for my_name, other_name in mapping.iteritems(): other_inputs.connect(other_name, self, my_name) def save(self, *args, **kwargs): From 08b991f8670680b97a51011048638cbd3f036025 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 12:23:49 +0100 Subject: [PATCH 046/191] Fixed inputs list tracking --- solar/solar/dblayer/solar_models.py | 32 ++++++++++++++++------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 59926a19..65e8bfb2 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -49,29 +49,33 @@ class InputsFieldWrp(IndexFieldWrp): other_inp_name) my_ind_val += types_mapping - for ind_name, ind_value in my_resource._riak_object.indexes: - if ind_name == my_ind_name: - mr, mn, _ = ind_value.split('|', 2) - if mr == my_resource.key and mn == my_inp_name: - my_resource._remove_index(ind_name, ind_value) - break + real_my_type = self._input_type(my_resource, my_inp_name) + if real_my_type == InputTypes.simple: + for ind_name, ind_value in my_resource._riak_object.indexes: + if ind_name == my_ind_name: + mr, mn, _ = ind_value.split('|', 2) + if mr == my_resource.key and mn == my_inp_name: + my_resource._remove_index(ind_name, ind_value) + break my_resource._add_index(my_ind_name, my_ind_val) return my_inp_name - def _connect_other_simple(self, my_resource, my_inp_name, other_resource, other_inp_name): + def _connect_other_simple(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): other_ind_name = '{}_emit_bin'.format(self.fname) other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, other_inp_name, my_resource.key, my_inp_name) - for ind_name, ind_value in my_resource._riak_object.indexes: - if ind_name == other_ind_name: - mr, mn = ind_value.rsplit('|')[2:] - if mr == my_resource.key and mn == my_inp_name: - my_resource._remove_index(ind_name, ind_value) - break + real_my_type = self._input_type(my_resource, my_inp_name) + if real_my_type == InputTypes.simple: + for ind_name, ind_value in my_resource._riak_object.indexes: + if ind_name == other_ind_name: + mr, mn = ind_value.rsplit('|')[2:] + if mr == my_resource.key and mn == my_inp_name: + my_resource._remove_index(ind_name, ind_value) + break my_resource._add_index(other_ind_name, other_ind_val) @@ -121,7 +125,7 @@ class InputsFieldWrp(IndexFieldWrp): # set other side other_meth = getattr(self, '_connect_other_{}'.format(other_type.name)) - other_meth(my_resource, my_inp_name, other_resource, other_inp_name) + other_meth(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) try: del self._cache[my_affected] From acf67249958b9792434cb5aec8dcacc52374c078 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 12:32:15 +0100 Subject: [PATCH 047/191] Resource show, resource show by tag, adding tags works --- solar/solar/cli/__init__.py | 7 +++++++ solar/solar/core/resource/resource.py | 15 +++++++++++---- solar/solar/dblayer/model.py | 12 +++++++----- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/solar/solar/cli/__init__.py b/solar/solar/cli/__init__.py index e69de29b..ab3ccfc0 100644 --- a/solar/solar/cli/__init__.py +++ b/solar/solar/cli/__init__.py @@ -0,0 +1,7 @@ +from solar.dblayer.model import ModelMeta + +import atexit + +ModelMeta.session_start() + +atexit.register(ModelMeta.session_end) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 7878c87a..8fb5c7e7 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -202,10 +202,14 @@ class Resource(object): return self.db_obj.tags def add_tags(self, *tags): - self.db_obj.add_tags(*tags) + for tag in tags: + self.db_obj.tags.set(tag) + self.db_obj.save_lazy() def remove_tags(self, *tags): - self.db_obj.remove_tags(*tags) + for tag in tags: + self.db_obj.tags.remove(tag) + self.db_obj.save_lazy() @property def connections(self): @@ -290,8 +294,11 @@ def load_all(): def load_by_tags(tags): tags = set(tags) - return [Resource(r) for r in orm.DBResource.load_all() - if tags.issubset(set(r.tags))] + candids_all = set() + for tag in tags: + candids = DBResource.tags.filter(tag) + candids_all.update(set(candids)) + return [Resource(r) for r in DBResource.multi_get(candids_all)] def validate_resources(): diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index bd6d5c73..c9012c23 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -38,7 +38,7 @@ class SingleClassCache(object): class ClassCache(object): def __get__(self, _, owner): - th = current_thread() + # th = current_thread() l = LOCAL # better don't duplicate class names cache_name = owner.__name__ @@ -47,9 +47,9 @@ class ClassCache(object): except AttributeError: cache_id = uuid.UUID(int=getrandbits(128), version=4).hex setattr(l, 'cache_id', cache_id) - if getattr(th, 'cache_id', None) != cache_id: + if getattr(l, 'cache_id_cmp', None) != cache_id: # new cache - setattr(th, 'cache_id', cache_id) + setattr(l, 'cache_id_cmp', cache_id) c = SingleClassCache(owner) setattr(l, '_model_caches', {}) l._model_caches[cache_name] = c @@ -64,9 +64,10 @@ class ClassCache(object): def clear_cache(): - th = current_thread() + # th = current_thread() + l = LOCAL cache_id = uuid.UUID(int=getrandbits(128), version=4).hex - setattr(th, 'cache_id', cache_id) + setattr(l, 'cache_id_cmp', cache_id) def get_bucket(_, owner, mcs): @@ -766,6 +767,7 @@ class Model(object): def save_lazy(self): self._c.lazy_save.add(self) + def delete(self): ls = self._c.lazy_save try: From 9c3d89465e1d0a0706a19009630e99ce5cec9211 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 12:37:16 +0100 Subject: [PATCH 048/191] load_all --- solar/solar/cli/resource.py | 3 ++- solar/solar/core/resource/resource.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/solar/solar/cli/resource.py b/solar/solar/cli/resource.py index d83e629f..3e4085d8 100644 --- a/solar/solar/cli/resource.py +++ b/solar/solar/cli/resource.py @@ -136,14 +136,15 @@ def create(args, base_path, name): @click.option('--json', default=False, is_flag=True) @click.option('--color', default=True, is_flag=True) def show(name, tag, json, color): + echo = click.echo_via_pager if name: resources = [sresource.load(name)] + echo = click.echo elif tag: resources = sresource.load_by_tags(set(tag)) else: resources = sresource.load_all() - echo = click.echo_via_pager if json: output = json.dumps([r.to_dict() for r in resources], indent=2) echo = click.echo diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 8fb5c7e7..b551fb32 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -31,6 +31,7 @@ from hashlib import md5 from solar.dblayer.solar_models import Resource as DBResource +from solar.dblayer.model import StrInt def read_meta(base_path): @@ -290,7 +291,8 @@ def load(name): # TODO def load_all(): - return [Resource(r) for r in orm.DBResource.load_all()] + candids = DBResource.updated.filter(StrInt.p_min(), StrInt.p_max()) + return [Resource(r) for r in DBResource.multi_get(candids)] def load_by_tags(tags): tags = set(tags) From 9c2a2ac15a718f86deca41bac9683d9116842ef4 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 12:41:48 +0100 Subject: [PATCH 049/191] Do not set duplicate tags --- solar/solar/dblayer/solar_models.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 65e8bfb2..cf92cc2e 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -320,15 +320,16 @@ class TagsFieldWrp(IndexFieldWrp): name, value = name.split('=', 1) if value is None: value = '' + full_value = '{}={}'.format(name, value) inst = self._instance - indexes = inst._riak_object.indexes.copy() # copy it - - inst._add_index('{}_bin'.format(self.fname), '{}~{}'.format(name, value)) try: fld = inst._data_container[self.fname] except IndexError: fld = inst._data_container[self.fname] = [] - full_value = '{}={}'.format(name, value) + if full_value in fld: + return + # indexes = inst._riak_object.indexes.copy() # copy it + inst._add_index('{}_bin'.format(self.fname), '{}~{}'.format(name, value)) try: fld.append(full_value) except KeyError: From cff6a0ec5679d35e3197434d00ea8269327b30ad Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 13:41:23 +0100 Subject: [PATCH 050/191] connection graph / edges (not working because of networknx) --- solar/solar/core/resource/resource.py | 15 +++++++-------- solar/solar/dblayer/solar_models.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index b551fb32..9f05ab11 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -28,6 +28,7 @@ from solar.events import api from uuid import uuid4 from hashlib import md5 +import networkx from solar.dblayer.solar_models import Resource as DBResource @@ -220,22 +221,20 @@ class Resource(object): [(emitter, emitter_input, receiver, receiver_input), ...] """ rst = [] - for emitter, receiver, meta in self.db_obj.graph().edges(data=True): + for (emitter_resource, emitter_input), (receiver_resource, receiver_input), meta in self.graph().edges(data=True): if meta: - receiver_input = '{}:{}|{}'.format(receiver.name, + receiver_input = '{}:{}|{}'.format(receiver_input, meta['destination_key'], meta['tag']) - else: - receiver_input = receiver.name rst.append( - [emitter.resource.name, emitter.name, - receiver.resource.name, receiver_input]) + [emitter_resource, emitter_input, + receiver_resource, receiver_input]) return rst def graph(self): mdg = networkx.MultiDiGraph() - for input_name, input_value in self.inputs: - mdg.add_edges_from(input.edges()) + for data in self.db_obj.inputs._edges(): + mdg.add_edges_from(data) return mdg def resource_inputs(self): diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index cf92cc2e..96ffc9e5 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -40,6 +40,33 @@ class InputsFieldWrp(IndexFieldWrp): return InputTypes.hash raise Exception("Unknown type") + def _edges(self): + inst = self._instance + start = inst.key + my_ind_name = '{}_recv_bin'.format(self.fname) + res = inst._get_index(my_ind_name, + startkey=start + '|', + endkey=start + '|~', + return_terms=True, + max_results=99999).results + vals = map(itemgetter(0), res) + for val in vals: + data = val.split('|') + dlen = len(data) + my_resource = data[0] + my_input = data[1] + other_resource = data[2] + other_input = data[3] + if dlen == 5: + meta = None + elif dlen == 7: + meta = {'destination_key': data[5], + 'tag': data[4]} + else: + raise Exception("Unsupported case") + yield (my_resource, my_input), (other_resource, other_input), meta + + def _connect_my_simple(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): types_mapping = '|{}_{}'.format(my_type.value, other_type.value) my_ind_name = '{}_recv_bin'.format(self.fname) From d589070c90508e412b6f10c28a8960e122d314e3 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 13:41:56 +0100 Subject: [PATCH 051/191] Added save_lazy to resource.py --- solar/solar/core/resource/resource.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 9f05ab11..575a3126 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -170,9 +170,7 @@ class Resource(object): for k, v in args.items(): self.db_obj.inputs[k] = v - # i = resource_inputs[k] - # i.value = v - # i.save() + self.db_obj.save_lazy() def delete(self): return self.db_obj.delete() @@ -182,19 +180,19 @@ class Resource(object): self.delete() else: self.db_obj.state = RESOURCE_STATE.removed.name - self.db_obj.save() + self.db_obj.save_lazy() def set_operational(self): self.db_obj.state = RESOURCE_STATE.operational.name - self.db_obj.save() + self.db_obj.save_lazy() def set_error(self): self.db_obj.state = RESOURCE_STATE.error.name - self.db_obj.save() + self.db_obj.save_lazy() def set_created(self): self.db_obj.state = RESOURCE_STATE.created.name - self.db_obj.save() + self.db_obj.save_lazy() def to_be_removed(self): return self.db_obj.state == RESOURCE_STATE.removed.name @@ -265,6 +263,7 @@ class Resource(object): def connect_with_events(self, receiver, mapping=None, events=None, use_defaults=False): self.db_obj.connect(receiver.db_obj, mapping=mapping) + self.db_obj.save_lazy() # signals.connect(self, receiver, mapping=mapping) # TODO: implement events return From f2a98738bcbe8ef106e7d95097388ebd86e26cb8 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 13:51:45 +0100 Subject: [PATCH 052/191] Added comment that connections requires fixes --- solar/solar/core/resource/resource.py | 1 + 1 file changed, 1 insertion(+) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 575a3126..6a1f84e9 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -219,6 +219,7 @@ class Resource(object): [(emitter, emitter_input, receiver, receiver_input), ...] """ rst = [] + # TODO: fix it for (emitter_resource, emitter_input), (receiver_resource, receiver_input), meta in self.graph().edges(data=True): if meta: receiver_input = '{}:{}|{}'.format(receiver_input, From e18c77f8bbfcf661a9f91d405c7160afd740afb3 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 13:59:36 +0100 Subject: [PATCH 053/191] as_list for inputs --- solar/solar/dblayer/solar_models.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 96ffc9e5..e2b2b59d 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -342,6 +342,12 @@ class TagsFieldWrp(IndexFieldWrp): def __iter__(self): return iter(self._instance._data_container[self.fname]) + def as_list(self): + try: + return self._instance._data_container[self.fname][:] + except KeyError: + return [] + def set(self, name, value=None): if '=' in name and value is None: name, value = name.split('=', 1) @@ -425,6 +431,7 @@ class TagsField(IndexField): return set(map(itemgetter(1), res)) + # class MetaInput(NestedModel): # name = Field(str) From b1dc3f62ae4154c9a4a5b30e272d9ff718879492 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 27 Oct 2015 17:01:07 +0200 Subject: [PATCH 054/191] Add Tasks model for riak db backend --- solar/solar/dblayer/solar_models.py | 57 +++++++++++++++++++ .../dblayer/test/test_execution_models.py | 42 ++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 solar/solar/dblayer/test/test_execution_models.py diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index e2b2b59d..ae4bbdce 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -470,3 +470,60 @@ class Resource(Model): if self.changed(): self.updated = StrInt() return super(Resource, self).save(*args, **kwargs) + +""" +Type of operations: + +- load all tasks + transitions +- load single task + childs + all parents of childs (and transitions between them) +""" + +class TasksFieldWrp(IndexFieldWrp): + + def add(self, task): + self._instance._add_index('{}_bin'.format(self.fname), task.key) + return True + + +class TasksField(IndexField): + + _wrp_class = TasksFieldWrp + + def __set__(self, obj, value): + wrp = getattr(obj, self.fname) + obj._data_container[self.fname] = self.default + for val in value: + wrp.add(val) + + +class ChildFieldWrp(TasksFieldWrp): + + def add(self, task): + task.parents.add(self._instance) + + +class ChildField(TasksField): + + _wrp_class = ChildFieldWrp + + +class Task(Model): + """Node object""" + + name = Field(str) + status = Field(str) + target = Field(str) + + execution = IndexedField(str) + parents = TasksField(default=list) + childs = ChildField(default=list) + + +# class Transition(Model): +# """Edge object""" + + # parent = TaskField() + # child = TaskField() + + # for now it is only state, e.g. transition based on success/error + # condition = Field(str) diff --git a/solar/solar/dblayer/test/test_execution_models.py b/solar/solar/dblayer/test/test_execution_models.py new file mode 100644 index 00000000..8cee861a --- /dev/null +++ b/solar/solar/dblayer/test/test_execution_models.py @@ -0,0 +1,42 @@ +import pytest + +from solar.dblayer.solar_models import Task + + +def test_tasks_selected_by_execution_id(rk): + execution = next(rk) + + for i in range(2): + t = Task.from_dict( + str(i)+execution, + {'name': str(i), + 'execution': execution}) + t.save() + another_execution = next(rk) + + for i in range(2): + t = Task.from_dict( + str(i)+another_execution, + {'name': str(i), + 'execution': another_execution}) + t.save() + + assert len(Task.execution.filter(execution)) == 2 + assert len(Task.execution.filter(another_execution)) == 2 + + +def test_parent_child(rk): + execution = next(rk) + + t1 = Task.from_dict( + '1'+execution, + {'name': '1', + 'execution': execution}) + t1.save() + t2 = Task.from_dict( + '2'+execution, + {'name': '2', + 'execution': execution}) + t2.save() + + t1.childs.add(t2) From 2b3ceb5b30a9decc3629fe72d28d9d4841fc7013 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Wed, 28 Oct 2015 12:15:59 +0200 Subject: [PATCH 055/191] Implement childs/parents tasks --- solar/solar/dblayer/solar_models.py | 55 ++++++++++++++----- solar/solar/dblayer/test/conftest.py | 8 ++- .../dblayer/test/test_execution_models.py | 23 ++++---- 3 files changed, 61 insertions(+), 25 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index ae4bbdce..b08647ba 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -474,16 +474,26 @@ class Resource(Model): """ Type of operations: -- load all tasks + transitions +- load all tasks - load single task + childs + all parents of childs (and transitions between them) """ class TasksFieldWrp(IndexFieldWrp): def add(self, task): - self._instance._add_index('{}_bin'.format(self.fname), task.key) return True + def all(self, postprocessor=None): + if postprocessor: + return map(postprocessor, self._instance._data_container[self.fname]) + return self._instance._data_container[self.fname] + + def all_names(self): + return self.all(lambda key: key.split('~')[1]) + + def all_tasks(self): + return self.all(Task.get) + class TasksField(IndexField): @@ -495,11 +505,19 @@ class TasksField(IndexField): for val in value: wrp.add(val) + def _parse_key(self, startkey): + return startkey + class ChildFieldWrp(TasksFieldWrp): def add(self, task): - task.parents.add(self._instance) + self._instance._data_container['childs'].append(task.key) + task._data_container['parents'].append(self._instance.key) + + task._add_index('childs_bin', self._instance.key) + self._instance._add_index('parents_bin', task.key) + return True class ChildField(TasksField): @@ -507,6 +525,22 @@ class ChildField(TasksField): _wrp_class = ChildFieldWrp +class ParentFieldWrp(TasksFieldWrp): + + def add(self, task): + self._instance._data_container['parents'].append(task.key) + task._data_container['childs'].append(self._instance.key) + + task._add_index('parents_bin', self._instance.key) + self._instance._add_index('childs_bin', task.key) + return True + + +class ParentField(TasksField): + + _wrp_class = ParentFieldWrp + + class Task(Model): """Node object""" @@ -515,15 +549,10 @@ class Task(Model): target = Field(str) execution = IndexedField(str) - parents = TasksField(default=list) + parents = ParentField(default=list) childs = ChildField(default=list) - -# class Transition(Model): -# """Edge object""" - - # parent = TaskField() - # child = TaskField() - - # for now it is only state, e.g. transition based on success/error - # condition = Field(str) + @classmethod + def new(self, data): + key = '%s~%s' % (data['execution'], data['name']) + return Task.from_dict(key, data) diff --git a/solar/solar/dblayer/test/conftest.py b/solar/solar/dblayer/test/conftest.py index 1c03e089..33cfcc9a 100644 --- a/solar/solar/dblayer/test/conftest.py +++ b/solar/solar/dblayer/test/conftest.py @@ -57,8 +57,12 @@ client = SqlClient(':memory:', threadlocals=False, autocommit=False) # autocommit=False, pragmas=(('journal_mode', 'WAL'), # ('synchronous', 'NORMAL'))) -# from solar.dblayer.riak_client import RiakClient -# client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) +#from solar.dblayer.riak_client import RiakClient +#client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) # client = RiakClient(protocol='http', host='10.0.0.3', http_port=18098) +@pytest.fixture +def riakc(): + return client + ModelMeta.setup(client) diff --git a/solar/solar/dblayer/test/test_execution_models.py b/solar/solar/dblayer/test/test_execution_models.py index 8cee861a..3e20414c 100644 --- a/solar/solar/dblayer/test/test_execution_models.py +++ b/solar/solar/dblayer/test/test_execution_models.py @@ -7,16 +7,14 @@ def test_tasks_selected_by_execution_id(rk): execution = next(rk) for i in range(2): - t = Task.from_dict( - str(i)+execution, + t = Task.new( {'name': str(i), 'execution': execution}) t.save() another_execution = next(rk) for i in range(2): - t = Task.from_dict( - str(i)+another_execution, + t = Task.new( {'name': str(i), 'execution': another_execution}) t.save() @@ -28,15 +26,20 @@ def test_tasks_selected_by_execution_id(rk): def test_parent_child(rk): execution = next(rk) - t1 = Task.from_dict( - '1'+execution, + t1 = Task.new( {'name': '1', 'execution': execution}) - t1.save() - t2 = Task.from_dict( - '2'+execution, + + t2 = Task.new( {'name': '2', 'execution': execution}) + t1.childs.add(t2) + t1.save() t2.save() - t1.childs.add(t2) + assert Task.childs.filter(t1.key) == {t2.key} + assert Task.parents.filter(t2.key) == {t1.key} + assert t1.childs.all_tasks() == [t2] + assert t2.parents.all_names() == [t1.name] + + From df5a1e0e447a70dbfc25669bcf26292c3bc0f3c6 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Wed, 28 Oct 2015 13:07:36 +0200 Subject: [PATCH 056/191] Conver save_graph/get_graph to riak client --- solar/solar/dblayer/solar_models.py | 27 +++++++++-------- solar/solar/orchestration/graph.py | 46 +++++++++++++++++++---------- 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index b08647ba..f4f8da19 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -474,7 +474,7 @@ class Resource(Model): """ Type of operations: -- load all tasks +- load all tasks for execution - load single task + childs + all parents of childs (and transitions between them) """ @@ -494,6 +494,14 @@ class TasksFieldWrp(IndexFieldWrp): def all_tasks(self): return self.all(Task.get) + def _add(self, parent, child): + parent._data_container['childs'].append(child.key) + child._data_container['parents'].append(parent.key) + + child._add_index('childs_bin', parent.key) + parent._add_index('parents_bin', child.key) + return True + class TasksField(IndexField): @@ -509,15 +517,11 @@ class TasksField(IndexField): return startkey + class ChildFieldWrp(TasksFieldWrp): def add(self, task): - self._instance._data_container['childs'].append(task.key) - task._data_container['parents'].append(self._instance.key) - - task._add_index('childs_bin', self._instance.key) - self._instance._add_index('parents_bin', task.key) - return True + return self._add(self._instance, task) class ChildField(TasksField): @@ -528,12 +532,7 @@ class ChildField(TasksField): class ParentFieldWrp(TasksFieldWrp): def add(self, task): - self._instance._data_container['parents'].append(task.key) - task._data_container['childs'].append(self._instance.key) - - task._add_index('parents_bin', self._instance.key) - self._instance._add_index('childs_bin', task.key) - return True + return self._add(task, self._instance) class ParentField(TasksField): @@ -547,6 +546,8 @@ class Task(Model): name = Field(str) status = Field(str) target = Field(str) + task_type = Field(str) + args = Field(list) execution = IndexedField(str) parents = ParentField(default=list) diff --git a/solar/solar/orchestration/graph.py b/solar/solar/orchestration/graph.py index 8370e289..b13f49ef 100644 --- a/solar/solar/orchestration/graph.py +++ b/solar/solar/orchestration/graph.py @@ -26,32 +26,48 @@ from collections import Counter from solar.interfaces.db import get_db +from solar.dblayer.model import ModelMeta +from solar.dblayer.riak_client import RiakClient +from solar.dblayer.solar_models import Task +client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) + +ModelMeta.setup(client) + db = get_db() def save_graph(graph): # maybe it is possible to store part of information in AsyncResult backend uid = graph.graph['uid'] - db.create(uid, graph.graph, db.COLLECTIONS.plan_graph) - for n in graph: - collection = db.COLLECTIONS.plan_node.name + ':' + uid - db.create(n, properties=graph.node[n], collection=collection) - db.create_relation_str(uid, n, type_=db.RELATION_TYPES.graph_to_node) - - for u, v, properties in graph.edges(data=True): - type_ = db.RELATION_TYPES.plan_edge.name + ':' + uid - db.create_relation_str(u, v, properties, type_=type_) + for n in nx.topological_sort(graph): + t = Task.new( + {'name': n, + 'execution': uid, + 'status': graph.node[n].get('status', ''), + 'target': str(graph.node[n].get('target', '')), + 'task_type': graph.node[n].get('type', ''), + 'args': graph.node[n].get('args')}) + graph.node[n]['task'] = t + for pred in graph.predecessors(n): + pred_task = graph.node[pred]['task'] + t.parents.add(pred_task) + pred_task.save() + t.save() def get_graph(uid): dg = nx.MultiDiGraph() - collection = db.COLLECTIONS.plan_node.name + ':' + uid - type_ = db.RELATION_TYPES.plan_edge.name + ':' + uid - dg.graph = db.get(uid, collection=db.COLLECTIONS.plan_graph).properties - dg.add_nodes_from([(n.uid, n.properties) for n in db.all(collection=collection)]) - dg.add_edges_from([(i['source'], i['dest'], i['properties']) - for i in db.all_relations(type_=type_, db_convert=False)]) + dg.graph['uid'] = uid + dg.graph['name'] = uid.split(':')[0] + tasks = map(Task.get, Task.execution.filter(uid)) + for t in tasks: + dg.add_node( + t.name, status=t.status, + type=t.task_type, args=t.args, + target=t.target) + for u in t.parents.all_names(): + dg.add_edge(u, t.name) return dg From 63ded9165d90993e84de3e16c398a51eb184db5e Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Wed, 28 Oct 2015 14:47:21 +0200 Subject: [PATCH 057/191] Fix defaults for graphviz --- solar/solar/dblayer/solar_models.py | 1 + solar/solar/events/controls.py | 8 ++++---- solar/solar/orchestration/graph.py | 8 +++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index f4f8da19..b289d92f 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -548,6 +548,7 @@ class Task(Model): target = Field(str) task_type = Field(str) args = Field(list) + errmsg = Field(str) execution = IndexedField(str) parents = ParentField(default=list) diff --git a/solar/solar/events/controls.py b/solar/solar/events/controls.py index 2a699d6f..f9bf65e1 100644 --- a/solar/solar/events/controls.py +++ b/solar/solar/events/controls.py @@ -102,13 +102,13 @@ class React(Event): loaded_resource = resource.load(self.child) except KeyError: # orm throws this error when we're NOT using resource there - location_id = None + location_id = '' else: location_id = loaded_resource.args['location_id'] changes_graph.add_node( self.child_node, status='PENDING', target=location_id, - errmsg=None, type='solar_resource', + errmsg='', type='solar_resource', args=[self.child, self.child_action]) changes_graph.add_edge( @@ -128,11 +128,11 @@ class StateChange(Event): loaded_resource = resource.load(self.parent) except KeyError: # orm throws this error when we're NOT using resource there - location_id = None + location_id = '' else: location_id = loaded_resource.args['location_id'] changes_graph.add_node( self.parent_node, status='PENDING', target=location_id, - errmsg=None, type='solar_resource', + errmsg='', type='solar_resource', args=[self.parent, self.parent_action]) diff --git a/solar/solar/orchestration/graph.py b/solar/solar/orchestration/graph.py index b13f49ef..9807b7e2 100644 --- a/solar/solar/orchestration/graph.py +++ b/solar/solar/orchestration/graph.py @@ -47,7 +47,8 @@ def save_graph(graph): 'status': graph.node[n].get('status', ''), 'target': str(graph.node[n].get('target', '')), 'task_type': graph.node[n].get('type', ''), - 'args': graph.node[n].get('args')}) + 'args': graph.node[n].get('args', []), + 'errmsg': graph.node[n].get('errmsg', '')}) graph.node[n]['task'] = t for pred in graph.predecessors(n): pred_task = graph.node[pred]['task'] @@ -65,7 +66,8 @@ def get_graph(uid): dg.add_node( t.name, status=t.status, type=t.task_type, args=t.args, - target=t.target) + target=t.target or None, + errmsg=t.errmsg or None) for u in t.parents.all_names(): dg.add_edge(u, t.name) return dg @@ -83,7 +85,7 @@ def parse_plan(plan_path): for task in plan['tasks']: defaults = { 'status': 'PENDING', - 'errmsg': None, + 'errmsg': '', } defaults.update(task['parameters']) dg.add_node( From ad2548a0995a8692bf624e03f3dd59e302d5d445 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Wed, 28 Oct 2015 16:49:29 +0200 Subject: [PATCH 058/191] Add field for events --- solar/solar/dblayer/model.py | 1 - solar/solar/dblayer/solar_models.py | 9 ++++-- solar/solar/dblayer/test/test_real.py | 11 +++++++ solar/solar/events/api.py | 41 +++++++++------------------ 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index c9012c23..be698626 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -398,7 +398,6 @@ class IndexField(FieldBase): for f_name, f_value in value.iteritems(): wrp[f_name] = f_value - def _parse_key(self, k): if '=' in k: val, subval = k.split('=', 1) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index b289d92f..ff824026 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -454,10 +454,12 @@ class Resource(Model): puppet_module = Field(str) # remove meta_inputs = Field(dict, default=dict) state = Field(str) # on_set/on_get would be useful + events = Field(list, default=list) inputs = InputsField(default=dict) tags = TagsField(default=list) + updated = IndexedField(StrInt) def connect(self, other, mapping): @@ -483,10 +485,13 @@ class TasksFieldWrp(IndexFieldWrp): def add(self, task): return True + def __iter__(self): + return iter(self._instance._data_container[self.fname]) + def all(self, postprocessor=None): if postprocessor: - return map(postprocessor, self._instance._data_container[self.fname]) - return self._instance._data_container[self.fname] + return map(postprocessor, self) + return list(self) def all_names(self): return self.all(lambda key: key.split('~')[1]) diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 364aba8e..659590c5 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -221,6 +221,7 @@ def test_updated_behaviour(rk): assert k1 in Resource.updated.filter(StrInt.p_min(), StrInt.p_max()) +<<<<<<< HEAD def test_list_inputs(rk): k1 = next(rk) @@ -366,3 +367,13 @@ def test_simple_to_listdict_inputs(rk): assert r2.inputs['input'] == [{u'input2': 1115, u'input1': 110}, {u'input2': 115, u'input1': 1110}, {u'input2': 15, u'input1': 10}] + +def test_events(rk): + k = next(rk) + r1 = Resource.from_dict(k, {'events': ['event1', 'event2']}) + r1.save() + assert r1.events == ['event1', 'event2'] + r1.events.pop() + r1.save() + assert r1.events == ['event1'] + diff --git a/solar/solar/events/api.py b/solar/solar/events/api.py index 937c3efc..3cd8e8d4 100644 --- a/solar/solar/events/api.py +++ b/solar/solar/events/api.py @@ -22,6 +22,14 @@ from solar.interfaces import orm from solar.events.controls import Dep, React, StateChange +from solar.dblayer.model import ModelMeta +from solar.dblayer.riak_client import RiakClient +from solar.dblayer.solar_models import Resource +client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) + +ModelMeta.setup(client) + + def create_event(event_dict): etype = event_dict['etype'] kwargs = {'child': event_dict['child'], @@ -52,11 +60,7 @@ def add_event(ev): if ev == rev: break else: - rst.append(ev) - resource_events = orm.DBResourceEvents.get_or_create(ev.parent) - event_db = orm.DBEvent(**ev.to_dict()) - event_db.save() - resource_events.events.add(event_db) + add_events(ev.parent, [ev]) def add_dep(parent, dep, actions, state='success'): @@ -76,34 +80,17 @@ def add_react(parent, dep, actions, state='success'): def add_events(resource, lst): - resource_events = orm.DBResourceEvents.get_or_create(resource) - for ev in lst: - event_db = orm.DBEvent(**ev.to_dict()) - event_db.save() - resource_events.events.add(event_db) - - -def set_events(resource, lst): - resource_events = orm.DBResourceEvents.get_or_create(resource) - for ev in resource_events.events.as_set(): - ev.delete() - for ev in lst: - event_db = orm.DBEvent(**ev.to_dict()) - event_db.save() - resource_events.events.add(event_db) + resource = Resource.get(resource) + resource.events.append([ev.to_dict() for ev in lst]) + resource.save() def remove_event(ev): - event_db = orm.DBEvent(**ev.to_dict()) - event_db.delete() + raise NotImplemented() def all_events(resource): - events = orm.DBResourceEvents.get_or_create(resource).events.as_set() - - if not events: - return [] - return [create_event(i.to_dict()) for i in events] + return [create_event(e) for e in Resource.get(resource).events] def bft_events_graph(start): From 46193e3947c930c903088cf4b4871c4aa9f0e7e9 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 29 Oct 2015 12:27:26 +0200 Subject: [PATCH 059/191] Add model for CompositeIndex and LogItems --- solar/solar/dblayer/model.py | 20 ++++++++++++ solar/solar/dblayer/solar_models.py | 47 ++++++++++++++++++++++++++-- solar/solar/dblayer/test/test_log.py | 42 +++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 solar/solar/dblayer/test/test_log.py diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index be698626..44e21e4e 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -424,6 +424,26 @@ class IndexField(FieldBase): return set(res) +class CompositeIndexFieldWrp(IndexFieldWrp): + + def reset(self): + index = [] + for f in self._field_obj.fields: + index.append(self._instance._data_container.get(f, '')) + index = '|'.join(index) + self._instance._add_index('%s_bin' % self.fname, index) + +class CompositeIndexField(IndexField): + + _wrp_class = CompositeIndexFieldWrp + + def __init__(self, fields=(), *args, **kwargs): + super(CompositeIndexField, self).__init__(*args, **kwargs) + self.fields = fields + + def _parse_key(self, startkey): + vals = [startkey[f] for f in self.fields if f in startkey] + return '|'.join(vals) + '*' class ModelMeta(type): diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index ff824026..91996bf4 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -3,15 +3,17 @@ from solar.dblayer.model import (Model, Field, IndexField, DBLayerException, requires_clean_state, check_state_for, StrInt, - IndexedField) + IndexedField, CompositeIndexField) from types import NoneType from operator import itemgetter from enum import Enum from itertools import groupby +from uuid import uuid4 InputTypes = Enum('InputTypes', 'simple list hash list_hash') + class DBLayerSolarException(DBLayerException): pass @@ -560,6 +562,47 @@ class Task(Model): childs = ChildField(default=list) @classmethod - def new(self, data): + def new(cls, data): key = '%s~%s' % (data['execution'], data['name']) return Task.from_dict(key, data) + + +""" +system log + +1. one bucket for all log items +2. separate logs for stage/history (using index) +3. last log item for resource in history +4. log item in staged log for resource.action +""" + +class LogItem(Model): + + uid = IndexedField(str, default=lambda: str(uuid4())) + resource = Field(str) + action = Field(str) + diff = Field(list) + connections_diff = Field(list) + state = Field(list) + base_path = Field(str) # remove me + + log = Field(str) # staged/history + + composite = CompositeIndexField(fields=('log', 'resource', 'action')) + + @property + def log_action(self): + return '.'.join((self.resource, self.action)) + + def save(self): + if any(f in self._modified_fields for f in LogItem.composite.fields): + self.composite.reset() + return super(LogItem, self).save() + + @classmethod + def new(cls, data): + vals = {} + if 'uid' not in vals: + vals['uid'] = cls.uid.default + vals.update(data) + return LogItem.from_dict(vals['uid'], vals) diff --git a/solar/solar/dblayer/test/test_log.py b/solar/solar/dblayer/test/test_log.py new file mode 100644 index 00000000..4bd7f228 --- /dev/null +++ b/solar/solar/dblayer/test/test_log.py @@ -0,0 +1,42 @@ +import pytest + +from solar.dblayer.solar_models import LogItem + + +def test_separate_logs(): + + history = 'history' + staged = 'staged' + history_uids = set() + staged_uids = set() + for i in range(2): + l = LogItem.new({'log': history}) + l.save() + history_uids.add(l.uid) + for i in range(3): + l = LogItem.new({'log': staged}) + l.save() + staged_uids.add(l.uid) + + assert LogItem.composite.filter({'log': history}) == history_uids + assert LogItem.composite.filter({'log': staged}) == staged_uids + + +def test_multiple_filter(): + + l1 = LogItem.new({'log': 'history', 'resource': 'a'}) + l2 = LogItem.new({'log': 'history', 'resource': 'b'}) + + l1.save() + l2.save() + + assert LogItem.composite.filter({'log': 'history', 'resource': 'a'}) == {l1.uid} + assert LogItem.composite.filter({'log': 'history', 'resource': 'b'}) == {l2.uid} + + +def test_changed_index(): + + l = LogItem.new({'log': 'staged', 'resource': 'a', 'action': 'run'}) + l.save() + + assert LogItem.composite.filter({'log': 'staged'}) == {l.uid} From d6e4e73d62df9c48cf5a80b4d1f6ef5576347c2a Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 29 Oct 2015 12:27:44 +0200 Subject: [PATCH 060/191] Add *.svg to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2e732827..6187e89f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ celery*.log *.dot *.png +*.svg resources_compiled.py # bootstrap From 2f62aa9c2ceda9092c9bcc9f99e25421ac98d474 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 29 Oct 2015 12:59:41 +0200 Subject: [PATCH 061/191] Change bucket for every test --- solar/solar/dblayer/test/conftest.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/solar/solar/dblayer/test/conftest.py b/solar/solar/dblayer/test/conftest.py index 33cfcc9a..54c1a31a 100644 --- a/solar/solar/dblayer/test/conftest.py +++ b/solar/solar/dblayer/test/conftest.py @@ -1,4 +1,4 @@ -from solar.dblayer.model import Model, ModelMeta +from solar.dblayer.model import Model, ModelMeta, get_bucket import pytest import time import string @@ -40,6 +40,12 @@ def rt(request): return obj +@pytest.fixture(autouse=True) +def setup(request): + + for model in ModelMeta._defined_models: + model.bucket = get_bucket(None, model, ModelMeta) + def pytest_runtest_teardown(item, nextitem): ModelMeta.session_end(result=True) @@ -57,12 +63,9 @@ client = SqlClient(':memory:', threadlocals=False, autocommit=False) # autocommit=False, pragmas=(('journal_mode', 'WAL'), # ('synchronous', 'NORMAL'))) -#from solar.dblayer.riak_client import RiakClient -#client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) +# from solar.dblayer.riak_client import RiakClient +# client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) # client = RiakClient(protocol='http', host='10.0.0.3', http_port=18098) -@pytest.fixture -def riakc(): - return client ModelMeta.setup(client) From 0cf38a9e1dd0d9c24958e286988835d170988e9b Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 29 Oct 2015 13:12:02 +0200 Subject: [PATCH 062/191] Auto-reindex composite field in case of changes --- solar/solar/dblayer/model.py | 10 ++++++++++ solar/solar/dblayer/test/test_log.py | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 44e21e4e..b4327d63 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -431,6 +431,16 @@ class CompositeIndexFieldWrp(IndexFieldWrp): for f in self._field_obj.fields: index.append(self._instance._data_container.get(f, '')) index = '|'.join(index) + + index_to_del = [] + for index_name, index_val in self._instance._riak_object.indexes: + if index_name == '%s_bin' % self.fname: + if index != index_val: + index_to_del.append((index_name, index_val)) + + for index_name, index_val in index_to_del: + self._instance._remove_index(index_name, index_val) + self._instance._add_index('%s_bin' % self.fname, index) class CompositeIndexField(IndexField): diff --git a/solar/solar/dblayer/test/test_log.py b/solar/solar/dblayer/test/test_log.py index 4bd7f228..a61b0bc7 100644 --- a/solar/solar/dblayer/test/test_log.py +++ b/solar/solar/dblayer/test/test_log.py @@ -40,3 +40,9 @@ def test_changed_index(): l.save() assert LogItem.composite.filter({'log': 'staged'}) == {l.uid} + + l.log = 'history' + l.save() + + assert LogItem.composite.filter({'log': 'staged'}) == set() + assert LogItem.composite.filter({'log': 'history'}) == {l.uid} From d29ac65a3042f65cf28b4adac3d4df6c579a36f9 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 29 Oct 2015 13:53:48 +0200 Subject: [PATCH 063/191] Add negative counter wrapper --- solar/solar/dblayer/model.py | 7 +++++++ solar/solar/dblayer/solar_models.py | 28 +++++++++++++++++++++++++++- solar/solar/dblayer/test/test_log.py | 19 ++++++++++++------- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index b4327d63..6228c635 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -720,6 +720,13 @@ class Model(object): def new(cls, key, data): return cls.from_dict(key, data) + @classmethod + def get_or_create(cls, key): + try: + return cls.get(key) + except DBLayerNotFound: + return cls.new(key, {}) + @classmethod def from_riakobj(cls, riak_obj): obj = cls(riak_obj.key) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 91996bf4..5de6939b 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -574,10 +574,35 @@ system log 2. separate logs for stage/history (using index) 3. last log item for resource in history 4. log item in staged log for resource.action +5. keep order of history """ +class NegativeCounter(Model): + + count = Field(int, default=int) + + def next(self): + self.count -= 1 + self.save() + return self.count + + +class NegativeCounterWrapper(object): + + def __init__(self, name): + self.name = name + self.backend_inst = None + + def next(self): + if self.backend_inst is None: + self.backend_inst = NegativeCounter.get_or_create(self.name) + return next(self.backend_inst) + + class LogItem(Model): + logs_counter = NegativeCounterWrapper('logs_counter') + uid = IndexedField(str, default=lambda: str(uuid4())) resource = Field(str) action = Field(str) @@ -605,4 +630,5 @@ class LogItem(Model): if 'uid' not in vals: vals['uid'] = cls.uid.default vals.update(data) - return LogItem.from_dict(vals['uid'], vals) + key = '{}~{}'.format(next(cls.logs_counter), vals['uid']) + return LogItem.from_dict(key, vals) diff --git a/solar/solar/dblayer/test/test_log.py b/solar/solar/dblayer/test/test_log.py index a61b0bc7..35007c94 100644 --- a/solar/solar/dblayer/test/test_log.py +++ b/solar/solar/dblayer/test/test_log.py @@ -1,6 +1,6 @@ import pytest -from solar.dblayer.solar_models import LogItem +from solar.dblayer.solar_models import LogItem, NegativeCounter def test_separate_logs(): @@ -12,11 +12,11 @@ def test_separate_logs(): for i in range(2): l = LogItem.new({'log': history}) l.save() - history_uids.add(l.uid) + history_uids.add(l.key) for i in range(3): l = LogItem.new({'log': staged}) l.save() - staged_uids.add(l.uid) + staged_uids.add(l.key) assert LogItem.composite.filter({'log': history}) == history_uids assert LogItem.composite.filter({'log': staged}) == staged_uids @@ -30,8 +30,8 @@ def test_multiple_filter(): l1.save() l2.save() - assert LogItem.composite.filter({'log': 'history', 'resource': 'a'}) == {l1.uid} - assert LogItem.composite.filter({'log': 'history', 'resource': 'b'}) == {l2.uid} + assert LogItem.composite.filter({'log': 'history', 'resource': 'a'}) == {l1.key} + assert LogItem.composite.filter({'log': 'history', 'resource': 'b'}) == {l2.key} def test_changed_index(): @@ -39,10 +39,15 @@ def test_changed_index(): l = LogItem.new({'log': 'staged', 'resource': 'a', 'action': 'run'}) l.save() - assert LogItem.composite.filter({'log': 'staged'}) == {l.uid} + assert LogItem.composite.filter({'log': 'staged'}) == {l.key} l.log = 'history' l.save() assert LogItem.composite.filter({'log': 'staged'}) == set() - assert LogItem.composite.filter({'log': 'history'}) == {l.uid} + assert LogItem.composite.filter({'log': 'history'}) == {l.key} + + +def test_negative_counter(): + nc = NegativeCounter.get_or_create('non_exist') + assert nc.count == 0 From 7d51ff5465e99d03c1b1d2943b5da4938155ba3f Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 29 Oct 2015 15:21:07 +0200 Subject: [PATCH 064/191] Add test for preserving order --- solar/solar/dblayer/model.py | 2 +- solar/solar/dblayer/solar_models.py | 3 +-- solar/solar/dblayer/test/test_log.py | 26 +++++++++++++++++--------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 6228c635..ad150ce1 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -421,7 +421,7 @@ class IndexField(FieldBase): startkey=startkey, endkey=endkey, **kwargs).results - return set(res) + return list(res) class CompositeIndexFieldWrp(IndexFieldWrp): diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 5de6939b..1ddc7eca 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -630,5 +630,4 @@ class LogItem(Model): if 'uid' not in vals: vals['uid'] = cls.uid.default vals.update(data) - key = '{}~{}'.format(next(cls.logs_counter), vals['uid']) - return LogItem.from_dict(key, vals) + return LogItem.from_dict(str(StrInt(next(cls.logs_counter))), vals) diff --git a/solar/solar/dblayer/test/test_log.py b/solar/solar/dblayer/test/test_log.py index 35007c94..ea957f0a 100644 --- a/solar/solar/dblayer/test/test_log.py +++ b/solar/solar/dblayer/test/test_log.py @@ -7,16 +7,16 @@ def test_separate_logs(): history = 'history' staged = 'staged' - history_uids = set() - staged_uids = set() + history_uids = [] + staged_uids = [] for i in range(2): l = LogItem.new({'log': history}) l.save() - history_uids.add(l.key) + history_uids.append(l.key) for i in range(3): l = LogItem.new({'log': staged}) l.save() - staged_uids.add(l.key) + staged_uids.append(l.key) assert LogItem.composite.filter({'log': history}) == history_uids assert LogItem.composite.filter({'log': staged}) == staged_uids @@ -30,8 +30,8 @@ def test_multiple_filter(): l1.save() l2.save() - assert LogItem.composite.filter({'log': 'history', 'resource': 'a'}) == {l1.key} - assert LogItem.composite.filter({'log': 'history', 'resource': 'b'}) == {l2.key} + assert LogItem.composite.filter({'log': 'history', 'resource': 'a'}) == [l1.key] + assert LogItem.composite.filter({'log': 'history', 'resource': 'b'}) == [l2.key] def test_changed_index(): @@ -39,15 +39,23 @@ def test_changed_index(): l = LogItem.new({'log': 'staged', 'resource': 'a', 'action': 'run'}) l.save() - assert LogItem.composite.filter({'log': 'staged'}) == {l.key} + assert LogItem.composite.filter({'log': 'staged'}) == [l.key] l.log = 'history' l.save() - assert LogItem.composite.filter({'log': 'staged'}) == set() - assert LogItem.composite.filter({'log': 'history'}) == {l.key} + assert LogItem.composite.filter({'log': 'staged'}) == [] + assert LogItem.composite.filter({'log': 'history'}) == [l.key] def test_negative_counter(): nc = NegativeCounter.get_or_create('non_exist') assert nc.count == 0 + + +def test_reversed_order_is_preserved(): + l1 = LogItem.new({'log': 'history'}) + l2 = LogItem.new({'log': 'history'}) + l1.save() + l2.save() + assert LogItem.composite.filter({'log': 'history'}, max_results=1) == [l2.key] From ab0a22ee05ced030a4fe422b39a99decb76c5081 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 29 Oct 2015 16:57:17 +0200 Subject: [PATCH 065/191] Add history field and preserve order by this field --- solar/solar/dblayer/model.py | 2 +- solar/solar/dblayer/solar_models.py | 22 +++------- .../dblayer/test/test_execution_models.py | 4 +- solar/solar/dblayer/test/test_log.py | 42 +++++++++++++------ 4 files changed, 39 insertions(+), 31 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index ad150ce1..787d7d97 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -332,7 +332,7 @@ class IndexedField(Field): def filter(self, *args, **kwargs): kwargs['return_terms'] = False res = self._filter(*args, **kwargs) - return set(res) + return res class IndexFieldWrp(object): diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 1ddc7eca..6a9dd016 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -573,7 +573,7 @@ system log 1. one bucket for all log items 2. separate logs for stage/history (using index) 3. last log item for resource in history -4. log item in staged log for resource.action +4. log item in staged log for resource|action 5. keep order of history """ @@ -587,22 +587,8 @@ class NegativeCounter(Model): return self.count -class NegativeCounterWrapper(object): - - def __init__(self, name): - self.name = name - self.backend_inst = None - - def next(self): - if self.backend_inst is None: - self.backend_inst = NegativeCounter.get_or_create(self.name) - return next(self.backend_inst) - - class LogItem(Model): - logs_counter = NegativeCounterWrapper('logs_counter') - uid = IndexedField(str, default=lambda: str(uuid4())) resource = Field(str) action = Field(str) @@ -611,6 +597,7 @@ class LogItem(Model): state = Field(list) base_path = Field(str) # remove me + history = IndexedField(StrInt) log = Field(str) # staged/history composite = CompositeIndexField(fields=('log', 'resource', 'action')) @@ -622,6 +609,9 @@ class LogItem(Model): def save(self): if any(f in self._modified_fields for f in LogItem.composite.fields): self.composite.reset() + + if 'log' in self._modified_fields and self.log == 'history': + self.history = StrInt(next(NegativeCounter.get_or_create('history'))) return super(LogItem, self).save() @classmethod @@ -630,4 +620,4 @@ class LogItem(Model): if 'uid' not in vals: vals['uid'] = cls.uid.default vals.update(data) - return LogItem.from_dict(str(StrInt(next(cls.logs_counter))), vals) + return LogItem.from_dict(vals['uid'], vals) diff --git a/solar/solar/dblayer/test/test_execution_models.py b/solar/solar/dblayer/test/test_execution_models.py index 3e20414c..644f81ff 100644 --- a/solar/solar/dblayer/test/test_execution_models.py +++ b/solar/solar/dblayer/test/test_execution_models.py @@ -37,8 +37,8 @@ def test_parent_child(rk): t1.save() t2.save() - assert Task.childs.filter(t1.key) == {t2.key} - assert Task.parents.filter(t2.key) == {t1.key} + assert Task.childs.filter(t1.key) == [t2.key] + assert Task.parents.filter(t2.key) == [t1.key] assert t1.childs.all_tasks() == [t2] assert t2.parents.all_names() == [t1.name] diff --git a/solar/solar/dblayer/test/test_log.py b/solar/solar/dblayer/test/test_log.py index ea957f0a..6e756f71 100644 --- a/solar/solar/dblayer/test/test_log.py +++ b/solar/solar/dblayer/test/test_log.py @@ -1,25 +1,26 @@ import pytest from solar.dblayer.solar_models import LogItem, NegativeCounter +from solar.dblayer.model import StrInt def test_separate_logs(): history = 'history' staged = 'staged' - history_uids = [] - staged_uids = [] + history_uids = set() + staged_uids = set() for i in range(2): l = LogItem.new({'log': history}) l.save() - history_uids.append(l.key) + history_uids.add(l.key) for i in range(3): l = LogItem.new({'log': staged}) l.save() - staged_uids.append(l.key) + staged_uids.add(l.key) - assert LogItem.composite.filter({'log': history}) == history_uids - assert LogItem.composite.filter({'log': staged}) == staged_uids + assert set(LogItem.composite.filter({'log': history})) == history_uids + assert set(LogItem.composite.filter({'log': staged})) == staged_uids def test_multiple_filter(): @@ -29,7 +30,6 @@ def test_multiple_filter(): l1.save() l2.save() - assert LogItem.composite.filter({'log': 'history', 'resource': 'a'}) == [l1.key] assert LogItem.composite.filter({'log': 'history', 'resource': 'b'}) == [l2.key] @@ -54,8 +54,26 @@ def test_negative_counter(): def test_reversed_order_is_preserved(): - l1 = LogItem.new({'log': 'history'}) - l2 = LogItem.new({'log': 'history'}) - l1.save() - l2.save() - assert LogItem.composite.filter({'log': 'history'}, max_results=1) == [l2.key] + added = [] + for i in range(4): + li = LogItem.new({'log': 'history'}) + li.save() + added.append(li.key) + added.reverse() + assert LogItem.history.filter( + StrInt.n_max(), StrInt.n_min(), max_results=2) == added[:2] + + +def test_staged_not_indexed(): + added = [] + for i in range(3): + li = LogItem.new({'log': 'staged'}) + li.save() + added.append(li) + + for li in added[:2]: + li.log = 'history' + li.save() + + assert set(LogItem.history.filter( + StrInt.n_max(), StrInt.n_min())) == {li.key for li in added[:2]} From 8f8beff98e1eeb183300aa2cca932efd437c536f Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 29 Oct 2015 17:43:28 +0200 Subject: [PATCH 066/191] Use new LogItem interface in system_log module --- solar/solar/dblayer/model.py | 2 +- solar/solar/dblayer/test/conftest.py | 8 +- solar/solar/system_log/change.py | 44 +++++----- solar/solar/system_log/consts.py | 3 + solar/solar/system_log/data.py | 117 ++++----------------------- solar/solar/system_log/operations.py | 14 ++-- 6 files changed, 46 insertions(+), 142 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 787d7d97..dab65dc6 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -810,4 +810,4 @@ class Model(object): ls.remove(self.key) except KeyError: pass - raise NotImplementedError() + self._riak_object.delete() diff --git a/solar/solar/dblayer/test/conftest.py b/solar/solar/dblayer/test/conftest.py index 54c1a31a..594be012 100644 --- a/solar/solar/dblayer/test/conftest.py +++ b/solar/solar/dblayer/test/conftest.py @@ -57,14 +57,14 @@ def pytest_runtest_call(item): Model.get_bucket_name = classmethod(patched_get_bucket_name) -from solar.dblayer.sql_client import SqlClient -client = SqlClient(':memory:', threadlocals=False, autocommit=False) +# from solar.dblayer.sql_client import SqlClient +# client = SqlClient(':memory:', threadlocals=False, autocommit=False) # client = SqlClient('/tmp/blah.db', threadlocals=True, # autocommit=False, pragmas=(('journal_mode', 'WAL'), # ('synchronous', 'NORMAL'))) -# from solar.dblayer.riak_client import RiakClient -# client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) +from solar.dblayer.riak_client import RiakClient +client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) # client = RiakClient(protocol='http', host='10.0.0.3', http_port=18098) diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index cfe6c02d..837a78a0 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -19,16 +19,14 @@ from solar.core.log import log from solar.core import signals from solar.core import resource from solar import utils -from solar.interfaces.db import get_db -from solar.system_log import data + from solar.orchestration import graph from solar.events import api as evapi -from solar.interfaces import orm from .consts import CHANGES from solar.core.resource.resource import RESOURCE_STATE from solar.errors import CannotFindID -db = get_db() +from solar.dblayer.solar_models import LogItem def guess_action(from_, to): @@ -47,14 +45,14 @@ def create_diff(staged, commited): def create_logitem(resource, action, diffed, connections_diffed, - base_path=None): - return data.LogItem( - utils.generate_uuid(), - resource, - action, - diffed, - connections_diffed, - base_path=base_path) + base_path=''): + return LogItem.new( + {'resource': resource, + 'action': action, + 'diff': diffed, + 'connections_diff': connections_diffed, + 'base_path': base_path, + 'log': 'staged'}) def create_sorted_diff(staged, commited): @@ -63,11 +61,8 @@ def create_sorted_diff(staged, commited): return create_diff(staged, commited) - def stage_changes(): - log = data.SL() - log.clean() - + staged_log = [] for resouce_obj in resource.load_all(): commited = resouce_obj.load_commited() base_path = resouce_obj.base_path @@ -92,14 +87,15 @@ def stage_changes(): # if new connection created it will be reflected in inputs # but using inputs to reverse connections is not possible if inputs_diff: - log_item = create_logitem( + li = create_logitem( resouce_obj.name, guess_action(commited_args, resource_args), inputs_diff, connections_diff, base_path=base_path) - log.append(log_item) - return log + li.save() + staged_log.append(li) + return staged_log def send_to_orchestration(): @@ -108,10 +104,10 @@ def send_to_orchestration(): changed_nodes = [] for logitem in data.SL(): - events[logitem.res] = evapi.all_events(logitem.res) - changed_nodes.append(logitem.res) + events[logitem.resource] = evapi.all_events(logitem.resource) + changed_nodes.append(logitem.resource) - state_change = evapi.StateChange(logitem.res, logitem.action) + state_change = evapi.StateChange(logitem.resource, logitem.action) state_change.insert(changed_nodes, dg) evapi.build_edges(dg, events) @@ -123,9 +119,7 @@ def send_to_orchestration(): def parameters(res, action, data): return {'args': [res, action], - 'type': 'solar_resource', - # unique identifier for a node should be passed - 'target': data.get('ip')} + 'type': 'solar_resource'} def check_uids_present(log, uids): diff --git a/solar/solar/system_log/consts.py b/solar/solar/system_log/consts.py index 3e24e97e..af2d5f76 100644 --- a/solar/solar/system_log/consts.py +++ b/solar/solar/system_log/consts.py @@ -18,3 +18,6 @@ CHANGES = Enum( 'Changes', 'run remove update' ) + + +STATES = Enum('States', 'error inprogress pending success') diff --git a/solar/solar/system_log/data.py b/solar/solar/system_log/data.py index 40551ef8..4f9dff1c 100644 --- a/solar/solar/system_log/data.py +++ b/solar/solar/system_log/data.py @@ -12,77 +12,29 @@ # License for the specific language governing permissions and limitations # under the License. -import collections -from functools import partial -from solar import utils -from solar.interfaces.db import get_db - -from enum import Enum +from solar.dblayer.solar_models import LogItem -db = get_db() -STATES = Enum('States', 'error inprogress pending success') + +def SL(): + rst = LogItem.composite.filter({'log': 'staged'}) + return map(LogItem.get, rst) + +def CL(): + rst = LogItem.composite.filter({'log': 'history'}) + return map(LogItem.get, rst) -def state_file(name): - if 'log' in name: - return Log(name) + +def compact(logitem): + return 'log task={} uid={}'.format(logitem.log_action, logitem.uid) -SL = partial(state_file, 'stage_log') -CL = partial(state_file, 'commit_log') - - -class LogItem(object): - - def __init__(self, uid, res, action, diff, - signals_diff, state=None, base_path=None): - self.uid = uid - self.res = res - self.log_action = '{}.{}'.format(res, action) - self.action = action - self.diff = diff - self.signals_diff = signals_diff - self.state = state or STATES.pending - self.base_path = base_path - - def to_yaml(self): - return utils.yaml_dump(self.to_dict()) - - def to_dict(self): - return {'uid': self.uid, - 'res': self.res, - 'diff': self.diff, - 'state': self.state.name, - 'signals_diff': self.signals_diff, - 'base_path': self.base_path, - 'action': self.action} - - @classmethod - def from_dict(cls, **kwargs): - state = getattr(STATES, kwargs.get('state', ''), STATES.pending) - kwargs['state'] = state - return cls(**kwargs) - - def __str__(self): - return self.compact - - def __repr__(self): - return self.compact - - @property - def compact(self): - return 'log task={} uid={}'.format(self.log_action, self.uid) - - @property - def details(self): - return details(self.diff) - - -def details(diff): +def details(logitem): + diff = logitem.diff rst = [] for type_, val, change in diff: if type_ == 'add': @@ -114,44 +66,3 @@ def unwrap_change_val(val): else: return val - -class Log(object): - - def __init__(self, path): - self.ordered_log = db.get_ordered_hash(path) - - def append(self, logitem): - self.ordered_log.add([(logitem.uid, logitem.to_dict())]) - - def pop(self, uid): - item = self.get(uid) - if not item: - return None - self.ordered_log.rem([uid]) - return item - - def update(self, logitem): - self.ordered_log.update(logitem.uid, logitem.to_dict()) - - def clean(self): - self.ordered_log.clean() - - def get(self, key): - item = self.ordered_log.get(key) - if item: - return LogItem.from_dict(**item) - return None - - def collection(self, n=0): - for item in self.ordered_log.reverse(n=n): - yield LogItem.from_dict(**item) - - def reverse(self, n=0): - for item in self.ordered_log.list(n=n): - yield LogItem.from_dict(**item) - - def __iter__(self): - return iter(self.collection()) - - def __len__(self): - return len(list(self.collection())) diff --git a/solar/solar/system_log/operations.py b/solar/solar/system_log/operations.py index b93d2350..4448709e 100644 --- a/solar/solar/system_log/operations.py +++ b/solar/solar/system_log/operations.py @@ -25,15 +25,15 @@ def set_error(log_action, *args, **kwargs): if item: resource_obj = resource.load(item.res) resource_obj.set_error() - item.state = data.STATES.error - sl.update(item) + item.state = 'error' + item.save() def move_to_commited(log_action, *args, **kwargs): sl = data.SL() item = next((i for i in sl if i.log_action == log_action), None) if item: - sl.pop(item.uid) + resource_obj = resource.load(item.res) commited = orm.DBCommitedState.get_or_create(item.res) @@ -49,9 +49,5 @@ def move_to_commited(log_action, *args, **kwargs): commited.connections = patch(item.signals_diff, sorted_connections) commited.base_path = item.base_path - commited.save() - cl = data.CL() - item.state = data.STATES.success - cl.append(item) - - + item.log = 'history' + item.state = 'success' From bea38c923c38d12e24d29327187253e3a67bb7b2 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 29 Oct 2015 17:59:14 +0200 Subject: [PATCH 067/191] Add CommitedResource model --- solar/solar/core/resource/resource.py | 3 ++- solar/solar/dblayer/solar_models.py | 10 ++++++++++ solar/solar/system_log/change.py | 5 ++++- solar/solar/system_log/data.py | 2 -- solar/solar/system_log/operations.py | 8 +++++--- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 6a1f84e9..2f523668 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -30,6 +30,7 @@ from uuid import uuid4 from hashlib import md5 import networkx +from solar.dblayer.solar_models import CommitedResource from solar.dblayer.solar_models import Resource as DBResource from solar.dblayer.model import StrInt @@ -259,7 +260,7 @@ class Resource(object): ) def load_commited(self): - return orm.DBCommitedState.get_or_create(self.name) + return CommitedResource.get_or_create(self.name) def connect_with_events(self, receiver, mapping=None, events=None, use_defaults=False): diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 6a9dd016..cc1ed034 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -475,6 +475,16 @@ class Resource(Model): self.updated = StrInt() return super(Resource, self).save(*args, **kwargs) + +class CommitedResource(Model): + + inputs = Field(dict, default=dict) + connections = Field(list, default=list) + base_path = Field(str) + tags = Field(list, default=list) + state = Field(str) + + """ Type of operations: diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index 837a78a0..939848a0 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -26,7 +26,7 @@ from .consts import CHANGES from solar.core.resource.resource import RESOURCE_STATE from solar.errors import CannotFindID -from solar.dblayer.solar_models import LogItem +from solar.dblayer.solar_models import LogItem, CommitedResource def guess_action(from_, to): @@ -62,6 +62,9 @@ def create_sorted_diff(staged, commited): def stage_changes(): + for li in data.SL(): + li.delete() + staged_log = [] for resouce_obj in resource.load_all(): commited = resouce_obj.load_commited() diff --git a/solar/solar/system_log/data.py b/solar/solar/system_log/data.py index 4f9dff1c..e04ffa14 100644 --- a/solar/solar/system_log/data.py +++ b/solar/solar/system_log/data.py @@ -17,8 +17,6 @@ from solar.dblayer.solar_models import LogItem - - def SL(): rst = LogItem.composite.filter({'log': 'staged'}) return map(LogItem.get, rst) diff --git a/solar/solar/system_log/operations.py b/solar/solar/system_log/operations.py index 4448709e..293aad39 100644 --- a/solar/solar/system_log/operations.py +++ b/solar/solar/system_log/operations.py @@ -13,6 +13,7 @@ # under the License. from solar.system_log import data +from solar.dblayer.solar_models import CommitedResource from dictdiffer import patch from solar.interfaces import orm from solar.core.resource import resource @@ -34,8 +35,8 @@ def move_to_commited(log_action, *args, **kwargs): item = next((i for i in sl if i.log_action == log_action), None) if item: - resource_obj = resource.load(item.res) - commited = orm.DBCommitedState.get_or_create(item.res) + resource_obj = resource.load(item.resource) + commited = CommitedResource.get_or_create(item.resource) if item.action == CHANGES.remove.name: resource_obj.delete() @@ -48,6 +49,7 @@ def move_to_commited(log_action, *args, **kwargs): sorted_connections = sorted(commited.connections) commited.connections = patch(item.signals_diff, sorted_connections) commited.base_path = item.base_path - + commited.save() item.log = 'history' item.state = 'success' + item.save() From 78f29a67e6738315605a8ca687ca87460c2bb6f9 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 30 Oct 2015 10:03:23 +0200 Subject: [PATCH 068/191] Fix several inconsistencies in interfaces --- examples/riak/riaks.py | 4 +++- solar/solar/cli/system_log.py | 23 ++++++++++++----------- solar/solar/dblayer/__init__.py | 6 ++++++ solar/solar/dblayer/model.py | 7 +++++++ solar/solar/dblayer/solar_models.py | 14 +++++++------- solar/solar/events/api.py | 15 ++++----------- solar/solar/orchestration/graph.py | 8 +------- solar/solar/system_log/change.py | 1 + solar/solar/system_log/operations.py | 2 +- 9 files changed, 42 insertions(+), 38 deletions(-) diff --git a/examples/riak/riaks.py b/examples/riak/riaks.py index 3fdf704c..0a02b184 100755 --- a/examples/riak/riaks.py +++ b/examples/riak/riaks.py @@ -20,6 +20,8 @@ from solar.core import validation from solar.core.resource import virtual_resource as vr from solar import errors +from solar.dblayer.model import ModelMeta + from solar.interfaces.db import get_db from solar.events.controls import React, Dep @@ -31,7 +33,7 @@ db = get_db() def setup_riak(): db.clear() - + ModelMeta.remove_all() resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 3}) nodes = [x for x in resources if x.name.startswith('node')] node1, node2, node3 = nodes diff --git a/solar/solar/cli/system_log.py b/solar/solar/cli/system_log.py index 2e257ade..890c5d64 100644 --- a/solar/solar/cli/system_log.py +++ b/solar/solar/cli/system_log.py @@ -42,24 +42,25 @@ def validate(): @changes.command() @click.option('-d', default=False, is_flag=True, help='detailed view') def stage(d): - log = list(change.stage_changes().reverse()) + log = change.stage_changes() + log.reverse() for item in log: - click.echo(item) + click.echo(data.compact(item)) if d: - for line in item.details: + for line in data.details(item): click.echo(' '*4+line) if not log: click.echo('No changes') @changes.command(name='staged-item') -@click.argument('log_action') -def staged_item(log_action): - item = data.SL().get(log_action) +@click.argument('uid') +def staged_item(uid): + item = data.LogItem.get(uid) if not item: click.echo('No staged changes for {}'.format(log_action)) else: - click.echo(item) - for line in item.details: + click.echo(data.compact(item)) + for line in data.details(item): click.echo(' '*4+line) @changes.command() @@ -80,15 +81,15 @@ def commit(uid): @click.option('-d', default=False, is_flag=True, help='detailed view') @click.option('-s', default=False, is_flag=True, help='short view, only uid') def history(n, d, s): - log = list(data.CL().collection(n)) + log = data.CL() for item in log: if s: click.echo(item.uid) continue - click.echo(item) + click.echo(data.compact(item)) if d: - for line in item.details: + for line in data.details(item): click.echo(' '*4+line) if not log: click.echo('No history') diff --git a/solar/solar/dblayer/__init__.py b/solar/solar/dblayer/__init__.py index e69de29b..5c07c60e 100644 --- a/solar/solar/dblayer/__init__.py +++ b/solar/solar/dblayer/__init__.py @@ -0,0 +1,6 @@ +from solar.dblayer.model import ModelMeta +from solar.dblayer.riak_client import RiakClient +client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) +# client = RiakClient(protocol='http', host='10.0.0.3', http_port=18098) + +ModelMeta.setup(client) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index dab65dc6..54cff251 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -497,6 +497,13 @@ class ModelMeta(type): def setup(mcs, riak_client): mcs.riak_client = riak_client + @classmethod + def remove_all(mcs): + for model in mcs._defined_models: + rst = model.bucket.get_index('$bucket', startkey='\x00', max_results=100000).results + for key in rst: + model.bucket.delete(key) + @classmethod def session_end(mcs, result=True): for cls in mcs._defined_models: diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index cc1ed034..c87ec11f 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -482,7 +482,7 @@ class CommitedResource(Model): connections = Field(list, default=list) base_path = Field(str) tags = Field(list, default=list) - state = Field(str) + state = Field(str, default=lambda: 'removed') """ @@ -599,16 +599,16 @@ class NegativeCounter(Model): class LogItem(Model): - uid = IndexedField(str, default=lambda: str(uuid4())) - resource = Field(str) - action = Field(str) + uid = IndexedField(basestring, default=lambda: str(uuid4())) + resource = Field(basestring) + action = Field(basestring) diff = Field(list) connections_diff = Field(list) - state = Field(list) - base_path = Field(str) # remove me + state = Field(basestring) + base_path = Field(basestring) # remove me history = IndexedField(StrInt) - log = Field(str) # staged/history + log = Field(basestring) # staged/history composite = CompositeIndexField(fields=('log', 'resource', 'action')) diff --git a/solar/solar/events/api.py b/solar/solar/events/api.py index 3cd8e8d4..3bb506cb 100644 --- a/solar/solar/events/api.py +++ b/solar/solar/events/api.py @@ -21,14 +21,7 @@ from solar.core.log import log from solar.interfaces import orm from solar.events.controls import Dep, React, StateChange - -from solar.dblayer.model import ModelMeta -from solar.dblayer.riak_client import RiakClient from solar.dblayer.solar_models import Resource -client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) - -ModelMeta.setup(client) - def create_event(event_dict): etype = event_dict['etype'] @@ -80,9 +73,9 @@ def add_react(parent, dep, actions, state='success'): def add_events(resource, lst): - resource = Resource.get(resource) - resource.events.append([ev.to_dict() for ev in lst]) - resource.save() + resource = Resource.get_or_create(resource) + resource.events.extend([ev.to_dict() for ev in lst]) + resource.save(force=True) def remove_event(ev): @@ -90,7 +83,7 @@ def remove_event(ev): def all_events(resource): - return [create_event(e) for e in Resource.get(resource).events] + return [create_event(e) for e in Resource.get_or_create(resource).events] def bft_events_graph(start): diff --git a/solar/solar/orchestration/graph.py b/solar/solar/orchestration/graph.py index 9807b7e2..85b225a8 100644 --- a/solar/solar/orchestration/graph.py +++ b/solar/solar/orchestration/graph.py @@ -23,17 +23,11 @@ from solar import errors from collections import Counter - -from solar.interfaces.db import get_db - -from solar.dblayer.model import ModelMeta -from solar.dblayer.riak_client import RiakClient from solar.dblayer.solar_models import Task -client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) + ModelMeta.setup(client) -db = get_db() def save_graph(graph): diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index 939848a0..638bc129 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -15,6 +15,7 @@ import dictdiffer import networkx as nx +from solar.system_log import data from solar.core.log import log from solar.core import signals from solar.core import resource diff --git a/solar/solar/system_log/operations.py b/solar/solar/system_log/operations.py index 293aad39..c83e2a38 100644 --- a/solar/solar/system_log/operations.py +++ b/solar/solar/system_log/operations.py @@ -47,7 +47,7 @@ def move_to_commited(log_action, *args, **kwargs): commited.inputs = patch(item.diff, commited.inputs) commited.tags = resource_obj.tags sorted_connections = sorted(commited.connections) - commited.connections = patch(item.signals_diff, sorted_connections) + commited.connections = patch(item.connections_diff, sorted_connections) commited.base_path = item.base_path commited.save() item.log = 'history' From 6eec0d5c992c7c7b321716307d6a331cba03ecc2 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 30 Oct 2015 11:26:08 +0200 Subject: [PATCH 069/191] Use update instead of save for already created graph --- solar/solar/cli/orch.py | 2 +- solar/solar/dblayer/solar_models.py | 12 ++++++------ solar/solar/orchestration/graph.py | 21 +++++++++++++-------- solar/solar/orchestration/tasks.py | 6 ++---- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/solar/solar/cli/orch.py b/solar/solar/cli/orch.py index e79454c4..c33dc85a 100755 --- a/solar/solar/cli/orch.py +++ b/solar/solar/cli/orch.py @@ -114,7 +114,7 @@ def filter(uid, start, end): errors = filters.filter(plan, start=start, end=end) if errors: raise click.ClickException('\n'.join(errors)) - graph.save_graph(uid, plan) + graph.update_graph(plan) utils.write_graph(plan) click.echo('Created {name}.png'.format(name=plan.graph['name'])) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index c87ec11f..ac2a42a9 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -560,14 +560,14 @@ class ParentField(TasksField): class Task(Model): """Node object""" - name = Field(str) - status = Field(str) - target = Field(str) - task_type = Field(str) + name = Field(basestring) + status = Field(basestring) + target = Field(basestring, default=str) + task_type = Field(basestring) args = Field(list) - errmsg = Field(str) + errmsg = Field(basestring, default=str) - execution = IndexedField(str) + execution = IndexedField(basestring) parents = ParentField(default=list) childs = ChildField(default=list) diff --git a/solar/solar/orchestration/graph.py b/solar/solar/orchestration/graph.py index 85b225a8..bc1346cf 100644 --- a/solar/solar/orchestration/graph.py +++ b/solar/solar/orchestration/graph.py @@ -26,10 +26,6 @@ from collections import Counter from solar.dblayer.solar_models import Task -ModelMeta.setup(client) - - - def save_graph(graph): # maybe it is possible to store part of information in AsyncResult backend uid = graph.graph['uid'] @@ -39,10 +35,10 @@ def save_graph(graph): {'name': n, 'execution': uid, 'status': graph.node[n].get('status', ''), - 'target': str(graph.node[n].get('target', '')), + 'target': graph.node[n].get('target', ''), 'task_type': graph.node[n].get('type', ''), 'args': graph.node[n].get('args', []), - 'errmsg': graph.node[n].get('errmsg', '')}) + 'errmsg': graph.node[n].get('errmsg', '') or ''}) graph.node[n]['task'] = t for pred in graph.predecessors(n): pred_task = graph.node[pred]['task'] @@ -51,6 +47,14 @@ def save_graph(graph): t.save() +def update_graph(graph): + for n in graph: + task = graph.node[n]['task'] + task.status = graph.node[n]['status'] + task.errmsg = graph.node[n]['errmsg'] or '' + task.save() + + def get_graph(uid): dg = nx.MultiDiGraph() dg.graph['uid'] = uid @@ -61,7 +65,8 @@ def get_graph(uid): t.name, status=t.status, type=t.task_type, args=t.args, target=t.target or None, - errmsg=t.errmsg or None) + errmsg=t.errmsg or None, + task=t) for u in t.parents.all_names(): dg.add_edge(u, t.name) return dg @@ -151,7 +156,7 @@ def reset(graph, state_list=None): for n in graph: if state_list is None or graph.node[n]['status'] in state_list: graph.node[n]['status'] = states.PENDING.name - save_graph(graph) + update_graph(graph) def reset_filtered(uid): diff --git a/solar/solar/orchestration/tasks.py b/solar/solar/orchestration/tasks.py index 671b52d7..7c558a2a 100644 --- a/solar/solar/orchestration/tasks.py +++ b/solar/solar/orchestration/tasks.py @@ -28,7 +28,6 @@ from solar.orchestration.traversal import traverse from solar.orchestration import limits from solar.orchestration import executor - r = redis.StrictRedis(host='10.0.0.2', port=6379, db=1) @@ -126,7 +125,7 @@ def schedule(plan_uid, dg): tasks) execution = executor.celery_executor( dg, limit_chain, control_tasks=('fault_tolerance',)) - graph.save_graph(dg) + graph.update_graph(dg) execution() @@ -147,8 +146,7 @@ def soft_stop(plan_uid): for n in dg: if dg.node[n]['status'] == 'PENDING': dg.node[n]['status'] = 'SKIPPED' - graph.save_graph(dg) - + graph.update_graph(dg) @app.task(name='schedule_next') def schedule_next(task_id, status, errmsg=None): From 523a91976112a4501e56074d7e08b8179198a137 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 30 Oct 2015 11:35:57 +0200 Subject: [PATCH 070/191] Fix item.res in operations.py --- solar/solar/system_log/operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/system_log/operations.py b/solar/solar/system_log/operations.py index c83e2a38..429ae3f9 100644 --- a/solar/solar/system_log/operations.py +++ b/solar/solar/system_log/operations.py @@ -24,7 +24,7 @@ def set_error(log_action, *args, **kwargs): sl = data.SL() item = next((i for i in sl if i.log_action == log_action), None) if item: - resource_obj = resource.load(item.res) + resource_obj = resource.load(item.resource) resource_obj.set_error() item.state = 'error' item.save() From 0b3173c1193b71926713e4f8563617bfe54d74f3 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 30 Oct 2015 12:13:08 +0200 Subject: [PATCH 071/191] Fix differences using sqlite backend --- solar/solar/dblayer/test/test_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solar/solar/dblayer/test/test_log.py b/solar/solar/dblayer/test/test_log.py index 6e756f71..93c87a85 100644 --- a/solar/solar/dblayer/test/test_log.py +++ b/solar/solar/dblayer/test/test_log.py @@ -60,8 +60,8 @@ def test_reversed_order_is_preserved(): li.save() added.append(li.key) added.reverse() - assert LogItem.history.filter( - StrInt.n_max(), StrInt.n_min(), max_results=2) == added[:2] + assert list(LogItem.history.filter( + StrInt.n_max(), StrInt.n_min(), max_results=2)) == added[:2] def test_staged_not_indexed(): From f4ab6dc0b92e2f172a93406758918e52699591dc Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 30 Oct 2015 12:47:11 +0200 Subject: [PATCH 072/191] Mock resource connections and fix some stuff in stage/process --- solar/solar/core/resource/__init__.py | 2 +- solar/solar/core/resource/resource.py | 8 ++++++++ solar/solar/dblayer/solar_models.py | 6 ++++++ solar/solar/dblayer/test/test_real.py | 2 -- solar/solar/orchestration/graph.py | 2 +- solar/solar/system_log/change.py | 6 +++--- 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/solar/solar/core/resource/__init__.py b/solar/solar/core/resource/__init__.py index 4ebb93e2..277cf2a4 100644 --- a/solar/solar/core/resource/__init__.py +++ b/solar/solar/core/resource/__init__.py @@ -12,4 +12,4 @@ # License for the specific language governing permissions and limitations # under the License. -from .resource import Resource, load, load_all, validate_resources, load_by_tags +from .resource import Resource, load, load_all, validate_resources, load_by_tags, load_updated diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 2f523668..87ed33f0 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -35,6 +35,8 @@ from solar.dblayer.solar_models import CommitedResource from solar.dblayer.solar_models import Resource as DBResource from solar.dblayer.model import StrInt +from solar.dblayer.model import StrInt + def read_meta(base_path): base_meta_file = os.path.join(base_path, 'meta.yaml') @@ -219,6 +221,7 @@ class Resource(object): stored as: [(emitter, emitter_input, receiver, receiver_input), ...] """ + return [] rst = [] # TODO: fix it for (emitter_resource, emitter_input), (receiver_resource, receiver_input), meta in self.graph().edges(data=True): @@ -289,11 +292,16 @@ def load(name): return Resource(r) +def load_updated(): + return [Resource(DBResource.get(r)) for r + in DBResource.updated.filter(StrInt.p_min(), StrInt.p_max())] + # TODO def load_all(): candids = DBResource.updated.filter(StrInt.p_min(), StrInt.p_max()) return [Resource(r) for r in DBResource.multi_get(candids)] + def load_by_tags(tags): tags = set(tags) candids_all = set() diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index ac2a42a9..c5f269b0 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -319,6 +319,12 @@ class InputsFieldWrp(IndexFieldWrp): self._cache[name] = value return True + def to_dict(self): + rst = {} + for key in self._instance._data_container[self.fname].keys(): + rst[key] = self[key] + return rst + class InputsField(IndexField): _wrp_class = InputsFieldWrp diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 659590c5..a55670c9 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -221,8 +221,6 @@ def test_updated_behaviour(rk): assert k1 in Resource.updated.filter(StrInt.p_min(), StrInt.p_max()) -<<<<<<< HEAD - def test_list_inputs(rk): k1 = next(rk) k2 = next(rk) diff --git a/solar/solar/orchestration/graph.py b/solar/solar/orchestration/graph.py index bc1346cf..3ffd5c85 100644 --- a/solar/solar/orchestration/graph.py +++ b/solar/solar/orchestration/graph.py @@ -35,7 +35,7 @@ def save_graph(graph): {'name': n, 'execution': uid, 'status': graph.node[n].get('status', ''), - 'target': graph.node[n].get('target', ''), + 'target': graph.node[n].get('target', '') or '', 'task_type': graph.node[n].get('type', ''), 'args': graph.node[n].get('args', []), 'errmsg': graph.node[n].get('errmsg', '') or ''}) diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index 638bc129..8442d5d2 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -29,7 +29,6 @@ from solar.errors import CannotFindID from solar.dblayer.solar_models import LogItem, CommitedResource - def guess_action(from_, to): # NOTE(dshulyak) imo the way to solve this - is dsl for orchestration, # something where this action will be excplicitly specified @@ -67,14 +66,15 @@ def stage_changes(): li.delete() staged_log = [] - for resouce_obj in resource.load_all(): + + for resouce_obj in resource.load_updated(): commited = resouce_obj.load_commited() base_path = resouce_obj.base_path if resouce_obj.to_be_removed(): resource_args = {} resource_connections = [] else: - resource_args = resouce_obj.args + resource_args = resouce_obj.to_dict() resource_connections = resouce_obj.connections if commited.state == RESOURCE_STATE.removed.name: From f0cf42dbada1234d4a317fd62ee873d139b45692 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 30 Oct 2015 14:41:36 +0200 Subject: [PATCH 073/191] Clear database using ModelMeta from new backend and unmock events --- solar/solar/cli/resource.py | 5 ++++- solar/solar/core/resource/resource.py | 9 ++++----- solar/solar/dblayer/__init__.py | 4 ++-- solar/solar/dblayer/test/conftest.py | 2 +- solar/solar/events/api.py | 4 ++-- solar/solar/system_log/change.py | 2 +- solar/solar/system_log/operations.py | 3 ++- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/solar/solar/cli/resource.py b/solar/solar/cli/resource.py index 3e4085d8..f6ff2eb0 100644 --- a/solar/solar/cli/resource.py +++ b/solar/solar/cli/resource.py @@ -108,8 +108,11 @@ def compile_all(): @resource.command() def clear_all(): + from solar.dblayer.model import ModelMeta click.echo('Clearing all resources and connections') - orm.db.clear() + ModelMeta.remove_all() + + # orm.db.clear() @resource.command() @click.argument('name') diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 87ed33f0..2b5ea7ed 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -271,11 +271,10 @@ class Resource(object): self.db_obj.save_lazy() # signals.connect(self, receiver, mapping=mapping) # TODO: implement events - return - # if use_defaults: - # api.add_default_events(self, receiver) - # if events: - # api.add_events(self.name, events) + if use_defaults: + api.add_default_events(self, receiver) + if events: + api.add_events(self.name, events) def connect(self, receiver, mapping=None, events=None): return self.connect_with_events( diff --git a/solar/solar/dblayer/__init__.py b/solar/solar/dblayer/__init__.py index 5c07c60e..ca0e1f3b 100644 --- a/solar/solar/dblayer/__init__.py +++ b/solar/solar/dblayer/__init__.py @@ -1,6 +1,6 @@ from solar.dblayer.model import ModelMeta from solar.dblayer.riak_client import RiakClient -client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) -# client = RiakClient(protocol='http', host='10.0.0.3', http_port=18098) +client = RiakClient(protocol='pbc', host='10.0.0.2', pb_port=8087) +# client = RiakClient(protocol='http', host='10.0.0.2', http_port=8098) ModelMeta.setup(client) diff --git a/solar/solar/dblayer/test/conftest.py b/solar/solar/dblayer/test/conftest.py index 594be012..2ba01da2 100644 --- a/solar/solar/dblayer/test/conftest.py +++ b/solar/solar/dblayer/test/conftest.py @@ -64,7 +64,7 @@ Model.get_bucket_name = classmethod(patched_get_bucket_name) # ('synchronous', 'NORMAL'))) from solar.dblayer.riak_client import RiakClient -client = RiakClient(protocol='pbc', host='10.0.0.3', pb_port=18087) +client = RiakClient(protocol='pbc', host='10.0.0.2', pb_port=8087) # client = RiakClient(protocol='http', host='10.0.0.3', http_port=18098) diff --git a/solar/solar/events/api.py b/solar/solar/events/api.py index 3bb506cb..0599dc1b 100644 --- a/solar/solar/events/api.py +++ b/solar/solar/events/api.py @@ -73,7 +73,7 @@ def add_react(parent, dep, actions, state='success'): def add_events(resource, lst): - resource = Resource.get_or_create(resource) + resource = Resource.get(resource) resource.events.extend([ev.to_dict() for ev in lst]) resource.save(force=True) @@ -83,7 +83,7 @@ def remove_event(ev): def all_events(resource): - return [create_event(e) for e in Resource.get_or_create(resource).events] + return [create_event(e) for e in Resource.get(resource).events] def bft_events_graph(start): diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index 8442d5d2..516ee9a3 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -74,7 +74,7 @@ def stage_changes(): resource_args = {} resource_connections = [] else: - resource_args = resouce_obj.to_dict() + resource_args = resouce_obj.args.to_dict() resource_connections = resouce_obj.connections if commited.state == RESOURCE_STATE.removed.name: diff --git a/solar/solar/system_log/operations.py b/solar/solar/system_log/operations.py index 429ae3f9..954aa330 100644 --- a/solar/solar/system_log/operations.py +++ b/solar/solar/system_log/operations.py @@ -45,7 +45,8 @@ def move_to_commited(log_action, *args, **kwargs): resource_obj.set_operational() commited.state = resource.RESOURCE_STATE.operational.name commited.inputs = patch(item.diff, commited.inputs) - commited.tags = resource_obj.tags + # TODO fix TagsWrp to return list + # commited.tags = resource_obj.tags sorted_connections = sorted(commited.connections) commited.connections = patch(item.connections_diff, sorted_connections) commited.base_path = item.base_path From f3f1f06a8181508fbdc47ee81bd4518f2ce760f3 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 14:20:29 +0100 Subject: [PATCH 074/191] WIP but dshulyak needs it --- examples/riak/riaks.py | 7 +++++-- solar/solar/dblayer/model.py | 17 ++++++++++++++++- solar/solar/dblayer/solar_models.py | 15 +++++++++++++-- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/examples/riak/riaks.py b/examples/riak/riaks.py index 3fdf704c..f7a0cbc8 100755 --- a/examples/riak/riaks.py +++ b/examples/riak/riaks.py @@ -25,12 +25,13 @@ from solar.interfaces.db import get_db from solar.events.controls import React, Dep from solar.events.api import add_event +from solar.dblayer.solar_models import Resource -db = get_db() +# db = get_db() def setup_riak(): - db.clear() + # db.clear() resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 3}) nodes = [x for x in resources if x.name.startswith('node')] @@ -43,6 +44,7 @@ def setup_riak(): r = vr.create('riak_service%d' % num, 'resources/riak_node', {'riak_self_name': 'riak%d' % num, + 'storage_backend': 'leveldb', 'riak_hostname': 'riak_server%d.solar' % num, 'riak_name': 'riak%d@riak_server%d.solar' % (num, num)})[0] riak_services.append(r) @@ -67,6 +69,7 @@ def setup_riak(): {'riak_hostname': 'hosts:name', 'ip': 'hosts:ip'}) + Resource.save_all_lazy() errors = resource.validate_resources() for r, error in errors: click.echo('ERROR: %s: %s' % (r.name, error)) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index c9012c23..bc4e53ec 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -468,8 +468,9 @@ class ModelMeta(type): def setup(mcs, riak_client): mcs.riak_client = riak_client + @classmethod - def session_end(mcs, result=True): + def save_all_lazy(mcs): for cls in mcs._defined_models: for to_save in cls._c.lazy_save: try: @@ -477,6 +478,10 @@ class ModelMeta(type): except DBLayerException: continue cls._c.lazy_save.clear() + + @classmethod + def session_end(mcs, result=True): + mcs.save_all_lazy() mcs.riak_client.session_end(result) @classmethod @@ -755,6 +760,16 @@ class Model(object): self._modified_fields.clear() self._indexes_hash = None + @classmethod + def save_all_lazy(cls): + for to_save in set(cls._c.lazy_save): + try: + to_save.save() + except DBLayerException: + continue + cls._c.lazy_save.clear() + + @clears_state_for('index') def save(self, force=False): if self.changed() or force or self._new: diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index e2b2b59d..e4c2f1ff 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -294,6 +294,15 @@ class InputsFieldWrp(IndexFieldWrp): self._instance._field_changed(self) return self._set_field_value(name, value) + def items(self): + return self._instance._data_container[self.fname].items() + + def get(self, name, default=None): + if self._has_own_input(name): + return self[name] + else: + return default + def _set_field_value(self, name, value): fname = self.fname my_name = self._instance.key @@ -305,8 +314,8 @@ class InputsFieldWrp(IndexFieldWrp): return_terms=True).results if recvs: recvs = recvs[0] - inp, emitter_name, emitter_inp = recvs[0].split('|', 3) - raise Exception("I'm connected with resource %r input %r" % (emitter_name, emitter_inp)) + res, inp, emitter_name, emitter_inp = recvs[0].split('|')[:4] + raise Exception("%r is connected with resource %r input %r" % (res, emitter_name, emitter_inp)) # inst = self._instance robj = self._instance._riak_object self._instance._add_index('%s_bin' % self.fname, '{}|{}'.format(my_name, name)) @@ -463,6 +472,8 @@ class Resource(Model): def connect(self, other, mapping): my_inputs = self.inputs other_inputs = other.inputs + if mapping is None: + return for my_name, other_name in mapping.iteritems(): other_inputs.connect(other_name, self, my_name) From 36f6d27012f6a4aa4a8e5305458aee69f493beed Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 17:23:06 +0100 Subject: [PATCH 075/191] Removed duplicate hosts file resouce creation --- examples/riak/riaks.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/examples/riak/riaks.py b/examples/riak/riaks.py index f7a0cbc8..8e0a4d22 100755 --- a/examples/riak/riaks.py +++ b/examples/riak/riaks.py @@ -36,6 +36,7 @@ def setup_riak(): resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 3}) nodes = [x for x in resources if x.name.startswith('node')] node1, node2, node3 = nodes + hosts_services = [x for x in resources if x.name.startswith('hosts_file')] riak_services = [] ips = '10.0.0.%d' @@ -55,14 +56,6 @@ def setup_riak(): for i, riak in enumerate(riak_services[1:]): riak_services[0].connect(riak, {'riak_name': 'join_to'}) - hosts_services = [] - for i, riak in enumerate(riak_services): - num = i + 1 - hosts_file = vr.create('hosts_file%d' % num, - 'resources/hosts_file', {})[0] - hosts_services.append(hosts_file) - nodes[i].connect(hosts_file) - for riak in riak_services: for hosts_file in hosts_services: riak.connect_with_events(hosts_file, From a99663438ba34cdb4f29834da65423af8868cf2e Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 17:24:10 +0100 Subject: [PATCH 076/191] More progress in new_db, still some minor things missing --- solar/solar/core/resource/resource.py | 10 +- solar/solar/core/signals.py | 160 ++++++++++++++------------ solar/solar/core/validation.py | 4 +- solar/solar/dblayer/model.py | 14 ++- solar/solar/dblayer/solar_models.py | 16 ++- 5 files changed, 121 insertions(+), 83 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 6a1f84e9..86c5bc18 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -33,6 +33,7 @@ import networkx from solar.dblayer.solar_models import Resource as DBResource from solar.dblayer.model import StrInt +from solar.core.signals import get_mapping def read_meta(base_path): @@ -261,10 +262,15 @@ class Resource(object): def load_commited(self): return orm.DBCommitedState.get_or_create(self.name) - def connect_with_events(self, receiver, mapping=None, events=None, - use_defaults=False): + def _connect_inputs(self, receiver, mapping): + print mapping self.db_obj.connect(receiver.db_obj, mapping=mapping) self.db_obj.save_lazy() + + def connect_with_events(self, receiver, mapping=None, events=None, + use_defaults=False): + mapping = get_mapping(self, receiver, mapping) + self._connect_inputs(receiver, mapping) # signals.connect(self, receiver, mapping=mapping) # TODO: implement events return diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index 0ddce5d6..685df861 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -17,6 +17,7 @@ import networkx from solar.core.log import log from solar.interfaces import orm +from solar.dblayer.solar_models import Resource as DBResource def guess_mapping(emitter, receiver): @@ -102,11 +103,13 @@ def location_and_transports(emitter, receiver, orig_mapping): # connect in other direction if emitter_single_reverse: if receiver_single_reverse: - connect_single(receiver, single, emitter, single) + # TODO: this should be moved to other place + receiver._connect_inputs(emitter, {single: single}) _remove_from_mapping(single) return if receiver_single_reverse: - connect_single(receiver, single, emitter, single) + # TODO: this should be moved to other place + receiver._connect_inputs(emitter, {single: single}) _remove_from_mapping(single) return if isinstance(orig_mapping, dict): @@ -114,9 +117,10 @@ def location_and_transports(emitter, receiver, orig_mapping): # XXX: that .args is slow on current backend # would be faster or another - inps_emitter = emitter.args - inps_receiver = receiver.args + inps_emitter = emitter.db_obj.inputs + inps_receiver = receiver.db_obj.inputs # XXX: should be somehow parametrized (input attribute?) + # with dirty_state_ok(DBResource, ('index', )): for single in ('transports_id', 'location_id'): if single in inps_emitter and inps_receiver: _single(single, emitter, receiver, inps_emitter[single], inps_receiver[single]) @@ -127,102 +131,108 @@ def location_and_transports(emitter, receiver, orig_mapping): return -def connect(emitter, receiver, mapping=None): +def get_mapping(emitter, receiver, mapping=None): if mapping is None: mapping = guess_mapping(emitter, receiver) - - # XXX: we didn't agree on that "reverse" there location_and_transports(emitter, receiver, mapping) + return mapping - if isinstance(mapping, set): - mapping = {src: src for src in mapping} +# def connect(emitter, receiver, mapping=None): +# if mapping is None: +# mapping = guess_mapping(emitter, receiver) - for src, dst in mapping.items(): - if not isinstance(dst, list): - dst = [dst] +# # XXX: we didn't agree on that "reverse" there +# location_and_transports(emitter, receiver, mapping) - for d in dst: - connect_single(emitter, src, receiver, d) +# if isinstance(mapping, set): +# mapping = {src: src for src in mapping} + +# for src, dst in mapping.items(): +# if not isinstance(dst, list): +# dst = [dst] + +# for d in dst: +# connect_single(emitter, src, receiver, d) -def connect_single(emitter, src, receiver, dst): - if ':' in dst: - return connect_multi(emitter, src, receiver, dst) +# def connect_single(emitter, src, receiver, dst): +# if ':' in dst: +# return connect_multi(emitter, src, receiver, dst) - # Disconnect all receiver inputs - # Check if receiver input is of list type first - emitter_input = emitter.resource_inputs()[src] - receiver_input = receiver.resource_inputs()[dst] +# # Disconnect all receiver inputs +# # Check if receiver input is of list type first +# emitter_input = emitter.resource_inputs()[src] +# receiver_input = receiver.resource_inputs()[dst] - if emitter_input.id == receiver_input.id: - raise Exception( - 'Trying to connect {} to itself, this is not possible'.format( - emitter_input.id) - ) +# if emitter_input.id == receiver_input.id: +# raise Exception( +# 'Trying to connect {} to itself, this is not possible'.format( +# emitter_input.id) +# ) - if not receiver_input.is_list: - receiver_input.receivers.delete_all_incoming(receiver_input) +# if not receiver_input.is_list: +# receiver_input.receivers.delete_all_incoming(receiver_input) - # Check for cycles - # TODO: change to get_paths after it is implemented in drivers - if emitter_input in receiver_input.receivers.as_set(): - raise Exception('Prevented creating a cycle on %s::%s' % (emitter.name, - emitter_input.name)) +# # Check for cycles +# # TODO: change to get_paths after it is implemented in drivers +# if emitter_input in receiver_input.receivers.as_set(): +# raise Exception('Prevented creating a cycle on %s::%s' % (emitter.name, +# emitter_input.name)) - log.debug('Connecting {}::{} -> {}::{}'.format( - emitter.name, emitter_input.name, receiver.name, receiver_input.name - )) - emitter_input.receivers.add(receiver_input) +# log.debug('Connecting {}::{} -> {}::{}'.format( +# emitter.name, emitter_input.name, receiver.name, receiver_input.name +# )) +# emitter_input.receivers.add(receiver_input) -def connect_multi(emitter, src, receiver, dst): - receiver_input_name, receiver_input_key = dst.split(':') - if '|' in receiver_input_key: - receiver_input_key, receiver_input_tag = receiver_input_key.split('|') - else: - receiver_input_tag = None +# def connect_multi(emitter, src, receiver, dst): +# receiver_input_name, receiver_input_key = dst.split(':') +# if '|' in receiver_input_key: +# receiver_input_key, receiver_input_tag = receiver_input_key.split('|') +# else: +# receiver_input_tag = None - emitter_input = emitter.resource_inputs()[src] - receiver_input = receiver.resource_inputs()[receiver_input_name] +# emitter_input = emitter.resource_inputs()[src] +# receiver_input = receiver.resource_inputs()[receiver_input_name] - if not receiver_input.is_list or receiver_input_tag: - receiver_input.receivers.delete_all_incoming( - receiver_input, - destination_key=receiver_input_key, - tag=receiver_input_tag - ) +# if not receiver_input.is_list or receiver_input_tag: +# receiver_input.receivers.delete_all_incoming( +# receiver_input, +# destination_key=receiver_input_key, +# tag=receiver_input_tag +# ) - # We can add default tag now - receiver_input_tag = receiver_input_tag or emitter.name +# # We can add default tag now +# receiver_input_tag = receiver_input_tag or emitter.name - # NOTE: make sure that receiver.args[receiver_input] is of dict type - if not receiver_input.is_hash: - raise Exception( - 'Receiver input {} must be a hash or a list of hashes'.format(receiver_input_name) - ) +# # NOTE: make sure that receiver.args[receiver_input] is of dict type +# if not receiver_input.is_hash: +# raise Exception( +# 'Receiver input {} must be a hash or a list of hashes'.format(receiver_input_name) +# ) - log.debug('Connecting {}::{} -> {}::{}[{}], tag={}'.format( - emitter.name, emitter_input.name, receiver.name, receiver_input.name, - receiver_input_key, - receiver_input_tag - )) - emitter_input.receivers.add_hash( - receiver_input, - receiver_input_key, - tag=receiver_input_tag - ) +# log.debug('Connecting {}::{} -> {}::{}[{}], tag={}'.format( +# emitter.name, emitter_input.name, receiver.name, receiver_input.name, +# receiver_input_key, +# receiver_input_tag +# )) +# emitter_input.receivers.add_hash( +# receiver_input, +# receiver_input_key, +# tag=receiver_input_tag +# ) -def disconnect_receiver_by_input(receiver, input_name): - input_node = receiver.resource_inputs()[input_name] +# def disconnect_receiver_by_input(receiver, input_name): +# input_node = receiver.resource_inputs()[input_name] - input_node.receivers.delete_all_incoming(input_node) +# input_node.receivers.delete_all_incoming(input_node) -def disconnect(emitter, receiver): - for emitter_input in emitter.resource_inputs().values(): - for receiver_input in receiver.resource_inputs().values(): - emitter_input.receivers.remove(receiver_input) +# def disconnect(emitter, receiver): +# for emitter_input in emitter.resource_inputs().values(): +# for receiver_input in receiver.resource_inputs().values(): +# emitter_input.receivers.remove(receiver_input) def detailed_connection_graph(start_with=None, end_with=None): diff --git a/solar/solar/core/validation.py b/solar/solar/core/validation.py index 91f6c565..59d5018d 100644 --- a/solar/solar/core/validation.py +++ b/solar/solar/core/validation.py @@ -174,11 +174,11 @@ def validate_resource(r): inputs = r.resource_inputs() args = r.args - for input_name, input_definition in inputs.items(): + for input_name, _ in inputs.items(): errors = validate_input( args.get(input_name), #jsonschema=input_definition.get('jsonschema'), - schema=input_definition.schema + schema=r.db_obj.meta_inputs[input_name]['schema'] ) if errors: ret[input_name] = errors diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index bc4e53ec..8d256051 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -4,6 +4,7 @@ import uuid from functools import wraps, total_ordering from operator import itemgetter import time +from contextlib import contextmanager LOCAL = local() @@ -26,7 +27,8 @@ class NONE: class SingleClassCache(object): - __slots__ = ['obj_cache', 'db_ch_state', 'lazy_save', 'origin_class'] + __slots__ = ['obj_cache', 'db_ch_state', + 'lazy_save', 'origin_class'] def __init__(self, origin_class): self.obj_cache = {} @@ -81,6 +83,7 @@ def changes_state_for(_type): @wraps(f) def _inner2(obj, *args, **kwargs): obj._c.db_ch_state['index'].add(obj.key) + obj.save_lazy() return f(obj, *args, **kwargs) return _inner2 return _inner1 @@ -112,7 +115,14 @@ def requires_clean_state(_type): def check_state_for(_type, obj): state = obj._c.db_ch_state.get(_type) if state: - raise Exception("Dirty state, save all objects first") + if True: + # TODO: solve it + orig_state = state + obj.save_all_lazy() + state = obj._c.db_ch_state.get(_type) + if not state: + return + raise Exception("Dirty state, save all %r objects first" % obj.__class__) @total_ordering diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index e4c2f1ff..364bf66d 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -109,6 +109,18 @@ class InputsFieldWrp(IndexFieldWrp): return other_inp_name + def __contains__(self, name): + try: + self._has_own_input(name) + except Exception: + return False + else: + return True + + def __iter__(self): + for name in self._instance._data_container: + yield name + def _connect_my_list(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): ret = self._connect_my_simple(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) return ret @@ -175,11 +187,11 @@ class InputsFieldWrp(IndexFieldWrp): def _get_field_val(self, name): # maybe it should be tco - check_state_for('index', self._instance) try: return self._cache[name] except KeyError: pass + check_state_for('index', self._instance) fname = self.fname my_name = self._instance.key self._has_own_input(name) @@ -315,7 +327,7 @@ class InputsFieldWrp(IndexFieldWrp): if recvs: recvs = recvs[0] res, inp, emitter_name, emitter_inp = recvs[0].split('|')[:4] - raise Exception("%r is connected with resource %r input %r" % (res, emitter_name, emitter_inp)) + raise Exception("%s:%s is connected with resource %s:%s" % (res, inp, emitter_name, emitter_inp)) # inst = self._instance robj = self._instance._riak_object self._instance._add_index('%s_bin' % self.fname, '{}|{}'.format(my_name, name)) From dd4e2526a0bbe67ff61cbeb06b43c01df65beae7 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 19:39:21 +0100 Subject: [PATCH 077/191] Added passthrough input test --- solar/solar/dblayer/test/test_real.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 364aba8e..05e4a53d 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -366,3 +366,29 @@ def test_simple_to_listdict_inputs(rk): assert r2.inputs['input'] == [{u'input2': 1115, u'input1': 110}, {u'input2': 115, u'input1': 1110}, {u'input2': 15, u'input1': 10}] + + +def test_passthrough_inputs(rk): + + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + r2 = create_resource(k2, {'name': 'first', + 'inputs': {'input1': None, + 'input2': None}}) + r3 = create_resource(k3, {'name': 'first', + 'inputs': {'input1': None, + 'input2': None}}) + + r2.connect(r3, {'input1': 'input1', + 'input2': 'input2'}) + r1.connect(r2, {'input1': 'input1', + 'input2': 'input2'}) + + + assert r3.inputs['input1'] == 10 + assert r3.inputs['input2'] == 15 From 899bd57a6c96764a9407c1a91d05b397385f10d5 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 22:35:58 +0100 Subject: [PATCH 078/191] Corrected __iter__ for inputs --- solar/solar/dblayer/solar_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 364bf66d..e143ad6e 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -118,7 +118,7 @@ class InputsFieldWrp(IndexFieldWrp): return True def __iter__(self): - for name in self._instance._data_container: + for name in self._instance._data_container[self.fname]: yield name def _connect_my_list(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): From d2b6a2888b4ac5a13ceecd08378976d227376b9d Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 30 Oct 2015 22:44:34 +0100 Subject: [PATCH 079/191] Corrected method order in solar_models --- solar/solar/dblayer/solar_models.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index e143ad6e..6774d2e1 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -66,6 +66,17 @@ class InputsFieldWrp(IndexFieldWrp): raise Exception("Unsupported case") yield (my_resource, my_input), (other_resource, other_input), meta + def __contains__(self, name): + try: + self._has_own_input(name) + except Exception: + return False + else: + return True + + def __iter__(self): + for name in self._instance._data_container[self.fname]: + yield name def _connect_my_simple(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): types_mapping = '|{}_{}'.format(my_type.value, other_type.value) @@ -108,19 +119,6 @@ class InputsFieldWrp(IndexFieldWrp): other_ind_val) return other_inp_name - - def __contains__(self, name): - try: - self._has_own_input(name) - except Exception: - return False - else: - return True - - def __iter__(self): - for name in self._instance._data_container[self.fname]: - yield name - def _connect_my_list(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): ret = self._connect_my_simple(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) return ret From f3f8d9b1a844e7e1f3ecdd23116c8bac1059ab53 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 18:09:43 +0100 Subject: [PATCH 080/191] removed debug print --- solar/solar/core/resource/resource.py | 1 - 1 file changed, 1 deletion(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index c2575ed2..f121bc3a 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -267,7 +267,6 @@ class Resource(object): return CommitedResource.get_or_create(self.name) def _connect_inputs(self, receiver, mapping): - print mapping self.db_obj.connect(receiver.db_obj, mapping=mapping) self.db_obj.save_lazy() From 8cca74fee2dea8395257d07109e654bb3e223cc9 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 18:09:58 +0100 Subject: [PATCH 081/191] Changed remove_all and implement it in sql_client --- solar/solar/dblayer/model.py | 2 +- solar/solar/dblayer/sql_client.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 0be0e38a..735f7fd0 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -511,7 +511,7 @@ class ModelMeta(type): @classmethod def remove_all(mcs): for model in mcs._defined_models: - rst = model.bucket.get_index('$bucket', startkey='\x00', max_results=100000).results + rst = model.bucket.get_index('$bucket', startkey='_', max_results=100000).results for key in rst: model.bucket.delete(key) diff --git a/solar/solar/dblayer/sql_client.py b/solar/solar/dblayer/sql_client.py index 5a95a5ce..03412639 100644 --- a/solar/solar/dblayer/sql_client.py +++ b/solar/solar/dblayer/sql_client.py @@ -314,9 +314,11 @@ class Bucket(object): ret.vclock = "new" return RiakObj(ret, new) - def get_index(self, index, startkey, endkey, return_terms=None, + def get_index(self, index, startkey, endkey=None, return_terms=None, max_results=None, continuation=None, timeout=None, fmt=None, term_regex=None): + if startkey and endkey is None: + endkey = startkey if startkey > endkey: startkey, endkey = endkey, startkey @@ -329,6 +331,16 @@ class Bucket(object): q = q.where( self._sql_model.key >= startkey, self._sql_model.key <= endkey ).order_by(self._sql_model.key) + elif index == '$bucket': + if return_terms: + q = self._sql_model.select( + self._sql_model.value, self._sql_model.key) + else: + q = self._sql_model.select(self._sql_model.key) + if not startkey == '_' and endkey == '_': + q = q.where( + self._sql_model.key >= startkey, self._sql_model.key <= endkey + ) else: if return_terms: q = self._sql_idx.select( From fe9f9dec042633adb8abb44a9807a3b8cc1d07ed Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 20:49:22 +0100 Subject: [PATCH 082/191] Added possibility to fetch other value (required for transports and ip) --- solar/solar/core/resource/resource.py | 11 +++------ solar/solar/dblayer/solar_models.py | 35 +++++++++++++++++---------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index f121bc3a..976fc18c 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -125,14 +125,12 @@ class Resource(object): inputs[inp]['value'] = md5(self.name + uuid4().hex).hexdigest() def transports(self): - inputs = self.resource_inputs() - transports_id = inputs['transports_id'] - return transports_id.backtrack_value(other_val='transports') + db_obj = self.db_obj + return db_obj.inputs._get_field_val('transports_id', other='transports') def ip(self): - inputs = self.resource_inputs() - transports_id = inputs['location_id'] - return transports_id.backtrack_value(other_val='ip') + db_obj = self.db_obj + return db_obj.inputs._get_field_val('location_id', other='ip') @property def actions(self): @@ -222,7 +220,6 @@ class Resource(object): stored as: [(emitter, emitter_input, receiver, receiver_input), ...] """ - return [] rst = [] # TODO: fix it for (emitter_resource, emitter_input), (receiver_resource, receiver_input), meta in self.graph().edges(data=True): diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index cf3f8288..6a8af9c5 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -185,10 +185,14 @@ class InputsFieldWrp(IndexFieldWrp): else: return True - def _get_field_val(self, name): + def _get_field_val(self, name, other=None): # maybe it should be tco + if other: + full_name = '{}_other_{}'.format(name, other) + else: + full_name = name try: - return self._cache[name] + return self._cache[full_name] except KeyError: pass check_state_for('index', self._instance) @@ -211,24 +215,29 @@ class InputsFieldWrp(IndexFieldWrp): if not recvs: _res = self._get_raw_field_val(name) self._cache[name] = _res + if other: + other_res = self._get_raw_field_val(other) + self._cache[full_name] = other_res + return other_res return _res my_meth = getattr(self, '_map_field_val_{}'.format(my_type.name)) - return my_meth(recvs, my_name) + return my_meth(recvs, my_name, other=other) - def _map_field_val_simple(self, recvs, name): + + def _map_field_val_simple(self, recvs, name, other=None): recvs = recvs[0] index_val, obj_key = recvs _, inp, emitter_key, emitter_inp, _mapping_type = index_val.split('|', 4) - res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) self._cache[name] = res return res - def _map_field_val_list(self, recvs, name): + def _map_field_val_list(self, recvs, name, other=None): if len(recvs) == 1: recv = recvs[0] index_val, obj_key = recv _, inp, emitter_key, emitter_inp, mapping_type = index_val.split('|', 4) - res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) if mapping_type != "{}_{}".format(InputTypes.simple.value, InputTypes.simple.value): res = [res] else: @@ -236,17 +245,17 @@ class InputsFieldWrp(IndexFieldWrp): for recv in recvs: index_val, obj_key = recv _, _, emitter_key, emitter_inp, mapping_type = index_val.split('|', 4) - cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) res.append(cres) self._cache[name] = res return res - def _map_field_val_hash(self, recvs, name): + def _map_field_val_hash(self, recvs, name, other=None): if len(recvs) == 1: recv = recvs[0] index_val, obj_key = recv _, inp, emitter_key, emitter_inp, mapping_type = index_val.split('|', 4) - res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) if mapping_type != "{}_{}".format(InputTypes.simple.value, InputTypes.simple.value): raise NotImplementedError() else: @@ -255,7 +264,7 @@ class InputsFieldWrp(IndexFieldWrp): for recv in recvs: index_val, obj_key = recv _, _, emitter_key, emitter_inp, my_tag, my_val, mapping_type = index_val.split('|', 6) - cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) items.append((my_tag, my_val, cres)) tags.add(my_tag) if len(tags) != 1: @@ -267,13 +276,13 @@ class InputsFieldWrp(IndexFieldWrp): self._cache[name] = res return res - def _map_field_val_list_hash(self, recvs, name): + def _map_field_val_list_hash(self, recvs, name, other=None): items = [] tags = set() for recv in recvs: index_val, obj_key = recv _, _, emitter_key, emitter_inp, my_tag, my_val, mapping_type = index_val.split('|', 6) - cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp) + cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) items.append((my_tag, my_val, cres)) tmp_res = {} for my_tag, my_val, value in items: From 019127d8c0b0c0c9847c804336395044fed3447c Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 20:53:51 +0100 Subject: [PATCH 083/191] sql_client support for delete --- solar/solar/dblayer/sql_client.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/solar/solar/dblayer/sql_client.py b/solar/solar/dblayer/sql_client.py index 03412639..04bfb2db 100644 --- a/solar/solar/dblayer/sql_client.py +++ b/solar/solar/dblayer/sql_client.py @@ -294,6 +294,15 @@ class Bucket(object): ret = self._sql_model(key=key, _new=new) return RiakObj(ret, new) + def delete(self, data, *args, **kwargs): + if isinstance(data, basestring): + key = data + else: + key = data.key + self._sql_model.delete().where(self._sql_model.key == key).execute() + self._sql_idx.delete().where(self._sql_idx.key == key).execute() + return self + def new(self, key, data=None, encoded_data=None, **kwargs): if key is not None: try: From 54794623804a8dd28e1f31e6bd534bbf8d8a30ad Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 21:29:14 +0100 Subject: [PATCH 084/191] Fixed other value tracking --- solar/solar/dblayer/solar_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 6a8af9c5..876ad282 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -216,7 +216,7 @@ class InputsFieldWrp(IndexFieldWrp): _res = self._get_raw_field_val(name) self._cache[name] = _res if other: - other_res = self._get_raw_field_val(other) + other_res = self._get_field_val(other) self._cache[full_name] = other_res return other_res return _res From 9843a0652ae15f8f1bf01dab976c5c8f46c6436c Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 21:39:47 +0100 Subject: [PATCH 085/191] Added as_dict to inputs --- solar/solar/dblayer/solar_models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 876ad282..cc767c62 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -80,6 +80,10 @@ class InputsFieldWrp(IndexFieldWrp): for name in self._instance._data_container[self.fname]: yield name + def as_dict(self): + # TODO: could be paralelized + return dict((name, self._get_field_val(name)) for name in self) + def _connect_my_simple(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): types_mapping = '|{}_{}'.format(my_type.value, other_type.value) my_ind_name = '{}_recv_bin'.format(self.fname) From a89e06737968d24367c04f913ed7141c14604c76 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 21:40:07 +0100 Subject: [PATCH 086/191] Riak example works --- examples/riak/riaks.py | 1 - solar/solar/core/resource/resource.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/riak/riaks.py b/examples/riak/riaks.py index 8905c102..fd5b9235 100755 --- a/examples/riak/riaks.py +++ b/examples/riak/riaks.py @@ -35,7 +35,6 @@ from solar.dblayer.solar_models import Resource def setup_riak(): # db.clear() - db.clear() ModelMeta.remove_all() resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 3}) nodes = [x for x in resources if x.name.startswith('node')] diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 976fc18c..d202e0fc 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -158,7 +158,7 @@ class Resource(object): @property def args(self): - return self.db_obj.inputs + return self.db_obj.inputs.as_dict() # ret = {} # for i in self.resource_inputs().values(): # ret[i.name] = i.backtrack_value() @@ -220,6 +220,7 @@ class Resource(object): stored as: [(emitter, emitter_input, receiver, receiver_input), ...] """ + return [] rst = [] # TODO: fix it for (emitter_resource, emitter_input), (receiver_resource, receiver_input), meta in self.graph().edges(data=True): From ccd7376d1e2b232f5d8d73702201ac537ccd8d43 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 22:37:45 +0100 Subject: [PATCH 087/191] Added dict_to_list inputs connection --- solar/solar/dblayer/solar_models.py | 3 +++ solar/solar/dblayer/test/test_real.py | 26 +++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index cc767c62..0d651b37 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -125,6 +125,9 @@ class InputsFieldWrp(IndexFieldWrp): other_ind_val) return other_inp_name + def _connect_other_hash(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): + return self._connect_other_simple(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) + def _connect_my_list(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): ret = self._connect_my_simple(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) return ret diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 4397a262..c166d70e 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -366,6 +366,31 @@ def test_simple_to_listdict_inputs(rk): {u'input2': 115, u'input1': 1110}, {u'input2': 15, u'input1': 10}] + +def test_dict_to_list_inputs(rk): + + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'modules': []}}) + r2 = create_resource(k2, {'name': 'second', + 'inputs': {'module': {'name': 'blah2'}}}) + r3 = create_resource(k3, {'name': 'third', + 'inputs': {'module': {'name': 'blah3'}}}) + + r2.connect(r1, {'module': 'modules'}) + r3.connect(r1, {'module': 'modules'}) + r1.save() + r2.save() + r3.save() + + assert r1.inputs['modules'] == [{'name': 'blah2'}, {'name': 'blah3'}] + + + + def test_passthrough_inputs(rk): k1 = next(rk) @@ -400,4 +425,3 @@ def test_events(rk): r1.events.pop() r1.save() assert r1.events == ['event1'] - From 1719d512bccdd8e426f81d905923ddc18837c43f Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 23:06:56 +0100 Subject: [PATCH 088/191] marked one test as xfail --- solar/solar/dblayer/test/test_real.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index c166d70e..7f036dab 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -21,7 +21,7 @@ def create_resource(key, data): data['meta_inputs'] = mi return Resource.from_dict(key, data) - +@pytest.mark.xfail(reason="Not YET decided how it should work") def test_changes_state(rk): key = next(rk) r = create_resource(key, {'name': 'a name'}) From 11775cb5ae94d8976608ed5f25a00044efdc369d Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 23:07:47 +0100 Subject: [PATCH 089/191] Handle one more connection type (dict to hash_list) --- solar/solar/core/resource/resource.py | 2 ++ solar/solar/dblayer/solar_models.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index d202e0fc..7c5dac20 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -265,6 +265,8 @@ class Resource(object): return CommitedResource.get_or_create(self.name) def _connect_inputs(self, receiver, mapping): + if isinstance(mapping, set): + mapping = dict((x, x) for x in mapping) self.db_obj.connect(receiver.db_obj, mapping=mapping) self.db_obj.save_lazy() diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 0d651b37..756180c0 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -164,6 +164,9 @@ class InputsFieldWrp(IndexFieldWrp): # if the type is the same map 1:1 my_type = InputTypes.simple other_type = InputTypes.simple + elif my_type == InputTypes.list_hash and other_type == InputTypes.hash: + # whole dict to a list with dicts + my_type = InputTypes.list # set my side my_meth = getattr(self, '_connect_my_{}'.format(my_type.name)) From 6e8f94d8f939a5d8326a74176f17bf1446a795b7 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 23:08:33 +0100 Subject: [PATCH 090/191] Temporary added signals.connect --- solar/solar/core/signals.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index 685df861..876c0d02 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -137,6 +137,10 @@ def get_mapping(emitter, receiver, mapping=None): location_and_transports(emitter, receiver, mapping) return mapping + +def connect(emitter, receiver, mapping=None): + emitter.connect(receiver, mapping) + # def connect(emitter, receiver, mapping=None): # if mapping is None: # mapping = guess_mapping(emitter, receiver) From 40630f4651180395ec35529f54ac08f7de173357 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 23:08:52 +0100 Subject: [PATCH 091/191] Fixed librarian modules input schema --- resources/librarian/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/librarian/meta.yaml b/resources/librarian/meta.yaml index d40334d2..a0929899 100644 --- a/resources/librarian/meta.yaml +++ b/resources/librarian/meta.yaml @@ -7,7 +7,7 @@ actions: remove: remove.yaml input: modules: - schema: [str] + schema: [{}] value: [] modules_path: schema: str! From a8fed8bb60b18a76ca35e630e338fbef26aa5792 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sat, 31 Oct 2015 23:14:42 +0100 Subject: [PATCH 092/191] support list for other_name in connect --- solar/solar/dblayer/solar_models.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 756180c0..9e7c7336 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -514,7 +514,12 @@ class Resource(Model): if mapping is None: return for my_name, other_name in mapping.iteritems(): - other_inputs.connect(other_name, self, my_name) + if isinstance(other_name, (list, tuple)): + # XXX: could be paralelized + for other in other_name: + other_inputs.connect(other, self, my_name) + else: + other_inputs.connect(other_name, self, my_name) def save(self, *args, **kwargs): if self.changed(): From 41c7138409e083b64c50cfff3a02af8a7fda094d Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sun, 1 Nov 2015 22:50:12 +0100 Subject: [PATCH 093/191] Solution for hash to listhash mapping --- solar/solar/dblayer/solar_models.py | 28 +++++++++++++++++++-------- solar/solar/dblayer/test/test_real.py | 2 +- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 9e7c7336..22ac7be6 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -133,6 +133,7 @@ class InputsFieldWrp(IndexFieldWrp): return ret def _connect_my_hash(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): + my_key, my_val = my_inp_name.split(':', 1) if '|' in my_val: my_val, my_tag = my_val.split('|', 1) @@ -165,7 +166,8 @@ class InputsFieldWrp(IndexFieldWrp): my_type = InputTypes.simple other_type = InputTypes.simple elif my_type == InputTypes.list_hash and other_type == InputTypes.hash: - # whole dict to a list with dicts + # whole dict to list with dicts + # TODO: solve this problem my_type = InputTypes.list # set my side @@ -291,15 +293,25 @@ class InputsFieldWrp(IndexFieldWrp): tags = set() for recv in recvs: index_val, obj_key = recv - _, _, emitter_key, emitter_inp, my_tag, my_val, mapping_type = index_val.split('|', 6) - cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) - items.append((my_tag, my_val, cres)) + splitted_val = index_val.split('|', 6) + if len(splitted_val) == 5: + # it was list hash but with whole dict mapping + _, _, emitter_key, emitter_inp, mapping_type = splitted_val + cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) + items.append((emitter_key, None, cres)) + else: + _, _, emitter_key, emitter_inp, my_tag, my_val, mapping_type = splitted_val + cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) + items.append((my_tag, my_val, cres)) tmp_res = {} for my_tag, my_val, value in items: - try: - tmp_res[my_tag][my_val] = value - except KeyError: - tmp_res[my_tag] = {my_val: value} + if my_val is None: + tmp_res[my_tag] = value + else: + try: + tmp_res[my_tag][my_val] = value + except KeyError: + tmp_res[my_tag] = {my_val: value} res = tmp_res.values() self._cache[name] = res return res diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 7f036dab..e7c31b9e 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -374,7 +374,7 @@ def test_dict_to_list_inputs(rk): k3 = next(rk) r1 = create_resource(k1, {'name': 'first', - 'inputs': {'modules': []}}) + 'inputs': {'modules': [{}]}}) r2 = create_resource(k2, {'name': 'second', 'inputs': {'module': {'name': 'blah2'}}}) r3 = create_resource(k3, {'name': 'third', From e0a60b92207a74d29e03b5909a97151c6b23d956 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sun, 1 Nov 2015 22:50:45 +0100 Subject: [PATCH 094/191] corrected usage of `as_dict` in system_log change --- solar/solar/system_log/change.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index 516ee9a3..d66a1766 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -74,7 +74,7 @@ def stage_changes(): resource_args = {} resource_connections = [] else: - resource_args = resouce_obj.args.to_dict() + resource_args = resouce_obj.args resource_connections = resouce_obj.connections if commited.state == RESOURCE_STATE.removed.name: From 7ebed2e164f1ae7cdab40b6dbb9eeb8c7fb49034 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Sun, 1 Nov 2015 23:36:09 +0100 Subject: [PATCH 095/191] Added disconnect possibility --- solar/solar/core/signals.py | 7 ++-- solar/solar/dblayer/solar_models.py | 17 ++++++++++ solar/solar/dblayer/test/test_real.py | 48 +++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index 876c0d02..75c5bb8d 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -227,10 +227,11 @@ def connect(emitter, receiver, mapping=None): # ) -# def disconnect_receiver_by_input(receiver, input_name): -# input_node = receiver.resource_inputs()[input_name] +def disconnect_receiver_by_input(receiver, input_name): + # input_node = receiver.resource_inputs()[input_name] -# input_node.receivers.delete_all_incoming(input_node) + # input_node.receivers.delete_all_incoming(input_node) + receiver.db_obj.inputs.disconnect(input_name) # def disconnect(emitter, receiver): diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 22ac7be6..8dc873ea 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -184,6 +184,23 @@ class InputsFieldWrp(IndexFieldWrp): pass return True + def disconnect(self, name): + # ind_name = '{}_recv_bin'.format(self.fname) + indexes = self._instance._riak_object.indexes + to_dels = [] + recvs = filter(lambda x: x[0] == '{}_recv_bin'.format(self.fname), indexes) + for recv in recvs: + _, ind_value = recv + if ind_value.startswith('{}|{}|'.format(self._instance.key, name)): + to_dels.append(recv) + emits = filter(lambda x: x[0] == '{}_emit_bin'.format(self.fname), indexes) + for emit in emits: + _, ind_value = recv + if ind_value.endswith('|{}|{}'.format(self._instance.key, name)): + to_dels.append(emit) + for to_del in to_dels: + self._instance._remove_index(*to_del) + def _has_own_input(self, name): try: return self._cache[name] diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index e7c31b9e..cbc84b79 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -412,11 +412,59 @@ def test_passthrough_inputs(rk): r1.connect(r2, {'input1': 'input1', 'input2': 'input2'}) + r1.save() + r2.save() + r3.save() + assert r3.inputs['input1'] == 10 assert r3.inputs['input2'] == 15 +def test_disconnect_by_input(rk): + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + r2 = create_resource(k2, {'name': 'first', + 'inputs': {'input1': None, + 'input2': None}}) + r3 = create_resource(k3, {'name': 'first', + 'inputs': {'input1': None, + 'input2': None}}) + + r2.connect(r3, {'input1': 'input1', + 'input2': 'input2'}) + r1.connect(r2, {'input1': 'input1', + 'input2': 'input2'}) + + r1.save() + r2.save() + r3.save() + + with pytest.raises(Exception): + r2.inputs['input1'] = 150 + + r2.inputs.disconnect('input1') + + r2.save() + + assert r2.inputs['input1'] is None + + r2.inputs['input1'] = 150 + + r2.save() + + assert r2.inputs['input1'] == 150 + assert r2.inputs['input2'] == 15 + + assert r3.inputs['input1'] == 150 + + + def test_events(rk): k = next(rk) r1 = Resource.from_dict(k, {'events': ['event1', 'event2']}) From 623af9ee30a9c4471082dc7439d2cad86fa851c3 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 2 Nov 2015 12:06:48 +0100 Subject: [PATCH 096/191] Gevent local implementation which tracks parent greenlet Slightly changed original gevent.local implementation --- solar/solar/dblayer/gevent_local.py | 172 ++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 solar/solar/dblayer/gevent_local.py diff --git a/solar/solar/dblayer/gevent_local.py b/solar/solar/dblayer/gevent_local.py new file mode 100644 index 00000000..c84877d0 --- /dev/null +++ b/solar/solar/dblayer/gevent_local.py @@ -0,0 +1,172 @@ +""" +This code is slight modification of gevent.local + +Original file is MIT licensed. + +For details please refer for gevent license +""" + +from copy import copy +from weakref import ref +from contextlib import contextmanager +from gevent.hub import getcurrent, PYPY +from gevent.lock import RLock + +__all__ = ["local"] + + +class _wrefdict(dict): + """A dict that can be weak referenced""" + + +class _localimpl(object): + """A class managing thread-local dicts""" + __slots__ = 'key', 'dicts', 'localargs', 'locallock', '__weakref__' + + def __init__(self): + # The key used in the Thread objects' attribute dicts. + # We keep it a string for speed but make it unlikely to clash with + # a "real" attribute. + self.key = '_threading_local._localimpl.' + str(id(self)) + # { id(Thread) -> (ref(Thread), thread-local dict) } + self.dicts = _wrefdict() + + def find_parent(self): + """ + Iterate to top most parent, and use it as a base + """ + c = getcurrent() + while 1: + tmp_c = getattr(c, '_nested_parent', c.parent) + if not tmp_c: + return c + c = tmp_c + + def get_dict(self): + """Return the dict for the current thread. Raises KeyError if none + defined.""" + # thread = getcurrent() + thread = self.find_parent() + return self.dicts[id(thread)][1] + + def create_dict(self): + """Create a new dict for the current thread, and return it.""" + localdict = {} + key = self.key + thread = self.find_parent() + idt = id(thread) + + # If we are working with a gevent.greenlet.Greenlet, we can + # pro-actively clear out with a link. Use rawlink to avoid + # spawning any more greenlets + try: + rawlink = thread.rawlink + except AttributeError: + # Otherwise we need to do it with weak refs + def local_deleted(_, key=key): + # When the localimpl is deleted, remove the thread attribute. + thread = wrthread() + if thread is not None: + del thread.__dict__[key] + + def thread_deleted(_, idt=idt): + # When the thread is deleted, remove the local dict. + # Note that this is suboptimal if the thread object gets + # caught in a reference loop. We would like to be called + # as soon as the OS-level thread ends instead. + _local = wrlocal() + if _local is not None: + _local.dicts.pop(idt, None) + wrlocal = ref(self, local_deleted) + wrthread = ref(thread, thread_deleted) + thread.__dict__[key] = wrlocal + else: + wrdicts = ref(self.dicts) + + def clear(_): + dicts = wrdicts() + if dicts: + dicts.pop(idt, None) + rawlink(clear) + wrthread = None + + self.dicts[idt] = wrthread, localdict + return localdict + + +@contextmanager +def _patch(self): + impl = object.__getattribute__(self, '_local__impl') + orig_dct = object.__getattribute__(self, '__dict__') + try: + dct = impl.get_dict() + except KeyError: + # it's OK to acquire the lock here and not earlier, because the above code won't switch out + # however, subclassed __init__ might switch, so we do need to acquire the lock here + dct = impl.create_dict() + args, kw = impl.localargs + with impl.locallock: + self.__init__(*args, **kw) + with impl.locallock: + object.__setattr__(self, '__dict__', dct) + yield + object.__setattr__(self, '__dict__', orig_dct) + + +class local(object): + __slots__ = '_local__impl', '__dict__' + + def __new__(cls, *args, **kw): + if args or kw: + if (PYPY and cls.__init__ == object.__init__) or (not PYPY and cls.__init__ is object.__init__): + raise TypeError("Initialization arguments are not supported") + self = object.__new__(cls) + impl = _localimpl() + impl.localargs = (args, kw) + impl.locallock = RLock() + object.__setattr__(self, '_local__impl', impl) + # We need to create the thread dict in anticipation of + # __init__ being called, to make sure we don't call it + # again ourselves. + impl.create_dict() + return self + + def __getattribute__(self, name): + with _patch(self): + return object.__getattribute__(self, name) + + def __setattr__(self, name, value): + if name == '__dict__': + raise AttributeError( + "%r object attribute '__dict__' is read-only" + % self.__class__.__name__) + with _patch(self): + return object.__setattr__(self, name, value) + + def __delattr__(self, name): + if name == '__dict__': + raise AttributeError( + "%r object attribute '__dict__' is read-only" + % self.__class__.__name__) + with _patch(self): + return object.__delattr__(self, name) + + def __copy__(self): + impl = object.__getattribute__(self, '_local__impl') + current = impl.find_parent() + currentId = id(current) + d = impl.get_dict() + duplicate = copy(d) + + cls = type(self) + if (PYPY and cls.__init__ != object.__init__) or (not PYPY and cls.__init__ is not object.__init__): + args, kw = impl.localargs + instance = cls(*args, **kw) + else: + instance = cls() + + new_impl = object.__getattribute__(instance, '_local__impl') + tpl = new_impl.dicts[currentId] + new_impl.dicts[currentId] = (tpl[0], duplicate) + + return instance From 5f0cf9d544b002387daca90d26654f7b8d23580f Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 2 Nov 2015 12:16:56 +0100 Subject: [PATCH 097/191] Added gevent pool that tracks parent (for cache) --- solar/solar/dblayer/gevent_helpers.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 solar/solar/dblayer/gevent_helpers.py diff --git a/solar/solar/dblayer/gevent_helpers.py b/solar/solar/dblayer/gevent_helpers.py new file mode 100644 index 00000000..9fbefa2f --- /dev/null +++ b/solar/solar/dblayer/gevent_helpers.py @@ -0,0 +1,15 @@ +from gevent.pool import Pool +import gevent + + +class DBLayerPool(Pool): + + def __init__(self, *args, **kwargs): + super(DBLayerPool, self).__init__(*args, **kwargs) + self.parent = gevent.getcurrent() + + def spawn(self, *args, **kwargs): + greenlet = self.greenlet_class(*args, **kwargs) + greenlet._nested_parent = self.parent + self.start(greenlet) + return greenlet From 9a3335a67e5bf70c54757a7882721544da6434fa Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 2 Nov 2015 15:18:25 +0100 Subject: [PATCH 098/191] Reworked stage a bit (it's twice as fast right now) --- solar/solar/cli/__init__.py | 6 +++ solar/solar/dblayer/gevent_helpers.py | 17 +++++++ solar/solar/dblayer/model.py | 7 ++- solar/solar/dblayer/solar_models.py | 20 ++++---- solar/solar/system_log/change.py | 68 ++++++++++++++------------- solar/solar/utils.py | 9 ++++ 6 files changed, 84 insertions(+), 43 deletions(-) diff --git a/solar/solar/cli/__init__.py b/solar/solar/cli/__init__.py index ab3ccfc0..1f558b0d 100644 --- a/solar/solar/cli/__init__.py +++ b/solar/solar/cli/__init__.py @@ -1,3 +1,9 @@ +from gevent import monkey +monkey.patch_all() + + +from solar.dblayer.gevent_patches import patch_all +patch_all() from solar.dblayer.model import ModelMeta import atexit diff --git a/solar/solar/dblayer/gevent_helpers.py b/solar/solar/dblayer/gevent_helpers.py index 9fbefa2f..a6245758 100644 --- a/solar/solar/dblayer/gevent_helpers.py +++ b/solar/solar/dblayer/gevent_helpers.py @@ -1,5 +1,6 @@ from gevent.pool import Pool import gevent +from solar.dblayer.solar_models import Resource class DBLayerPool(Pool): @@ -13,3 +14,19 @@ class DBLayerPool(Pool): greenlet._nested_parent = self.parent self.start(greenlet) return greenlet + + +@classmethod +def multi_get(obj, keys): + pool = DBLayerPool(10) + return pool.map(obj.get, keys) + + +def solar_map(funct, args, concurrency=5): + dp = DBLayerPool(concurrency) + return dp.map(funct, args) + + +def get_local(): + from solar.dblayer.gevent_local import local + return local diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 735f7fd0..8dad9a53 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -1,4 +1,6 @@ -from threading import local, current_thread +# from threading import local, current_thread +# from solar.dblayer.gevent_local import local +from solar.utils import get_local from random import getrandbits import uuid from functools import wraps, total_ordering @@ -6,7 +8,8 @@ from operator import itemgetter import time from contextlib import contextmanager -LOCAL = local() +LOCAL = get_local() + class DBLayerException(Exception): pass diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 8dc873ea..d8abe55e 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -279,6 +279,17 @@ class InputsFieldWrp(IndexFieldWrp): self._cache[name] = res return res + def _map_field_val_hash_single(self, recvs, other): + items = [] + tags = set() + for recv in recvs: + index_val, obj_key = recv + _, _, emitter_key, emitter_inp, my_tag, my_val, mapping_type = index_val.split('|', 6) + cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) + items.append((my_tag, my_val, cres)) + tags.add(my_tag) + return items, tags + def _map_field_val_hash(self, recvs, name, other=None): if len(recvs) == 1: recv = recvs[0] @@ -288,14 +299,7 @@ class InputsFieldWrp(IndexFieldWrp): if mapping_type != "{}_{}".format(InputTypes.simple.value, InputTypes.simple.value): raise NotImplementedError() else: - items = [] - tags = set() - for recv in recvs: - index_val, obj_key = recv - _, _, emitter_key, emitter_inp, my_tag, my_val, mapping_type = index_val.split('|', 6) - cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) - items.append((my_tag, my_val, cres)) - tags.add(my_tag) + items, tags = self._map_field_val_hash_single(recvs, other) if len(tags) != 1: # TODO: add it also for during connecting raise Exception("Detected dict with different tags") diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index d66a1766..f7910b51 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -61,44 +61,46 @@ def create_sorted_diff(staged, commited): return create_diff(staged, commited) +def make_single_stage_item(resource_obj): + commited = resource_obj.load_commited() + base_path = resource_obj.base_path + if resource_obj.to_be_removed(): + resource_args = {} + resource_connections = [] + else: + resource_args = resource_obj.args + resource_connections = resource_obj.connections + + if commited.state == RESOURCE_STATE.removed.name: + commited_args = {} + commited_connections = [] + else: + commited_args = commited.inputs + commited_connections = commited.connections + + inputs_diff = create_diff(resource_args, commited_args) + connections_diff = create_sorted_diff( + resource_connections, commited_connections) + + # if new connection created it will be reflected in inputs + # but using inputs to reverse connections is not possible + if inputs_diff: + li = create_logitem( + resource_obj.name, + guess_action(commited_args, resource_args), + inputs_diff, + connections_diff, + base_path=base_path) + li.save() + return li + + def stage_changes(): for li in data.SL(): li.delete() - staged_log = [] + staged_log = utils.solar_map(make_single_stage_item, resource.load_updated()) - for resouce_obj in resource.load_updated(): - commited = resouce_obj.load_commited() - base_path = resouce_obj.base_path - if resouce_obj.to_be_removed(): - resource_args = {} - resource_connections = [] - else: - resource_args = resouce_obj.args - resource_connections = resouce_obj.connections - - if commited.state == RESOURCE_STATE.removed.name: - commited_args = {} - commited_connections = [] - else: - commited_args = commited.inputs - commited_connections = commited.connections - - inputs_diff = create_diff(resource_args, commited_args) - connections_diff = create_sorted_diff( - resource_connections, commited_connections) - - # if new connection created it will be reflected in inputs - # but using inputs to reverse connections is not possible - if inputs_diff: - li = create_logitem( - resouce_obj.name, - guess_action(commited_args, resource_args), - inputs_diff, - connections_diff, - base_path=base_path) - li.save() - staged_log.append(li) return staged_log diff --git a/solar/solar/utils.py b/solar/solar/utils.py index 2e8f4678..f67a86ad 100644 --- a/solar/solar/utils.py +++ b/solar/solar/utils.py @@ -127,3 +127,12 @@ def save_to_config_file(key, data): with open(fpath, 'w') as f: encoder = ext_encoder(fpath) encoder.dump(data, f) + + +def solar_map(funct, args, **kwargs): + return map(funct, args) + + +def get_local(): + from threading import local + return local From 6550deebc558e553431551cb1e612a13ac04008a Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 2 Nov 2015 15:28:08 +0100 Subject: [PATCH 099/191] smart gevent patch import --- solar/solar/cli/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/solar/solar/cli/__init__.py b/solar/solar/cli/__init__.py index 1f558b0d..9b795591 100644 --- a/solar/solar/cli/__init__.py +++ b/solar/solar/cli/__init__.py @@ -1,9 +1,12 @@ -from gevent import monkey -monkey.patch_all() +try: + from gevent import monkey +except ImportError: + pass +else: + monkey.patch_all() + from solar.dblayer.gevent_patches import patch_all + patch_all() - -from solar.dblayer.gevent_patches import patch_all -patch_all() from solar.dblayer.model import ModelMeta import atexit From 63450408eb7720b54f4bd3d3b1193ef33e982ddf Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 2 Nov 2015 16:02:35 +0100 Subject: [PATCH 100/191] Parallel input connect --- solar/solar/dblayer/solar_models.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index d8abe55e..9c9d07fb 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -10,6 +10,8 @@ from enum import Enum from itertools import groupby from uuid import uuid4 +from solar.utils import solar_map + InputTypes = Enum('InputTypes', 'simple list hash list_hash') @@ -81,8 +83,8 @@ class InputsFieldWrp(IndexFieldWrp): yield name def as_dict(self): - # TODO: could be paralelized - return dict((name, self._get_field_val(name)) for name in self) + items = solar_map(lambda x: (x, self._get_field_val(x)), [x for x in self], concurrency=3) + return dict(items) def _connect_my_simple(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): types_mapping = '|{}_{}'.format(my_type.value, other_type.value) @@ -541,18 +543,21 @@ class Resource(Model): updated = IndexedField(StrInt) + def _connect_single(self, other_inputs, other_name, my_name): + if isinstance(other_name, (list, tuple)): + # XXX: could be paralelized + for other in other_name: + other_inputs.connect(other, self, my_name) + else: + other_inputs.connect(other_name, self, my_name) + def connect(self, other, mapping): my_inputs = self.inputs other_inputs = other.inputs if mapping is None: return - for my_name, other_name in mapping.iteritems(): - if isinstance(other_name, (list, tuple)): - # XXX: could be paralelized - for other in other_name: - other_inputs.connect(other, self, my_name) - else: - other_inputs.connect(other_name, self, my_name) + solar_map(lambda (my_name, other_name): self._connect_single(other_inputs, other_name, my_name), + mapping.iteritems(), concurrency=2) def save(self, *args, **kwargs): if self.changed(): From f51eb0e301c1a1a79de13cc83c3356548d81c846 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 2 Nov 2015 16:03:12 +0100 Subject: [PATCH 101/191] Fixed local usage --- solar/solar/dblayer/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 8dad9a53..44054c28 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -8,7 +8,7 @@ from operator import itemgetter import time from contextlib import contextmanager -LOCAL = get_local() +LOCAL = get_local()() class DBLayerException(Exception): From 72786b99af04b1be9c5190e64ad0d7003dc3f052 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 2 Nov 2015 16:06:32 +0100 Subject: [PATCH 102/191] Set 10 concurrency in stage --- solar/solar/system_log/change.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index f7910b51..c576d952 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -99,7 +99,7 @@ def stage_changes(): for li in data.SL(): li.delete() - staged_log = utils.solar_map(make_single_stage_item, resource.load_updated()) + staged_log = utils.solar_map(make_single_stage_item, resource.load_updated(), concurrency=10) return staged_log From 3aa959518efb3c1cdc798add90dfb0231adfca48 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 2 Nov 2015 16:59:49 +0100 Subject: [PATCH 103/191] Propert multi_get in system_log --- solar/solar/system_log/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solar/solar/system_log/data.py b/solar/solar/system_log/data.py index e04ffa14..0b0b7db2 100644 --- a/solar/solar/system_log/data.py +++ b/solar/solar/system_log/data.py @@ -19,11 +19,11 @@ from solar.dblayer.solar_models import LogItem def SL(): rst = LogItem.composite.filter({'log': 'staged'}) - return map(LogItem.get, rst) + return LogItem.multi_get(rst) def CL(): rst = LogItem.composite.filter({'log': 'history'}) - return map(LogItem.get, rst) + return LogItem.multi_get(rst) From 8ae0cb1452afe73a97a522359db5fa6abbddb542 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 2 Nov 2015 17:00:04 +0100 Subject: [PATCH 104/191] Lowered concurrency for multiget --- solar/solar/dblayer/gevent_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/dblayer/gevent_helpers.py b/solar/solar/dblayer/gevent_helpers.py index a6245758..a7ff4159 100644 --- a/solar/solar/dblayer/gevent_helpers.py +++ b/solar/solar/dblayer/gevent_helpers.py @@ -18,7 +18,7 @@ class DBLayerPool(Pool): @classmethod def multi_get(obj, keys): - pool = DBLayerPool(10) + pool = DBLayerPool(5) return pool.map(obj.get, keys) From 737e2d0cddc6dcfe28282baf564c7625e4501bed Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 2 Nov 2015 17:03:51 +0100 Subject: [PATCH 105/191] Added licenses to gevent stuff + added missing file --- solar/solar/dblayer/gevent_helpers.py | 15 +++++++++++ solar/solar/dblayer/gevent_patches.py | 39 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 solar/solar/dblayer/gevent_patches.py diff --git a/solar/solar/dblayer/gevent_helpers.py b/solar/solar/dblayer/gevent_helpers.py index a7ff4159..948936a2 100644 --- a/solar/solar/dblayer/gevent_helpers.py +++ b/solar/solar/dblayer/gevent_helpers.py @@ -1,3 +1,18 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + from gevent.pool import Pool import gevent from solar.dblayer.solar_models import Resource diff --git a/solar/solar/dblayer/gevent_patches.py b/solar/solar/dblayer/gevent_patches.py new file mode 100644 index 00000000..fac1bcce --- /dev/null +++ b/solar/solar/dblayer/gevent_patches.py @@ -0,0 +1,39 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def _patch(obj, name, target): + orig = getattr(obj, name) + setattr(obj, '_orig_%s' % name, orig) + setattr(obj, name, target) + + + +def patch_all(): + from solar.dblayer.model import ModelMeta + if ModelMeta._defined_models: + raise RuntimeError("You should run patch_multi_get before defining models") + from solar.dblayer.model import Model + from solar.dblayer.solar_models import InputsFieldWrp + + from solar.dblayer.gevent_helpers import (multi_get, + solar_map, + get_local) + from solar import utils + + + _patch(Model, 'multi_get', multi_get) + + _patch(utils, 'solar_map', solar_map) + _patch(utils, 'get_local', get_local) From 2402fb1f1851f339595e6d0c142dd0861ac98e7a Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 3 Nov 2015 13:30:14 +0200 Subject: [PATCH 106/191] Add filter for Resource that will fetch only updated entities --- solar/solar/core/resource/resource.py | 10 +++++++--- solar/solar/dblayer/model.py | 8 +++++++- solar/solar/dblayer/solar_models.py | 8 ++++++++ solar/solar/dblayer/test/test_log.py | 14 ++++++++++++++ solar/solar/dblayer/test/test_real.py | 8 ++++++++ solar/solar/system_log/change.py | 10 ++++++---- solar/solar/system_log/operations.py | 3 +++ 7 files changed, 53 insertions(+), 8 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 7c5dac20..bab764b7 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -296,9 +296,13 @@ def load(name): return Resource(r) -def load_updated(): - return [Resource(DBResource.get(r)) for r - in DBResource.updated.filter(StrInt.p_min(), StrInt.p_max())] +def load_updated(since=None): + if since is None: + startkey = StrInt.p_min() + else: + startkey = since + candids = DBResource.updated.filter(startkey, StrInt.p_max()) + return [Resource(r) for r in DBResource.multi_get(candids)] # TODO def load_all(): diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 44054c28..664f29a6 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -167,6 +167,12 @@ class StrInt(object): ret = self.__class__(-time_) return ret + @classmethod + def greater(cls, inst): + if isinstance(inst, cls): + return cls(inst._val + 'g') + return cls(inst + 'g') + @classmethod def to_hex(cls, value): char = cls.positive_char @@ -814,7 +820,7 @@ class Model(object): def _reset_state(self): self._new = False self._modified_fields.clear() - self._indexes_hash = None + self._indexes_changed = False @classmethod def save_all_lazy(cls): diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 9c9d07fb..968da589 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -695,6 +695,7 @@ class LogItem(Model): connections_diff = Field(list) state = Field(basestring) base_path = Field(basestring) # remove me + updated = Field(StrInt) history = IndexedField(StrInt) log = Field(basestring) # staged/history @@ -705,6 +706,13 @@ class LogItem(Model): def log_action(self): return '.'.join((self.resource, self.action)) + @classmethod + def history_last(cls): + items = cls.history.filter(StrInt.n_max(), StrInt.n_min(), max_results=1) + if not items: + return None + return cls.get(items[0]) + def save(self): if any(f in self._modified_fields for f in LogItem.composite.fields): self.composite.reset() diff --git a/solar/solar/dblayer/test/test_log.py b/solar/solar/dblayer/test/test_log.py index 93c87a85..2d1a77b9 100644 --- a/solar/solar/dblayer/test/test_log.py +++ b/solar/solar/dblayer/test/test_log.py @@ -77,3 +77,17 @@ def test_staged_not_indexed(): assert set(LogItem.history.filter( StrInt.n_max(), StrInt.n_min())) == {li.key for li in added[:2]} + + +def test_history_last_filter(): + for i in range(4): + li = LogItem.new({'log': 'history'}) + li.save() + last = li + + assert LogItem.history_last() == last + + +def test_history_last_returns_none(): + assert LogItem.history_last() == None + diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index cbc84b79..9d34f035 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -221,6 +221,14 @@ def test_updated_behaviour(rk): assert k1 in Resource.updated.filter(StrInt.p_min(), StrInt.p_max()) +def test_updated_only_last(rk): + + for i in range(3): + r = create_resource(next(rk), {'name': str(i)}) + r.save() + assert Resource.updated.filter(r.updated, StrInt.p_max()) == [r.key] + + def test_list_inputs(rk): k1 = next(rk) k2 = next(rk) diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index c576d952..9f80807e 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -27,7 +27,7 @@ from .consts import CHANGES from solar.core.resource.resource import RESOURCE_STATE from solar.errors import CannotFindID -from solar.dblayer.solar_models import LogItem, CommitedResource +from solar.dblayer.solar_models import LogItem, CommitedResource, StrInt def guess_action(from_, to): # NOTE(dshulyak) imo the way to solve this - is dsl for orchestration, @@ -44,7 +44,7 @@ def create_diff(staged, commited): return list(dictdiffer.diff(commited, staged)) -def create_logitem(resource, action, diffed, connections_diffed, +def create_logitem(resource, action, diffed, connections_diffed, updated, base_path=''): return LogItem.new( {'resource': resource, @@ -99,8 +99,10 @@ def stage_changes(): for li in data.SL(): li.delete() - staged_log = utils.solar_map(make_single_stage_item, resource.load_updated(), concurrency=10) - + last = LogItem.history_last() + since = StrInt.greater(last.updated) if last else None + staged_log = utils.solar_map(make_single_stage_item, + resource.load_updated(since), concurrency=10) return staged_log diff --git a/solar/solar/system_log/operations.py b/solar/solar/system_log/operations.py index 954aa330..f166ff3d 100644 --- a/solar/solar/system_log/operations.py +++ b/solar/solar/system_log/operations.py @@ -50,7 +50,10 @@ def move_to_commited(log_action, *args, **kwargs): sorted_connections = sorted(commited.connections) commited.connections = patch(item.connections_diff, sorted_connections) commited.base_path = item.base_path + # required to update `updated` field + resource_obj.db_obj.save() commited.save() item.log = 'history' item.state = 'success' + item.updated = resource_obj.db_obj.updated item.save() From e1d7903096a60e0c11bff017e3ab1920648c86f4 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 3 Nov 2015 15:57:51 +0200 Subject: [PATCH 107/191] Fetch all nested childs using depth first traversal --- solar/solar/core/resource/resource.py | 4 +++- solar/solar/dblayer/solar_models.py | 27 +++++++++++++++++++++++++++ solar/solar/dblayer/test/test_real.py | 25 ++++++++++++++++++++++++- solar/solar/system_log/change.py | 4 ++-- 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index bab764b7..5a2df211 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -296,12 +296,14 @@ def load(name): return Resource(r) -def load_updated(since=None): +def load_updated(since=None, with_childs=True): if since is None: startkey = StrInt.p_min() else: startkey = since candids = DBResource.updated.filter(startkey, StrInt.p_max()) + if with_childs: + candids = DBResource.childs(candids) return [Resource(r) for r in DBResource.multi_get(candids)] # TODO diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 968da589..f516de00 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -9,6 +9,7 @@ from operator import itemgetter from enum import Enum from itertools import groupby from uuid import uuid4 +from collections import defaultdict from solar.utils import solar_map @@ -564,6 +565,32 @@ class Resource(Model): self.updated = StrInt() return super(Resource, self).save(*args, **kwargs) + @classmethod + def childs(cls, parents): + all_indexes = cls.bucket.get_index( + 'inputs_recv_bin', + startkey='', + endkey='~', + return_terms=True, + max_results=999999) + + tmp = defaultdict(set) + to_visit = parents[:] + visited = [] + + for item in all_indexes.results: + data = item[0].split('|') + em, rcv = data[0], data[2] + tmp[rcv].add(em) + + while to_visit: + n = to_visit.pop() + for child in tmp[n]: + if child not in visited: + to_visit.append(child) + visited.append(n) + return visited + class CommitedResource(Model): diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 9d34f035..c8acf8ce 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -424,7 +424,6 @@ def test_passthrough_inputs(rk): r2.save() r3.save() - assert r3.inputs['input1'] == 10 assert r3.inputs['input2'] == 15 @@ -472,6 +471,30 @@ def test_disconnect_by_input(rk): assert r3.inputs['input1'] == 150 +def test_resource_childs(rk): + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + r2 = create_resource(k2, {'name': 'first', + 'inputs': {'input1': None, + 'input2': None}}) + r3 = create_resource(k3, {'name': 'first', + 'inputs': {'input1': None, + 'input2': None}}) + + r2.connect(r3, {'input1': 'input1'}) + r1.connect(r2, {'input1': 'input1'}) + + r1.save() + r2.save() + r3.save() + + assert set(Resource.childs([r1.key])) == {r1.key, r2.key, r3.key} + def test_events(rk): k = next(rk) diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index 9f80807e..e0ce7fb1 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -27,7 +27,7 @@ from .consts import CHANGES from solar.core.resource.resource import RESOURCE_STATE from solar.errors import CannotFindID -from solar.dblayer.solar_models import LogItem, CommitedResource, StrInt +from solar.dblayer.solar_models import Resource, LogItem, CommitedResource, StrInt def guess_action(from_, to): # NOTE(dshulyak) imo the way to solve this - is dsl for orchestration, @@ -44,7 +44,7 @@ def create_diff(staged, commited): return list(dictdiffer.diff(commited, staged)) -def create_logitem(resource, action, diffed, connections_diffed, updated, +def create_logitem(resource, action, diffed, connections_diffed, base_path=''): return LogItem.new( {'resource': resource, From 54b8e28a0114a6f760f05354f9c298ddcb62acef Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 3 Nov 2015 15:58:31 +0100 Subject: [PATCH 108/191] Filter None values in changes --- solar/solar/system_log/change.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index e0ce7fb1..ad3ad902 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -92,7 +92,8 @@ def make_single_stage_item(resource_obj): connections_diff, base_path=base_path) li.save() - return li + return li + return None def stage_changes(): @@ -103,6 +104,7 @@ def stage_changes(): since = StrInt.greater(last.updated) if last else None staged_log = utils.solar_map(make_single_stage_item, resource.load_updated(since), concurrency=10) + staged_log = filter(None, staged_log) return staged_log From 793327ac8549b309236b0f3de0fe62417f4b2cc4 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 3 Nov 2015 18:16:13 +0100 Subject: [PATCH 109/191] Commented some debug stuff in singnals.py --- solar/solar/core/signals.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index 75c5bb8d..0256ecfe 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -69,10 +69,12 @@ def location_and_transports(emitter, receiver, orig_mapping): # will be deleted too if inps_emitter and inps_receiver: if not inps_emitter == inps_receiver: - log.warning("Different %r defined %r => %r", single, emitter.name, receiver.name) + if not '::' in inps_receiver: + pass + # log.warning("Different %r defined %r => %r", single, emitter.name, receiver.name) return else: - log.debug("The same %r defined for %r => %r, skipping", single, emitter.name, receiver.name) + # log.debug("The same %r defined for %r => %r, skipping", single, emitter.name, receiver.name) return emitter_single = emitter.db_obj.meta_inputs[single] receiver_single = receiver.db_obj.meta_inputs[single] @@ -93,7 +95,7 @@ def location_and_transports(emitter, receiver, orig_mapping): # like adding ssh_transport for solard_transport and we don't want then # transports_id to be messed # it forbids passing this value around - log.debug("Disabled %r mapping for %r", single, emitter.name) + # log.debug("Disabled %r mapping for %r", single, emitter.name) return if receiver_single.get('is_own') is False: # this case is when we connect resource which has location_id but that is From 17130995ef968102a1fe5ea3e790fac4a6ac0575 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 3 Nov 2015 22:51:53 +0100 Subject: [PATCH 110/191] Destroy input cache in disconnect --- solar/solar/dblayer/solar_models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index f516de00..dd603a82 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -204,6 +204,11 @@ class InputsFieldWrp(IndexFieldWrp): for to_del in to_dels: self._instance._remove_index(*to_del) + try: + del self._cache[name] + except KeyError: + pass + def _has_own_input(self, name): try: return self._cache[name] From 84103ec820b028f2e36e953891b0e4ee4b6e05b1 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Wed, 4 Nov 2015 10:46:41 +0200 Subject: [PATCH 111/191] Load location_id instead of all args in controls.py --- solar/solar/events/controls.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/solar/solar/events/controls.py b/solar/solar/events/controls.py index f9bf65e1..537bc5cb 100644 --- a/solar/solar/events/controls.py +++ b/solar/solar/events/controls.py @@ -31,6 +31,7 @@ trigger action even if no changes noticed on dependent resource. - parent:update -> ok -> dependent:update """ +from solar.dblayer.solar_models import Resource class Event(object): @@ -96,15 +97,7 @@ class React(Event): if self.parent_node in changes_graph: if self.child_node not in changes_graph: - # TODO: solve this circular import problem - from solar.core import resource - try: - loaded_resource = resource.load(self.child) - except KeyError: - # orm throws this error when we're NOT using resource there - location_id = '' - else: - location_id = loaded_resource.args['location_id'] + location_id = Resource.get(self.child).inputs['location_id'] changes_graph.add_node( self.child_node, status='PENDING', target=location_id, @@ -113,7 +106,7 @@ class React(Event): changes_graph.add_edge( self.parent_node, self.child_node, state=self.state) - changed_resources.append(self.child_node) + changed_resources.append(self.child) class StateChange(Event): @@ -122,15 +115,8 @@ class StateChange(Event): def insert(self, changed_resources, changes_graph): changed_resources.append(self.parent) - # TODO: solve this circular import problem - from solar.core import resource - try: - loaded_resource = resource.load(self.parent) - except KeyError: - # orm throws this error when we're NOT using resource there - location_id = '' - else: - location_id = loaded_resource.args['location_id'] + + location_id = Resource.get(self.parent).inputs['location_id'] changes_graph.add_node( self.parent_node, status='PENDING', target=location_id, From 460003bab9bf2489814b3f97e1c07265bbcd18a5 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 4 Nov 2015 11:07:52 +0100 Subject: [PATCH 112/191] Don't force save events --- solar/solar/events/api.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/solar/solar/events/api.py b/solar/solar/events/api.py index 0599dc1b..75a6952c 100644 --- a/solar/solar/events/api.py +++ b/solar/solar/events/api.py @@ -74,8 +74,12 @@ def add_react(parent, dep, actions, state='success'): def add_events(resource, lst): resource = Resource.get(resource) - resource.events.extend([ev.to_dict() for ev in lst]) - resource.save(force=True) + events = resource.events + # TODO: currently we don't track mutable objects + events.extend([ev.to_dict() for ev in lst]) + resource.events = events + # import pdb; pdb.settrace() + resource.save_lazy() def remove_event(ev): From 3e564e45336163d87fdc0d8d406f0fdb805d231d Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 4 Nov 2015 19:06:47 +0100 Subject: [PATCH 113/191] Improved local cache usage (initialization), fixed for gevent case --- solar/solar/dblayer/gevent_patches.py | 1 + solar/solar/dblayer/model.py | 59 ++++++++++++++------------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/solar/solar/dblayer/gevent_patches.py b/solar/solar/dblayer/gevent_patches.py index fac1bcce..fdf80993 100644 --- a/solar/solar/dblayer/gevent_patches.py +++ b/solar/solar/dblayer/gevent_patches.py @@ -37,3 +37,4 @@ def patch_all(): _patch(utils, 'solar_map', solar_map) _patch(utils, 'get_local', get_local) + _patch(Model, '_local', get_local()()) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 664f29a6..1aa03c4d 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -1,5 +1,3 @@ -# from threading import local, current_thread -# from solar.dblayer.gevent_local import local from solar.utils import get_local from random import getrandbits import uuid @@ -7,8 +5,7 @@ from functools import wraps, total_ordering from operator import itemgetter import time from contextlib import contextmanager - -LOCAL = get_local()() +from threading import RLock class DBLayerException(Exception): @@ -42,35 +39,39 @@ class SingleClassCache(object): class ClassCache(object): - def __get__(self, _, owner): + def __init__(self, *args, **kwargs): + self._l = RLock() + + def __get__(self, inst, owner): # th = current_thread() - l = LOCAL - # better don't duplicate class names - cache_name = owner.__name__ - try: - cache_id = l.cache_id - except AttributeError: - cache_id = uuid.UUID(int=getrandbits(128), version=4).hex - setattr(l, 'cache_id', cache_id) - if getattr(l, 'cache_id_cmp', None) != cache_id: - # new cache - setattr(l, 'cache_id_cmp', cache_id) - c = SingleClassCache(owner) - setattr(l, '_model_caches', {}) - l._model_caches[cache_name] = c - try: - # already had this owner in cache - return l._model_caches[cache_name] - except KeyError: - # old cache but first time this owner - c = SingleClassCache(owner) - l._model_caches[cache_name] = c - return c + with self._l: + l = Model._local + # better don't duplicate class names + cache_name = owner.__name__ + try: + cache_id = l.cache_id + except AttributeError: + cache_id = uuid.UUID(int=getrandbits(128), version=4).hex + setattr(l, 'cache_id', cache_id) + if getattr(l, 'cache_id_cmp', None) != cache_id: + # new cache + setattr(l, 'cache_id_cmp', cache_id) + c = SingleClassCache(owner) + setattr(l, '_model_caches', {}) + l._model_caches[cache_name] = c + try: + # already had this owner in cache + return l._model_caches[cache_name] + except KeyError: + # old cache but first time this owner + c = SingleClassCache(owner) + l._model_caches[cache_name] = c + return c def clear_cache(): # th = current_thread() - l = LOCAL + l = Model._local cache_id = uuid.UUID(int=getrandbits(128), version=4).hex setattr(l, 'cache_id_cmp', cache_id) @@ -666,6 +667,8 @@ class Model(object): _changed = False + _local = get_local()() + def __init__(self, key=None): self._modified_fields = set() # TODO: that _indexes_changed should be smarter From a20356999cb919964ad397628136b5762344eaa4 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 4 Nov 2015 15:20:12 +0100 Subject: [PATCH 114/191] Added index cache, fetching inputs values is faster --- solar/solar/dblayer/model.py | 39 ++++++++++++++++++++++++++ solar/solar/dblayer/solar_models.py | 43 +++++++++++++++++++++-------- 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 1aa03c4d..5decc98a 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -25,6 +25,45 @@ class NONE: pass +class SingleIndexCache(object): + + def __init__(self): + self.lock = RLock() + self.cached_vals = [] + + def __enter__(self): + self.lock.acquire() + return self + + def fill(self, values): + self.cached_vals = values + + def wipe(self): + self.cached_vals = [] + + def get_index(self, real_funct, ind_name, **kwargs): + kwargs.setdefault('max_results', 999999) + if not self.cached_vals: + recvs = real_funct(ind_name, **kwargs).results + self.fill(recvs) + + def filter(self, startkey, endkey, max_results=1): + c = self.cached_vals + for (curr_val, obj_key) in c: + if max_results == 0: + break + if curr_val >= startkey: + if curr_val <= endkey: + max_results -= 1 + yield (curr_val, obj_key) + else: + break + + def __exit__(self, *args, **kwargs): + self.lock.release() + + + class SingleClassCache(object): __slots__ = ['obj_cache', 'db_ch_state', diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index dd603a82..436349bb 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -2,7 +2,7 @@ from solar.dblayer.model import (Model, Field, IndexField, IndexFieldWrp, DBLayerException, requires_clean_state, check_state_for, - StrInt, + StrInt, SingleIndexCache, IndexedField, CompositeIndexField) from types import NoneType from operator import itemgetter @@ -28,6 +28,7 @@ class InputsFieldWrp(IndexFieldWrp): def __init__(self, *args, **kwargs): super(InputsFieldWrp, self).__init__(*args, **kwargs) # TODO: add cache for lookup + self.inputs_index_cache = SingleIndexCache() self._cache = {} def _input_type(self, resource, name): @@ -185,6 +186,10 @@ class InputsFieldWrp(IndexFieldWrp): del self._cache[my_affected] except KeyError: pass + + with self.inputs_index_cache as c: + c.wipe() + return True def disconnect(self, name): @@ -209,6 +214,9 @@ class InputsFieldWrp(IndexFieldWrp): except KeyError: pass + with self.inputs_index_cache as c: + c.wipe() + def _has_own_input(self, name): try: return self._cache[name] @@ -239,16 +247,27 @@ class InputsFieldWrp(IndexFieldWrp): ind_name = '{}_recv_bin'.format(fname) # XXX: possible optimization # get all values for resource and cache it (use dirty to check) - kwargs = dict(startkey='{}|{}|'.format(my_name, name), - endkey='{}|{}|~'.format(my_name, name), - return_terms=True) - my_type = self._input_type(self._instance, name) - if my_type == InputTypes.simple: - kwargs['max_results'] = 1 - else: - kwargs['max_results'] = 99999 - recvs = self._instance._get_index(ind_name, - **kwargs).results + with self.inputs_index_cache as c: + kwargs = dict(startkey='{}|'.format(my_name), + endkey='{}|~'.format(my_name), + return_terms=True) + my_type = self._input_type(self._instance, name) + if my_type == InputTypes.simple: + max_results = 1 + else: + max_results = 99999 + c.get_index(self._instance._get_index, ind_name, **kwargs) + # recvs = self._instance._get_index(ind_name, + # **kwargs).results + recvs = tuple(c.filter(startkey="{}|{}|".format(my_name, name), + endkey="{}|{}|~".format(my_name, name), + max_results=max_results)) + # kwargs['max_results'] = max_results + # kwargs['startkey'] = "{}|{}|".format(my_name, name) + # kwargs['endkey'] = "{}|{}|~".format(my_name, name) + # recvs2 = self._instance._get_index(ind_name, + # **kwargs).results + # assert recvs == recvs2 if not recvs: _res = self._get_raw_field_val(name) self._cache[name] = _res @@ -395,6 +414,8 @@ class InputsFieldWrp(IndexFieldWrp): robj.data[self.fname][name] = value except KeyError: robj.data[self.fname] = {name: value} + with self.inputs_index_cache as c: + c.wipe() self._cache[name] = value return True From 5123eed101670549afd24a2b28e67f681a72fe36 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 4 Nov 2015 15:40:04 +0100 Subject: [PATCH 115/191] Removed comments --- solar/solar/dblayer/solar_models.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 436349bb..5e820ad2 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -245,8 +245,6 @@ class InputsFieldWrp(IndexFieldWrp): my_name = self._instance.key self._has_own_input(name) ind_name = '{}_recv_bin'.format(fname) - # XXX: possible optimization - # get all values for resource and cache it (use dirty to check) with self.inputs_index_cache as c: kwargs = dict(startkey='{}|'.format(my_name), endkey='{}|~'.format(my_name), @@ -257,17 +255,9 @@ class InputsFieldWrp(IndexFieldWrp): else: max_results = 99999 c.get_index(self._instance._get_index, ind_name, **kwargs) - # recvs = self._instance._get_index(ind_name, - # **kwargs).results recvs = tuple(c.filter(startkey="{}|{}|".format(my_name, name), endkey="{}|{}|~".format(my_name, name), max_results=max_results)) - # kwargs['max_results'] = max_results - # kwargs['startkey'] = "{}|{}|".format(my_name, name) - # kwargs['endkey'] = "{}|{}|~".format(my_name, name) - # recvs2 = self._instance._get_index(ind_name, - # **kwargs).results - # assert recvs == recvs2 if not recvs: _res = self._get_raw_field_val(name) self._cache[name] = _res From 4a380217d74567a34763c473dd732f5bb3d98649 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 5 Nov 2015 16:59:28 +0200 Subject: [PATCH 116/191] Fix discard --- solar/solar/cli/system_log.py | 3 +- solar/solar/core/resource/resource.py | 5 ++- solar/solar/dblayer/model.py | 9 +++-- solar/solar/dblayer/solar_models.py | 25 ++++++++++++-- solar/solar/dblayer/test/test_real.py | 50 +++++++++++++++++++++++++++ solar/solar/system_log/change.py | 27 ++++++++++----- 6 files changed, 99 insertions(+), 20 deletions(-) diff --git a/solar/solar/cli/system_log.py b/solar/solar/cli/system_log.py index 890c5d64..962e9d43 100644 --- a/solar/solar/cli/system_log.py +++ b/solar/solar/cli/system_log.py @@ -142,8 +142,7 @@ def test(name): @changes.command(name='clean-history') def clean_history(): - data.CL().clean() - data.CD().clean() + change.clear_history() @changes.command(help='USE ONLY FOR TESTING') def commit(): diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 5a2df211..75e6615c 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -220,7 +220,6 @@ class Resource(object): stored as: [(emitter, emitter_input, receiver, receiver_input), ...] """ - return [] rst = [] # TODO: fix it for (emitter_resource, emitter_input), (receiver_resource, receiver_input), meta in self.graph().edges(data=True): @@ -235,8 +234,8 @@ class Resource(object): def graph(self): mdg = networkx.MultiDiGraph() - for data in self.db_obj.inputs._edges(): - mdg.add_edges_from(data) + for u, v, data in self.db_obj.inputs._edges(): + mdg.add_edge(u, v, attr_dict=data) return mdg def resource_inputs(self): diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 664f29a6..f8caa57a 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -520,9 +520,7 @@ class ModelMeta(type): @classmethod def remove_all(mcs): for model in mcs._defined_models: - rst = model.bucket.get_index('$bucket', startkey='_', max_results=100000).results - for key in rst: - model.bucket.delete(key) + model.delete_all() @classmethod def save_all_lazy(mcs, result=True): @@ -844,6 +842,11 @@ class Model(object): def save_lazy(self): self._c.lazy_save.add(self) + @classmethod + def delete_all(cls): + rst = cls.bucket.get_index('$bucket', startkey='_', max_results=100000).results + for key in rst: + cls.bucket.delete(key) def delete(self): ls = self._c.lazy_save diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index f516de00..ca5e4dbf 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -69,7 +69,7 @@ class InputsFieldWrp(IndexFieldWrp): 'tag': data[4]} else: raise Exception("Unsupported case") - yield (my_resource, my_input), (other_resource, other_input), meta + yield (other_resource, other_input), (my_resource, my_input), meta def __contains__(self, name): try: @@ -194,11 +194,14 @@ class InputsFieldWrp(IndexFieldWrp): recvs = filter(lambda x: x[0] == '{}_recv_bin'.format(self.fname), indexes) for recv in recvs: _, ind_value = recv - if ind_value.startswith('{}|{}|'.format(self._instance.key, name)): + recv_name = name + if ':' in recv_name: + recv_name = recv_name.split(':')[0] + if ind_value.startswith('{}|{}|'.format(self._instance.key, recv_name)): to_dels.append(recv) emits = filter(lambda x: x[0] == '{}_emit_bin'.format(self.fname), indexes) for emit in emits: - _, ind_value = recv + _, ind_value = emit if ind_value.endswith('|{}|{}'.format(self._instance.key, name)): to_dels.append(emit) for to_del in to_dels: @@ -567,6 +570,7 @@ class Resource(Model): @classmethod def childs(cls, parents): + all_indexes = cls.bucket.get_index( 'inputs_recv_bin', startkey='', @@ -591,6 +595,21 @@ class Resource(Model): visited.append(n) return visited + def delete(self): + inputs_index = self.bucket.get_index( + 'inputs_emit_bin', + startkey=self.key, + endkey=self.key+'~', + return_terms=True, + max_results=999999) + + for emit_bin in inputs_index.results: + index_vals = emit_bin[0].split('|') + + my_res, my_key, other_res, other_key = index_vals[:4] + Resource.get(other_res).inputs.disconnect(other_key) + super(Resource, self).delete() + class CommitedResource(Model): diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index c8acf8ce..a1db5e53 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -504,3 +504,53 @@ def test_events(rk): r1.events.pop() r1.save() assert r1.events == ['event1'] + + +def test_delete(rk): + k1 = next(rk) + k2 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + r2 = create_resource(k2, {'name': 'first', + 'inputs': {'input1': None, + 'input2': None}}) + + r1.connect(r2, {'input1': 'input1'}) + r1.save() + r2.save() + + r1.delete() + + recv_emit_bin = [] + for index in r2._riak_object.indexes: + if 'recv' in index[0] or 'emit' in index[0]: + recv_emit_bin.append(index) + assert recv_emit_bin == [] + + +def test_delete_hash(rk): + k1 = next(rk) + k2 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'input1': 10, + 'input2': 15}}) + r2 = create_resource(k2, {'name': 'second', + 'inputs': {'input': {'input1': None, + 'input2': None}}}) + + + r1.connect(r2, {'input1': 'input:input1', + 'input2': 'input:input2'}) + + r1.save() + r2.save() + + r1.delete() + recv_emit_bin = [] + for index in r2._riak_object.indexes: + if 'recv' in index[0] or 'emit' in index[0]: + recv_emit_bin.append(index) + assert recv_emit_bin == [] diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index ad3ad902..4ecda84b 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -135,7 +135,7 @@ def parameters(res, action, data): def check_uids_present(log, uids): not_valid = [] for uid in uids: - if log.get(uid) is None: + if LogItem.get(uid) is None: not_valid.append(uid) if not_valid: raise CannotFindID('UIDS: {} not in history.'.format(not_valid)) @@ -149,7 +149,7 @@ def revert_uids(uids): check_uids_present(history, uids) for uid in uids: - item = history.get(uid) + item = LogItem.get(uid) if item.action == CHANGES.update.name: _revert_update(item) @@ -224,27 +224,32 @@ def revert(uid): def _discard_remove(item): - resource_obj = resource.load(item.res) + resource_obj = resource.load(item.resource) resource_obj.set_created() def _discard_update(item): - resource_obj = resource.load(item.res) + resource_obj = resource.load(item.resource) old_connections = resource_obj.connections - new_connections = dictdiffer.revert(item.signals_diff, sorted(old_connections)) + new_connections = dictdiffer.revert(item.connections_diff, sorted(old_connections)) args = dictdiffer.revert(item.diff, resource_obj.args) + inherited = [i[3].split(':') for i in new_connections] + args_to_update = { + key:args[key] for key in args + if key not in inherited + } _update_inputs_connections( - resource_obj, args, old_connections, new_connections) + resource_obj, args_to_update, old_connections, new_connections) def _discard_run(item): - resource.load(item.res).remove(force=True) + resource.load(item.resource).remove(force=True) def discard_uids(uids): staged_log = data.SL() check_uids_present(staged_log, uids) for uid in uids: - item = staged_log.get(uid) + item = LogItem.get(uid) if item.action == CHANGES.update.name: _discard_update(item) elif item.action == CHANGES.remove.name: @@ -254,7 +259,7 @@ def discard_uids(uids): else: log.debug('Action %s for resource %s is a side' ' effect of another action', item.action, item.res) - staged_log.pop(uid) + item.delete() def discard_uid(uid): @@ -271,3 +276,7 @@ def commit_all(): from .operations import move_to_commited for item in data.SL(): move_to_commited(item.log_action) + +def clear_history(): + LogItem.delete_all() + CommitedResource.delete_all() From ffa7e786f62016c12895825098b15c62419b2c26 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 5 Nov 2015 18:45:18 +0200 Subject: [PATCH 117/191] Fix revert --- solar/solar/core/resource/resource.py | 10 +++++----- solar/solar/system_log/change.py | 27 +++++++++++++++++++-------- solar/solar/system_log/operations.py | 16 ++++++++-------- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 75e6615c..2447190f 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -220,17 +220,17 @@ class Resource(object): stored as: [(emitter, emitter_input, receiver, receiver_input), ...] """ - rst = [] + rst = set() # TODO: fix it for (emitter_resource, emitter_input), (receiver_resource, receiver_input), meta in self.graph().edges(data=True): if meta: receiver_input = '{}:{}|{}'.format(receiver_input, meta['destination_key'], meta['tag']) - rst.append( - [emitter_resource, emitter_input, - receiver_resource, receiver_input]) - return rst + rst.add( + (emitter_resource, emitter_input, + receiver_resource, receiver_input)) + return [list(i) for i in rst] def graph(self): mdg = networkx.MultiDiGraph() diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index 4ecda84b..f56e6e08 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -165,10 +165,16 @@ def revert_uids(uids): def _revert_remove(logitem): """Resource should be created with all previous connections """ - commited = orm.DBCommitedState.load(logitem.res) + commited = CommitedResource.get(logitem.resource) args = dictdiffer.revert(logitem.diff, commited.inputs) - connections = dictdiffer.revert(logitem.signals_diff, sorted(commited.connections)) - resource.Resource(logitem.res, logitem.base_path, args=args, tags=commited.tags) + connections = dictdiffer.revert(logitem.connections_diff, sorted(commited.connections)) + inherited = [i[3].split(':')[0] for i in connections] + args_to_update = { + key:args[key] for key in args + if key not in inherited + } + + resource.Resource(logitem.resource, logitem.base_path, args=args_to_update, tags=commited.tags) for emitter, emitter_input, receiver, receiver_input in connections: emmiter_obj = resource.load(emitter) receiver_obj = resource.load(receiver) @@ -204,18 +210,23 @@ def _update_inputs_connections(res_obj, args, old_connections, new_connections): def _revert_update(logitem): """Revert of update should update inputs and connections """ - res_obj = resource.load(logitem.res) + res_obj = resource.load(logitem.resource) commited = res_obj.load_commited() - args_to_update = dictdiffer.revert(logitem.diff, commited.inputs) - connections = dictdiffer.revert(logitem.signals_diff, sorted(commited.connections)) + connections = dictdiffer.revert(logitem.connections_diff, sorted(commited.connections)) + args = dictdiffer.revert(logitem.diff, commited.inputs) + inherited = [i[3].split(':')[0] for i in connections] + args_to_update = { + key:args[key] for key in args + if key not in inherited + } _update_inputs_connections( res_obj, args_to_update, commited.connections, connections) def _revert_run(logitem): - res_obj = resource.load(logitem.res) + res_obj = resource.load(logitem.resource) res_obj.remove() @@ -233,7 +244,7 @@ def _discard_update(item): old_connections = resource_obj.connections new_connections = dictdiffer.revert(item.connections_diff, sorted(old_connections)) args = dictdiffer.revert(item.diff, resource_obj.args) - inherited = [i[3].split(':') for i in new_connections] + inherited = [i[3].split(':')[0] for i in new_connections] args_to_update = { key:args[key] for key in args if key not in inherited diff --git a/solar/solar/system_log/operations.py b/solar/solar/system_log/operations.py index f166ff3d..112ae483 100644 --- a/solar/solar/system_log/operations.py +++ b/solar/solar/system_log/operations.py @@ -34,26 +34,26 @@ def move_to_commited(log_action, *args, **kwargs): sl = data.SL() item = next((i for i in sl if i.log_action == log_action), None) if item: - resource_obj = resource.load(item.resource) commited = CommitedResource.get_or_create(item.resource) - + updated = resource_obj.db_obj.updated if item.action == CHANGES.remove.name: resource_obj.delete() commited.state = resource.RESOURCE_STATE.removed.name else: resource_obj.set_operational() commited.state = resource.RESOURCE_STATE.operational.name - commited.inputs = patch(item.diff, commited.inputs) - # TODO fix TagsWrp to return list - # commited.tags = resource_obj.tags - sorted_connections = sorted(commited.connections) - commited.connections = patch(item.connections_diff, sorted_connections) commited.base_path = item.base_path + updated = resource_obj.db_obj.updated # required to update `updated` field resource_obj.db_obj.save() + commited.inputs = patch(item.diff, commited.inputs) + # TODO fix TagsWrp to return list + # commited.tags = resource_obj.tags + sorted_connections = sorted(commited.connections) + commited.connections = patch(item.connections_diff, sorted_connections) commited.save() item.log = 'history' item.state = 'success' - item.updated = resource_obj.db_obj.updated + item.updated = updated item.save() From 6e4ae64d480ea440c97dd8e492e15113c0db11cb Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 6 Nov 2015 11:40:57 +0200 Subject: [PATCH 118/191] Fix duplicates in update filter and delete with lazy_save --- solar/solar/dblayer/model.py | 2 +- solar/solar/dblayer/solar_models.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index f8caa57a..24c7e155 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -851,7 +851,7 @@ class Model(object): def delete(self): ls = self._c.lazy_save try: - ls.remove(self.key) + ls.remove(self) except KeyError: pass self._riak_object.delete() diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index ca5e4dbf..67d5f6c6 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -580,7 +580,7 @@ class Resource(Model): tmp = defaultdict(set) to_visit = parents[:] - visited = [] + visited = set() for item in all_indexes.results: data = item[0].split('|') @@ -592,7 +592,7 @@ class Resource(Model): for child in tmp[n]: if child not in visited: to_visit.append(child) - visited.append(n) + visited.add(n) return visited def delete(self): @@ -607,7 +607,8 @@ class Resource(Model): index_vals = emit_bin[0].split('|') my_res, my_key, other_res, other_key = index_vals[:4] - Resource.get(other_res).inputs.disconnect(other_key) + emit_obj = Resource.get(other_res) + emit_obj.inputs.disconnect(other_key) super(Resource, self).delete() From 5ea07c658a634b56c40a1f283234803797301694 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 6 Nov 2015 12:38:14 +0200 Subject: [PATCH 119/191] Move args_to_update into separate function --- solar/solar/system_log/change.py | 33 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index f56e6e08..5c03c45e 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -141,6 +141,16 @@ def check_uids_present(log, uids): raise CannotFindID('UIDS: {} not in history.'.format(not_valid)) +def _get_args_to_update(args, connections): + """For each resource we can update only args that are not provided + by connections + """ + inherited = [i[3].split(':')[0] for i in connections] + return { + key:args[key] for key in args + if key not in inherited + } + def revert_uids(uids): """ :param uids: iterable not generator @@ -168,13 +178,9 @@ def _revert_remove(logitem): commited = CommitedResource.get(logitem.resource) args = dictdiffer.revert(logitem.diff, commited.inputs) connections = dictdiffer.revert(logitem.connections_diff, sorted(commited.connections)) - inherited = [i[3].split(':')[0] for i in connections] - args_to_update = { - key:args[key] for key in args - if key not in inherited - } - resource.Resource(logitem.resource, logitem.base_path, args=args_to_update, tags=commited.tags) + resource.Resource(logitem.resource, logitem.base_path, + args=_get_args_to_update(args, connections), tags=commited.tags) for emitter, emitter_input, receiver, receiver_input in connections: emmiter_obj = resource.load(emitter) receiver_obj = resource.load(receiver) @@ -215,14 +221,9 @@ def _revert_update(logitem): connections = dictdiffer.revert(logitem.connections_diff, sorted(commited.connections)) args = dictdiffer.revert(logitem.diff, commited.inputs) - inherited = [i[3].split(':')[0] for i in connections] - args_to_update = { - key:args[key] for key in args - if key not in inherited - } _update_inputs_connections( - res_obj, args_to_update, commited.connections, connections) + res_obj, _get_args_to_update(args, connections), commited.connections, connections) def _revert_run(logitem): @@ -244,13 +245,9 @@ def _discard_update(item): old_connections = resource_obj.connections new_connections = dictdiffer.revert(item.connections_diff, sorted(old_connections)) args = dictdiffer.revert(item.diff, resource_obj.args) - inherited = [i[3].split(':')[0] for i in new_connections] - args_to_update = { - key:args[key] for key in args - if key not in inherited - } + _update_inputs_connections( - resource_obj, args_to_update, old_connections, new_connections) + resource_obj, _get_args_to_update(args, new_connections), old_connections, new_connections) def _discard_run(item): resource.load(item.resource).remove(force=True) From bd21f902279e00716919ff7340e86d0a0788090d Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 6 Nov 2015 13:05:53 +0200 Subject: [PATCH 120/191] Use multi_get in discard and revert --- solar/solar/system_log/change.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index 5c03c45e..ecff4df6 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -132,15 +132,6 @@ def parameters(res, action, data): 'type': 'solar_resource'} -def check_uids_present(log, uids): - not_valid = [] - for uid in uids: - if LogItem.get(uid) is None: - not_valid.append(uid) - if not_valid: - raise CannotFindID('UIDS: {} not in history.'.format(not_valid)) - - def _get_args_to_update(args, connections): """For each resource we can update only args that are not provided by connections @@ -155,11 +146,9 @@ def revert_uids(uids): """ :param uids: iterable not generator """ - history = data.CL() - check_uids_present(history, uids) + items = LogItem.multi_get(uids) - for uid in uids: - item = LogItem.get(uid) + for item in items: if item.action == CHANGES.update.name: _revert_update(item) @@ -254,10 +243,8 @@ def _discard_run(item): def discard_uids(uids): - staged_log = data.SL() - check_uids_present(staged_log, uids) - for uid in uids: - item = LogItem.get(uid) + items = LogItem.multi_get(uids) + for item in items: if item.action == CHANGES.update.name: _discard_update(item) elif item.action == CHANGES.remove.name: From 4d7efd4ae70002b4c09633db69aedabae320b53d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Mon, 9 Nov 2015 14:52:54 +0100 Subject: [PATCH 121/191] Make puppet run on the new_db --- resources/apache_puppet/actions/run.pp | 112 +++++++------- resources/cinder_api_puppet/actions/run.pp | 38 ++--- resources/cinder_api_puppet/actions/update.pp | 38 ++--- resources/cinder_glance_puppet/actions/run.pp | 14 +- resources/cinder_puppet/actions/run.pp | 114 +++++++------- .../cinder_scheduler_puppet/actions/run.pp | 4 +- .../cinder_scheduler_puppet/actions/update.pp | 4 +- resources/cinder_volume_puppet/actions/run.pp | 12 +- .../cinder_volume_puppet/actions/update.pp | 12 +- resources/glance_puppet/actions/run.pp | 90 +++++------ resources/glance_puppet/actions/update.pp | 90 +++++------ .../glance_registry_puppet/actions/run.pp | 70 ++++----- .../glance_registry_puppet/actions/update.pp | 70 ++++----- resources/keystone_puppet/actions/run.pp | 18 +-- resources/keystone_puppet/actions/update.pp | 18 +-- .../neutron_agents_dhcp_puppet/actions/run.pp | 26 ++-- .../neutron_agents_l3_puppet/actions/run.pp | 44 +++--- .../actions/run.pp | 34 ++--- .../actions/run.pp | 32 ++-- .../neutron_plugins_ml2_puppet/actions/run.pp | 26 ++-- resources/neutron_puppet/actions/run.pp | 114 +++++++------- .../neutron_server_puppet/actions/run.pp | 84 +++++------ resources/node_network_puppet/actions/run.pp | 24 +-- resources/nova_api_puppet/actions/run.pp | 60 ++++---- resources/nova_api_puppet/actions/update.pp | 60 ++++---- .../actions/run.pp | 28 ++-- .../actions/update.pp | 28 ++-- resources/nova_compute_puppet/actions/run.pp | 42 +++--- .../nova_compute_puppet/actions/update.pp | 42 +++--- .../nova_conductor_puppet/actions/run.pp | 4 +- .../nova_conductor_puppet/actions/update.pp | 4 +- .../actions/remove.pp | 6 +- .../actions/run.pp | 8 +- .../actions/update.pp | 8 +- resources/nova_neutron_puppet/actions/run.pp | 48 +++--- resources/nova_puppet/actions/run.pp | 140 +++++++++--------- resources/rabbitmq_service/actions/run.pp | 4 +- solar/solar/core/handlers/puppet.py | 4 +- 38 files changed, 787 insertions(+), 787 deletions(-) diff --git a/resources/apache_puppet/actions/run.pp b/resources/apache_puppet/actions/run.pp index ee2379da..52d642de 100644 --- a/resources/apache_puppet/actions/run.pp +++ b/resources/apache_puppet/actions/run.pp @@ -1,61 +1,61 @@ $resource = hiera($::resource_name) -$apache_name = $resource['input']['apache_name']['value'] -$service_name = $resource['input']['service_name']['value'] -$default_mods = $resource['input']['default_mods']['value'] -$default_vhost = $resource['input']['default_vhost']['value'] -$default_charset = $resource['input']['default_charset']['value'] -$default_confd_files = $resource['input']['default_confd_files']['value'] -$default_ssl_vhost = $resource['input']['default_ssl_vhost']['value'] -$default_ssl_cert = $resource['input']['default_ssl_cert']['value'] -$default_ssl_key = $resource['input']['default_ssl_key']['value'] -$default_ssl_chain = $resource['input']['default_ssl_chain']['value'] -$default_ssl_ca = $resource['input']['default_ssl_ca']['value'] -$default_ssl_crl_path = $resource['input']['default_ssl_crl_path']['value'] -$default_ssl_crl = $resource['input']['default_ssl_crl']['value'] -$default_ssl_crl_check = $resource['input']['default_ssl_crl_check']['value'] -$default_type = $resource['input']['default_type']['value'] -$ip = $resource['input']['ip']['value'] -$service_restart = $resource['input']['service_restart']['value'] -$purge_configs = $resource['input']['purge_configs']['value'] -$purge_vhost_dir = $resource['input']['purge_vhost_dir']['value'] -$purge_vdir = $resource['input']['purge_vdir']['value'] -$serveradmin = $resource['input']['serveradmin']['value'] -$sendfile = $resource['input']['sendfile']['value'] -$error_documents = $resource['input']['error_documents']['value'] -$timeout = $resource['input']['timeout']['value'] -$httpd_dir = $resource['input']['httpd_dir']['value'] -$server_root = $resource['input']['server_root']['value'] -$conf_dir = $resource['input']['conf_dir']['value'] -$confd_dir = $resource['input']['confd_dir']['value'] -$vhost_dir = $resource['input']['vhost_dir']['value'] -$vhost_enable_dir = $resource['input']['vhost_enable_dir']['value'] -$mod_dir = $resource['input']['mod_dir']['value'] -$mod_enable_dir = $resource['input']['mod_enable_dir']['value'] -$mpm_module = $resource['input']['mpm_module']['value'] -$lib_path = $resource['input']['lib_path']['value'] -$conf_template = $resource['input']['conf_template']['value'] -$servername = $resource['input']['servername']['value'] -$manage_user = $resource['input']['manage_user']['value'] -$manage_group = $resource['input']['manage_group']['value'] -$user = $resource['input']['user']['value'] -$group = $resource['input']['group']['value'] -$keepalive = $resource['input']['keepalive']['value'] -$keepalive_timeout = $resource['input']['keepalive_timeout']['value'] -$max_keepalive_requests = $resource['input']['max_keepalive_requests']['value'] -$logroot = $resource['input']['logroot']['value'] -$logroot_mode = $resource['input']['logroot_mode']['value'] -$log_level = $resource['input']['log_level']['value'] -$log_formats = $resource['input']['log_formats']['value'] -$ports_file = $resource['input']['ports_file']['value'] -$docroot = $resource['input']['docroot']['value'] -$apache_version = $resource['input']['apache_version']['value'] -$server_tokens = $resource['input']['server_tokens']['value'] -$server_signature = $resource['input']['server_signature']['value'] -$trace_enable = $resource['input']['trace_enable']['value'] -$allow_encoded_slashes = $resource['input']['allow_encoded_slashes']['value'] -$package_ensure = $resource['input']['package_ensure']['value'] -$use_optional_includes = $resource['input']['use_optional_includes']['value'] +$apache_name = $resource['input']['apache_name'] +$service_name = $resource['input']['service_name'] +$default_mods = $resource['input']['default_mods'] +$default_vhost = $resource['input']['default_vhost'] +$default_charset = $resource['input']['default_charset'] +$default_confd_files = $resource['input']['default_confd_files'] +$default_ssl_vhost = $resource['input']['default_ssl_vhost'] +$default_ssl_cert = $resource['input']['default_ssl_cert'] +$default_ssl_key = $resource['input']['default_ssl_key'] +$default_ssl_chain = $resource['input']['default_ssl_chain'] +$default_ssl_ca = $resource['input']['default_ssl_ca'] +$default_ssl_crl_path = $resource['input']['default_ssl_crl_path'] +$default_ssl_crl = $resource['input']['default_ssl_crl'] +$default_ssl_crl_check = $resource['input']['default_ssl_crl_check'] +$default_type = $resource['input']['default_type'] +$ip = $resource['input']['ip'] +$service_restart = $resource['input']['service_restart'] +$purge_configs = $resource['input']['purge_configs'] +$purge_vhost_dir = $resource['input']['purge_vhost_dir'] +$purge_vdir = $resource['input']['purge_vdir'] +$serveradmin = $resource['input']['serveradmin'] +$sendfile = $resource['input']['sendfile'] +$error_documents = $resource['input']['error_documents'] +$timeout = $resource['input']['timeout'] +$httpd_dir = $resource['input']['httpd_dir'] +$server_root = $resource['input']['server_root'] +$conf_dir = $resource['input']['conf_dir'] +$confd_dir = $resource['input']['confd_dir'] +$vhost_dir = $resource['input']['vhost_dir'] +$vhost_enable_dir = $resource['input']['vhost_enable_dir'] +$mod_dir = $resource['input']['mod_dir'] +$mod_enable_dir = $resource['input']['mod_enable_dir'] +$mpm_module = $resource['input']['mpm_module'] +$lib_path = $resource['input']['lib_path'] +$conf_template = $resource['input']['conf_template'] +$servername = $resource['input']['servername'] +$manage_user = $resource['input']['manage_user'] +$manage_group = $resource['input']['manage_group'] +$user = $resource['input']['user'] +$group = $resource['input']['group'] +$keepalive = $resource['input']['keepalive'] +$keepalive_timeout = $resource['input']['keepalive_timeout'] +$max_keepalive_requests = $resource['input']['max_keepalive_requests'] +$logroot = $resource['input']['logroot'] +$logroot_mode = $resource['input']['logroot_mode'] +$log_level = $resource['input']['log_level'] +$log_formats = $resource['input']['log_formats'] +$ports_file = $resource['input']['ports_file'] +$docroot = $resource['input']['docroot'] +$apache_version = $resource['input']['apache_version'] +$server_tokens = $resource['input']['server_tokens'] +$server_signature = $resource['input']['server_signature'] +$trace_enable = $resource['input']['trace_enable'] +$allow_encoded_slashes = $resource['input']['allow_encoded_slashes'] +$package_ensure = $resource['input']['package_ensure'] +$use_optional_includes = $resource['input']['use_optional_includes'] class {'apache': apache_name => $apache_name, diff --git a/resources/cinder_api_puppet/actions/run.pp b/resources/cinder_api_puppet/actions/run.pp index 857cbca1..f7d18786 100644 --- a/resources/cinder_api_puppet/actions/run.pp +++ b/resources/cinder_api_puppet/actions/run.pp @@ -1,24 +1,24 @@ $resource = hiera($::resource_name) -$keystone_password = $resource['input']['keystone_password']['value'] -$keystone_enabled = $resource['input']['keystone_enabled']['value'] -$keystone_tenant = $resource['input']['keystone_tenant']['value'] -$keystone_user = $resource['input']['keystone_user']['value'] -$keystone_auth_host = $resource['input']['keystone_auth_host']['value'] -$keystone_auth_port = $resource['input']['keystone_auth_port']['value'] -$keystone_auth_protocol = $resource['input']['keystone_auth_protocol']['value'] -$keystone_auth_admin_prefix = $resource['input']['keystone_auth_admin_prefix']['value'] -$keystone_auth_uri = $resource['input']['keystone_auth_uri']['value'] -$os_region_name = $resource['input']['os_region_name']['value'] -$service_port = $resource['input']['service_port']['value'] -$service_workers = $resource['input']['service_workers']['value'] -$package_ensure = $resource['input']['package_ensure']['value'] -$bind_host = $resource['input']['bind_host']['value'] -$ratelimits = $resource['input']['ratelimits']['value'] -$default_volume_type = $resource['input']['default_volume_type']['value'] -$ratelimits_factory = $resource['input']['ratelimits_factory']['value'] -$validate = $resource['input']['validate']['value'] -$validation_options = $resource['input']['validation_options']['value'] +$keystone_password = $resource['input']['keystone_password'] +$keystone_enabled = $resource['input']['keystone_enabled'] +$keystone_tenant = $resource['input']['keystone_tenant'] +$keystone_user = $resource['input']['keystone_user'] +$keystone_auth_host = $resource['input']['keystone_auth_host'] +$keystone_auth_port = $resource['input']['keystone_auth_port'] +$keystone_auth_protocol = $resource['input']['keystone_auth_protocol'] +$keystone_auth_admin_prefix = $resource['input']['keystone_auth_admin_prefix'] +$keystone_auth_uri = $resource['input']['keystone_auth_uri'] +$os_region_name = $resource['input']['os_region_name'] +$service_port = $resource['input']['service_port'] +$service_workers = $resource['input']['service_workers'] +$package_ensure = $resource['input']['package_ensure'] +$bind_host = $resource['input']['bind_host'] +$ratelimits = $resource['input']['ratelimits'] +$default_volume_type = $resource['input']['default_volume_type'] +$ratelimits_factory = $resource['input']['ratelimits_factory'] +$validate = $resource['input']['validate'] +$validation_options = $resource['input']['validation_options'] include cinder::params diff --git a/resources/cinder_api_puppet/actions/update.pp b/resources/cinder_api_puppet/actions/update.pp index f0eae3f4..3486c9a2 100644 --- a/resources/cinder_api_puppet/actions/update.pp +++ b/resources/cinder_api_puppet/actions/update.pp @@ -1,24 +1,24 @@ $resource = hiera($::resource_name) -$keystone_password = $resource['input']['keystone_password']['value'] -$keystone_enabled = $resource['input']['keystone_enabled']['value'] -$keystone_tenant = $resource['input']['keystone_tenant']['value'] -$keystone_user = $resource['input']['keystone_user']['value'] -$keystone_auth_host = $resource['input']['keystone_auth_host']['value'] -$keystone_auth_port = $resource['input']['keystone_auth_port']['value'] -$keystone_auth_protocol = $resource['input']['keystone_auth_protocol']['value'] -$keystone_auth_admin_prefix = $resource['input']['keystone_auth_admin_prefix']['value'] -$keystone_auth_uri = $resource['input']['keystone_auth_uri']['value'] -$os_region_name = $resource['input']['os_region_name']['value'] -$service_port = $resource['input']['service_port']['value'] -$service_workers = $resource['input']['service_workers']['value'] -$package_ensure = $resource['input']['package_ensure']['value'] -$bind_host = $resource['input']['bind_host']['value'] -$ratelimits = $resource['input']['ratelimits']['value'] -$default_volume_type = $resource['input']['default_volume_type']['value'] -$ratelimits_factory = $resource['input']['ratelimits_factory']['value'] -$validate = $resource['input']['validate']['value'] -$validation_options = $resource['input']['validation_options']['value'] +$keystone_password = $resource['input']['keystone_password'] +$keystone_enabled = $resource['input']['keystone_enabled'] +$keystone_tenant = $resource['input']['keystone_tenant'] +$keystone_user = $resource['input']['keystone_user'] +$keystone_auth_host = $resource['input']['keystone_auth_host'] +$keystone_auth_port = $resource['input']['keystone_auth_port'] +$keystone_auth_protocol = $resource['input']['keystone_auth_protocol'] +$keystone_auth_admin_prefix = $resource['input']['keystone_auth_admin_prefix'] +$keystone_auth_uri = $resource['input']['keystone_auth_uri'] +$os_region_name = $resource['input']['os_region_name'] +$service_port = $resource['input']['service_port'] +$service_workers = $resource['input']['service_workers'] +$package_ensure = $resource['input']['package_ensure'] +$bind_host = $resource['input']['bind_host'] +$ratelimits = $resource['input']['ratelimits'] +$default_volume_type = $resource['input']['default_volume_type'] +$ratelimits_factory = $resource['input']['ratelimits_factory'] +$validate = $resource['input']['validate'] +$validation_options = $resource['input']['validation_options'] include cinder::params diff --git a/resources/cinder_glance_puppet/actions/run.pp b/resources/cinder_glance_puppet/actions/run.pp index c5b030d7..fdbe9618 100644 --- a/resources/cinder_glance_puppet/actions/run.pp +++ b/resources/cinder_glance_puppet/actions/run.pp @@ -1,12 +1,12 @@ $resource = hiera($::resource_name) -$glance_api_version = $resource['input']['glance_api_version']['value'] -$glance_num_retries = $resource['input']['glance_num_retries']['value'] -$glance_api_insecure = $resource['input']['glance_api_insecure']['value'] -$glance_api_ssl_compression = $resource['input']['glance_api_ssl_compression']['value'] -$glance_request_timeout = $resource['input']['glance_request_timeout']['value'] -$glance_api_servers_host = $resource['input']['glance_api_servers_host']['value'] -$glance_api_servers_port = $resource['input']['glance_api_servers_port']['value'] +$glance_api_version = $resource['input']['glance_api_version'] +$glance_num_retries = $resource['input']['glance_num_retries'] +$glance_api_insecure = $resource['input']['glance_api_insecure'] +$glance_api_ssl_compression = $resource['input']['glance_api_ssl_compression'] +$glance_request_timeout = $resource['input']['glance_request_timeout'] +$glance_api_servers_host = $resource['input']['glance_api_servers_host'] +$glance_api_servers_port = $resource['input']['glance_api_servers_port'] class {'cinder::glance': glance_api_servers => "${glance_api_servers_host}:${glance_api_servers_port}", diff --git a/resources/cinder_puppet/actions/run.pp b/resources/cinder_puppet/actions/run.pp index d8daffd9..cb456062 100644 --- a/resources/cinder_puppet/actions/run.pp +++ b/resources/cinder_puppet/actions/run.pp @@ -1,65 +1,65 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$db_user = $resource['input']['db_user']['value'] -$db_password = $resource['input']['db_password']['value'] -$db_name = $resource['input']['db_name']['value'] -$db_host = $resource['input']['db_host']['value'] -$db_port = $resource['input']['db_port']['value'] +$db_user = $resource['input']['db_user'] +$db_password = $resource['input']['db_password'] +$db_name = $resource['input']['db_name'] +$db_host = $resource['input']['db_host'] +$db_port = $resource['input']['db_port'] -$database_connection = $resource['input']['database_connection']['value'] -$database_idle_timeout = $resource['input']['database_idle_timeout']['value'] -$database_min_pool_size = $resource['input']['database_min_pool_size']['value'] -$database_max_pool_size = $resource['input']['database_max_pool_size']['value'] -$database_max_retries = $resource['input']['database_max_retries']['value'] -$database_retry_interval = $resource['input']['database_retry_interval']['value'] -$database_max_overflow = $resource['input']['database_max_overflow']['value'] -$rpc_backend = $resource['input']['rpc_backend']['value'] -$control_exchange = $resource['input']['control_exchange']['value'] -$rabbit_host = $resource['input']['rabbit_host']['value'] -$rabbit_port = $resource['input']['rabbit_port']['value'] -$rabbit_hosts = $resource['input']['rabbit_hosts']['value'] -$rabbit_virtual_host = $resource['input']['rabbit_virtual_host']['value'] -$rabbit_userid = $resource['input']['rabbit_userid']['value'] -$rabbit_password = $resource['input']['rabbit_password']['value'] -$rabbit_use_ssl = $resource['input']['rabbit_use_ssl']['value'] -$kombu_ssl_ca_certs = $resource['input']['kombu_ssl_ca_certs']['value'] -$kombu_ssl_certfile = $resource['input']['kombu_ssl_certfile']['value'] -$kombu_ssl_keyfile = $resource['input']['kombu_ssl_keyfile']['value'] -$kombu_ssl_version = $resource['input']['kombu_ssl_version']['value'] -$amqp_durable_queues = $resource['input']['amqp_durable_queues']['value'] -$qpid_hostname = $resource['input']['qpid_hostname']['value'] -$qpid_port = $resource['input']['qpid_port']['value'] -$qpid_username = $resource['input']['qpid_username']['value'] -$qpid_password = $resource['input']['qpid_password']['value'] -$qpid_sasl_mechanisms = $resource['input']['qpid_sasl_mechanisms']['value'] -$qpid_reconnect = $resource['input']['qpid_reconnect']['value'] -$qpid_reconnect_timeout = $resource['input']['qpid_reconnect_timeout']['value'] -$qpid_reconnect_limit = $resource['input']['qpid_reconnect_limit']['value'] -$qpid_reconnect_interval_min = $resource['input']['qpid_reconnect_interval_min']['value'] -$qpid_reconnect_interval_max = $resource['input']['qpid_reconnect_interval_max']['value'] -$qpid_reconnect_interval = $resource['input']['qpid_reconnect_interval']['value'] -$qpid_heartbeat = $resource['input']['qpid_heartbeat']['value'] -$qpid_protocol = $resource['input']['qpid_protocol']['value'] -$qpid_tcp_nodelay = $resource['input']['qpid_tcp_nodelay']['value'] -$package_ensure = $resource['input']['package_ensure']['value'] -$use_ssl = $resource['input']['use_ssl']['value'] -$ca_file = $resource['input']['ca_file']['value'] -$cert_file = $resource['input']['cert_file']['value'] -$key_file = $resource['input']['key_file']['value'] -$api_paste_config = $resource['input']['api_paste_config']['value'] -$use_syslog = $resource['input']['use_syslog']['value'] -$log_facility = $resource['input']['log_facility']['value'] -$log_dir = $resource['input']['log_dir']['value'] -$verbose = $resource['input']['verbose']['value'] -$debug = $resource['input']['debug']['value'] -$storage_availability_zone = $resource['input']['storage_availability_zone']['value'] -$default_availability_zone = $resource['input']['default_availability_zone']['value'] -$mysql_module = $resource['input']['mysql_module']['value'] +$database_connection = $resource['input']['database_connection'] +$database_idle_timeout = $resource['input']['database_idle_timeout'] +$database_min_pool_size = $resource['input']['database_min_pool_size'] +$database_max_pool_size = $resource['input']['database_max_pool_size'] +$database_max_retries = $resource['input']['database_max_retries'] +$database_retry_interval = $resource['input']['database_retry_interval'] +$database_max_overflow = $resource['input']['database_max_overflow'] +$rpc_backend = $resource['input']['rpc_backend'] +$control_exchange = $resource['input']['control_exchange'] +$rabbit_host = $resource['input']['rabbit_host'] +$rabbit_port = $resource['input']['rabbit_port'] +$rabbit_hosts = $resource['input']['rabbit_hosts'] +$rabbit_virtual_host = $resource['input']['rabbit_virtual_host'] +$rabbit_userid = $resource['input']['rabbit_userid'] +$rabbit_password = $resource['input']['rabbit_password'] +$rabbit_use_ssl = $resource['input']['rabbit_use_ssl'] +$kombu_ssl_ca_certs = $resource['input']['kombu_ssl_ca_certs'] +$kombu_ssl_certfile = $resource['input']['kombu_ssl_certfile'] +$kombu_ssl_keyfile = $resource['input']['kombu_ssl_keyfile'] +$kombu_ssl_version = $resource['input']['kombu_ssl_version'] +$amqp_durable_queues = $resource['input']['amqp_durable_queues'] +$qpid_hostname = $resource['input']['qpid_hostname'] +$qpid_port = $resource['input']['qpid_port'] +$qpid_username = $resource['input']['qpid_username'] +$qpid_password = $resource['input']['qpid_password'] +$qpid_sasl_mechanisms = $resource['input']['qpid_sasl_mechanisms'] +$qpid_reconnect = $resource['input']['qpid_reconnect'] +$qpid_reconnect_timeout = $resource['input']['qpid_reconnect_timeout'] +$qpid_reconnect_limit = $resource['input']['qpid_reconnect_limit'] +$qpid_reconnect_interval_min = $resource['input']['qpid_reconnect_interval_min'] +$qpid_reconnect_interval_max = $resource['input']['qpid_reconnect_interval_max'] +$qpid_reconnect_interval = $resource['input']['qpid_reconnect_interval'] +$qpid_heartbeat = $resource['input']['qpid_heartbeat'] +$qpid_protocol = $resource['input']['qpid_protocol'] +$qpid_tcp_nodelay = $resource['input']['qpid_tcp_nodelay'] +$package_ensure = $resource['input']['package_ensure'] +$use_ssl = $resource['input']['use_ssl'] +$ca_file = $resource['input']['ca_file'] +$cert_file = $resource['input']['cert_file'] +$key_file = $resource['input']['key_file'] +$api_paste_config = $resource['input']['api_paste_config'] +$use_syslog = $resource['input']['use_syslog'] +$log_facility = $resource['input']['log_facility'] +$log_dir = $resource['input']['log_dir'] +$verbose = $resource['input']['verbose'] +$debug = $resource['input']['debug'] +$storage_availability_zone = $resource['input']['storage_availability_zone'] +$default_availability_zone = $resource['input']['default_availability_zone'] +$mysql_module = $resource['input']['mysql_module'] # Do not apply the legacy stuff -#$sql_connection = $resource['input']['sql_connection']['value'] -$sql_idle_timeout = $resource['input']['sql_idle_timeout']['value'] +#$sql_connection = $resource['input']['sql_connection'] +$sql_idle_timeout = $resource['input']['sql_idle_timeout'] class {'cinder': database_connection => "mysql://${db_user}:${db_password}@${db_host}:${db_port}/${db_name}", diff --git a/resources/cinder_scheduler_puppet/actions/run.pp b/resources/cinder_scheduler_puppet/actions/run.pp index 5d963a6d..8a354de1 100644 --- a/resources/cinder_scheduler_puppet/actions/run.pp +++ b/resources/cinder_scheduler_puppet/actions/run.pp @@ -1,7 +1,7 @@ $resource = hiera($::resource_name) -$scheduler_driver = $resource['input']['scheduler_driver']['value'] -$package_ensure = $resource['input']['package_ensure']['value'] +$scheduler_driver = $resource['input']['scheduler_driver'] +$package_ensure = $resource['input']['package_ensure'] include cinder::params diff --git a/resources/cinder_scheduler_puppet/actions/update.pp b/resources/cinder_scheduler_puppet/actions/update.pp index de3c4109..7b67bfd3 100644 --- a/resources/cinder_scheduler_puppet/actions/update.pp +++ b/resources/cinder_scheduler_puppet/actions/update.pp @@ -1,7 +1,7 @@ $resource = hiera($::resource_name) -$scheduler_driver = $resource['input']['scheduler_driver']['value'] -$package_ensure = $resource['input']['package_ensure']['value'] +$scheduler_driver = $resource['input']['scheduler_driver'] +$package_ensure = $resource['input']['package_ensure'] include cinder::params diff --git a/resources/cinder_volume_puppet/actions/run.pp b/resources/cinder_volume_puppet/actions/run.pp index 91e9b5d3..58a7ea4e 100644 --- a/resources/cinder_volume_puppet/actions/run.pp +++ b/resources/cinder_volume_puppet/actions/run.pp @@ -1,12 +1,12 @@ $resource = hiera($::resource_name) -$package_ensure = $resource['input']['package_ensure']['value'] -$use_iscsi_backend = $resource['input']['use_iscsi_backend']['value'] +$package_ensure = $resource['input']['package_ensure'] +$use_iscsi_backend = $resource['input']['use_iscsi_backend'] -$iscsi_ip_address = $resource['input']['iscsi_ip_address']['value'] -$volume_driver = $resource['input']['volume_driver']['value'] -$volume_group = $resource['input']['volume_group']['value'] -$iscsi_helper = $resource['input']['iscsi_helper']['value'] +$iscsi_ip_address = $resource['input']['iscsi_ip_address'] +$volume_driver = $resource['input']['volume_driver'] +$volume_group = $resource['input']['volume_group'] +$iscsi_helper = $resource['input']['iscsi_helper'] include cinder::params diff --git a/resources/cinder_volume_puppet/actions/update.pp b/resources/cinder_volume_puppet/actions/update.pp index 22964e05..b8f23629 100644 --- a/resources/cinder_volume_puppet/actions/update.pp +++ b/resources/cinder_volume_puppet/actions/update.pp @@ -1,12 +1,12 @@ $resource = hiera($::resource_name) -$package_ensure = $resource['input']['package_ensure']['value'] -$use_iscsi_backend = $resource['input']['use_iscsi_backend']['value'] +$package_ensure = $resource['input']['package_ensure'] +$use_iscsi_backend = $resource['input']['use_iscsi_backend'] -$iscsi_ip_address = $resource['input']['iscsi_ip_address']['value'] -$volume_driver = $resource['input']['volume_driver']['value'] -$volume_group = $resource['input']['volume_group']['value'] -$iscsi_helper = $resource['input']['iscsi_helper']['value'] +$iscsi_ip_address = $resource['input']['iscsi_ip_address'] +$volume_driver = $resource['input']['volume_driver'] +$volume_group = $resource['input']['volume_group'] +$iscsi_helper = $resource['input']['iscsi_helper'] include cinder::params diff --git a/resources/glance_puppet/actions/run.pp b/resources/glance_puppet/actions/run.pp index 10b10b19..ed2e3d13 100644 --- a/resources/glance_puppet/actions/run.pp +++ b/resources/glance_puppet/actions/run.pp @@ -1,53 +1,53 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$db_user = $resource['input']['db_user']['value'] -$db_password = $resource['input']['db_password']['value'] -$db_name = $resource['input']['db_name']['value'] -$db_host = $resource['input']['db_host']['value'] -$db_port = $resource['input']['db_port']['value'] +$db_user = $resource['input']['db_user'] +$db_password = $resource['input']['db_password'] +$db_name = $resource['input']['db_name'] +$db_host = $resource['input']['db_host'] +$db_port = $resource['input']['db_port'] -$filesystem_store_datadir = $resource['input']['filesystem_store_datadir']['value'] +$filesystem_store_datadir = $resource['input']['filesystem_store_datadir'] -$keystone_password = $resource['input']['keystone_password']['value'] -$verbose = $resource['input']['verbose']['value'] -$debug = $resource['input']['debug']['value'] -$bind_host = $resource['input']['bind_host']['value'] -$bind_port = $resource['input']['bind_port']['value'] -$backlog = $resource['input']['backlog']['value'] -$workers = $resource['input']['workers']['value'] -$log_file = $resource['input']['log_file']['value'] -$log_dir = $resource['input']['log_dir']['value'] -$registry_host = $resource['input']['registry_host']['value'] -$registry_port = $resource['input']['registry_port']['value'] -$registry_client_protocol = $resource['input']['registry_client_protocol']['value'] -$auth_type = $resource['input']['auth_type']['value'] -$auth_host = $resource['input']['auth_host']['value'] -$auth_url = $resource['input']['auth_url']['value'] -$auth_port = $resource['input']['auth_port']['value'] -$auth_uri = $resource['input']['auth_uri']['value'] -$auth_admin_prefix = $resource['input']['auth_admin_prefix']['value'] -$auth_protocol = $resource['input']['auth_protocol']['value'] -$pipeline = $resource['input']['pipeline']['value'] -$keystone_tenant = $resource['input']['keystone_tenant']['value'] -$keystone_user = $resource['input']['keystone_user']['value'] -$use_syslog = $resource['input']['use_syslog']['value'] -$log_facility = $resource['input']['log_facility']['value'] -$show_image_direct_url = $resource['input']['show_image_direct_url']['value'] -$purge_config = $resource['input']['purge_config']['value'] -$cert_file = $resource['input']['cert_file']['value'] -$key_file = $resource['input']['key_file']['value'] -$ca_file = $resource['input']['ca_file']['value'] -$known_stores = $resource['input']['known_stores']['value'] -$database_connection = $resource['input']['database_connection']['value'] -$database_idle_timeout = $resource['input']['database_idle_timeout']['value'] -$image_cache_dir = $resource['input']['image_cache_dir']['value'] -$os_region_name = $resource['input']['os_region_name']['value'] -$validate = $resource['input']['validate']['value'] -$validation_options = $resource['input']['validation_options']['value'] -$mysql_module = $resource['input']['mysql_module']['value'] -$sql_idle_timeout = $resource['input']['sql_idle_timeout']['value'] +$keystone_password = $resource['input']['keystone_password'] +$verbose = $resource['input']['verbose'] +$debug = $resource['input']['debug'] +$bind_host = $resource['input']['bind_host'] +$bind_port = $resource['input']['bind_port'] +$backlog = $resource['input']['backlog'] +$workers = $resource['input']['workers'] +$log_file = $resource['input']['log_file'] +$log_dir = $resource['input']['log_dir'] +$registry_host = $resource['input']['registry_host'] +$registry_port = $resource['input']['registry_port'] +$registry_client_protocol = $resource['input']['registry_client_protocol'] +$auth_type = $resource['input']['auth_type'] +$auth_host = $resource['input']['auth_host'] +$auth_url = $resource['input']['auth_url'] +$auth_port = $resource['input']['auth_port'] +$auth_uri = $resource['input']['auth_uri'] +$auth_admin_prefix = $resource['input']['auth_admin_prefix'] +$auth_protocol = $resource['input']['auth_protocol'] +$pipeline = $resource['input']['pipeline'] +$keystone_tenant = $resource['input']['keystone_tenant'] +$keystone_user = $resource['input']['keystone_user'] +$use_syslog = $resource['input']['use_syslog'] +$log_facility = $resource['input']['log_facility'] +$show_image_direct_url = $resource['input']['show_image_direct_url'] +$purge_config = $resource['input']['purge_config'] +$cert_file = $resource['input']['cert_file'] +$key_file = $resource['input']['key_file'] +$ca_file = $resource['input']['ca_file'] +$known_stores = $resource['input']['known_stores'] +$database_connection = $resource['input']['database_connection'] +$database_idle_timeout = $resource['input']['database_idle_timeout'] +$image_cache_dir = $resource['input']['image_cache_dir'] +$os_region_name = $resource['input']['os_region_name'] +$validate = $resource['input']['validate'] +$validation_options = $resource['input']['validation_options'] +$mysql_module = $resource['input']['mysql_module'] +$sql_idle_timeout = $resource['input']['sql_idle_timeout'] class {'glance': package_ensure => 'present', diff --git a/resources/glance_puppet/actions/update.pp b/resources/glance_puppet/actions/update.pp index 10b10b19..ed2e3d13 100644 --- a/resources/glance_puppet/actions/update.pp +++ b/resources/glance_puppet/actions/update.pp @@ -1,53 +1,53 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$db_user = $resource['input']['db_user']['value'] -$db_password = $resource['input']['db_password']['value'] -$db_name = $resource['input']['db_name']['value'] -$db_host = $resource['input']['db_host']['value'] -$db_port = $resource['input']['db_port']['value'] +$db_user = $resource['input']['db_user'] +$db_password = $resource['input']['db_password'] +$db_name = $resource['input']['db_name'] +$db_host = $resource['input']['db_host'] +$db_port = $resource['input']['db_port'] -$filesystem_store_datadir = $resource['input']['filesystem_store_datadir']['value'] +$filesystem_store_datadir = $resource['input']['filesystem_store_datadir'] -$keystone_password = $resource['input']['keystone_password']['value'] -$verbose = $resource['input']['verbose']['value'] -$debug = $resource['input']['debug']['value'] -$bind_host = $resource['input']['bind_host']['value'] -$bind_port = $resource['input']['bind_port']['value'] -$backlog = $resource['input']['backlog']['value'] -$workers = $resource['input']['workers']['value'] -$log_file = $resource['input']['log_file']['value'] -$log_dir = $resource['input']['log_dir']['value'] -$registry_host = $resource['input']['registry_host']['value'] -$registry_port = $resource['input']['registry_port']['value'] -$registry_client_protocol = $resource['input']['registry_client_protocol']['value'] -$auth_type = $resource['input']['auth_type']['value'] -$auth_host = $resource['input']['auth_host']['value'] -$auth_url = $resource['input']['auth_url']['value'] -$auth_port = $resource['input']['auth_port']['value'] -$auth_uri = $resource['input']['auth_uri']['value'] -$auth_admin_prefix = $resource['input']['auth_admin_prefix']['value'] -$auth_protocol = $resource['input']['auth_protocol']['value'] -$pipeline = $resource['input']['pipeline']['value'] -$keystone_tenant = $resource['input']['keystone_tenant']['value'] -$keystone_user = $resource['input']['keystone_user']['value'] -$use_syslog = $resource['input']['use_syslog']['value'] -$log_facility = $resource['input']['log_facility']['value'] -$show_image_direct_url = $resource['input']['show_image_direct_url']['value'] -$purge_config = $resource['input']['purge_config']['value'] -$cert_file = $resource['input']['cert_file']['value'] -$key_file = $resource['input']['key_file']['value'] -$ca_file = $resource['input']['ca_file']['value'] -$known_stores = $resource['input']['known_stores']['value'] -$database_connection = $resource['input']['database_connection']['value'] -$database_idle_timeout = $resource['input']['database_idle_timeout']['value'] -$image_cache_dir = $resource['input']['image_cache_dir']['value'] -$os_region_name = $resource['input']['os_region_name']['value'] -$validate = $resource['input']['validate']['value'] -$validation_options = $resource['input']['validation_options']['value'] -$mysql_module = $resource['input']['mysql_module']['value'] -$sql_idle_timeout = $resource['input']['sql_idle_timeout']['value'] +$keystone_password = $resource['input']['keystone_password'] +$verbose = $resource['input']['verbose'] +$debug = $resource['input']['debug'] +$bind_host = $resource['input']['bind_host'] +$bind_port = $resource['input']['bind_port'] +$backlog = $resource['input']['backlog'] +$workers = $resource['input']['workers'] +$log_file = $resource['input']['log_file'] +$log_dir = $resource['input']['log_dir'] +$registry_host = $resource['input']['registry_host'] +$registry_port = $resource['input']['registry_port'] +$registry_client_protocol = $resource['input']['registry_client_protocol'] +$auth_type = $resource['input']['auth_type'] +$auth_host = $resource['input']['auth_host'] +$auth_url = $resource['input']['auth_url'] +$auth_port = $resource['input']['auth_port'] +$auth_uri = $resource['input']['auth_uri'] +$auth_admin_prefix = $resource['input']['auth_admin_prefix'] +$auth_protocol = $resource['input']['auth_protocol'] +$pipeline = $resource['input']['pipeline'] +$keystone_tenant = $resource['input']['keystone_tenant'] +$keystone_user = $resource['input']['keystone_user'] +$use_syslog = $resource['input']['use_syslog'] +$log_facility = $resource['input']['log_facility'] +$show_image_direct_url = $resource['input']['show_image_direct_url'] +$purge_config = $resource['input']['purge_config'] +$cert_file = $resource['input']['cert_file'] +$key_file = $resource['input']['key_file'] +$ca_file = $resource['input']['ca_file'] +$known_stores = $resource['input']['known_stores'] +$database_connection = $resource['input']['database_connection'] +$database_idle_timeout = $resource['input']['database_idle_timeout'] +$image_cache_dir = $resource['input']['image_cache_dir'] +$os_region_name = $resource['input']['os_region_name'] +$validate = $resource['input']['validate'] +$validation_options = $resource['input']['validation_options'] +$mysql_module = $resource['input']['mysql_module'] +$sql_idle_timeout = $resource['input']['sql_idle_timeout'] class {'glance': package_ensure => 'present', diff --git a/resources/glance_registry_puppet/actions/run.pp b/resources/glance_registry_puppet/actions/run.pp index 02c34aa7..1bfad2f8 100644 --- a/resources/glance_registry_puppet/actions/run.pp +++ b/resources/glance_registry_puppet/actions/run.pp @@ -1,42 +1,42 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$db_user = $resource['input']['db_user']['value'] -$db_password = $resource['input']['db_password']['value'] -$db_name = $resource['input']['db_name']['value'] -$db_host = $resource['input']['db_host']['value'] -$db_port = $resource['input']['db_port']['value'] +$db_user = $resource['input']['db_user'] +$db_password = $resource['input']['db_password'] +$db_name = $resource['input']['db_name'] +$db_host = $resource['input']['db_host'] +$db_port = $resource['input']['db_port'] -$keystone_password = $resource['input']['keystone_password']['value'] -$package_ensure = $resource['input']['package_ensure']['value'] -$verbose = $resource['input']['verbose']['value'] -$debug = $resource['input']['debug']['value'] -$bind_host = $resource['input']['bind_host']['value'] -$bind_port = $resource['input']['bind_port']['value'] -$log_file = $resource['input']['log_file']['value'] -$log_dir = $resource['input']['log_dir']['value'] -$database_connection = $resource['input']['database_connection']['value'] -$database_idle_timeout = $resource['input']['database_idle_timeout']['value'] -$auth_type = $resource['input']['auth_type']['value'] -$auth_host = $resource['input']['auth_host']['value'] -$auth_port = $resource['input']['auth_port']['value'] -$auth_admin_prefix = $resource['input']['auth_admin_prefix']['value'] -$auth_uri = $resource['input']['auth_uri']['value'] -$auth_protocol = $resource['input']['auth_protocol']['value'] -$keystone_tenant = $resource['input']['keystone_tenant']['value'] -$keystone_user = $resource['input']['keystone_user']['value'] -$pipeline = $resource['input']['pipeline']['value'] -$use_syslog = $resource['input']['use_syslog']['value'] -$log_facility = $resource['input']['log_facility']['value'] -$purge_config = $resource['input']['purge_config']['value'] -$cert_file = $resource['input']['cert_file']['value'] -$key_file = $resource['input']['key_file']['value'] -$ca_file = $resource['input']['ca_file']['value'] -$sync_db = $resource['input']['sync_db']['value'] -$mysql_module = $resource['input']['mysql_module']['value'] -$sql_idle_timeout = $resource['input']['sql_idle_timeout']['value'] -$sql_connection = $resource['input']['sql_connection']['value'] +$keystone_password = $resource['input']['keystone_password'] +$package_ensure = $resource['input']['package_ensure'] +$verbose = $resource['input']['verbose'] +$debug = $resource['input']['debug'] +$bind_host = $resource['input']['bind_host'] +$bind_port = $resource['input']['bind_port'] +$log_file = $resource['input']['log_file'] +$log_dir = $resource['input']['log_dir'] +$database_connection = $resource['input']['database_connection'] +$database_idle_timeout = $resource['input']['database_idle_timeout'] +$auth_type = $resource['input']['auth_type'] +$auth_host = $resource['input']['auth_host'] +$auth_port = $resource['input']['auth_port'] +$auth_admin_prefix = $resource['input']['auth_admin_prefix'] +$auth_uri = $resource['input']['auth_uri'] +$auth_protocol = $resource['input']['auth_protocol'] +$keystone_tenant = $resource['input']['keystone_tenant'] +$keystone_user = $resource['input']['keystone_user'] +$pipeline = $resource['input']['pipeline'] +$use_syslog = $resource['input']['use_syslog'] +$log_facility = $resource['input']['log_facility'] +$purge_config = $resource['input']['purge_config'] +$cert_file = $resource['input']['cert_file'] +$key_file = $resource['input']['key_file'] +$ca_file = $resource['input']['ca_file'] +$sync_db = $resource['input']['sync_db'] +$mysql_module = $resource['input']['mysql_module'] +$sql_idle_timeout = $resource['input']['sql_idle_timeout'] +$sql_connection = $resource['input']['sql_connection'] include glance::params diff --git a/resources/glance_registry_puppet/actions/update.pp b/resources/glance_registry_puppet/actions/update.pp index 1e21403c..7169a3b5 100644 --- a/resources/glance_registry_puppet/actions/update.pp +++ b/resources/glance_registry_puppet/actions/update.pp @@ -1,42 +1,42 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$db_user = $resource['input']['db_user']['value'] -$db_password = $resource['input']['db_password']['value'] -$db_name = $resource['input']['db_name']['value'] -$db_host = $resource['input']['db_host']['value'] -$db_port = $resource['input']['db_port']['value'] +$db_user = $resource['input']['db_user'] +$db_password = $resource['input']['db_password'] +$db_name = $resource['input']['db_name'] +$db_host = $resource['input']['db_host'] +$db_port = $resource['input']['db_port'] -$keystone_password = $resource['input']['keystone_password']['value'] -$package_ensure = $resource['input']['package_ensure']['value'] -$verbose = $resource['input']['verbose']['value'] -$debug = $resource['input']['debug']['value'] -$bind_host = $resource['input']['bind_host']['value'] -$bind_port = $resource['input']['bind_port']['value'] -$log_file = $resource['input']['log_file']['value'] -$log_dir = $resource['input']['log_dir']['value'] -$database_connection = $resource['input']['database_connection']['value'] -$database_idle_timeout = $resource['input']['database_idle_timeout']['value'] -$auth_type = $resource['input']['auth_type']['value'] -$auth_host = $resource['input']['auth_host']['value'] -$auth_port = $resource['input']['auth_port']['value'] -$auth_admin_prefix = $resource['input']['auth_admin_prefix']['value'] -$auth_uri = $resource['input']['auth_uri']['value'] -$auth_protocol = $resource['input']['auth_protocol']['value'] -$keystone_tenant = $resource['input']['keystone_tenant']['value'] -$keystone_user = $resource['input']['keystone_user']['value'] -$pipeline = $resource['input']['pipeline']['value'] -$use_syslog = $resource['input']['use_syslog']['value'] -$log_facility = $resource['input']['log_facility']['value'] -$purge_config = $resource['input']['purge_config']['value'] -$cert_file = $resource['input']['cert_file']['value'] -$key_file = $resource['input']['key_file']['value'] -$ca_file = $resource['input']['ca_file']['value'] -$sync_db = $resource['input']['sync_db']['value'] -$mysql_module = $resource['input']['mysql_module']['value'] -$sql_idle_timeout = $resource['input']['sql_idle_timeout']['value'] -$sql_connection = $resource['input']['sql_connection']['value'] +$keystone_password = $resource['input']['keystone_password'] +$package_ensure = $resource['input']['package_ensure'] +$verbose = $resource['input']['verbose'] +$debug = $resource['input']['debug'] +$bind_host = $resource['input']['bind_host'] +$bind_port = $resource['input']['bind_port'] +$log_file = $resource['input']['log_file'] +$log_dir = $resource['input']['log_dir'] +$database_connection = $resource['input']['database_connection'] +$database_idle_timeout = $resource['input']['database_idle_timeout'] +$auth_type = $resource['input']['auth_type'] +$auth_host = $resource['input']['auth_host'] +$auth_port = $resource['input']['auth_port'] +$auth_admin_prefix = $resource['input']['auth_admin_prefix'] +$auth_uri = $resource['input']['auth_uri'] +$auth_protocol = $resource['input']['auth_protocol'] +$keystone_tenant = $resource['input']['keystone_tenant'] +$keystone_user = $resource['input']['keystone_user'] +$pipeline = $resource['input']['pipeline'] +$use_syslog = $resource['input']['use_syslog'] +$log_facility = $resource['input']['log_facility'] +$purge_config = $resource['input']['purge_config'] +$cert_file = $resource['input']['cert_file'] +$key_file = $resource['input']['key_file'] +$ca_file = $resource['input']['ca_file'] +$sync_db = $resource['input']['sync_db'] +$mysql_module = $resource['input']['mysql_module'] +$sql_idle_timeout = $resource['input']['sql_idle_timeout'] +$sql_connection = $resource['input']['sql_connection'] include glance::params diff --git a/resources/keystone_puppet/actions/run.pp b/resources/keystone_puppet/actions/run.pp index 087ad4d7..73c818af 100644 --- a/resources/keystone_puppet/actions/run.pp +++ b/resources/keystone_puppet/actions/run.pp @@ -1,14 +1,14 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] -$admin_token = $resource['input']['admin_token']['value'] -$db_user = $resource['input']['db_user']['value'] -$db_host = $resource['input']['db_host']['value'] -$db_password = $resource['input']['db_password']['value'] -$db_name = $resource['input']['db_name']['value'] -$db_port = $resource['input']['db_port']['value'] -$admin_port = $resource['input']['admin_port']['value'] -$port = $resource['input']['port']['value'] +$ip = $resource['input']['ip'] +$admin_token = $resource['input']['admin_token'] +$db_user = $resource['input']['db_user'] +$db_host = $resource['input']['db_host'] +$db_password = $resource['input']['db_password'] +$db_name = $resource['input']['db_name'] +$db_port = $resource['input']['db_port'] +$admin_port = $resource['input']['admin_port'] +$port = $resource['input']['port'] class {'keystone': package_ensure => 'present', diff --git a/resources/keystone_puppet/actions/update.pp b/resources/keystone_puppet/actions/update.pp index 5b51a370..c295c3a7 100644 --- a/resources/keystone_puppet/actions/update.pp +++ b/resources/keystone_puppet/actions/update.pp @@ -1,14 +1,14 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] -$admin_token = $resource['input']['admin_token']['value'] -$db_user = $resource['input']['db_user']['value'] -$db_host = $resource['input']['db_host']['value'] -$db_password = $resource['input']['db_password']['value'] -$db_name = $resource['input']['db_name']['value'] -$db_port = $resource['input']['db_port']['value'] -$admin_port = $resource['input']['admin_port']['value'] -$port = $resource['input']['port']['value'] +$ip = $resource['input']['ip'] +$admin_token = $resource['input']['admin_token'] +$db_user = $resource['input']['db_user'] +$db_host = $resource['input']['db_host'] +$db_password = $resource['input']['db_password'] +$db_name = $resource['input']['db_name'] +$db_port = $resource['input']['db_port'] +$admin_port = $resource['input']['admin_port'] +$port = $resource['input']['port'] class {'keystone': package_ensure => 'present', diff --git a/resources/neutron_agents_dhcp_puppet/actions/run.pp b/resources/neutron_agents_dhcp_puppet/actions/run.pp index c0c9cae1..b33e087d 100644 --- a/resources/neutron_agents_dhcp_puppet/actions/run.pp +++ b/resources/neutron_agents_dhcp_puppet/actions/run.pp @@ -1,19 +1,19 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$package_ensure = $resource['input']['package_ensure']['value'] -$debug = $resource['input']['debug']['value'] -$state_path = $resource['input']['state_path']['value'] -$resync_interval = $resource['input']['resync_interval']['value'] -$interface_driver = $resource['input']['interface_driver']['value'] -$dhcp_driver = $resource['input']['dhcp_driver']['value'] -$root_helper = $resource['input']['root_helper']['value'] -$use_namespaces = $resource['input']['use_namespaces']['value'] -$dnsmasq_config_file = $resource['input']['dnsmasq_config_file']['value'] -$dhcp_delete_namespaces = $resource['input']['dhcp_delete_namespaces']['value'] -$enable_isolated_metadata = $resource['input']['enable_isolated_metadata']['value'] -$enable_metadata_network = $resource['input']['enable_metadata_network']['value'] +$package_ensure = $resource['input']['package_ensure'] +$debug = $resource['input']['debug'] +$state_path = $resource['input']['state_path'] +$resync_interval = $resource['input']['resync_interval'] +$interface_driver = $resource['input']['interface_driver'] +$dhcp_driver = $resource['input']['dhcp_driver'] +$root_helper = $resource['input']['root_helper'] +$use_namespaces = $resource['input']['use_namespaces'] +$dnsmasq_config_file = $resource['input']['dnsmasq_config_file'] +$dhcp_delete_namespaces = $resource['input']['dhcp_delete_namespaces'] +$enable_isolated_metadata = $resource['input']['enable_isolated_metadata'] +$enable_metadata_network = $resource['input']['enable_metadata_network'] class { 'neutron::agents::dhcp': enabled => true, diff --git a/resources/neutron_agents_l3_puppet/actions/run.pp b/resources/neutron_agents_l3_puppet/actions/run.pp index 66971b1d..0871d88c 100644 --- a/resources/neutron_agents_l3_puppet/actions/run.pp +++ b/resources/neutron_agents_l3_puppet/actions/run.pp @@ -1,28 +1,28 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$package_ensure = $resource['input']['package_ensure']['value'] -$debug = $resource['input']['debug']['value'] -$external_network_bridge = $resource['input']['external_network_bridge']['value'] -$use_namespaces = $resource['input']['use_namespaces']['value'] -$interface_driver = $resource['input']['interface_driver']['value'] -$router_id = $resource['input']['router_id']['value'] -$gateway_external_network_id = $resource['input']['gateway_external_network_id']['value'] -$handle_internal_only_routers = $resource['input']['handle_internal_only_routers']['value'] -$metadata_port = $resource['input']['metadata_port']['value'] -$send_arp_for_ha = $resource['input']['send_arp_for_ha']['value'] -$periodic_interval = $resource['input']['periodic_interval']['value'] -$periodic_fuzzy_delay = $resource['input']['periodic_fuzzy_delay']['value'] -$enable_metadata_proxy = $resource['input']['enable_metadata_proxy']['value'] -$network_device_mtu = $resource['input']['network_device_mtu']['value'] -$router_delete_namespaces = $resource['input']['router_delete_namespaces']['value'] -$ha_enabled = $resource['input']['ha_enabled']['value'] -$ha_vrrp_auth_type = $resource['input']['ha_vrrp_auth_type']['value'] -$ha_vrrp_auth_password = $resource['input']['ha_vrrp_auth_password']['value'] -$ha_vrrp_advert_int = $resource['input']['ha_vrrp_advert_int']['value'] -$agent_mode = $resource['input']['agent_mode']['value'] -$allow_automatic_l3agent_failover = $resource['input']['allow_automatic_l3agent_failover']['value'] +$package_ensure = $resource['input']['package_ensure'] +$debug = $resource['input']['debug'] +$external_network_bridge = $resource['input']['external_network_bridge'] +$use_namespaces = $resource['input']['use_namespaces'] +$interface_driver = $resource['input']['interface_driver'] +$router_id = $resource['input']['router_id'] +$gateway_external_network_id = $resource['input']['gateway_external_network_id'] +$handle_internal_only_routers = $resource['input']['handle_internal_only_routers'] +$metadata_port = $resource['input']['metadata_port'] +$send_arp_for_ha = $resource['input']['send_arp_for_ha'] +$periodic_interval = $resource['input']['periodic_interval'] +$periodic_fuzzy_delay = $resource['input']['periodic_fuzzy_delay'] +$enable_metadata_proxy = $resource['input']['enable_metadata_proxy'] +$network_device_mtu = $resource['input']['network_device_mtu'] +$router_delete_namespaces = $resource['input']['router_delete_namespaces'] +$ha_enabled = $resource['input']['ha_enabled'] +$ha_vrrp_auth_type = $resource['input']['ha_vrrp_auth_type'] +$ha_vrrp_auth_password = $resource['input']['ha_vrrp_auth_password'] +$ha_vrrp_advert_int = $resource['input']['ha_vrrp_advert_int'] +$agent_mode = $resource['input']['agent_mode'] +$allow_automatic_l3agent_failover = $resource['input']['allow_automatic_l3agent_failover'] class { 'neutron::agents::l3': enabled => true, diff --git a/resources/neutron_agents_metadata_puppet/actions/run.pp b/resources/neutron_agents_metadata_puppet/actions/run.pp index ab99d768..d96955e0 100644 --- a/resources/neutron_agents_metadata_puppet/actions/run.pp +++ b/resources/neutron_agents_metadata_puppet/actions/run.pp @@ -1,24 +1,24 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$auth_host = $resource['input']['auth_host']['value'] -$auth_port = $resource['input']['auth_port']['value'] +$auth_host = $resource['input']['auth_host'] +$auth_port = $resource['input']['auth_port'] -$auth_password = $resource['input']['auth_password']['value'] -$shared_secret = $resource['input']['shared_secret']['value'] -$package_ensure = $resource['input']['package_ensure']['value'] -$debug = $resource['input']['debug']['value'] -$auth_tenant = $resource['input']['auth_tenant']['value'] -$auth_user = $resource['input']['auth_user']['value'] -$auth_insecure = $resource['input']['auth_insecure']['value'] -$auth_ca_cert = $resource['input']['auth_ca_cert']['value'] -$auth_region = $resource['input']['auth_region']['value'] -$metadata_ip = $resource['input']['metadata_ip']['value'] -$metadata_port = $resource['input']['metadata_port']['value'] -$metadata_workers = $resource['input']['metadata_workers']['value'] -$metadata_backlog = $resource['input']['metadata_backlog']['value'] -$metadata_memory_cache_ttl = $resource['input']['metadata_memory_cache_ttl']['value'] +$auth_password = $resource['input']['auth_password'] +$shared_secret = $resource['input']['shared_secret'] +$package_ensure = $resource['input']['package_ensure'] +$debug = $resource['input']['debug'] +$auth_tenant = $resource['input']['auth_tenant'] +$auth_user = $resource['input']['auth_user'] +$auth_insecure = $resource['input']['auth_insecure'] +$auth_ca_cert = $resource['input']['auth_ca_cert'] +$auth_region = $resource['input']['auth_region'] +$metadata_ip = $resource['input']['metadata_ip'] +$metadata_port = $resource['input']['metadata_port'] +$metadata_workers = $resource['input']['metadata_workers'] +$metadata_backlog = $resource['input']['metadata_backlog'] +$metadata_memory_cache_ttl = $resource['input']['metadata_memory_cache_ttl'] class { 'neutron::agents::metadata': enabled => true, diff --git a/resources/neutron_agents_ml2_ovs_puppet/actions/run.pp b/resources/neutron_agents_ml2_ovs_puppet/actions/run.pp index f9941be7..115e1ce8 100644 --- a/resources/neutron_agents_ml2_ovs_puppet/actions/run.pp +++ b/resources/neutron_agents_ml2_ovs_puppet/actions/run.pp @@ -1,22 +1,22 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$package_ensure = $resource['input']['package_ensure']['value'] -$enabled = $resource['input']['enabled']['value'] -$bridge_uplinks = $resource['input']['bridge_uplinks']['value'] -$bridge_mappings = $resource['input']['bridge_mappings']['value'] -$integration_bridge = $resource['input']['integration_bridge']['value'] -$enable_tunneling = $resource['input']['enable_tunneling']['value'] -$tunnel_types = $resource['input']['tunnel_types']['value'] -$local_ip = $resource['input']['local_ip']['value'] -$tunnel_bridge = $resource['input']['tunnel_bridge']['value'] -$vxlan_udp_port = $resource['input']['vxlan_udp_port']['value'] -$polling_interval = $resource['input']['polling_interval']['value'] -$l2_population = $resource['input']['l2_population']['value'] -$arp_responder = $resource['input']['arp_responder']['value'] -$firewall_driver = $resource['input']['firewall_driver']['value'] -$enable_distributed_routing = $resource['input']['enable_distributed_routing']['value'] +$package_ensure = $resource['input']['package_ensure'] +$enabled = $resource['input']['enabled'] +$bridge_uplinks = $resource['input']['bridge_uplinks'] +$bridge_mappings = $resource['input']['bridge_mappings'] +$integration_bridge = $resource['input']['integration_bridge'] +$enable_tunneling = $resource['input']['enable_tunneling'] +$tunnel_types = $resource['input']['tunnel_types'] +$local_ip = $resource['input']['local_ip'] +$tunnel_bridge = $resource['input']['tunnel_bridge'] +$vxlan_udp_port = $resource['input']['vxlan_udp_port'] +$polling_interval = $resource['input']['polling_interval'] +$l2_population = $resource['input']['l2_population'] +$arp_responder = $resource['input']['arp_responder'] +$firewall_driver = $resource['input']['firewall_driver'] +$enable_distributed_routing = $resource['input']['enable_distributed_routing'] class { 'neutron::agents::ml2::ovs': enabled => true, diff --git a/resources/neutron_plugins_ml2_puppet/actions/run.pp b/resources/neutron_plugins_ml2_puppet/actions/run.pp index 33478a37..347c0623 100644 --- a/resources/neutron_plugins_ml2_puppet/actions/run.pp +++ b/resources/neutron_plugins_ml2_puppet/actions/run.pp @@ -1,19 +1,19 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$type_drivers = $resource['input']['type_drivers']['value'] -$tenant_network_types = $resource['input']['tenant_network_types']['value'] -$mechanism_drivers = $resource['input']['mechanism_drivers']['value'] -$flat_networks = $resource['input']['flat_networks']['value'] -$network_vlan_ranges = $resource['input']['network_vlan_ranges']['value'] -$tunnel_id_ranges = $resource['input']['tunnel_id_ranges']['value'] -$vxlan_group = $resource['input']['vxlan_group']['value'] -$vni_ranges = $resource['input']['vni_ranges']['value'] -$enable_security_group = $resource['input']['enable_security_group']['value'] -$package_ensure = $resource['input']['package_ensure']['value'] -$supported_pci_vendor_devs = $resource['input']['supported_pci_vendor_devs']['value'] -$sriov_agent_required = $resource['input']['sriov_agent_required']['value'] +$type_drivers = $resource['input']['type_drivers'] +$tenant_network_types = $resource['input']['tenant_network_types'] +$mechanism_drivers = $resource['input']['mechanism_drivers'] +$flat_networks = $resource['input']['flat_networks'] +$network_vlan_ranges = $resource['input']['network_vlan_ranges'] +$tunnel_id_ranges = $resource['input']['tunnel_id_ranges'] +$vxlan_group = $resource['input']['vxlan_group'] +$vni_ranges = $resource['input']['vni_ranges'] +$enable_security_group = $resource['input']['enable_security_group'] +$package_ensure = $resource['input']['package_ensure'] +$supported_pci_vendor_devs = $resource['input']['supported_pci_vendor_devs'] +$sriov_agent_required = $resource['input']['sriov_agent_required'] # LP1490438 file {'/etc/default/neutron-server': diff --git a/resources/neutron_puppet/actions/run.pp b/resources/neutron_puppet/actions/run.pp index a0ee911d..84a009bc 100644 --- a/resources/neutron_puppet/actions/run.pp +++ b/resources/neutron_puppet/actions/run.pp @@ -1,63 +1,63 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$package_ensure = $resource['input']['package_ensure']['value'] -$verbose = $resource['input']['verbose']['value'] -$debug = $resource['input']['debug']['value'] -$bind_host = $resource['input']['bind_host']['value'] -$bind_port = $resource['input']['bind_port']['value'] -$core_plugin = $resource['input']['core_plugin']['value'] -$service_plugins = $resource['input']['service_plugins']['value'] -$auth_strategy = $resource['input']['auth_strategy']['value'] -$base_mac = $resource['input']['base_mac']['value'] -$mac_generation_retries = $resource['input']['mac_generation_retries']['value'] -$dhcp_lease_duration = $resource['input']['dhcp_lease_duration']['value'] -$dhcp_agents_per_network = $resource['input']['dhcp_agents_per_network']['value'] -$network_device_mtu = $resource['input']['network_device_mtu']['value'] -$dhcp_agent_notification = $resource['input']['dhcp_agent_notification']['value'] -$allow_bulk = $resource['input']['allow_bulk']['value'] -$allow_pagination = $resource['input']['allow_pagination']['value'] -$allow_sorting = $resource['input']['allow_sorting']['value'] -$allow_overlapping_ips = $resource['input']['allow_overlapping_ips']['value'] -$api_extensions_path = $resource['input']['api_extensions_path']['value'] -$root_helper = $resource['input']['root_helper']['value'] -$report_interval = $resource['input']['report_interval']['value'] -$control_exchange = $resource['input']['control_exchange']['value'] -$rpc_backend = $resource['input']['rpc_backend']['value'] -$rabbit_password = $resource['input']['rabbit_password']['value'] -$rabbit_host = $resource['input']['rabbit_host']['value'] -$rabbit_hosts = $resource['input']['rabbit_hosts']['value'] -$rabbit_port = $resource['input']['rabbit_port']['value'] -$rabbit_user = $resource['input']['rabbit_user']['value'] -$rabbit_virtual_host = $resource['input']['rabbit_virtual_host']['value'] -$rabbit_use_ssl = $resource['input']['rabbit_use_ssl']['value'] -$kombu_ssl_ca_certs = $resource['input']['kombu_ssl_ca_certs']['value'] -$kombu_ssl_certfile = $resource['input']['kombu_ssl_certfile']['value'] -$kombu_ssl_keyfile = $resource['input']['kombu_ssl_keyfile']['value'] -$kombu_ssl_version = $resource['input']['kombu_ssl_version']['value'] -$kombu_reconnect_delay = $resource['input']['kombu_reconnect_delay']['value'] -$qpid_hostname = $resource['input']['qpid_hostname']['value'] -$qpid_port = $resource['input']['qpid_port']['value'] -$qpid_username = $resource['input']['qpid_username']['value'] -$qpid_password = $resource['input']['qpid_password']['value'] -$qpid_heartbeat = $resource['input']['qpid_heartbeat']['value'] -$qpid_protocol = $resource['input']['qpid_protocol']['value'] -$qpid_tcp_nodelay = $resource['input']['qpid_tcp_nodelay']['value'] -$qpid_reconnect = $resource['input']['qpid_reconnect']['value'] -$qpid_reconnect_timeout = $resource['input']['qpid_reconnect_timeout']['value'] -$qpid_reconnect_limit = $resource['input']['qpid_reconnect_limit']['value'] -$qpid_reconnect_interval_min = $resource['input']['qpid_reconnect_interval_min']['value'] -$qpid_reconnect_interval_max = $resource['input']['qpid_reconnect_interval_max']['value'] -$qpid_reconnect_interval = $resource['input']['qpid_reconnect_interval']['value'] -$use_ssl = $resource['input']['use_ssl']['value'] -$cert_file = $resource['input']['cert_file']['value'] -$key_file = $resource['input']['key_file']['value'] -$ca_file = $resource['input']['ca_file']['value'] -$use_syslog = $resource['input']['use_syslog']['value'] -$log_facility = $resource['input']['log_facility']['value'] -$log_file = $resource['input']['log_file']['value'] -$log_dir = $resource['input']['log_dir']['value'] +$package_ensure = $resource['input']['package_ensure'] +$verbose = $resource['input']['verbose'] +$debug = $resource['input']['debug'] +$bind_host = $resource['input']['bind_host'] +$bind_port = $resource['input']['bind_port'] +$core_plugin = $resource['input']['core_plugin'] +$service_plugins = $resource['input']['service_plugins'] +$auth_strategy = $resource['input']['auth_strategy'] +$base_mac = $resource['input']['base_mac'] +$mac_generation_retries = $resource['input']['mac_generation_retries'] +$dhcp_lease_duration = $resource['input']['dhcp_lease_duration'] +$dhcp_agents_per_network = $resource['input']['dhcp_agents_per_network'] +$network_device_mtu = $resource['input']['network_device_mtu'] +$dhcp_agent_notification = $resource['input']['dhcp_agent_notification'] +$allow_bulk = $resource['input']['allow_bulk'] +$allow_pagination = $resource['input']['allow_pagination'] +$allow_sorting = $resource['input']['allow_sorting'] +$allow_overlapping_ips = $resource['input']['allow_overlapping_ips'] +$api_extensions_path = $resource['input']['api_extensions_path'] +$root_helper = $resource['input']['root_helper'] +$report_interval = $resource['input']['report_interval'] +$control_exchange = $resource['input']['control_exchange'] +$rpc_backend = $resource['input']['rpc_backend'] +$rabbit_password = $resource['input']['rabbit_password'] +$rabbit_host = $resource['input']['rabbit_host'] +$rabbit_hosts = $resource['input']['rabbit_hosts'] +$rabbit_port = $resource['input']['rabbit_port'] +$rabbit_user = $resource['input']['rabbit_user'] +$rabbit_virtual_host = $resource['input']['rabbit_virtual_host'] +$rabbit_use_ssl = $resource['input']['rabbit_use_ssl'] +$kombu_ssl_ca_certs = $resource['input']['kombu_ssl_ca_certs'] +$kombu_ssl_certfile = $resource['input']['kombu_ssl_certfile'] +$kombu_ssl_keyfile = $resource['input']['kombu_ssl_keyfile'] +$kombu_ssl_version = $resource['input']['kombu_ssl_version'] +$kombu_reconnect_delay = $resource['input']['kombu_reconnect_delay'] +$qpid_hostname = $resource['input']['qpid_hostname'] +$qpid_port = $resource['input']['qpid_port'] +$qpid_username = $resource['input']['qpid_username'] +$qpid_password = $resource['input']['qpid_password'] +$qpid_heartbeat = $resource['input']['qpid_heartbeat'] +$qpid_protocol = $resource['input']['qpid_protocol'] +$qpid_tcp_nodelay = $resource['input']['qpid_tcp_nodelay'] +$qpid_reconnect = $resource['input']['qpid_reconnect'] +$qpid_reconnect_timeout = $resource['input']['qpid_reconnect_timeout'] +$qpid_reconnect_limit = $resource['input']['qpid_reconnect_limit'] +$qpid_reconnect_interval_min = $resource['input']['qpid_reconnect_interval_min'] +$qpid_reconnect_interval_max = $resource['input']['qpid_reconnect_interval_max'] +$qpid_reconnect_interval = $resource['input']['qpid_reconnect_interval'] +$use_ssl = $resource['input']['use_ssl'] +$cert_file = $resource['input']['cert_file'] +$key_file = $resource['input']['key_file'] +$ca_file = $resource['input']['ca_file'] +$use_syslog = $resource['input']['use_syslog'] +$log_facility = $resource['input']['log_facility'] +$log_file = $resource['input']['log_file'] +$log_dir = $resource['input']['log_dir'] class { 'neutron': enabled => true, diff --git a/resources/neutron_server_puppet/actions/run.pp b/resources/neutron_server_puppet/actions/run.pp index 1d7d8315..f47974da 100644 --- a/resources/neutron_server_puppet/actions/run.pp +++ b/resources/neutron_server_puppet/actions/run.pp @@ -1,49 +1,49 @@ $resource = hiera($::resource_name) -$ip = $resource['input']['ip']['value'] +$ip = $resource['input']['ip'] -$db_user = $resource['input']['db_user']['value'] -$db_host = $resource['input']['db_host']['value'] -$db_port = $resource['input']['db_port']['value'] -$db_password = $resource['input']['db_password']['value'] -$db_name = $resource['input']['db_name']['value'] +$db_user = $resource['input']['db_user'] +$db_host = $resource['input']['db_host'] +$db_port = $resource['input']['db_port'] +$db_password = $resource['input']['db_password'] +$db_name = $resource['input']['db_name'] -$package_ensure = $resource['input']['package_ensure']['value'] -$auth_password = $resource['input']['auth_password']['value'] -$auth_type = $resource['input']['auth_type']['value'] -$auth_host = $resource['input']['auth_host']['value'] -$auth_port = $resource['input']['auth_port']['value'] -$auth_admin_prefix = $resource['input']['auth_admin_prefix']['value'] -$auth_tenant = $resource['input']['auth_tenant']['value'] -$auth_user = $resource['input']['auth_user']['value'] -$auth_protocol = $resource['input']['auth_protocol']['value'] -$auth_uri = $resource['input']['auth_uri']['value'] -$database_max_retries = $resource['input']['database_max_retries']['value'] -$database_idle_timeout = $resource['input']['database_idle_timeout']['value'] -$database_retry_interval = $resource['input']['database_retry_interval']['value'] -$database_min_pool_size = $resource['input']['database_min_pool_size']['value'] -$database_max_pool_size = $resource['input']['database_max_pool_size']['value'] -$database_max_overflow = $resource['input']['database_max_overflow']['value'] -$sync_db = $resource['input']['sync_db']['value'] -$api_workers = $resource['input']['api_workers']['value'] -$rpc_workers = $resource['input']['rpc_workers']['value'] -$agent_down_time = $resource['input']['agent_down_time']['value'] -$router_scheduler_driver = $resource['input']['router_scheduler_driver']['value'] -$router_distributed = $resource['input']['router_distributed']['value'] -$l3_ha = $resource['input']['l3_ha']['value'] -$max_l3_agents_per_router = $resource['input']['max_l3_agents_per_router']['value'] -$min_l3_agents_per_router = $resource['input']['min_l3_agents_per_router']['value'] -$l3_ha_net_cidr = $resource['input']['l3_ha_net_cidr']['value'] -$mysql_module = $resource['input']['mysql_module']['value'] -$sql_max_retries = $resource['input']['sql_max_retries']['value'] -$max_retries = $resource['input']['max_retries']['value'] -$sql_idle_timeout = $resource['input']['sql_idle_timeout']['value'] -$idle_timeout = $resource['input']['idle_timeout']['value'] -$sql_reconnect_interval = $resource['input']['sql_reconnect_interval']['value'] -$retry_interval = $resource['input']['retry_interval']['value'] -$log_dir = $resource['input']['log_dir']['value'] -$log_file = $resource['input']['log_file']['value'] -$report_interval = $resource['input']['report_interval']['value'] +$package_ensure = $resource['input']['package_ensure'] +$auth_password = $resource['input']['auth_password'] +$auth_type = $resource['input']['auth_type'] +$auth_host = $resource['input']['auth_host'] +$auth_port = $resource['input']['auth_port'] +$auth_admin_prefix = $resource['input']['auth_admin_prefix'] +$auth_tenant = $resource['input']['auth_tenant'] +$auth_user = $resource['input']['auth_user'] +$auth_protocol = $resource['input']['auth_protocol'] +$auth_uri = $resource['input']['auth_uri'] +$database_max_retries = $resource['input']['database_max_retries'] +$database_idle_timeout = $resource['input']['database_idle_timeout'] +$database_retry_interval = $resource['input']['database_retry_interval'] +$database_min_pool_size = $resource['input']['database_min_pool_size'] +$database_max_pool_size = $resource['input']['database_max_pool_size'] +$database_max_overflow = $resource['input']['database_max_overflow'] +$sync_db = $resource['input']['sync_db'] +$api_workers = $resource['input']['api_workers'] +$rpc_workers = $resource['input']['rpc_workers'] +$agent_down_time = $resource['input']['agent_down_time'] +$router_scheduler_driver = $resource['input']['router_scheduler_driver'] +$router_distributed = $resource['input']['router_distributed'] +$l3_ha = $resource['input']['l3_ha'] +$max_l3_agents_per_router = $resource['input']['max_l3_agents_per_router'] +$min_l3_agents_per_router = $resource['input']['min_l3_agents_per_router'] +$l3_ha_net_cidr = $resource['input']['l3_ha_net_cidr'] +$mysql_module = $resource['input']['mysql_module'] +$sql_max_retries = $resource['input']['sql_max_retries'] +$max_retries = $resource['input']['max_retries'] +$sql_idle_timeout = $resource['input']['sql_idle_timeout'] +$idle_timeout = $resource['input']['idle_timeout'] +$sql_reconnect_interval = $resource['input']['sql_reconnect_interval'] +$retry_interval = $resource['input']['retry_interval'] +$log_dir = $resource['input']['log_dir'] +$log_file = $resource['input']['log_file'] +$report_interval = $resource['input']['report_interval'] class { 'neutron::server': enabled => true, diff --git a/resources/node_network_puppet/actions/run.pp b/resources/node_network_puppet/actions/run.pp index 86d4b483..800605c2 100644 --- a/resources/node_network_puppet/actions/run.pp +++ b/resources/node_network_puppet/actions/run.pp @@ -1,17 +1,17 @@ $resource = hiera($::resource_name) -$ensure_package = $resource['input']['ensure_package']['value'] -$use_lnx = $resource['input']['use_lnx']['value'] -$use_ovs = $resource['input']['use_ovs']['value'] -$install_ovs = $resource['input']['install_ovs']['value'] -$install_brtool = $resource['input']['install_brtool']['value'] -$install_ethtool = $resource['input']['install_ethtool']['value'] -$install_bondtool = $resource['input']['install_bondtool']['value'] -$install_vlantool = $resource['input']['install_vlantool']['value'] -$ovs_modname = $resource['input']['ovs_modname']['value'] -$ovs_datapath_package_name = $resource['input']['ovs_datapath_package_name']['value'] -$ovs_common_package_name = $resource['input']['ovs_common_package_name']['value'] -$network_scheme = $resource['input']['network_scheme']['value'] +$ensure_package = $resource['input']['ensure_package'] +$use_lnx = $resource['input']['use_lnx'] +$use_ovs = $resource['input']['use_ovs'] +$install_ovs = $resource['input']['install_ovs'] +$install_brtool = $resource['input']['install_brtool'] +$install_ethtool = $resource['input']['install_ethtool'] +$install_bondtool = $resource['input']['install_bondtool'] +$install_vlantool = $resource['input']['install_vlantool'] +$ovs_modname = $resource['input']['ovs_modname'] +$ovs_datapath_package_name = $resource['input']['ovs_datapath_package_name'] +$ovs_common_package_name = $resource['input']['ovs_common_package_name'] +$network_scheme = $resource['input']['network_scheme'] class {'l23network': ensure_package => $ensure_package, diff --git a/resources/nova_api_puppet/actions/run.pp b/resources/nova_api_puppet/actions/run.pp index 46a2eb78..95885e81 100644 --- a/resources/nova_api_puppet/actions/run.pp +++ b/resources/nova_api_puppet/actions/run.pp @@ -1,35 +1,35 @@ $resource = hiera($::resource_name) -$ensure_package = $resource['input']['ensure_package']['value'] -$auth_strategy = $resource['input']['auth_strategy']['value'] -$auth_host = $resource['input']['auth_host']['value'] -$auth_port = $resource['input']['auth_port']['value'] -$auth_protocol = $resource['input']['auth_protocol']['value'] -$auth_uri = $resource['input']['auth_uri']['value'] -$auth_admin_prefix = $resource['input']['auth_admin_prefix']['value'] -$auth_version = $resource['input']['auth_version']['value'] -$admin_tenant_name = $resource['input']['admin_tenant_name']['value'] -$admin_user = $resource['input']['admin_user']['value'] -$admin_password = $resource['input']['admin_password']['value'] -$api_bind_address = $resource['input']['api_bind_address']['value'] -$metadata_listen = $resource['input']['metadata_listen']['value'] -$enabled_apis = $resource['input']['enabled_apis']['value'] -$keystone_ec2_url = $resource['input']['keystone_ec2_url']['value'] -$volume_api_class = $resource['input']['volume_api_class']['value'] -$use_forwarded_for = $resource['input']['use_forwarded_for']['value'] -$osapi_compute_workers = $resource['input']['osapi_compute_workers']['value'] -$ec2_workers = $resource['input']['ec2_workers']['value'] -$metadata_workers = $resource['input']['metadata_workers']['value'] -$sync_db = $resource['input']['sync_db']['value'] -$neutron_metadata_proxy_shared_secret = $resource['input']['neutron_metadata_proxy_shared_secret']['value'] -$osapi_v3 = $resource['input']['osapi_v3']['value'] -$pci_alias = $resource['input']['pci_alias']['value'] -$ratelimits = $resource['input']['ratelimits']['value'] -$ratelimits_factory = $resource['input']['ratelimits_factory']['value'] -$validate = $resource['input']['validate']['value'] -$validation_options = $resource['input']['validation_options']['value'] -$workers = $resource['input']['workers']['value'] -$conductor_workers = $resource['input']['conductor_workers']['value'] +$ensure_package = $resource['input']['ensure_package'] +$auth_strategy = $resource['input']['auth_strategy'] +$auth_host = $resource['input']['auth_host'] +$auth_port = $resource['input']['auth_port'] +$auth_protocol = $resource['input']['auth_protocol'] +$auth_uri = $resource['input']['auth_uri'] +$auth_admin_prefix = $resource['input']['auth_admin_prefix'] +$auth_version = $resource['input']['auth_version'] +$admin_tenant_name = $resource['input']['admin_tenant_name'] +$admin_user = $resource['input']['admin_user'] +$admin_password = $resource['input']['admin_password'] +$api_bind_address = $resource['input']['api_bind_address'] +$metadata_listen = $resource['input']['metadata_listen'] +$enabled_apis = $resource['input']['enabled_apis'] +$keystone_ec2_url = $resource['input']['keystone_ec2_url'] +$volume_api_class = $resource['input']['volume_api_class'] +$use_forwarded_for = $resource['input']['use_forwarded_for'] +$osapi_compute_workers = $resource['input']['osapi_compute_workers'] +$ec2_workers = $resource['input']['ec2_workers'] +$metadata_workers = $resource['input']['metadata_workers'] +$sync_db = $resource['input']['sync_db'] +$neutron_metadata_proxy_shared_secret = $resource['input']['neutron_metadata_proxy_shared_secret'] +$osapi_v3 = $resource['input']['osapi_v3'] +$pci_alias = $resource['input']['pci_alias'] +$ratelimits = $resource['input']['ratelimits'] +$ratelimits_factory = $resource['input']['ratelimits_factory'] +$validate = $resource['input']['validate'] +$validation_options = $resource['input']['validation_options'] +$workers = $resource['input']['workers'] +$conductor_workers = $resource['input']['conductor_workers'] exec { 'post-nova_config': command => '/bin/echo "Nova config has changed"', diff --git a/resources/nova_api_puppet/actions/update.pp b/resources/nova_api_puppet/actions/update.pp index 092deb01..90e409e5 100644 --- a/resources/nova_api_puppet/actions/update.pp +++ b/resources/nova_api_puppet/actions/update.pp @@ -1,35 +1,35 @@ $resource = hiera($::resource_name) -$ensure_package = $resource['input']['ensure_package']['value'] -$auth_strategy = $resource['input']['auth_strategy']['value'] -$auth_host = $resource['input']['auth_host']['value'] -$auth_port = $resource['input']['auth_port']['value'] -$auth_protocol = $resource['input']['auth_protocol']['value'] -$auth_uri = $resource['input']['auth_uri']['value'] -$auth_admin_prefix = $resource['input']['auth_admin_prefix']['value'] -$auth_version = $resource['input']['auth_version']['value'] -$admin_tenant_name = $resource['input']['admin_tenant_name']['value'] -$admin_user = $resource['input']['admin_user']['value'] -$admin_password = $resource['input']['admin_password']['value'] -$api_bind_address = $resource['input']['api_bind_address']['value'] -$metadata_listen = $resource['input']['metadata_listen']['value'] -$enabled_apis = $resource['input']['enabled_apis']['value'] -$keystone_ec2_url = $resource['input']['keystone_ec2_url']['value'] -$volume_api_class = $resource['input']['volume_api_class']['value'] -$use_forwarded_for = $resource['input']['use_forwarded_for']['value'] -$osapi_compute_workers = $resource['input']['osapi_compute_workers']['value'] -$ec2_workers = $resource['input']['ec2_workers']['value'] -$metadata_workers = $resource['input']['metadata_workers']['value'] -$sync_db = $resource['input']['sync_db']['value'] -$neutron_metadata_proxy_shared_secret = $resource['input']['neutron_metadata_proxy_shared_secret']['value'] -$osapi_v3 = $resource['input']['osapi_v3']['value'] -$pci_alias = $resource['input']['pci_alias']['value'] -$ratelimits = $resource['input']['ratelimits']['value'] -$ratelimits_factory = $resource['input']['ratelimits_factory']['value'] -$validate = $resource['input']['validate']['value'] -$validation_options = $resource['input']['validation_options']['value'] -$workers = $resource['input']['workers']['value'] -$conductor_workers = $resource['input']['conductor_workers']['value'] +$ensure_package = $resource['input']['ensure_package'] +$auth_strategy = $resource['input']['auth_strategy'] +$auth_host = $resource['input']['auth_host'] +$auth_port = $resource['input']['auth_port'] +$auth_protocol = $resource['input']['auth_protocol'] +$auth_uri = $resource['input']['auth_uri'] +$auth_admin_prefix = $resource['input']['auth_admin_prefix'] +$auth_version = $resource['input']['auth_version'] +$admin_tenant_name = $resource['input']['admin_tenant_name'] +$admin_user = $resource['input']['admin_user'] +$admin_password = $resource['input']['admin_password'] +$api_bind_address = $resource['input']['api_bind_address'] +$metadata_listen = $resource['input']['metadata_listen'] +$enabled_apis = $resource['input']['enabled_apis'] +$keystone_ec2_url = $resource['input']['keystone_ec2_url'] +$volume_api_class = $resource['input']['volume_api_class'] +$use_forwarded_for = $resource['input']['use_forwarded_for'] +$osapi_compute_workers = $resource['input']['osapi_compute_workers'] +$ec2_workers = $resource['input']['ec2_workers'] +$metadata_workers = $resource['input']['metadata_workers'] +$sync_db = $resource['input']['sync_db'] +$neutron_metadata_proxy_shared_secret = $resource['input']['neutron_metadata_proxy_shared_secret'] +$osapi_v3 = $resource['input']['osapi_v3'] +$pci_alias = $resource['input']['pci_alias'] +$ratelimits = $resource['input']['ratelimits'] +$ratelimits_factory = $resource['input']['ratelimits_factory'] +$validate = $resource['input']['validate'] +$validation_options = $resource['input']['validation_options'] +$workers = $resource['input']['workers'] +$conductor_workers = $resource['input']['conductor_workers'] exec { 'post-nova_config': command => '/bin/echo "Nova config has changed"', diff --git a/resources/nova_compute_libvirt_puppet/actions/run.pp b/resources/nova_compute_libvirt_puppet/actions/run.pp index a33a7761..868c4532 100644 --- a/resources/nova_compute_libvirt_puppet/actions/run.pp +++ b/resources/nova_compute_libvirt_puppet/actions/run.pp @@ -1,19 +1,19 @@ $resource = hiera($::resource_name) -$libvirt_virt_type = $resource['input']['libvirt_virt_type']['value'] -$vncserver_listen = $resource['input']['vncserver_listen']['value'] -$migration_support = $resource['input']['migration_support']['value'] -$libvirt_cpu_mode = $resource['input']['libvirt_cpu_mode']['value'] -$libvirt_disk_cachemodes = $resource['input']['libvirt_disk_cachemodes']['value'] -$libvirt_inject_password = $resource['input']['libvirt_inject_password']['value'] -$libvirt_inject_key = $resource['input']['libvirt_inject_key']['value'] -$libvirt_inject_partition = $resource['input']['libvirt_inject_partition']['value'] -$remove_unused_base_images = $resource['input']['remove_unused_base_images']['value'] -$remove_unused_kernels = $resource['input']['remove_unused_kernels']['value'] -$remove_unused_resized_minimum_age_seconds = $resource['input']['remove_unused_resized_minimum_age_seconds']['value'] -$remove_unused_original_minimum_age_seconds = $resource['input']['remove_unused_original_minimum_age_seconds']['value'] -$libvirt_service_name = $resource['input']['libvirt_service_name']['value'] -$libvirt_type = $resource['input']['libvirt_type']['value'] +$libvirt_virt_type = $resource['input']['libvirt_virt_type'] +$vncserver_listen = $resource['input']['vncserver_listen'] +$migration_support = $resource['input']['migration_support'] +$libvirt_cpu_mode = $resource['input']['libvirt_cpu_mode'] +$libvirt_disk_cachemodes = $resource['input']['libvirt_disk_cachemodes'] +$libvirt_inject_password = $resource['input']['libvirt_inject_password'] +$libvirt_inject_key = $resource['input']['libvirt_inject_key'] +$libvirt_inject_partition = $resource['input']['libvirt_inject_partition'] +$remove_unused_base_images = $resource['input']['remove_unused_base_images'] +$remove_unused_kernels = $resource['input']['remove_unused_kernels'] +$remove_unused_resized_minimum_age_seconds = $resource['input']['remove_unused_resized_minimum_age_seconds'] +$remove_unused_original_minimum_age_seconds = $resource['input']['remove_unused_original_minimum_age_seconds'] +$libvirt_service_name = $resource['input']['libvirt_service_name'] +$libvirt_type = $resource['input']['libvirt_type'] class { 'nova::compute::libvirt': libvirt_virt_type => $libvirt_virt_type, diff --git a/resources/nova_compute_libvirt_puppet/actions/update.pp b/resources/nova_compute_libvirt_puppet/actions/update.pp index 9fe8f147..33dbcd88 100644 --- a/resources/nova_compute_libvirt_puppet/actions/update.pp +++ b/resources/nova_compute_libvirt_puppet/actions/update.pp @@ -1,19 +1,19 @@ $resource = hiera($::resource_name) -$libvirt_virt_type = $resource['input']['libvirt_virt_type']['value'] -$vncserver_listen = $resource['input']['vncserver_listen']['value'] -$migration_support = $resource['input']['migration_support']['value'] -$libvirt_cpu_mode = $resource['input']['libvirt_cpu_mode']['value'] -$libvirt_disk_cachemodes = $resource['input']['libvirt_disk_cachemodes']['value'] -$libvirt_inject_password = $resource['input']['libvirt_inject_password']['value'] -$libvirt_inject_key = $resource['input']['libvirt_inject_key']['value'] -$libvirt_inject_partition = $resource['input']['libvirt_inject_partition']['value'] -$remove_unused_base_images = $resource['input']['remove_unused_base_images']['value'] -$remove_unused_kernels = $resource['input']['remove_unused_kernels']['value'] -$remove_unused_resized_minimum_age_seconds = $resource['input']['remove_unused_resized_minimum_age_seconds']['value'] -$remove_unused_original_minimum_age_seconds = $resource['input']['remove_unused_original_minimum_age_seconds']['value'] -$libvirt_service_name = $resource['input']['libvirt_service_name']['value'] -$libvirt_type = $resource['input']['libvirt_type']['value'] +$libvirt_virt_type = $resource['input']['libvirt_virt_type'] +$vncserver_listen = $resource['input']['vncserver_listen'] +$migration_support = $resource['input']['migration_support'] +$libvirt_cpu_mode = $resource['input']['libvirt_cpu_mode'] +$libvirt_disk_cachemodes = $resource['input']['libvirt_disk_cachemodes'] +$libvirt_inject_password = $resource['input']['libvirt_inject_password'] +$libvirt_inject_key = $resource['input']['libvirt_inject_key'] +$libvirt_inject_partition = $resource['input']['libvirt_inject_partition'] +$remove_unused_base_images = $resource['input']['remove_unused_base_images'] +$remove_unused_kernels = $resource['input']['remove_unused_kernels'] +$remove_unused_resized_minimum_age_seconds = $resource['input']['remove_unused_resized_minimum_age_seconds'] +$remove_unused_original_minimum_age_seconds = $resource['input']['remove_unused_original_minimum_age_seconds'] +$libvirt_service_name = $resource['input']['libvirt_service_name'] +$libvirt_type = $resource['input']['libvirt_type'] class { 'nova::compute::libvirt': libvirt_virt_type => $libvirt_virt_type, diff --git a/resources/nova_compute_puppet/actions/run.pp b/resources/nova_compute_puppet/actions/run.pp index 8c33baa1..2a8810be 100644 --- a/resources/nova_compute_puppet/actions/run.pp +++ b/resources/nova_compute_puppet/actions/run.pp @@ -1,26 +1,26 @@ $resource = hiera($::resource_name) -$ensure_package = $resource['input']['ensure_package']['value'] -$vnc_enabled = $resource['input']['vnc_enabled']['value'] -$vncserver_proxyclient_address = $resource['input']['vncserver_proxyclient_address']['value'] -$vncproxy_host = $resource['input']['vncproxy_host']['value'] -$vncproxy_protocol = $resource['input']['vncproxy_protocol']['value'] -$vncproxy_port = $resource['input']['vncproxy_port']['value'] -$vncproxy_path = $resource['input']['vncproxy_path']['value'] -$vnc_keymap = $resource['input']['vnc_keymap']['value'] -$force_config_drive = $resource['input']['force_config_drive']['value'] -$virtio_nic = $resource['input']['virtio_nic']['value'] -$neutron_enabled = $resource['input']['neutron_enabled']['value'] -$network_device_mtu = $resource['input']['network_device_mtu']['value'] -$instance_usage_audit = $resource['input']['instance_usage_audit']['value'] -$instance_usage_audit_period = $resource['input']['instance_usage_audit_period']['value'] -$force_raw_images = $resource['input']['force_raw_images']['value'] -$reserved_host_memory = $resource['input']['reserved_host_memory']['value'] -$compute_manager = $resource['input']['compute_manager']['value'] -$pci_passthrough = $resource['input']['pci_passthrough']['value'] -$default_availability_zone = $resource['input']['default_availability_zone']['value'] -$default_schedule_zone = $resource['input']['default_schedule_zone']['value'] -$internal_service_availability_zone = $resource['input']['internal_service_availability_zone']['value'] +$ensure_package = $resource['input']['ensure_package'] +$vnc_enabled = $resource['input']['vnc_enabled'] +$vncserver_proxyclient_address = $resource['input']['vncserver_proxyclient_address'] +$vncproxy_host = $resource['input']['vncproxy_host'] +$vncproxy_protocol = $resource['input']['vncproxy_protocol'] +$vncproxy_port = $resource['input']['vncproxy_port'] +$vncproxy_path = $resource['input']['vncproxy_path'] +$vnc_keymap = $resource['input']['vnc_keymap'] +$force_config_drive = $resource['input']['force_config_drive'] +$virtio_nic = $resource['input']['virtio_nic'] +$neutron_enabled = $resource['input']['neutron_enabled'] +$network_device_mtu = $resource['input']['network_device_mtu'] +$instance_usage_audit = $resource['input']['instance_usage_audit'] +$instance_usage_audit_period = $resource['input']['instance_usage_audit_period'] +$force_raw_images = $resource['input']['force_raw_images'] +$reserved_host_memory = $resource['input']['reserved_host_memory'] +$compute_manager = $resource['input']['compute_manager'] +$pci_passthrough = $resource['input']['pci_passthrough'] +$default_availability_zone = $resource['input']['default_availability_zone'] +$default_schedule_zone = $resource['input']['default_schedule_zone'] +$internal_service_availability_zone = $resource['input']['internal_service_availability_zone'] class { 'nova::compute': enabled => true, diff --git a/resources/nova_compute_puppet/actions/update.pp b/resources/nova_compute_puppet/actions/update.pp index 7837909f..ba31d806 100644 --- a/resources/nova_compute_puppet/actions/update.pp +++ b/resources/nova_compute_puppet/actions/update.pp @@ -1,26 +1,26 @@ $resource = hiera($::resource_name) -$ensure_package = $resource['input']['ensure_package']['value'] -$vnc_enabled = $resource['input']['vnc_enabled']['value'] -$vncserver_proxyclient_address = $resource['input']['vncserver_proxyclient_address']['value'] -$vncproxy_host = $resource['input']['vncproxy_host']['value'] -$vncproxy_protocol = $resource['input']['vncproxy_protocol']['value'] -$vncproxy_port = $resource['input']['vncproxy_port']['value'] -$vncproxy_path = $resource['input']['vncproxy_path']['value'] -$vnc_keymap = $resource['input']['vnc_keymap']['value'] -$force_config_drive = $resource['input']['force_config_drive']['value'] -$virtio_nic = $resource['input']['virtio_nic']['value'] -$neutron_enabled = $resource['input']['neutron_enabled']['value'] -$network_device_mtu = $resource['input']['network_device_mtu']['value'] -$instance_usage_audit = $resource['input']['instance_usage_audit']['value'] -$instance_usage_audit_period = $resource['input']['instance_usage_audit_period']['value'] -$force_raw_images = $resource['input']['force_raw_images']['value'] -$reserved_host_memory = $resource['input']['reserved_host_memory']['value'] -$compute_manager = $resource['input']['compute_manager']['value'] -$pci_passthrough = $resource['input']['pci_passthrough']['value'] -$default_availability_zone = $resource['input']['default_availability_zone']['value'] -$default_schedule_zone = $resource['input']['default_schedule_zone']['value'] -$internal_service_availability_zone = $resource['input']['internal_service_availability_zone']['value'] +$ensure_package = $resource['input']['ensure_package'] +$vnc_enabled = $resource['input']['vnc_enabled'] +$vncserver_proxyclient_address = $resource['input']['vncserver_proxyclient_address'] +$vncproxy_host = $resource['input']['vncproxy_host'] +$vncproxy_protocol = $resource['input']['vncproxy_protocol'] +$vncproxy_port = $resource['input']['vncproxy_port'] +$vncproxy_path = $resource['input']['vncproxy_path'] +$vnc_keymap = $resource['input']['vnc_keymap'] +$force_config_drive = $resource['input']['force_config_drive'] +$virtio_nic = $resource['input']['virtio_nic'] +$neutron_enabled = $resource['input']['neutron_enabled'] +$network_device_mtu = $resource['input']['network_device_mtu'] +$instance_usage_audit = $resource['input']['instance_usage_audit'] +$instance_usage_audit_period = $resource['input']['instance_usage_audit_period'] +$force_raw_images = $resource['input']['force_raw_images'] +$reserved_host_memory = $resource['input']['reserved_host_memory'] +$compute_manager = $resource['input']['compute_manager'] +$pci_passthrough = $resource['input']['pci_passthrough'] +$default_availability_zone = $resource['input']['default_availability_zone'] +$default_schedule_zone = $resource['input']['default_schedule_zone'] +$internal_service_availability_zone = $resource['input']['internal_service_availability_zone'] class { 'nova::compute': enabled => true, diff --git a/resources/nova_conductor_puppet/actions/run.pp b/resources/nova_conductor_puppet/actions/run.pp index 93daf765..f0aa53e7 100644 --- a/resources/nova_conductor_puppet/actions/run.pp +++ b/resources/nova_conductor_puppet/actions/run.pp @@ -1,7 +1,7 @@ $resource = hiera($::resource_name) -$ensure_package = $resource['input']['ensure_package']['value'] -$workers = $resource['input']['workers']['value'] +$ensure_package = $resource['input']['ensure_package'] +$workers = $resource['input']['workers'] exec { 'post-nova_config': command => '/bin/echo "Nova config has changed"', diff --git a/resources/nova_conductor_puppet/actions/update.pp b/resources/nova_conductor_puppet/actions/update.pp index 9ab0f15b..ed258675 100644 --- a/resources/nova_conductor_puppet/actions/update.pp +++ b/resources/nova_conductor_puppet/actions/update.pp @@ -1,7 +1,7 @@ $resource = hiera($::resource_name) -$ensure_package = $resource['input']['ensure_package']['value'] -$workers = $resource['input']['workers']['value'] +$ensure_package = $resource['input']['ensure_package'] +$workers = $resource['input']['workers'] exec { 'post-nova_config': command => '/bin/echo "Nova config has changed"', diff --git a/resources/nova_generic_service_puppet/actions/remove.pp b/resources/nova_generic_service_puppet/actions/remove.pp index 964f79c9..d848408a 100644 --- a/resources/nova_generic_service_puppet/actions/remove.pp +++ b/resources/nova_generic_service_puppet/actions/remove.pp @@ -1,6 +1,6 @@ -$service_title = $resource['input']['title']['value'] -$package_name = $resource['input']['package_name']['value'] -$service_name = $resource['input']['service_name']['value'] +$service_title = $resource['input']['title'] +$package_name = $resource['input']['package_name'] +$service_name = $resource['input']['service_name'] exec { 'post-nova_config': command => '/bin/echo "Nova config has changed"', diff --git a/resources/nova_generic_service_puppet/actions/run.pp b/resources/nova_generic_service_puppet/actions/run.pp index 9e696f4d..ab47b8cc 100644 --- a/resources/nova_generic_service_puppet/actions/run.pp +++ b/resources/nova_generic_service_puppet/actions/run.pp @@ -1,9 +1,9 @@ $resource = hiera($::resource_name) -$service_title = $resource['input']['title']['value'] -$package_name = $resource['input']['package_name']['value'] -$service_name = $resource['input']['service_name']['value'] -$ensure_package = $resource['input']['ensure_package']['value'] +$service_title = $resource['input']['title'] +$package_name = $resource['input']['package_name'] +$service_name = $resource['input']['service_name'] +$ensure_package = $resource['input']['ensure_package'] exec { 'post-nova_config': command => '/bin/echo "Nova config has changed"', diff --git a/resources/nova_generic_service_puppet/actions/update.pp b/resources/nova_generic_service_puppet/actions/update.pp index 2aa7d142..49301e58 100644 --- a/resources/nova_generic_service_puppet/actions/update.pp +++ b/resources/nova_generic_service_puppet/actions/update.pp @@ -1,9 +1,9 @@ $resource = hiera($::resource_name) -$service_title = $resource['input']['title']['value'] -$package_name = $resource['input']['package_name']['value'] -$service_name = $resource['input']['service_name']['value'] -$ensure_package = $resource['input']['ensure_package']['value'] +$service_title = $resource['input']['title'] +$package_name = $resource['input']['package_name'] +$service_name = $resource['input']['service_name'] +$ensure_package = $resource['input']['ensure_package'] exec { 'post-nova_config': command => '/bin/echo "Nova config has changed"', diff --git a/resources/nova_neutron_puppet/actions/run.pp b/resources/nova_neutron_puppet/actions/run.pp index 4a0db5b8..5589032c 100644 --- a/resources/nova_neutron_puppet/actions/run.pp +++ b/resources/nova_neutron_puppet/actions/run.pp @@ -1,30 +1,30 @@ $resource = hiera($::resource_name) -$auth_host = $resource['input']['auth_host']['value'] -$auth_port = $resource['input']['auth_port']['value'] -$auth_protocol = $resource['input']['auth_protocol']['value'] -$neutron_endpoint_host = $resource['input']['neutron_endpoint_host']['value'] -$neutron_endpoint_port = $resource['input']['neutron_endpoint_port']['value'] -$neutron_endpoint_protocol = $resource['input']['neutron_endpoint_protocol']['value'] +$auth_host = $resource['input']['auth_host'] +$auth_port = $resource['input']['auth_port'] +$auth_protocol = $resource['input']['auth_protocol'] +$neutron_endpoint_host = $resource['input']['neutron_endpoint_host'] +$neutron_endpoint_port = $resource['input']['neutron_endpoint_port'] +$neutron_endpoint_protocol = $resource['input']['neutron_endpoint_protocol'] -$libvirt_vif_driver = $resource['input']['libvirt_vif_driver']['value'] -$force_snat_range = $resource['input']['force_snat_range']['value'] -$neutron_admin_password = $resource['input']['neutron_admin_password']['value'] -$neutron_auth_strategy = $resource['input']['neutron_auth_strategy']['value'] -$neutron_url_timeout = $resource['input']['neutron_url_timeout']['value'] -$neutron_admin_tenant_name = $resource['input']['neutron_admin_tenant_name']['value'] -$neutron_default_tenant_id = $resource['input']['neutron_default_tenant_id']['value'] -$neutron_region_name = $resource['input']['neutron_region_name']['value'] -$neutron_admin_username = $resource['input']['neutron_admin_username']['value'] -$neutron_ovs_bridge = $resource['input']['neutron_ovs_bridge']['value'] -$neutron_extension_sync_interval = $resource['input']['neutron_extension_sync_interval']['value'] -$neutron_ca_certificates_file = $resource['input']['neutron_ca_certificates_file']['value'] -$network_api_class = $resource['input']['network_api_class']['value'] -$security_group_api = $resource['input']['security_group_api']['value'] -$firewall_driver = $resource['input']['firewall_driver']['value'] -$vif_plugging_is_fatal = $resource['input']['vif_plugging_is_fatal']['value'] -$vif_plugging_timeout = $resource['input']['vif_plugging_timeout']['value'] -$dhcp_domain = $resource['input']['dhcp_domain']['value'] +$libvirt_vif_driver = $resource['input']['libvirt_vif_driver'] +$force_snat_range = $resource['input']['force_snat_range'] +$neutron_admin_password = $resource['input']['neutron_admin_password'] +$neutron_auth_strategy = $resource['input']['neutron_auth_strategy'] +$neutron_url_timeout = $resource['input']['neutron_url_timeout'] +$neutron_admin_tenant_name = $resource['input']['neutron_admin_tenant_name'] +$neutron_default_tenant_id = $resource['input']['neutron_default_tenant_id'] +$neutron_region_name = $resource['input']['neutron_region_name'] +$neutron_admin_username = $resource['input']['neutron_admin_username'] +$neutron_ovs_bridge = $resource['input']['neutron_ovs_bridge'] +$neutron_extension_sync_interval = $resource['input']['neutron_extension_sync_interval'] +$neutron_ca_certificates_file = $resource['input']['neutron_ca_certificates_file'] +$network_api_class = $resource['input']['network_api_class'] +$security_group_api = $resource['input']['security_group_api'] +$firewall_driver = $resource['input']['firewall_driver'] +$vif_plugging_is_fatal = $resource['input']['vif_plugging_is_fatal'] +$vif_plugging_timeout = $resource['input']['vif_plugging_timeout'] +$dhcp_domain = $resource['input']['dhcp_domain'] class { 'nova::compute::neutron': diff --git a/resources/nova_puppet/actions/run.pp b/resources/nova_puppet/actions/run.pp index 46bf2b42..34003538 100644 --- a/resources/nova_puppet/actions/run.pp +++ b/resources/nova_puppet/actions/run.pp @@ -1,76 +1,76 @@ $resource = hiera($::resource_name) -$db_user = $resource['input']['db_user']['value'] -$db_password = $resource['input']['db_password']['value'] -$db_name = $resource['input']['db_name']['value'] -$db_host = $resource['input']['db_host']['value'] -$db_port = $resource['input']['db_port']['value'] -$glance_api_servers_host = $resource['input']['glance_api_servers_host']['value'] -$glance_api_servers_port = $resource['input']['glance_api_servers_port']['value'] +$db_user = $resource['input']['db_user'] +$db_password = $resource['input']['db_password'] +$db_name = $resource['input']['db_name'] +$db_host = $resource['input']['db_host'] +$db_port = $resource['input']['db_port'] +$glance_api_servers_host = $resource['input']['glance_api_servers_host'] +$glance_api_servers_port = $resource['input']['glance_api_servers_port'] -$ensure_package = $resource['input']['ensure_package']['value'] -$database_connection = $resource['input']['database_connection']['value'] -$slave_connection = $resource['input']['slave_connection']['value'] -$database_idle_timeout = $resource['input']['database_idle_timeout']['value'] -$rpc_backend = $resource['input']['rpc_backend']['value'] -$image_service = $resource['input']['image_service']['value'] -$glance_api_servers = $resource['input']['glance_api_servers']['value'] -$memcached_servers = $resource['input']['memcached_servers']['value'] -$rabbit_host = $resource['input']['rabbit_host']['value'] -$rabbit_hosts = $resource['input']['rabbit_hosts']['value'] -$rabbit_password = $resource['input']['rabbit_password']['value'] -$rabbit_port = $resource['input']['rabbit_port']['value'] -$rabbit_userid = $resource['input']['rabbit_userid']['value'] -$rabbit_virtual_host = $resource['input']['rabbit_virtual_host']['value'] -$rabbit_use_ssl = $resource['input']['rabbit_use_ssl']['value'] -$rabbit_ha_queues = $resource['input']['rabbit_ha_queues']['value'] -$kombu_ssl_ca_certs = $resource['input']['kombu_ssl_ca_certs']['value'] -$kombu_ssl_certfile = $resource['input']['kombu_ssl_certfile']['value'] -$kombu_ssl_keyfile = $resource['input']['kombu_ssl_keyfile']['value'] -$kombu_ssl_version = $resource['input']['kombu_ssl_version']['value'] -$amqp_durable_queues = $resource['input']['amqp_durable_queues']['value'] -$qpid_hostname = $resource['input']['qpid_hostname']['value'] -$qpid_port = $resource['input']['qpid_port']['value'] -$qpid_username = $resource['input']['qpid_username']['value'] -$qpid_password = $resource['input']['qpid_password']['value'] -$qpid_sasl_mechanisms = $resource['input']['qpid_sasl_mechanisms']['value'] -$qpid_heartbeat = $resource['input']['qpid_heartbeat']['value'] -$qpid_protocol = $resource['input']['qpid_protocol']['value'] -$qpid_tcp_nodelay = $resource['input']['qpid_tcp_nodelay']['value'] -$auth_strategy = $resource['input']['auth_strategy']['value'] -$service_down_time = $resource['input']['service_down_time']['value'] -$log_dir = $resource['input']['log_dir']['value'] -$state_path = $resource['input']['state_path']['value'] -$lock_path = $resource['input']['lock_path']['value'] -$verbose = $resource['input']['verbose']['value'] -$debug = $resource['input']['debug']['value'] -$periodic_interval = $resource['input']['periodic_interval']['value'] -$report_interval = $resource['input']['report_interval']['value'] -$rootwrap_config = $resource['input']['rootwrap_config']['value'] -$use_ssl = $resource['input']['use_ssl']['value'] -$enabled_ssl_apis = $resource['input']['enabled_ssl_apis']['value'] -$ca_file = $resource['input']['ca_file']['value'] -$cert_file = $resource['input']['cert_file']['value'] -$key_file = $resource['input']['key_file']['value'] -$nova_user_id = $resource['input']['nova_user_id']['value'] -$nova_group_id = $resource['input']['nova_group_id']['value'] -$nova_public_key = $resource['input']['nova_public_key']['value'] -$nova_private_key = $resource['input']['nova_private_key']['value'] -$nova_shell = $resource['input']['nova_shell']['value'] -$monitoring_notifications = $resource['input']['monitoring_notifications']['value'] -$use_syslog = $resource['input']['use_syslog']['value'] -$log_facility = $resource['input']['log_facility']['value'] -$install_utilities = $resource['input']['install_utilities']['value'] -$notification_driver = $resource['input']['notification_driver']['value'] -$notification_topics = $resource['input']['notification_topics']['value'] -$notify_api_faults = $resource['input']['notify_api_faults']['value'] -$notify_on_state_change = $resource['input']['notify_on_state_change']['value'] -$mysql_module = $resource['input']['mysql_module']['value'] -$nova_cluster_id = $resource['input']['nova_cluster_id']['value'] -$sql_connection = $resource['input']['sql_connection']['value'] -$sql_idle_timeout = $resource['input']['sql_idle_timeout']['value'] -$logdir = $resource['input']['logdir']['value'] -$os_region_name = $resource['input']['os_region_name']['value'] +$ensure_package = $resource['input']['ensure_package'] +$database_connection = $resource['input']['database_connection'] +$slave_connection = $resource['input']['slave_connection'] +$database_idle_timeout = $resource['input']['database_idle_timeout'] +$rpc_backend = $resource['input']['rpc_backend'] +$image_service = $resource['input']['image_service'] +$glance_api_servers = $resource['input']['glance_api_servers'] +$memcached_servers = $resource['input']['memcached_servers'] +$rabbit_host = $resource['input']['rabbit_host'] +$rabbit_hosts = $resource['input']['rabbit_hosts'] +$rabbit_password = $resource['input']['rabbit_password'] +$rabbit_port = $resource['input']['rabbit_port'] +$rabbit_userid = $resource['input']['rabbit_userid'] +$rabbit_virtual_host = $resource['input']['rabbit_virtual_host'] +$rabbit_use_ssl = $resource['input']['rabbit_use_ssl'] +$rabbit_ha_queues = $resource['input']['rabbit_ha_queues'] +$kombu_ssl_ca_certs = $resource['input']['kombu_ssl_ca_certs'] +$kombu_ssl_certfile = $resource['input']['kombu_ssl_certfile'] +$kombu_ssl_keyfile = $resource['input']['kombu_ssl_keyfile'] +$kombu_ssl_version = $resource['input']['kombu_ssl_version'] +$amqp_durable_queues = $resource['input']['amqp_durable_queues'] +$qpid_hostname = $resource['input']['qpid_hostname'] +$qpid_port = $resource['input']['qpid_port'] +$qpid_username = $resource['input']['qpid_username'] +$qpid_password = $resource['input']['qpid_password'] +$qpid_sasl_mechanisms = $resource['input']['qpid_sasl_mechanisms'] +$qpid_heartbeat = $resource['input']['qpid_heartbeat'] +$qpid_protocol = $resource['input']['qpid_protocol'] +$qpid_tcp_nodelay = $resource['input']['qpid_tcp_nodelay'] +$auth_strategy = $resource['input']['auth_strategy'] +$service_down_time = $resource['input']['service_down_time'] +$log_dir = $resource['input']['log_dir'] +$state_path = $resource['input']['state_path'] +$lock_path = $resource['input']['lock_path'] +$verbose = $resource['input']['verbose'] +$debug = $resource['input']['debug'] +$periodic_interval = $resource['input']['periodic_interval'] +$report_interval = $resource['input']['report_interval'] +$rootwrap_config = $resource['input']['rootwrap_config'] +$use_ssl = $resource['input']['use_ssl'] +$enabled_ssl_apis = $resource['input']['enabled_ssl_apis'] +$ca_file = $resource['input']['ca_file'] +$cert_file = $resource['input']['cert_file'] +$key_file = $resource['input']['key_file'] +$nova_user_id = $resource['input']['nova_user_id'] +$nova_group_id = $resource['input']['nova_group_id'] +$nova_public_key = $resource['input']['nova_public_key'] +$nova_private_key = $resource['input']['nova_private_key'] +$nova_shell = $resource['input']['nova_shell'] +$monitoring_notifications = $resource['input']['monitoring_notifications'] +$use_syslog = $resource['input']['use_syslog'] +$log_facility = $resource['input']['log_facility'] +$install_utilities = $resource['input']['install_utilities'] +$notification_driver = $resource['input']['notification_driver'] +$notification_topics = $resource['input']['notification_topics'] +$notify_api_faults = $resource['input']['notify_api_faults'] +$notify_on_state_change = $resource['input']['notify_on_state_change'] +$mysql_module = $resource['input']['mysql_module'] +$nova_cluster_id = $resource['input']['nova_cluster_id'] +$sql_connection = $resource['input']['sql_connection'] +$sql_idle_timeout = $resource['input']['sql_idle_timeout'] +$logdir = $resource['input']['logdir'] +$os_region_name = $resource['input']['os_region_name'] class { 'nova': database_connection => "mysql://${db_user}:${db_password}@${db_host}:${db_port}/${db_name}?charset=utf8", diff --git a/resources/rabbitmq_service/actions/run.pp b/resources/rabbitmq_service/actions/run.pp index 2ea22054..02ed9cff 100644 --- a/resources/rabbitmq_service/actions/run.pp +++ b/resources/rabbitmq_service/actions/run.pp @@ -1,7 +1,7 @@ $resource = hiera($::resource_name) -$port = "${resource['input']['port']['value']}" -$management_port = "${resource['input']['management_port']['value']}" +$port = "${resource['input']['port']}" +$management_port = "${resource['input']['management_port']}" class { '::rabbitmq': service_manage => true, diff --git a/solar/solar/core/handlers/puppet.py b/solar/solar/core/handlers/puppet.py index 8ac2fbcb..28f75e23 100644 --- a/solar/solar/core/handlers/puppet.py +++ b/solar/solar/core/handlers/puppet.py @@ -25,7 +25,7 @@ from solar import errors # - puppet is installed class Puppet(TempFileHandler): def action(self, resource, action_name): - log.debug('Executing Puppet manifest %s %s', action_name, resource) + log.debug('Executing Puppet manifest %s %s', action_name, resource.name) action_file = self._compile_action_file(resource, action_name) log.debug('action_file: %s', action_file) @@ -62,7 +62,7 @@ class Puppet(TempFileHandler): return cmd def _make_args(self, resource): - return {resource.name: resource.to_dict()} + return {resource.name: {'input': resource.args}} def upload_hiera_resource(self, resource): src = '/tmp/puppet_{}.yaml'.format(resource.name) From 5e8d04ddbd4c13462f7f0136f0cb8fc4cdc0956e Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 9 Nov 2015 14:59:48 +0100 Subject: [PATCH 122/191] Don't add input index when it already exists --- solar/solar/dblayer/solar_models.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 5e820ad2..e77069d9 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -399,11 +399,10 @@ class InputsFieldWrp(IndexFieldWrp): raise Exception("%s:%s is connected with resource %s:%s" % (res, inp, emitter_name, emitter_inp)) # inst = self._instance robj = self._instance._riak_object - self._instance._add_index('%s_bin' % self.fname, '{}|{}'.format(my_name, name)) - try: - robj.data[self.fname][name] = value - except KeyError: - robj.data[self.fname] = {name: value} + if name not in robj.data[self.fname]: + self._instance._add_index('%s_bin' % self.fname, '{}|{}'.format(my_name, name)) + robj.data[self.fname][name] = value + with self.inputs_index_cache as c: c.wipe() self._cache[name] = value From a27fd11a6e6765204d3cd8b46eab9bca3bf92ac0 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 12 Nov 2015 10:29:39 +0100 Subject: [PATCH 123/191] resource.to_dict could return proper inputs values. fixes solar resource show --- solar/solar/cli/resource.py | 2 +- solar/solar/core/resource/resource.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/solar/solar/cli/resource.py b/solar/solar/cli/resource.py index f6ff2eb0..b297f0b0 100644 --- a/solar/solar/cli/resource.py +++ b/solar/solar/cli/resource.py @@ -149,7 +149,7 @@ def show(name, tag, json, color): resources = sresource.load_all() if json: - output = json.dumps([r.to_dict() for r in resources], indent=2) + output = json.dumps([r.to_dict(inputs=True) for r in resources], indent=2) echo = click.echo else: if color: diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 2447190f..70f0fc62 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -241,8 +241,10 @@ class Resource(object): def resource_inputs(self): return self.db_obj.inputs - def to_dict(self): + def to_dict(self, inputs=False): ret = self.db_obj.to_dict() + if inputs: + ret['inputs'] = self.db_obj.inputs.as_dict() return ret def color_repr(self): From a7827d19a899d9b630d0b12d0ebd4feb7c47dd68 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 12 Nov 2015 11:53:58 +0100 Subject: [PATCH 124/191] Fixed solar resource show --- solar/solar/cli/resource.py | 8 ++++---- solar/solar/core/resource/resource.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/solar/solar/cli/resource.py b/solar/solar/cli/resource.py index b297f0b0..0c96fa71 100644 --- a/solar/solar/cli/resource.py +++ b/solar/solar/cli/resource.py @@ -136,9 +136,9 @@ def create(args, base_path, name): @resource.command() @click.option('--name', '-n', default=None) @click.option('--tag', '-t', multiple=True) -@click.option('--json', default=False, is_flag=True) +@click.option('--as_json', default=False, is_flag=True) @click.option('--color', default=True, is_flag=True) -def show(name, tag, json, color): +def show(name, tag, as_json, color): echo = click.echo_via_pager if name: resources = [sresource.load(name)] @@ -148,12 +148,12 @@ def show(name, tag, json, color): else: resources = sresource.load_all() - if json: + if as_json: output = json.dumps([r.to_dict(inputs=True) for r in resources], indent=2) echo = click.echo else: if color: - formatter = lambda r: r.color_repr() + formatter = lambda r: r.color_repr(inputs=True) else: formatter = lambda r: unicode(r) output = '\n'.join(formatter(r) for r in resources) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 70f0fc62..4e978585 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -247,7 +247,7 @@ class Resource(object): ret['inputs'] = self.db_obj.inputs.as_dict() return ret - def color_repr(self): + def color_repr(self, inputs=False): import click arg_color = 'yellow' @@ -259,7 +259,7 @@ class Resource(object): base_path_s=click.style('base_path', fg=arg_color, bold=True), args_s=click.style('args', fg=arg_color, bold=True), tags_s=click.style('tags', fg=arg_color, bold=True), - **self.to_dict() + **self.to_dict(inputs) ) def load_commited(self): From 1c28ca2fddf695f6118d1195e448f85b588b6be0 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 12 Nov 2015 12:42:58 +0100 Subject: [PATCH 125/191] Created standalone_session_wrapper. Use it in standalone scripts --- solar/solar/cli/__init__.py | 17 +------------- .../dblayer/standalone_session_wrapper.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 16 deletions(-) create mode 100644 solar/solar/dblayer/standalone_session_wrapper.py diff --git a/solar/solar/cli/__init__.py b/solar/solar/cli/__init__.py index 9b795591..54174935 100644 --- a/solar/solar/cli/__init__.py +++ b/solar/solar/cli/__init__.py @@ -1,16 +1 @@ -try: - from gevent import monkey -except ImportError: - pass -else: - monkey.patch_all() - from solar.dblayer.gevent_patches import patch_all - patch_all() - -from solar.dblayer.model import ModelMeta - -import atexit - -ModelMeta.session_start() - -atexit.register(ModelMeta.session_end) +from solar.dblayer import standalone_session_wrapper diff --git a/solar/solar/dblayer/standalone_session_wrapper.py b/solar/solar/dblayer/standalone_session_wrapper.py new file mode 100644 index 00000000..8ad3fc0a --- /dev/null +++ b/solar/solar/dblayer/standalone_session_wrapper.py @@ -0,0 +1,22 @@ +""" +Starts single seession, and ends it with `atexit` +can be used from cli / examples +shouldn't be used from long running processes (workers etc) +""" + +try: + from gevent import monkey +except ImportError: + pass +else: + monkey.patch_all() + from solar.dblayer.gevent_patches import patch_all + patch_all() + +from solar.dblayer.model import ModelMeta + +import atexit + +ModelMeta.session_start() + +atexit.register(ModelMeta.session_end) From 33080c52118d8889925e8c84f761f20c61fdfd97 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 12 Nov 2015 13:50:11 +0100 Subject: [PATCH 126/191] add standalone_session_wrapper to all standalone python executions --- solar/solar/dblayer/__init__.py | 4 ++++ .../dblayer/standalone_session_wrapper.py | 20 +++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/solar/solar/dblayer/__init__.py b/solar/solar/dblayer/__init__.py index ca0e1f3b..7ed377c1 100644 --- a/solar/solar/dblayer/__init__.py +++ b/solar/solar/dblayer/__init__.py @@ -1,6 +1,10 @@ from solar.dblayer.model import ModelMeta from solar.dblayer.riak_client import RiakClient + client = RiakClient(protocol='pbc', host='10.0.0.2', pb_port=8087) # client = RiakClient(protocol='http', host='10.0.0.2', http_port=8098) ModelMeta.setup(client) + +from solar.dblayer import standalone_session_wrapper +standalone_session_wrapper.create_all() diff --git a/solar/solar/dblayer/standalone_session_wrapper.py b/solar/solar/dblayer/standalone_session_wrapper.py index 8ad3fc0a..04cc3ea4 100644 --- a/solar/solar/dblayer/standalone_session_wrapper.py +++ b/solar/solar/dblayer/standalone_session_wrapper.py @@ -2,6 +2,7 @@ Starts single seession, and ends it with `atexit` can be used from cli / examples shouldn't be used from long running processes (workers etc) + """ try: @@ -13,10 +14,21 @@ else: from solar.dblayer.gevent_patches import patch_all patch_all() -from solar.dblayer.model import ModelMeta +def create_all(): -import atexit + import sys + if sys.executable.startswith(('python', )): + # auto add session to only standalone python runs + return -ModelMeta.session_start() + from solar.dblayer.model import ModelMeta -atexit.register(ModelMeta.session_end) + import atexit + + ModelMeta.session_start() + + atexit.register(ModelMeta.session_end) + + + +create_all() From 7da2cf3f6a1ec6fb598b717e246397d06337bccc Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 12 Nov 2015 15:04:33 +0100 Subject: [PATCH 127/191] added backtrack value to cli --- solar/solar/cli/resource.py | 50 +++++++++++++++++------------ solar/solar/dblayer/solar_models.py | 36 +++++++++++++++------ 2 files changed, 55 insertions(+), 31 deletions(-) diff --git a/solar/solar/cli/resource.py b/solar/solar/cli/resource.py index 0c96fa71..d89cd48c 100644 --- a/solar/solar/cli/resource.py +++ b/solar/solar/cli/resource.py @@ -64,32 +64,40 @@ def action(dry_run_mapping, dry_run, action, resource): )) @resource.command() +@click.option('-v', '--values', default=False, is_flag=True) +@click.option('-i', '--input', default=None) @click.argument('resource') -def backtrack_inputs(resource): +def backtrack_inputs(resource, input, values): r = sresource.load(resource) - inputs = [] + db_obj = r.db_obj + def single(resource, name, get_val=False): + db_obj = sresource.load(resource).db_obj + se = db_obj.inputs._single_edge(name) + se = tuple(se) + if not se: + if get_val: + return dict(resource=resource, name=name, value=db_obj.inputs[name]) + else: + return dict(resource=resource, name=name) + l = [] + for (rname, rinput), _, meta in se: + l.append(dict(resource=resource, name=name)) + val = single(rname, rinput, get_val) + if meta: + val['meta'] = meta + l.append(val) + return l - def backtrack(i): - def format_input(i): - return '{}::{}'.format(i.resource.name, i.name) + inps = {} + if input: + inps[input] = single(resource, input, values) + else: + for _inp in db_obj.inputs: + inps[_inp] = single(resource, _inp, values) - if isinstance(i, list): - return [backtrack(bi) for bi in i] - - if isinstance(i, dict): - return { - k: backtrack(bi) for k, bi in i.items() - } - - bi = i.backtrack_value_emitter(level=1) - if isinstance(i, orm.DBResourceInput) and isinstance(bi, orm.DBResourceInput) and i == bi: - return (format_input(i), ) - - return (format_input(i), backtrack(bi)) - - for i in r.resource_inputs().values(): - click.echo(yaml.safe_dump({i.name: backtrack(i)}, default_flow_style=False)) + for name, values in inps.iteritems(): + click.echo(yaml.safe_dump({name: values}, default_flow_style=False)) @resource.command() def compile_all(): diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 60a5819f..d638ddef 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -46,16 +46,7 @@ class InputsFieldWrp(IndexFieldWrp): return InputTypes.hash raise Exception("Unknown type") - def _edges(self): - inst = self._instance - start = inst.key - my_ind_name = '{}_recv_bin'.format(self.fname) - res = inst._get_index(my_ind_name, - startkey=start + '|', - endkey=start + '|~', - return_terms=True, - max_results=99999).results - vals = map(itemgetter(0), res) + def _edges_fmt(self, vals): for val in vals: data = val.split('|') dlen = len(data) @@ -72,6 +63,31 @@ class InputsFieldWrp(IndexFieldWrp): raise Exception("Unsupported case") yield (other_resource, other_input), (my_resource, my_input), meta + def _edges(self): + inst = self._instance + start = inst.key + my_ind_name = '{}_recv_bin'.format(self.fname) + res = inst._get_index(my_ind_name, + startkey=start + '|', + endkey=start + '|~', + return_terms=True, + max_results=99999).results + vals = map(itemgetter(0), res) + return self._edges_fmt(vals) + + def _single_edge(self, name): + inst = self._instance + self._has_own_input(name) + start = '{}|{}'.format(inst.key, name) + my_ind_name = '{}_recv_bin'.format(self.fname) + res = inst._get_index(my_ind_name, + startkey=start + '|', + endkey=start + '|~', + return_terms=True, + max_results=99999).results + vals = map(itemgetter(0), res) + return self._edges_fmt(vals) + def __contains__(self, name): try: self._has_own_input(name) From 2e363ca4880a020325b39177d2f90465d91fcd00 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 12 Nov 2015 16:09:41 +0100 Subject: [PATCH 128/191] Better delete_all for riak (loops and waits while) --- solar/solar/dblayer/model.py | 7 ++++--- solar/solar/dblayer/riak_client.py | 12 ++++++++++++ solar/solar/dblayer/sql_client.py | 6 ++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 7ec6abda..f9b87a37 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -886,9 +886,10 @@ class Model(object): @classmethod def delete_all(cls): - rst = cls.bucket.get_index('$bucket', startkey='_', max_results=100000).results - for key in rst: - cls.bucket.delete(key) + cls.riak_client.delete_all(cls) + # rst = cls.bucket.get_index('$bucket', startkey='_', max_results=100000).results + # for key in rst: + # cls.bucket.delete(key) def delete(self): ls = self._c.lazy_save diff --git a/solar/solar/dblayer/riak_client.py b/solar/solar/dblayer/riak_client.py index 6d3f4231..43506f8d 100644 --- a/solar/solar/dblayer/riak_client.py +++ b/solar/solar/dblayer/riak_client.py @@ -1,4 +1,5 @@ from riak import RiakClient as OrigRiakClient +import time from solar.dblayer.model import clear_cache @@ -11,3 +12,14 @@ class RiakClient(OrigRiakClient): def session_end(self, result=True): # ignore result clear_cache() + + def delete_all(self, cls): + for _ in xrange(10): + # riak dislikes deletes without dvv + rst = cls.bucket.get_index('$bucket', startkey='_', max_results=100000).results + for key in rst: + cls.bucket.delete(key) + else: + return + time.sleep(0.5) + diff --git a/solar/solar/dblayer/sql_client.py b/solar/solar/dblayer/sql_client.py index 04bfb2db..e6c59a81 100644 --- a/solar/solar/dblayer/sql_client.py +++ b/solar/solar/dblayer/sql_client.py @@ -428,3 +428,9 @@ class SqlClient(object): else: sess.rollback() clear_cache() + + def delete_all(self, cls): + # naive way for SQL, we could delete whole table contents + rst = cls.bucket.get_index('$bucket', startkey='_', max_results=100000).results + for key in rst: + cls.bucket.delete(key) From 6af32be2f226e3173c5d4d7508ba0e468a436119 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 12 Nov 2015 16:11:09 +0100 Subject: [PATCH 129/191] Removed commented code --- solar/solar/dblayer/model.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index f9b87a37..be4ff05f 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -887,9 +887,6 @@ class Model(object): @classmethod def delete_all(cls): cls.riak_client.delete_all(cls) - # rst = cls.bucket.get_index('$bucket', startkey='_', max_results=100000).results - # for key in rst: - # cls.bucket.delete(key) def delete(self): ls = self._c.lazy_save From 8af576952584be3d54364f98f7ea2bc8745f60ee Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 12 Nov 2015 18:23:13 +0100 Subject: [PATCH 130/191] solar connections show for new_db --- solar/solar/cli/main.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/solar/solar/cli/main.py b/solar/solar/cli/main.py index e8205a23..00ea15c4 100644 --- a/solar/solar/cli/main.py +++ b/solar/solar/cli/main.py @@ -25,6 +25,7 @@ import os import sys import tabulate import yaml +from collections import defaultdict from solar.core import actions from solar.core import resource as sresource @@ -45,27 +46,24 @@ from solar.cli.resource import resource as cli_resource # HELPERS -def format_resource_input(resource_input): +def format_resource_input(resource_name, resource_input): return '{}::{}'.format( - #click.style(resource_name, fg='white', bold=True), - resource_input.resource.name, - click.style(resource_input.name, fg='yellow') + resource_name, + click.style(resource_input, fg='yellow') ) -def show_emitter_connections(emitter): - for emitter_input in emitter.resource_inputs().values(): - click.echo( - '{} -> {}'.format( - format_resource_input(emitter_input), - '[{}]'.format( - ', '.join( - format_resource_input(r) - for r in emitter_input.receivers.as_set() - ) - ) - ) - ) +def show_emitter_connections(res): + db_obj = res.db_obj + d = defaultdict(list) + for emitter, receiver, _meta in db_obj.inputs._edges(): + d[emitter].append(receiver) + + for emitter, receivers in d.iteritems(): + click.echo("{} -> {}".format( + format_resource_input(*emitter), + '[{}]'.format(', '.join( + format_resource_input(*recv) for recv in receivers)))) @click.group(cls=base.AliasedGroup) From c3c042466fbf793bbb6d5b6353ba3e0c4ad37537 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 12 Nov 2015 18:26:18 +0100 Subject: [PATCH 131/191] connect and disconnect from CLI in new_db --- solar/solar/cli/main.py | 2 +- solar/solar/core/resource/resource.py | 6 ++++++ solar/solar/dblayer/solar_models.py | 11 +++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/solar/solar/cli/main.py b/solar/solar/cli/main.py index e8205a23..74339f54 100644 --- a/solar/solar/cli/main.py +++ b/solar/solar/cli/main.py @@ -133,7 +133,7 @@ def init_cli_connect(): receiver = sresource.load(receiver) click.echo(emitter) click.echo(receiver) - signals.disconnect(emitter, receiver) + emitter.disconnect(receiver) show_emitter_connections(emitter) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 4e978585..f5877f8f 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -286,6 +286,12 @@ class Resource(object): return self.connect_with_events( receiver, mapping=mapping, events=events, use_defaults=True) + def disconnect(self, receiver): + inputs = self.db_obj.inputs.keys() + self.db_obj.disconnect(other=receiver.db_obj, inputs=inputs) + receiver.db_obj.save_lazy() + self.db_obj.save_lazy() + def load(name): diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index d638ddef..5ee48b84 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -100,6 +100,9 @@ class InputsFieldWrp(IndexFieldWrp): for name in self._instance._data_container[self.fname]: yield name + def keys(self): + return list(self.__iter__()) + def as_dict(self): items = solar_map(lambda x: (x, self._get_field_val(x)), [x for x in self], concurrency=3) return dict(items) @@ -594,6 +597,14 @@ class Resource(Model): solar_map(lambda (my_name, other_name): self._connect_single(other_inputs, other_name, my_name), mapping.iteritems(), concurrency=2) + def disconnect(self, other, inputs): + other_key = other.key + edges = other.inputs._edges() + edges = filter(lambda (receiver, emitter, _): emitter[0] == other_key and receiver[1] in inputs, + edges) + inputs = [x[1][1] for x in edges] + solar_map(other.inputs.disconnect, inputs, concurrency=2) + def save(self, *args, **kwargs): if self.changed(): self.updated = StrInt() From 3b4b13a80997d45aa7e21ab9496b0f810e0474eb Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 13 Nov 2015 12:55:24 +0100 Subject: [PATCH 132/191] solar connections graph for new db --- solar/solar/cli/main.py | 6 ++- solar/solar/core/resource/resource.py | 1 - solar/solar/core/signals.py | 64 ++++++++++++++------------- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/solar/solar/cli/main.py b/solar/solar/cli/main.py index e8205a23..0c87d474 100644 --- a/solar/solar/cli/main.py +++ b/solar/solar/cli/main.py @@ -152,9 +152,11 @@ def init_cli_connections(): @connections.command() @click.option('--start-with', default=None) @click.option('--end-with', default=None) - def graph(start_with, end_with): + @click.option('--details', is_flag=True, default=False) + def graph(start_with, end_with, details): g = signals.detailed_connection_graph(start_with=start_with, - end_with=end_with) + end_with=end_with, + details=details) nx.write_dot(g, 'graph.dot') fabric_api.local('dot -Tsvg graph.dot -o graph.svg') diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 4e978585..08a51a80 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -221,7 +221,6 @@ class Resource(object): [(emitter, emitter_input, receiver, receiver_input), ...] """ rst = set() - # TODO: fix it for (emitter_resource, emitter_input), (receiver_resource, receiver_input), meta in self.graph().edges(data=True): if meta: receiver_input = '{}:{}|{}'.format(receiver_input, diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index 0256ecfe..1faecf6f 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -16,7 +16,7 @@ import networkx from solar.core.log import log -from solar.interfaces import orm +# from solar.interfaces import orm from solar.dblayer.solar_models import Resource as DBResource @@ -242,38 +242,40 @@ def disconnect_receiver_by_input(receiver, input_name): # emitter_input.receivers.remove(receiver_input) -def detailed_connection_graph(start_with=None, end_with=None): - resource_inputs_graph = orm.DBResource.inputs.graph() - inputs_graph = orm.DBResourceInput.receivers.graph() +def detailed_connection_graph(start_with=None, end_with=None, details=False): + from solar.core.resource import Resource, load_all - def node_attrs(n): - if isinstance(n, orm.DBResource): - return { - 'color': 'yellowgreen', - 'style': 'filled', - } - elif isinstance(n, orm.DBResourceInput): - return { - 'color': 'lightskyblue', - 'style': 'filled, rounded', - } + if details: + def format_for_edge(resource, input): + return '"{}/{}"'.format(resource, input) + else: + def format_for_edge(resource, input): + input = input.split(':', 1)[0] + return '"{}/{}"'.format(resource, input) - def format_name(i): - if isinstance(i, orm.DBResource): - return '"{}"'.format(i.name) - elif isinstance(i, orm.DBResourceInput): - return '{}/{}'.format(i.resource.name, i.name) + res_props = {'color': 'yellowgreen', + 'style': 'filled'} + inp_props = {'color': 'lightskyblue', + 'style': 'filled, rounded'} - for r, i in resource_inputs_graph.edges(): - inputs_graph.add_edge(r, i) + graph = networkx.DiGraph() - ret = networkx.MultiDiGraph() + resources = load_all() - for u, v in inputs_graph.edges(): - u_n = format_name(u) - v_n = format_name(v) - ret.add_edge(u_n, v_n) - ret.node[u_n] = node_attrs(u) - ret.node[v_n] = node_attrs(v) - - return ret + for resource in resources: + res_node = '{}'.format(resource.name) + for name in resource.db_obj.meta_inputs: + resource_input = format_for_edge(resource.name, name) + graph.add_edge(resource.name, resource_input) + graph.node[resource_input] = inp_props + conns = resource.connections + for (emitter_resource, emitter_input, receiver_resource, receiver_input) in conns: + e = format_for_edge(emitter_resource, emitter_input) + r = format_for_edge(receiver_resource, receiver_input) + graph.add_edge(emitter_resource, e) + graph.add_edge(receiver_resource, r) + graph.add_edge(e, r) + graph.node[e] = inp_props + graph.node[r] = inp_props + graph.node[res_node] = res_props + return graph From 98fe839bddb4e82e025cef8ee7f4061b15ab1b3f Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 13 Nov 2015 14:24:41 +0100 Subject: [PATCH 133/191] Removed duplicate create_all() --- solar/solar/dblayer/standalone_session_wrapper.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/solar/solar/dblayer/standalone_session_wrapper.py b/solar/solar/dblayer/standalone_session_wrapper.py index 04cc3ea4..dad1dc89 100644 --- a/solar/solar/dblayer/standalone_session_wrapper.py +++ b/solar/solar/dblayer/standalone_session_wrapper.py @@ -28,7 +28,3 @@ def create_all(): ModelMeta.session_start() atexit.register(ModelMeta.session_end) - - - -create_all() From 1a1da81601c09a3b0d220472504666539f333bdd Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 16 Nov 2015 13:38:28 +0100 Subject: [PATCH 134/191] test_resource works --- solar/solar/test/base.py | 19 +++++++++++++++---- solar/solar/test/test_resource.py | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/solar/solar/test/base.py b/solar/solar/test/base.py index 7d949b11..0586d901 100644 --- a/solar/solar/test/base.py +++ b/solar/solar/test/base.py @@ -17,20 +17,30 @@ import shutil import tempfile import unittest import yaml +import time from solar.core.resource import virtual_resource as vr -from solar.interfaces.db import get_db +from solar.dblayer.model import Model, Replacer, ModelMeta, get_bucket -db = get_db() + +def patched_get_bucket_name(cls): + return cls.__name__ + str(int(time.time() * 10000)) + +Model.get_bucket_name = classmethod(patched_get_bucket_name) class BaseResourceTest(unittest.TestCase): + def setUp(self): self.storage_dir = tempfile.mkdtemp() + for model in ModelMeta._defined_models: + model.bucket = get_bucket(None, model, ModelMeta) + ModelMeta.session_start() def tearDown(self): shutil.rmtree(self.storage_dir) - db.clear() + ModelMeta.session_end() + def make_resource_meta(self, meta_yaml): meta = yaml.load(meta_yaml) @@ -48,5 +58,6 @@ class BaseResourceTest(unittest.TestCase): return path - def create_resource(self, name, src, args={}): + def create_resource(self, name, src, args=None): + args = args or {} return vr.create(name, src, args=args)[0] diff --git a/solar/solar/test/test_resource.py b/solar/solar/test/test_resource.py index a3af0256..9252c8a0 100644 --- a/solar/solar/test/test_resource.py +++ b/solar/solar/test/test_resource.py @@ -59,7 +59,7 @@ input: 'sample1', sample_meta_dir, {'value': 1} ) sample2 = self.create_resource( - 'sample2', sample_meta_dir, {} + 'sample2', sample_meta_dir, ) signals.connect(sample1, sample2) self.assertEqual(sample1.args['value'], sample2.args['value']) @@ -92,7 +92,7 @@ input: sample_l = resource.load('sample') self.assertDictEqual(sample.args, sample_l.args) - self.assertListEqual(sample.tags, sample_l.tags) + self.assertListEqual(list(sample.tags), list(sample_l.tags)) def test_removal(self): """Test that connection removed with resource.""" From e084e5b3fe52e450b7cd8d9ccaf60cbbd2e9280c Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 16 Nov 2015 15:20:16 +0100 Subject: [PATCH 135/191] inmemory sqlite for tests --- solar/solar/test/conftest.py | 39 +++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/solar/solar/test/conftest.py b/solar/solar/test/conftest.py index 73080c57..bcc02a12 100644 --- a/solar/solar/test/conftest.py +++ b/solar/solar/test/conftest.py @@ -14,9 +14,12 @@ import os import pytest +import time from solar.core.resource import Resource -from solar.interfaces import db +# from solar.interfaces import db + +from solar.dblayer.model import get_bucket, ModelMeta, Model @pytest.fixture def resources(): @@ -36,19 +39,27 @@ def resources(): } -def pytest_configure(): - if db.CURRENT_DB == 'redis_graph_db': - db.DB = db.get_db(backend='fakeredis_graph_db') - elif db.CURRENT_DB == 'redis_db': - db.DB = db.get_db(backend='fakeredis_db') - else: - db.DB = db.get_db(backend=db.CURRENT_DB) - - @pytest.fixture(autouse=True) -def cleanup(request): +def setup(request): - def fin(): - db.get_db().clear() + for model in ModelMeta._defined_models: + model.bucket = get_bucket(None, model, ModelMeta) - request.addfinalizer(fin) + +def pytest_runtest_teardown(item, nextitem): + ModelMeta.session_end(result=True) + return nextitem + +def pytest_runtest_call(item): + ModelMeta.session_start() + +def patched_get_bucket_name(cls): + return cls.__name__ + str(time.time()) + + +Model.get_bucket_name = classmethod(patched_get_bucket_name) + +from solar.dblayer.sql_client import SqlClient +client = SqlClient(':memory:', threadlocals=True, autocommit=False) + +ModelMeta.setup(client) From 4e2c6a944dbb3e0f223cd3a6e9e2dedb64aee4e1 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 16 Nov 2015 16:27:53 +0100 Subject: [PATCH 136/191] Added test for cache logic --- solar/solar/dblayer/test/test_basic.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index cd7c03ec..2823f565 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -80,6 +80,28 @@ def test_lazy(rk): assert m1.f1 == 'blah' +def test_cache_logic(rk): + k = next(rk) + M1.session_start() + m1 = M1.from_dict(k, {'f1': 'blah', 'f2': 150}) + m1.save() + M1.session_end() + + M1.session_start() + assert M1._c.obj_cache == {} + m11 = M1.get(k) + pid = id(M1._c) + assert M1._c.obj_cache == {k: m11} + M1.session_end() + + M1.session_start() + assert M1._c.obj_cache == {} + m12 = M1.get(k) + aid = id(M1._c) + + assert pid != aid + + def test_normal_index(rk): key = next(rk) key2 = next(rk) From 4be95eb3d53df3acda8bbbcf49949fb4bcc1feee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Nowak?= Date: Mon, 16 Nov 2015 17:28:17 +0100 Subject: [PATCH 137/191] Added assert for ensuring that session cache is always clear --- solar/solar/dblayer/test/test_basic.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index 2823f565..9f946c82 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -83,6 +83,8 @@ def test_lazy(rk): def test_cache_logic(rk): k = next(rk) M1.session_start() + assert M1._c.obj_cache == {} + m1 = M1.from_dict(k, {'f1': 'blah', 'f2': 150}) m1.save() M1.session_end() From 4984e1fa0db6a9bd922752c15974fba37f976632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Mon, 16 Nov 2015 17:52:44 +0100 Subject: [PATCH 138/191] Fix unit tests for VR --- solar/solar/test/test_virtual_resource.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/solar/solar/test/test_virtual_resource.py b/solar/solar/test/test_virtual_resource.py index b9b3fc98..de6f5d71 100644 --- a/solar/solar/test/test_virtual_resource.py +++ b/solar/solar/test/test_virtual_resource.py @@ -128,23 +128,23 @@ def test_parse_bad_event(bad_event_type): def test_add_connections(mocker, resources): - mocked_signals = mocker.patch('solar.core.resource.resource.signals') + mocked_signals = mocker.patch('solar.core.resource.resource.Resource.connect_with_events') args = {'ip': 'node1::ip', 'servers': ['node1::ip', 'node2::ip'], 'alias': 'ser1' } vr.update_inputs('service1', args) - assert mocked_signals.connect.call_count == 3 + assert mocked_signals.call_count == 3 def test_add_list_values(mocker, resources): - mocked_signals = mocker.patch('solar.core.resource.resource.signals') + mocked_signals = mocker.patch('solar.core.resource.resource.Resource.connect_with_events') args = {'ip': 'node1::ip', 'servers': ['server1', 'server2'], 'alias': 'ser1' } vr.update_inputs('service1', args) - assert mocked_signals.connect.call_count == 1 + assert mocked_signals.call_count == 1 def test_parse_connection(): From 654039859632caf23cace77f6855581277d76120 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 16 Nov 2015 17:57:59 +0100 Subject: [PATCH 139/191] After delete object is purged from cache --- solar/solar/dblayer/model.py | 5 +++++ solar/solar/dblayer/test/test_basic.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index be4ff05f..f30505bc 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -894,4 +894,9 @@ class Model(object): ls.remove(self) except KeyError: pass + try: + del self._c.obj_cache[self.key] + except KeyError: + pass self._riak_object.delete() + return self diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index cd7c03ec..9ce7c79f 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -195,3 +195,17 @@ def test_strint_comparsions(): assert isinstance(c, basestring) assert a > b assert a > c + + +def test_delete_cache_behaviour(rk): + key1 = next(rk) + + m1 = M1.from_dict(key1, {'f1': 'm1'}) + + m1.save() + + clear_cache() + + M1.get(key1).delete() + with pytest.raises(DBLayerNotFound): + m12 = M1.get(key1) From dec25184dbf3457ec9db974d1a37507b609bd0cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Mon, 16 Nov 2015 19:00:04 +0100 Subject: [PATCH 140/191] Clear cache before running fixtures for tests, not after --- solar/solar/test/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/solar/solar/test/conftest.py b/solar/solar/test/conftest.py index bcc02a12..6d209971 100644 --- a/solar/solar/test/conftest.py +++ b/solar/solar/test/conftest.py @@ -50,7 +50,13 @@ def pytest_runtest_teardown(item, nextitem): ModelMeta.session_end(result=True) return nextitem +# It will run before all fixtures +def pytest_runtest_setup(item): + ModelMeta.session_start() + +# it will run after fixtures but before test def pytest_runtest_call(item): + ModelMeta.session_end() ModelMeta.session_start() def patched_get_bucket_name(cls): From c5e70d9ae95c7f200bce2e8a320696bd2bbabe6c Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 16 Nov 2015 20:01:22 +0100 Subject: [PATCH 141/191] Added naive conflict resolution. --- solar/solar/dblayer/conflict_resolution.py | 19 +++++++++++++++++++ solar/solar/dblayer/model.py | 2 ++ solar/solar/dblayer/test/test_basic.py | 12 ++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 solar/solar/dblayer/conflict_resolution.py diff --git a/solar/solar/dblayer/conflict_resolution.py b/solar/solar/dblayer/conflict_resolution.py new file mode 100644 index 00000000..2ad9a57a --- /dev/null +++ b/solar/solar/dblayer/conflict_resolution.py @@ -0,0 +1,19 @@ +from collections import Counter + + +def naive_resolver(riak_object): + # for now we support deleted vs existing object + siblings = riak_object.siblings + siblings_len = map(lambda sibling: (len(sibling._get_encoded_data()), sibling), siblings) + siblings_len.sort() + c = Counter((x[0] for x in siblings_len)) + if len(c) > 2: + raise RuntimeError("Too many different siblings, not sure what to do with siblings") + if not 0 in c: + raise RuntimeError("No empty object for resolution, not sure what to do with siblings") + selected = max(siblings_len) + # TODO: pass info to obj save_lazy too + riak_object.siblings = [selected[1]] + + +dblayer_conflict_resolver = naive_resolver diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index f30505bc..9849ccfa 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -6,6 +6,7 @@ from operator import itemgetter import time from contextlib import contextmanager from threading import RLock +from solar.dblayer.conflict_resolution import dblayer_conflict_resolver class DBLayerException(Exception): @@ -118,6 +119,7 @@ def clear_cache(): def get_bucket(_, owner, mcs): name = owner.get_bucket_name() bucket = mcs.riak_client.bucket(name) + bucket.resolver = dblayer_conflict_resolver return bucket diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index 9ce7c79f..c7cfbc94 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -209,3 +209,15 @@ def test_delete_cache_behaviour(rk): M1.get(key1).delete() with pytest.raises(DBLayerNotFound): m12 = M1.get(key1) + + +def test_fast_delete(rk): + key1 = next(rk) + + m1 = M1.from_dict(key1, {'f1': 'm1'}) + m1.save() + m1.delete() + M1.session_start() + m12 = M1.from_dict(key1, {'f1': 'm12'}) + m12.save() + assert m12.f1 == 'm12' From b8180f29bc9a61123cb04576be8a033c36eef628 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Mon, 16 Nov 2015 15:34:29 +0200 Subject: [PATCH 142/191] Fix system log api tests --- solar/solar/core/resource/__init__.py | 2 +- solar/solar/core/resource/resource.py | 6 +- solar/solar/core/signals.py | 8 +- solar/solar/dblayer/model.py | 1 + solar/solar/dblayer/solar_models.py | 1 + solar/solar/system_log/change.py | 19 ++- solar/solar/system_log/operations.py | 1 + solar/solar/test/conftest.py | 45 +++--- solar/solar/test/test_system_log_api.py | 183 +++++++++++++++--------- 9 files changed, 159 insertions(+), 107 deletions(-) diff --git a/solar/solar/core/resource/__init__.py b/solar/solar/core/resource/__init__.py index 277cf2a4..0e288405 100644 --- a/solar/solar/core/resource/__init__.py +++ b/solar/solar/core/resource/__init__.py @@ -12,4 +12,4 @@ # License for the specific language governing permissions and limitations # under the License. -from .resource import Resource, load, load_all, validate_resources, load_by_tags, load_updated +from .resource import Resource, load, load_all, validate_resources, load_by_tags, load_updated, RESOURCE_STATE diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 48a035ad..50aa3191 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -81,7 +81,6 @@ class Resource(object): inputs = metadata.get('input', {}) self.auto_extend_inputs(inputs) - self.db_obj = DBResource.from_dict( name, { @@ -98,11 +97,11 @@ class Resource(object): 'tags': tags, 'state': RESOURCE_STATE.created.name }) - self.create_inputs(args) self.db_obj.save() + # Load @dispatch(DBResource) def __init__(self, resource_db): @@ -269,6 +268,8 @@ class Resource(object): mapping = dict((x, x) for x in mapping) self.db_obj.connect(receiver.db_obj, mapping=mapping) self.db_obj.save_lazy() + receiver.db_obj.save_lazy() + def connect_with_events(self, receiver, mapping=None, events=None, use_defaults=False): @@ -287,6 +288,7 @@ class Resource(object): def disconnect(self, receiver): inputs = self.db_obj.inputs.keys() + inputs += ['location_id', 'transports_id'] self.db_obj.disconnect(other=receiver.db_obj, inputs=inputs) receiver.db_obj.save_lazy() self.db_obj.save_lazy() diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index 1faecf6f..cffd0faf 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -39,10 +39,10 @@ def guess_mapping(emitter, receiver): :return: """ guessed = {} - for key in emitter.args: - if key in receiver.args: - guessed[key] = key + for key in emitter.db_obj.meta_inputs: + if key in receiver.db_obj.meta_inputs: + guessed[key] = key return guessed @@ -124,7 +124,7 @@ def location_and_transports(emitter, receiver, orig_mapping): # XXX: should be somehow parametrized (input attribute?) # with dirty_state_ok(DBResource, ('index', )): for single in ('transports_id', 'location_id'): - if single in inps_emitter and inps_receiver: + if single in inps_emitter and single in inps_receiver: _single(single, emitter, receiver, inps_emitter[single], inps_receiver[single]) else: log.warning('Unable to create connection for %s with' diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 9849ccfa..600f7e72 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -823,6 +823,7 @@ class Model(object): riak_obj = cls.bucket.new(key, data={}) obj = cls.from_riakobj(riak_obj) obj._new = True + for field in cls._model_fields: # if field is cls._pkey_field: # continue # pkey already set diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 5ee48b84..ea5217e1 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -228,6 +228,7 @@ class InputsFieldWrp(IndexFieldWrp): _, ind_value = emit if ind_value.endswith('|{}|{}'.format(self._instance.key, name)): to_dels.append(emit) + for to_del in to_dels: self._instance._remove_index(*to_del) diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index ecff4df6..84182489 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -64,6 +64,7 @@ def create_sorted_diff(staged, commited): def make_single_stage_item(resource_obj): commited = resource_obj.load_commited() base_path = resource_obj.base_path + if resource_obj.to_be_removed(): resource_args = {} resource_connections = [] @@ -177,8 +178,6 @@ def _revert_remove(logitem): def _update_inputs_connections(res_obj, args, old_connections, new_connections): - res_obj.update(args) - removed = [] for item in old_connections: @@ -190,16 +189,22 @@ def _update_inputs_connections(res_obj, args, old_connections, new_connections): if item not in old_connections: added.append(item) - for emitter, _, receiver, _ in removed: + if removed or added: emmiter_obj = resource.load(emitter) receiver_obj = resource.load(receiver) - signals.disconnect(emmiter_obj, receiver_obj) + for emitter, _, receiver, _ in removed: + emmiter_obj.disconnect(receiver_obj) for emitter, emitter_input, receiver, receiver_input in added: - emmiter_obj = resource.load(emitter) - receiver_obj = resource.load(receiver) - signals.connect(emmiter_obj, receiver_obj, {emitter_input: receiver_input}) + emmiter_obj.connect(receiver_obj, {emitter_input: receiver_input}) + + if removed or added: + # without save we will get error that some values can not be updated + # even if connection was removed + receiver_obj.db_obj.save() + + res_obj.update(args) def _revert_update(logitem): diff --git a/solar/solar/system_log/operations.py b/solar/solar/system_log/operations.py index 112ae483..1ce131f4 100644 --- a/solar/solar/system_log/operations.py +++ b/solar/solar/system_log/operations.py @@ -38,6 +38,7 @@ def move_to_commited(log_action, *args, **kwargs): commited = CommitedResource.get_or_create(item.resource) updated = resource_obj.db_obj.updated if item.action == CHANGES.remove.name: + resource_obj.delete() commited.state = resource.RESOURCE_STATE.removed.name else: diff --git a/solar/solar/test/conftest.py b/solar/solar/test/conftest.py index 6d209971..b8d62bbf 100644 --- a/solar/solar/test/conftest.py +++ b/solar/solar/test/conftest.py @@ -11,32 +11,14 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import os - +from solar.dblayer.model import Model, ModelMeta, get_bucket import pytest import time -from solar.core.resource import Resource -# from solar.interfaces import db -from solar.dblayer.model import get_bucket, ModelMeta, Model +def patched_get_bucket_name(cls): + return cls.__name__ + str(time.time()) -@pytest.fixture -def resources(): - base_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'resource_fixtures') - - node_path = os.path.join(base_path, 'node') - node1 = Resource('node1', node_path, args={'ip':'10.0.0.1'}) - node2 = Resource('node2', node_path, args={'ip':'10.0.0.2'}) - - base_service_path = os.path.join(base_path, 'base_service') - service1 = Resource('service1', base_service_path) - return {'node1' : node1, - 'node2' : node2, - 'service1': service1 - } @pytest.fixture(autouse=True) @@ -46,6 +28,12 @@ def setup(request): model.bucket = get_bucket(None, model, ModelMeta) +@pytest.fixture(autouse=True) +def setup(request): + + for model in ModelMeta._defined_models: + model.bucket = get_bucket(None, model, ModelMeta) + def pytest_runtest_teardown(item, nextitem): ModelMeta.session_end(result=True) return nextitem @@ -59,13 +47,18 @@ def pytest_runtest_call(item): ModelMeta.session_end() ModelMeta.session_start() -def patched_get_bucket_name(cls): - return cls.__name__ + str(time.time()) - Model.get_bucket_name = classmethod(patched_get_bucket_name) -from solar.dblayer.sql_client import SqlClient -client = SqlClient(':memory:', threadlocals=True, autocommit=False) +# from solar.dblayer.sql_client import SqlClient +# client = SqlClient(':memory:', threadlocals=False, autocommit=False) +# client = SqlClient('/tmp/blah.db', threadlocals=True, +# autocommit=False, pragmas=(('journal_mode', 'WAL'), +# ('synchronous', 'NORMAL'))) + +from solar.dblayer.riak_client import RiakClient +client = RiakClient(protocol='pbc', host='10.0.0.2', pb_port=8087) +# client = RiakClient(protocol='http', host='10.0.0.3', http_port=18098) + ModelMeta.setup(client) diff --git a/solar/solar/test/test_system_log_api.py b/solar/solar/test/test_system_log_api.py index a23ef90c..2c7db519 100644 --- a/solar/solar/test/test_system_log_api.py +++ b/solar/solar/test/test_system_log_api.py @@ -13,25 +13,28 @@ # under the License. import mock - from pytest import fixture from pytest import mark + from solar.system_log import change from solar.system_log import data from solar.system_log import operations from solar.core import signals -from solar.core.resource import resource -from solar.interfaces import orm +from solar.core.resource import resource, RESOURCE_STATE +from solar.dblayer.solar_models import Resource as DBResource +from solar.dblayer.solar_models import CommitedResource +from solar.dblayer.model import ModelMeta def test_revert_update(): commit = {'a': '10'} previous = {'a': '9'} - res = orm.DBResource(id='test1', name='test1', base_path='x') + res = DBResource.from_dict('test1', + {'name': 'test1', 'base_path': 'x', + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res.save() - res.add_input('a', 'str', '9') action = 'update' - + res.inputs['a'] = '9' resource_obj = resource.load(res.name) assert resource_obj.args == previous @@ -52,76 +55,97 @@ def test_revert_update(): def test_revert_update_connected(): - res1 = orm.DBResource(id='test1', name='test1', base_path='x') - res1.save() - res1.add_input('a', 'str', '9') + res1 = DBResource.from_dict('test1', + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + res1.inputs['a'] = '9' + res1.save_lazy() - res2 = orm.DBResource(id='test2', name='test2', base_path='x') - res2.save() - res2.add_input('a', 'str', 0) + res2 = DBResource.from_dict('test2', + {'name': 'test2', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + res2.inputs['a'] = '' + res2.save_lazy() - res3 = orm.DBResource(id='test3', name='test3', base_path='x') - res3.save() - res3.add_input('a', 'str', 0) + res3 = DBResource.from_dict('test3', + {'name': 'test3', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + res3.inputs['a'] = '' + res3.save_lazy() res1 = resource.load('test1') res2 = resource.load('test2') res3 = resource.load('test3') - signals.connect(res1, res2) - signals.connect(res2, res3) + res1.connect(res2) + res2.connect(res3) + ModelMeta.save_all_lazy() staged_log = change.stage_changes() assert len(staged_log) == 3 + for item in staged_log: + assert item.action == 'run' operations.move_to_commited(item.log_action) - assert len(staged_log) == 0 - - signals.disconnect(res1, res2) + assert len(change.stage_changes()) == 0 + res1.disconnect(res2) staged_log = change.stage_changes() assert len(staged_log) == 2 to_revert = [] + for item in staged_log: + assert item.action == 'update' operations.move_to_commited(item.log_action) to_revert.append(item.uid) change.revert_uids(sorted(to_revert, reverse=True)) + ModelMeta.save_all_lazy() + staged_log = change.stage_changes() + assert len(staged_log) == 2 for item in staged_log: - assert item.diff == [['change', 'a', [0, '9']]] + assert item.diff == [['change', 'a', ['', '9']]] def test_revert_removal(): - res = orm.DBResource(id='test1', name='test1', base_path='x') - res.save() - res.add_input('a', 'str', '9') - res.add_input('location_id', 'str', '1') - res.add_input('transports_id', 'str', '1') + res = DBResource.from_dict('test1', + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + res.inputs['a'] = '9' + res.save_lazy() - commited = orm.DBCommitedState.get_or_create('test1') - commited.inputs = {'a': '9', 'location_id': '1', 'transports_id': '1'} - commited.save() + commited = CommitedResource.from_dict('test1', + {'inputs': {'a': '9'}, + 'state': 'operational'}) + commited.save_lazy() - logitem =change.create_logitem( - res.name, 'remove', change.create_diff({}, {'a': '9'}), [], - base_path=res.base_path) - log = data.SL() - log.append(logitem) resource_obj = resource.load(res.name) resource_obj.remove() - operations.move_to_commited(logitem.log_action) + ModelMeta.save_all_lazy() - resources = orm.DBResource.load_all() + changes = change.stage_changes() + assert len(changes) == 1 + assert changes[0].diff == [['remove', '', [['a', '9']]]] + operations.move_to_commited(changes[0].log_action) - assert resources == [] - assert logitem.diff == [('remove', '', [('a', '9')])] + ModelMeta.session_start() + assert DBResource._c.obj_cache == {} + assert DBResource.bucket.get('test1').siblings == [] with mock.patch.object(resource, 'read_meta') as mread: mread.return_value = {'input': {'a': {'schema': 'str!'}}, 'id': 'mocked'} - change.revert(logitem.uid) + change.revert(changes[0].uid) + ModelMeta.save_all_lazy() + assert len(DBResource.bucket.get('test1').siblings) == 1 + resource_obj = resource.load('test1') - assert resource_obj.args == {'a': '9', 'location_id': '1', 'transports_id': '1'} + assert resource_obj.args == { + 'a': '9', 'location_id': '', 'transports_id': ''} @mark.xfail(reason='With current approach child will be notice changes after parent is removed') @@ -158,18 +182,21 @@ def test_revert_removed_child(): def test_revert_create(): - res = orm.DBResource(id='test1', name='test1', base_path='x') - res.save() - res.add_input('a', 'str', '9') + res = DBResource.from_dict('test1', + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + res.inputs['a'] = '9' + res.save_lazy() + ModelMeta.save_all_lazy() staged_log = change.stage_changes() assert len(staged_log) == 1 - logitem = next(staged_log.collection()) - + logitem = staged_log[0] operations.move_to_commited(logitem.log_action) assert logitem.diff == [['add', '', [['a', '9']]]] - commited = orm.DBCommitedState.load('test1') + commited = CommitedResource.get('test1') assert commited.inputs == {'a': '9'} change.revert(logitem.uid) @@ -178,17 +205,24 @@ def test_revert_create(): assert len(staged_log) == 1 for item in staged_log: operations.move_to_commited(item.log_action) - assert orm.DBResource.load_all() == [] + assert resource.load_all() == [] def test_discard_all_pending_changes_resources_created(): - res1 = orm.DBResource(id='test1', name='test1', base_path='x') - res1.save() - res1.add_input('a', 'str', '9') + res1 = DBResource.from_dict('test1', + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + res1.inputs['a'] = '9' + res1.save_lazy() - res2 = orm.DBResource(id='test2', name='test2', base_path='x') - res2.save() - res2.add_input('a', 'str', 0) + res2 = DBResource.from_dict('test2', + {'name': 'test2', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + res2.inputs['a'] = '0' + res2.save_lazy() + ModelMeta.save_all_lazy() staged_log = change.stage_changes() assert len(staged_log) == 2 @@ -196,17 +230,24 @@ def test_discard_all_pending_changes_resources_created(): change.discard_all() staged_log = change.stage_changes() assert len(staged_log) == 0 - assert orm.DBResource.load_all() == [] + assert resource.load_all() == [] def test_discard_connection(): - res1 = orm.DBResource(id='test1', name='test1', base_path='x') - res1.save() - res1.add_input('a', 'str', '9') + res1 = DBResource.from_dict('test1', + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + res1.inputs['a'] = '9' + res1.save_lazy() - res2 = orm.DBResource(id='test2', name='test2', base_path='x') - res2.save() - res2.add_input('a', 'str', '0') + res2 = DBResource.from_dict('test2', + {'name': 'test2', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + res2.inputs['a'] = '0' + res2.save_lazy() + ModelMeta.save_all_lazy() staged_log = change.stage_changes() for item in staged_log: @@ -214,7 +255,7 @@ def test_discard_connection(): res1 = resource.load('test1') res2 = resource.load('test2') - signals.connect(res1, res2) + res1.connect(res2, {'a': 'a'}) staged_log = change.stage_changes() assert len(staged_log) == 1 assert res2.args == {'a': '9'} @@ -224,9 +265,13 @@ def test_discard_connection(): def test_discard_removed(): - res1 = orm.DBResource(id='test1', name='test1', base_path='x') - res1.save() - res1.add_input('a', 'str', '9') + res1 = DBResource.from_dict('test1', + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + res1.inputs['a'] = '9' + res1.save_lazy() + ModelMeta.save_all_lazy() staged_log = change.stage_changes() for item in staged_log: operations.move_to_commited(item.log_action) @@ -242,9 +287,13 @@ def test_discard_removed(): def test_discard_update(): - res1 = orm.DBResource(id='test1', name='test1', base_path='x') - res1.save() - res1.add_input('a', 'str', '9') + res1 = DBResource.from_dict('test1', + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + res1.inputs['a'] = '9' + res1.save_lazy() + ModelMeta.save_all_lazy() staged_log = change.stage_changes() for item in staged_log: operations.move_to_commited(item.log_action) From 637596343b2ff17d33023aec5188f5b234bd06bc Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 17 Nov 2015 11:41:44 +0200 Subject: [PATCH 143/191] Fix interface for generating pretty diff --- solar/solar/cli/system_log.py | 6 +++--- solar/solar/system_log/data.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/solar/solar/cli/system_log.py b/solar/solar/cli/system_log.py index 962e9d43..fc202d6c 100644 --- a/solar/solar/cli/system_log.py +++ b/solar/solar/cli/system_log.py @@ -47,7 +47,7 @@ def stage(d): for item in log: click.echo(data.compact(item)) if d: - for line in data.details(item): + for line in data.details(item.diff): click.echo(' '*4+line) if not log: click.echo('No changes') @@ -60,7 +60,7 @@ def staged_item(uid): click.echo('No staged changes for {}'.format(log_action)) else: click.echo(data.compact(item)) - for line in data.details(item): + for line in data.details(item.diff): click.echo(' '*4+line) @changes.command() @@ -89,7 +89,7 @@ def history(n, d, s): click.echo(data.compact(item)) if d: - for line in data.details(item): + for line in data.details(item.diff): click.echo(' '*4+line) if not log: click.echo('No history') diff --git a/solar/solar/system_log/data.py b/solar/solar/system_log/data.py index 0b0b7db2..d4561978 100644 --- a/solar/solar/system_log/data.py +++ b/solar/solar/system_log/data.py @@ -31,8 +31,7 @@ def compact(logitem): return 'log task={} uid={}'.format(logitem.log_action, logitem.uid) -def details(logitem): - diff = logitem.diff +def details(diff): rst = [] for type_, val, change in diff: if type_ == 'add': From ac3cb4d6c01d3e24b1b2018c37c438f24f99d58e Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 17 Nov 2015 11:43:35 +0200 Subject: [PATCH 144/191] Remove location_id/transports_id from disconnect --- solar/solar/core/resource/resource.py | 1 - solar/solar/system_log/change.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 50aa3191..985dcf16 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -288,7 +288,6 @@ class Resource(object): def disconnect(self, receiver): inputs = self.db_obj.inputs.keys() - inputs += ['location_id', 'transports_id'] self.db_obj.disconnect(other=receiver.db_obj, inputs=inputs) receiver.db_obj.save_lazy() self.db_obj.save_lazy() diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index 84182489..6d57cf36 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -189,18 +189,18 @@ def _update_inputs_connections(res_obj, args, old_connections, new_connections): if item not in old_connections: added.append(item) - if removed or added: + for emitter, _, receiver, _ in removed: emmiter_obj = resource.load(emitter) receiver_obj = resource.load(receiver) - - for emitter, _, receiver, _ in removed: emmiter_obj.disconnect(receiver_obj) for emitter, emitter_input, receiver, receiver_input in added: + emmiter_obj = resource.load(emitter) + receiver_obj = resource.load(receiver) emmiter_obj.connect(receiver_obj, {emitter_input: receiver_input}) if removed or added: - # without save we will get error that some values can not be updated + # TODO without save we will get error that some values can not be updated # even if connection was removed receiver_obj.db_obj.save() From f842452ac00c5c33c52afa6869c8db299bd628ea Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 17 Nov 2015 11:38:07 +0100 Subject: [PATCH 145/191] remove event implementation --- solar/solar/events/api.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/solar/solar/events/api.py b/solar/solar/events/api.py index 75a6952c..fbeabd4a 100644 --- a/solar/solar/events/api.py +++ b/solar/solar/events/api.py @@ -83,7 +83,14 @@ def add_events(resource, lst): def remove_event(ev): - raise NotImplemented() + to_remove = ev.to_dict() + resource = ev.parent + resource = Resource.get(resource) + # TODO: currently we don't track mutable objects + events = resource.events + events.remove(to_remove) + resource.events = events + resource.save_lazy() def all_events(resource): From 31ddddcc72c758bd19765199c2bca70598776868 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 17 Nov 2015 11:38:59 +0100 Subject: [PATCH 146/191] Fixed events tests --- solar/solar/test/test_events.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/solar/solar/test/test_events.py b/solar/solar/test/test_events.py index 48a4843d..84860b26 100644 --- a/solar/solar/test/test_events.py +++ b/solar/solar/test/test_events.py @@ -17,7 +17,7 @@ import networkx as nx from pytest import fixture from solar.events import api as evapi -from solar.interfaces import orm +from solar.dblayer.solar_models import Resource from .base import BaseResourceTest @@ -32,24 +32,15 @@ def events_example(): def test_add_events(events_example): - r = orm.DBResource(id='e1', name='e1', base_path='x') + r = Resource.from_dict(dict(key='e1', name='e1', base_path='x')) r.save() evapi.add_events('e1', events_example) assert set(evapi.all_events('e1')) == set(events_example) -def test_set_events(events_example): - r = orm.DBResource(id='e1', name='e1', base_path='x') - r.save() - partial = events_example[:2] - evapi.add_events('e1', events_example[:2]) - evapi.set_events('e1', events_example[2:]) - assert evapi.all_events('e1') == events_example[2:] - - def test_remove_events(events_example): - r = orm.DBResource(id='e1', name='e1', base_path='x') + r = Resource.from_dict(dict(key='e1', name='e1', base_path='x')) r.save() to_be_removed = events_example[2] evapi.add_events('e1', events_example) @@ -58,7 +49,7 @@ def test_remove_events(events_example): def test_single_event(events_example): - r = orm.DBResource(id='e1', name='e1', base_path='x') + r = Resource.from_dict(dict(key='e1', name='e1', base_path='x')) r.save() evapi.add_events('e1', events_example[:2]) evapi.add_event(events_example[2]) From da89aa54786b4c2e0f17c6ccfecefaa3fb5b1e4f Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 17 Nov 2015 11:39:45 +0100 Subject: [PATCH 147/191] Workaround for reading location_id for not existing resource in controls --- solar/solar/events/controls.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/solar/solar/events/controls.py b/solar/solar/events/controls.py index 537bc5cb..dc35e57d 100644 --- a/solar/solar/events/controls.py +++ b/solar/solar/events/controls.py @@ -32,6 +32,7 @@ trigger action even if no changes noticed on dependent resource. """ from solar.dblayer.solar_models import Resource +from solar.dblayer.model import DBLayerNotFound class Event(object): @@ -97,7 +98,10 @@ class React(Event): if self.parent_node in changes_graph: if self.child_node not in changes_graph: - location_id = Resource.get(self.child).inputs['location_id'] + try: + location_id = Resource.get(self.child).inputs['location_id'] + except DBLayerNotFound: + location_id = None changes_graph.add_node( self.child_node, status='PENDING', target=location_id, From be979fe2c006ac3f4386cd5ea0e095f1ef97b1c7 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 17 Nov 2015 12:20:55 +0100 Subject: [PATCH 148/191] Correct node add in controls --- solar/solar/events/controls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/events/controls.py b/solar/solar/events/controls.py index dc35e57d..1d00c9e2 100644 --- a/solar/solar/events/controls.py +++ b/solar/solar/events/controls.py @@ -110,7 +110,7 @@ class React(Event): changes_graph.add_edge( self.parent_node, self.child_node, state=self.state) - changed_resources.append(self.child) + changed_resources.append(self.child_node) class StateChange(Event): From 6a14db8b303d65e685f4052a24a7221a6b459a63 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 17 Nov 2015 13:09:11 +0200 Subject: [PATCH 149/191] Remove update from yaml features --- solar/solar/cli/orch.py | 7 ------- solar/solar/events/api.py | 1 - solar/solar/events/controls.py | 8 +++++--- solar/solar/orchestration/graph.py | 18 ------------------ solar/solar/test/test_graph_api.py | 22 ---------------------- 5 files changed, 5 insertions(+), 51 deletions(-) diff --git a/solar/solar/cli/orch.py b/solar/solar/cli/orch.py index c33dc85a..4f28e737 100755 --- a/solar/solar/cli/orch.py +++ b/solar/solar/cli/orch.py @@ -48,13 +48,6 @@ def create(plan): click.echo(uid) -@orchestration.command() -@click.argument('uid', type=SOLARUID) -@click.argument('plan') -def update(uid, plan): - graph.update_plan(uid, plan) - - def wait_report(uid, timeout, interval=3): try: if timeout: diff --git a/solar/solar/events/api.py b/solar/solar/events/api.py index fbeabd4a..c9f2da8a 100644 --- a/solar/solar/events/api.py +++ b/solar/solar/events/api.py @@ -152,6 +152,5 @@ def build_edges(changes_graph, events): for parent, child, data in events_graph.edges(event_name, data=True): succ_ev = data['event'] succ_ev.insert(stack, changes_graph) - visited.add(event_name) return changes_graph diff --git a/solar/solar/events/controls.py b/solar/solar/events/controls.py index 1d00c9e2..6d204de0 100644 --- a/solar/solar/events/controls.py +++ b/solar/solar/events/controls.py @@ -118,9 +118,11 @@ class StateChange(Event): etype = 'state_change' def insert(self, changed_resources, changes_graph): - changed_resources.append(self.parent) - - location_id = Resource.get(self.parent).inputs['location_id'] + changed_resources.append(self.parent_node) + try: + location_id = Resource.get(self.parent).inputs['location_id'] + except DBLayerNotFound: + location_id = None changes_graph.add_node( self.parent_node, status='PENDING', target=location_id, diff --git a/solar/solar/orchestration/graph.py b/solar/solar/orchestration/graph.py index 3ffd5c85..fd01e8cd 100644 --- a/solar/solar/orchestration/graph.py +++ b/solar/solar/orchestration/graph.py @@ -128,24 +128,6 @@ def create_plan(plan_path, save=True): return create_plan_from_graph(dg, save=save) -def update_plan(uid, plan_path): - """update preserves old status of tasks if they werent removed - """ - - new = parse_plan(plan_path) - old = get_graph(uid) - return update_plan_from_graph(new, old).graph['uid'] - - -def update_plan_from_graph(new, old): - new.graph = old.graph - for n in new: - if n in old: - new.node[n]['status'] = old.node[n]['status'] - - save_graph(new) - return new - def reset_by_uid(uid, state_list=None): dg = get_graph(uid) diff --git a/solar/solar/test/test_graph_api.py b/solar/solar/test/test_graph_api.py index 80726f95..6f0df36e 100644 --- a/solar/solar/test/test_graph_api.py +++ b/solar/solar/test/test_graph_api.py @@ -34,28 +34,6 @@ def test_simple_plan_created_and_loaded(simple): plan = graph.get_plan(simple.graph['uid']) assert set(plan.nodes()) == {'just_fail', 'echo_stuff'} - -def test_update_plan_with_new_node(simple): - new = deepcopy(simple) - new.add_node('one_more', {}) - graph.update_plan_from_graph(new, simple) - updated = graph.get_plan(new.graph['uid']) - assert set(updated.nodes()) == {'one_more', 'just_fail', 'echo_stuff'} - - -def test_status_preserved_on_update(simple): - new = deepcopy(simple) - task_under_test = 'echo_stuff' - - assert new.node[task_under_test]['status'] == states.PENDING.name - - simple.node[task_under_test]['status'] = states.SUCCESS.name - graph.update_plan_from_graph(new, simple) - - updated = graph.get_plan(new.graph['uid']) - assert new.node[task_under_test]['status'] == states.SUCCESS.name - - def test_reset_all_states(simple): for n in simple: simple.node[n]['status'] == states.ERROR.name From f5d7dcc80a40d4616a070295747740af92e16622 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 17 Nov 2015 13:27:12 +0200 Subject: [PATCH 150/191] wait_finish routing will clear cache before updating state --- solar/solar/orchestration/graph.py | 4 ++++ solar/solar/test/test_graph_api.py | 9 ++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/solar/solar/orchestration/graph.py b/solar/solar/orchestration/graph.py index fd01e8cd..bc3a47a8 100644 --- a/solar/solar/orchestration/graph.py +++ b/solar/solar/orchestration/graph.py @@ -24,6 +24,7 @@ from solar import errors from collections import Counter from solar.dblayer.solar_models import Task +from solar.dblayer.model import ModelMeta def save_graph(graph): @@ -169,6 +170,8 @@ def wait_finish(uid, timeout): start_time = time.time() while start_time + timeout >= time.time(): + # need to clear cache before fetching updated status + ModelMeta.session_start() dg = get_graph(uid) summary = Counter() summary.update({s.name: 0 for s in states}) @@ -176,6 +179,7 @@ def wait_finish(uid, timeout): yield summary if summary[states.PENDING.name] + summary[states.INPROGRESS.name] == 0: return + else: raise errors.ExecutionTimeout( 'Run %s wasnt able to finish' % uid) diff --git a/solar/solar/test/test_graph_api.py b/solar/solar/test/test_graph_api.py index 6f0df36e..65c488ce 100644 --- a/solar/solar/test/test_graph_api.py +++ b/solar/solar/test/test_graph_api.py @@ -13,12 +13,12 @@ # under the License. import os -from copy import deepcopy from pytest import fixture from solar.orchestration import graph from solar.orchestration.traversal import states +from solar.dblayer.model import ModelMeta @fixture @@ -56,18 +56,17 @@ def test_reset_only_provided(simple): def test_wait_finish(simple): for n in simple: simple.node[n]['status'] = states.SUCCESS.name - graph.save_graph(simple) - + graph.update_graph(simple) assert next(graph.wait_finish(simple.graph['uid'], 10)) == {'SKIPPED': 0, 'SUCCESS': 2, 'NOOP': 0, 'ERROR': 0, 'INPROGRESS': 0, 'PENDING': 0} def test_several_updates(simple): simple.node['just_fail']['status'] = states.ERROR.name - graph.save_graph(simple) + graph.update_graph(simple) assert next(graph.wait_finish(simple.graph['uid'], 10)) == {'SKIPPED': 0, 'SUCCESS': 0, 'NOOP': 0, 'ERROR': 1, 'INPROGRESS': 0, 'PENDING': 1} simple.node['echo_stuff']['status'] = states.ERROR.name - graph.save_graph(simple) + graph.update_graph(simple) assert next(graph.wait_finish(simple.graph['uid'], 10)) == {'SKIPPED': 0, 'SUCCESS': 0, 'NOOP': 0, 'ERROR': 2, 'INPROGRESS': 0, 'PENDING': 0} From 1f64016664ae49fc8ce035ea9994321043e24194 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 17 Nov 2015 13:27:45 +0200 Subject: [PATCH 151/191] Fix tests for load_by_tags command --- solar/solar/orchestration/graph.py | 4 +-- solar/solar/test/test_operations_with_tags.py | 25 +++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/solar/solar/orchestration/graph.py b/solar/solar/orchestration/graph.py index bc3a47a8..28e88047 100644 --- a/solar/solar/orchestration/graph.py +++ b/solar/solar/orchestration/graph.py @@ -24,7 +24,7 @@ from solar import errors from collections import Counter from solar.dblayer.solar_models import Task -from solar.dblayer.model import ModelMeta +from solar.dblayer.model import clear_cache def save_graph(graph): @@ -171,7 +171,7 @@ def wait_finish(uid, timeout): while start_time + timeout >= time.time(): # need to clear cache before fetching updated status - ModelMeta.session_start() + clear_cache() dg = get_graph(uid) summary = Counter() summary.update({s.name: 0 for s in states}) diff --git a/solar/solar/test/test_operations_with_tags.py b/solar/solar/test/test_operations_with_tags.py index 039762f2..ea9bfe5c 100644 --- a/solar/solar/test/test_operations_with_tags.py +++ b/solar/solar/test/test_operations_with_tags.py @@ -15,20 +15,31 @@ from pytest import fixture from solar.core import resource +from solar.dblayer.solar_models import Resource +from solar.dblayer.model import ModelMeta @fixture -def tagged_resources(resources): - assert len(resources) == 3 - for res in resources.values(): - res.add_tags('n1', 'n2', 'n3') - return resources +def tagged_resources(): + tags = ['n1', 'n2', 'n3'] + t1 = Resource.from_dict('t1', + {'name': 't1', 'tags': tags, 'base_path': 'x'}) + t1.save_lazy() + t2 = Resource.from_dict('t2', + {'name': 't2', 'tags': tags, 'base_path': 'x'}) + t2.save_lazy() + t3 = Resource.from_dict('t3', + {'name': 't3', 'tags': tags, 'base_path': 'x'}) + t3.save_lazy() + ModelMeta.save_all_lazy() + return [t1, t2, t3] def test_add_remove_tags(tagged_resources): - assert len(resource.load_by_tags({'n1', 'n2'})) == 3 + loaded = resource.load_by_tags({'n1', 'n2'}) + assert len(loaded) == 3 - for res in tagged_resources.values(): + for res in loaded: res.remove_tags('n1') assert len(resource.load_by_tags(set(['n1']))) == 0 From 1807219376c295ecd165e596b625a68a6ba05358 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 17 Nov 2015 13:01:21 +0100 Subject: [PATCH 152/191] Removed old db implementation --- examples/bootstrap/example-bootstrap.py | 19 +- .../example-compiled-resources.py | 16 +- examples/hosts_file/hosts.py | 8 +- examples/library_ceph/ceph.py | 6 +- examples/lxc/example-lxc.py | 8 +- examples/openstack/openstack.py | 12 +- examples/riak/riaks-template.py | 7 +- examples/riak/riaks.py | 5 - examples/solard/example.py | 9 +- examples/torrent/example.py | 8 +- solar/solar/cli/main.py | 34 +- solar/solar/cli/resource.py | 2 - solar/solar/core/resource/resource.py | 1 - solar/solar/core/signals.py | 92 --- solar/solar/events/api.py | 1 - solar/solar/interfaces/__init__.py | 0 solar/solar/interfaces/db/__init__.py | 38 - solar/solar/interfaces/db/base.py | 236 ------ solar/solar/interfaces/db/neo4j.py | 205 ----- solar/solar/interfaces/db/redis_db.py | 156 ---- solar/solar/interfaces/db/redis_graph_db.py | 300 ------- solar/solar/interfaces/orm.py | 735 ------------------ solar/solar/system_log/operations.py | 1 - solar/solar/test/test_orm.py | 488 ------------ 24 files changed, 44 insertions(+), 2343 deletions(-) mode change 100644 => 100755 examples/bootstrap/example-bootstrap.py mode change 100644 => 100755 examples/compiled-resources/example-compiled-resources.py mode change 100644 => 100755 examples/lxc/example-lxc.py mode change 100644 => 100755 examples/riak/riaks-template.py delete mode 100644 solar/solar/interfaces/__init__.py delete mode 100644 solar/solar/interfaces/db/__init__.py delete mode 100644 solar/solar/interfaces/db/base.py delete mode 100644 solar/solar/interfaces/db/neo4j.py delete mode 100644 solar/solar/interfaces/db/redis_db.py delete mode 100644 solar/solar/interfaces/db/redis_graph_db.py delete mode 100644 solar/solar/interfaces/orm.py delete mode 100644 solar/solar/test/test_orm.py diff --git a/examples/bootstrap/example-bootstrap.py b/examples/bootstrap/example-bootstrap.py old mode 100644 new mode 100755 index 7b45fa93..20f474bf --- a/examples/bootstrap/example-bootstrap.py +++ b/examples/bootstrap/example-bootstrap.py @@ -10,11 +10,7 @@ from solar.core import signals from solar.core import validation from solar.core.resource import virtual_resource as vr from solar import errors - -from solar.interfaces.db import get_db - - -db = get_db() +from solar.dblayer.model import ModelMeta @click.group() @@ -23,9 +19,7 @@ def main(): def setup_resources(): - db.clear() - - signals.Connections.clear() + ModelMeta.remove_all() node2 = vr.create('node2', 'resources/ro_node/', { 'ip': '10.0.0.4', @@ -61,7 +55,7 @@ def deploy(): setup_resources() # run - resources = map(resource.wrap_resource, db.get_list(collection=db.COLLECTIONS.resource)) + resources = resource.load_all() resources = {r.name: r for r in resources} for name in resources_to_run: @@ -76,7 +70,7 @@ def deploy(): @click.command() def undeploy(): - resources = map(resource.wrap_resource, db.get_list(collection=db.COLLECTIONS.resource)) + resources = resource.load_all() resources = {r.name: r for r in resources} for name in reversed(resources_to_run): @@ -85,10 +79,7 @@ def undeploy(): except errors.SolarError as e: print 'WARNING: %s' % str(e) - db.clear() - - signals.Connections.clear() - + ModelMeta.remove_all() main.add_command(deploy) main.add_command(undeploy) diff --git a/examples/compiled-resources/example-compiled-resources.py b/examples/compiled-resources/example-compiled-resources.py old mode 100644 new mode 100755 index 503aea68..e34ce024 --- a/examples/compiled-resources/example-compiled-resources.py +++ b/examples/compiled-resources/example-compiled-resources.py @@ -19,11 +19,9 @@ from solar.core import actions from solar.core.resource import virtual_resource as vr from solar.core import resource from solar.core import signals - -from solar.interfaces.db import get_db +from solar.dblayer.model import ModelMeta from solar.core.resource_provider import GitProvider, RemoteZipProvider - import resources_compiled @@ -34,9 +32,7 @@ def main(): @click.command() def deploy(): - db = get_db() - db.clear() - + ModelMeta.remove_all() signals.Connections.clear() node1 = resources_compiled.RoNodeResource('node1', None, {}) @@ -75,18 +71,16 @@ def deploy(): @click.command() def undeploy(): - db = get_db() + ModelMeta.remove_all() - resources = map(resource.wrap_resource, db.get_list(collection=db.COLLECTIONS.resource)) + resources = resource.load_all() resources = {r.name: r for r in resources} actions.resource_action(resources['openstack_rabbitmq_user'], 'remove') actions.resource_action(resources['openstack_vhost'], 'remove') actions.resource_action(resources['rabbitmq_service1'], 'remove') - db.clear() - - signals.Connections.clear() + ModelMeta.remove_all() main.add_command(deploy) diff --git a/examples/hosts_file/hosts.py b/examples/hosts_file/hosts.py index f6aba5d3..54d04583 100644 --- a/examples/hosts_file/hosts.py +++ b/examples/hosts_file/hosts.py @@ -4,15 +4,11 @@ import time from solar.core import signals from solar.core.resource import virtual_resource as vr - -from solar.interfaces.db import get_db - - -db = get_db() +from solar.dblayer.model import ModelMeta def run(): - db.clear() + ModelMeta.remove_all() resources = vr.create('nodes', 'templates/nodes_with_transports.yaml', {'count': 2}) nodes = [x for x in resources if x.name.startswith('node')] diff --git a/examples/library_ceph/ceph.py b/examples/library_ceph/ceph.py index 0e0f7361..78d93e5c 100644 --- a/examples/library_ceph/ceph.py +++ b/examples/library_ceph/ceph.py @@ -1,10 +1,8 @@ from solar.core.resource import virtual_resource as vr -from solar.interfaces.db import get_db - +from solar.dblayer.model import ModelMeta import yaml -db = get_db() STORAGE = {'objects_ceph': True, 'osd_pool_size': 2, @@ -34,7 +32,7 @@ NETWORK_METADATA = yaml.load(""" def deploy(): - db.clear() + ModelMeta.remove_all() resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 2}) first_node, second_node = [x for x in resources if x.name.startswith('node')] first_transp = next(x for x in resources if x.name.startswith('transport')) diff --git a/examples/lxc/example-lxc.py b/examples/lxc/example-lxc.py old mode 100644 new mode 100755 index 1c898ca9..01d6c7b7 --- a/examples/lxc/example-lxc.py +++ b/examples/lxc/example-lxc.py @@ -12,10 +12,10 @@ import click from solar.core import signals from solar.core.resource import virtual_resource as vr -from solar.interfaces.db import get_db - from solar.system_log import change from solar.cli import orch +from solar.dblayer.model import ModelMeta + @click.group() def main(): @@ -43,9 +43,7 @@ def lxc_template(idx): @click.command() def deploy(): - db = get_db() - db.clear() - signals.Connections.clear() + ModelMeta.remove_all() node1 = vr.create('nodes', 'templates/nodes.yaml', {})[0] seed = vr.create('nodes', 'templates/seed_node.yaml', {})[0] diff --git a/examples/openstack/openstack.py b/examples/openstack/openstack.py index f850d09c..8f0d28d3 100755 --- a/examples/openstack/openstack.py +++ b/examples/openstack/openstack.py @@ -8,9 +8,7 @@ from solar.core import signals from solar.core import validation from solar.core.resource import virtual_resource as vr from solar import events as evapi - -from solar.interfaces.db import get_db - +from solar.dblayer.model import ModelMeta PROFILE = False #PROFILE = True @@ -35,8 +33,6 @@ if PROFILE: # Official puppet manifests, not fuel-library -db = get_db() - @click.group() def main(): @@ -247,7 +243,7 @@ def setup_neutron(node, librarian, rabbitmq_service, openstack_rabbitmq_user, op return {'neutron_puppet': neutron_puppet} def setup_neutron_api(node, mariadb_service, admin_user, keystone_puppet, services_tenant, neutron_puppet): - # NEUTRON PLUGIN AND NEUTRON API (SERVER) + # NEUTRON PLUGIN AND NEUTRON API (SERVER) neutron_plugins_ml2 = vr.create('neutron_plugins_ml2', 'resources/neutron_plugins_ml2_puppet', {})[0] node.connect(neutron_plugins_ml2) @@ -830,7 +826,7 @@ def create_compute(node): @click.command() def create_all(): - db.clear() + ModelMeta.remove_all() r = prepare_nodes(2) r.update(create_controller('node0')) r.update(create_compute('node1')) @@ -856,7 +852,7 @@ def add_controller(node): @click.command() def clear(): - db.clear() + ModelMeta.remove_all() if __name__ == '__main__': diff --git a/examples/riak/riaks-template.py b/examples/riak/riaks-template.py old mode 100644 new mode 100755 index 479aad3f..13e49ed6 --- a/examples/riak/riaks-template.py +++ b/examples/riak/riaks-template.py @@ -8,16 +8,13 @@ import click import sys from solar.core import resource -from solar.interfaces.db import get_db from solar import template - - -db = get_db() +from solar.dblayer.model import ModelMeta def setup_riak(): - db.clear() + ModelMeta.remove_all() nodes = template.nodes_from('templates/riak_nodes.yaml') riak_services = nodes.on_each( diff --git a/examples/riak/riaks.py b/examples/riak/riaks.py index fd5b9235..9819bf92 100755 --- a/examples/riak/riaks.py +++ b/examples/riak/riaks.py @@ -22,18 +22,13 @@ from solar import errors from solar.dblayer.model import ModelMeta -from solar.interfaces.db import get_db - from solar.events.controls import React, Dep from solar.events.api import add_event from solar.dblayer.solar_models import Resource -# db = get_db() - def setup_riak(): - # db.clear() ModelMeta.remove_all() resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 3}) diff --git a/examples/solard/example.py b/examples/solard/example.py index 74726442..d58af9ca 100644 --- a/examples/solard/example.py +++ b/examples/solard/example.py @@ -5,16 +5,11 @@ import time from solar.core import resource from solar.core import signals from solar.core.resource import virtual_resource as vr - -from solar.interfaces.db import get_db - - -db = get_db() - +from solar.dblayer.model import ModelMeta def run(): - db.clear() + ModelMeta.remove_all() node = vr.create('node', 'resources/ro_node', {'name': 'first' + str(time.time()), 'ip': '10.0.0.3', diff --git a/examples/torrent/example.py b/examples/torrent/example.py index 3719105e..5043baab 100644 --- a/examples/torrent/example.py +++ b/examples/torrent/example.py @@ -2,15 +2,11 @@ import time from solar.core.resource import virtual_resource as vr from solar import errors - -from solar.interfaces.db import get_db - - -db = get_db() +from solar.dblayer.model import ModelMeta def run(): - db.clear() + ModelMeta.remove_all() node = vr.create('node', 'resources/ro_node', {'name': 'first' + str(time.time()), 'ip': '10.0.0.3', diff --git a/solar/solar/cli/main.py b/solar/solar/cli/main.py index e8e6acc7..9cac54a7 100644 --- a/solar/solar/cli/main.py +++ b/solar/solar/cli/main.py @@ -34,7 +34,6 @@ from solar.core.tags_set_parser import Expression from solar.core.resource import virtual_resource as vr from solar.core.log import log from solar import errors -from solar.interfaces import orm from solar import utils from solar.cli import base @@ -78,25 +77,26 @@ def init_actions(): @click.option('-d', '--dry-run', default=False, is_flag=True) @click.option('-m', '--dry-run-mapping', default='{}') def run(dry_run_mapping, dry_run, action, tags): - if dry_run: - dry_run_executor = executors.DryRunExecutor(mapping=json.loads(dry_run_mapping)) + raise NotImplementedError("Not yet implemented") + # if dry_run: + # dry_run_executor = executors.DryRunExecutor(mapping=json.loads(dry_run_mapping)) - resources = filter( - lambda r: Expression(tags, r.tags).evaluate(), - orm.DBResource.all() - ) + # resources = filter( + # lambda r: Expression(tags, r.tags).evaluate(), + # orm.DBResource.all() + # ) - for r in resources: - resource_obj = sresource.load(r['id']) - actions.resource_action(resource_obj, action) + # for r in resources: + # resource_obj = sresource.load(r['id']) + # actions.resource_action(resource_obj, action) - if dry_run: - click.echo('EXECUTED:') - for key in dry_run_executor.executed: - click.echo('{}: {}'.format( - click.style(dry_run_executor.compute_hash(key), fg='green'), - str(key) - )) + # if dry_run: + # click.echo('EXECUTED:') + # for key in dry_run_executor.executed: + # click.echo('{}: {}'.format( + # click.style(dry_run_executor.compute_hash(key), fg='green'), + # str(key) + # )) def init_cli_connect(): diff --git a/solar/solar/cli/resource.py b/solar/solar/cli/resource.py index d89cd48c..199ecc07 100644 --- a/solar/solar/cli/resource.py +++ b/solar/solar/cli/resource.py @@ -25,7 +25,6 @@ from solar.core import resource as sresource from solar.core.resource import virtual_resource as vr from solar.core.log import log from solar import errors -from solar.interfaces import orm from solar import utils from solar.cli import executors @@ -120,7 +119,6 @@ def clear_all(): click.echo('Clearing all resources and connections') ModelMeta.remove_all() - # orm.db.clear() @resource.command() @click.argument('name') diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 985dcf16..81720a18 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -22,7 +22,6 @@ import os from solar import utils from solar.core import validation -from solar.interfaces import orm from solar.core import signals from solar.events import api diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index cffd0faf..9a92f8c0 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -16,7 +16,6 @@ import networkx from solar.core.log import log -# from solar.interfaces import orm from solar.dblayer.solar_models import Resource as DBResource @@ -143,91 +142,6 @@ def get_mapping(emitter, receiver, mapping=None): def connect(emitter, receiver, mapping=None): emitter.connect(receiver, mapping) -# def connect(emitter, receiver, mapping=None): -# if mapping is None: -# mapping = guess_mapping(emitter, receiver) - -# # XXX: we didn't agree on that "reverse" there -# location_and_transports(emitter, receiver, mapping) - -# if isinstance(mapping, set): -# mapping = {src: src for src in mapping} - -# for src, dst in mapping.items(): -# if not isinstance(dst, list): -# dst = [dst] - -# for d in dst: -# connect_single(emitter, src, receiver, d) - - -# def connect_single(emitter, src, receiver, dst): -# if ':' in dst: -# return connect_multi(emitter, src, receiver, dst) - -# # Disconnect all receiver inputs -# # Check if receiver input is of list type first -# emitter_input = emitter.resource_inputs()[src] -# receiver_input = receiver.resource_inputs()[dst] - -# if emitter_input.id == receiver_input.id: -# raise Exception( -# 'Trying to connect {} to itself, this is not possible'.format( -# emitter_input.id) -# ) - -# if not receiver_input.is_list: -# receiver_input.receivers.delete_all_incoming(receiver_input) - -# # Check for cycles -# # TODO: change to get_paths after it is implemented in drivers -# if emitter_input in receiver_input.receivers.as_set(): -# raise Exception('Prevented creating a cycle on %s::%s' % (emitter.name, -# emitter_input.name)) - -# log.debug('Connecting {}::{} -> {}::{}'.format( -# emitter.name, emitter_input.name, receiver.name, receiver_input.name -# )) -# emitter_input.receivers.add(receiver_input) - - -# def connect_multi(emitter, src, receiver, dst): -# receiver_input_name, receiver_input_key = dst.split(':') -# if '|' in receiver_input_key: -# receiver_input_key, receiver_input_tag = receiver_input_key.split('|') -# else: -# receiver_input_tag = None - -# emitter_input = emitter.resource_inputs()[src] -# receiver_input = receiver.resource_inputs()[receiver_input_name] - -# if not receiver_input.is_list or receiver_input_tag: -# receiver_input.receivers.delete_all_incoming( -# receiver_input, -# destination_key=receiver_input_key, -# tag=receiver_input_tag -# ) - -# # We can add default tag now -# receiver_input_tag = receiver_input_tag or emitter.name - -# # NOTE: make sure that receiver.args[receiver_input] is of dict type -# if not receiver_input.is_hash: -# raise Exception( -# 'Receiver input {} must be a hash or a list of hashes'.format(receiver_input_name) -# ) - -# log.debug('Connecting {}::{} -> {}::{}[{}], tag={}'.format( -# emitter.name, emitter_input.name, receiver.name, receiver_input.name, -# receiver_input_key, -# receiver_input_tag -# )) -# emitter_input.receivers.add_hash( -# receiver_input, -# receiver_input_key, -# tag=receiver_input_tag -# ) - def disconnect_receiver_by_input(receiver, input_name): # input_node = receiver.resource_inputs()[input_name] @@ -236,12 +150,6 @@ def disconnect_receiver_by_input(receiver, input_name): receiver.db_obj.inputs.disconnect(input_name) -# def disconnect(emitter, receiver): -# for emitter_input in emitter.resource_inputs().values(): -# for receiver_input in receiver.resource_inputs().values(): -# emitter_input.receivers.remove(receiver_input) - - def detailed_connection_graph(start_with=None, end_with=None, details=False): from solar.core.resource import Resource, load_all diff --git a/solar/solar/events/api.py b/solar/solar/events/api.py index fbeabd4a..2b0c68de 100644 --- a/solar/solar/events/api.py +++ b/solar/solar/events/api.py @@ -18,7 +18,6 @@ __all__ = ['add_dep', 'add_react', 'Dep', 'React', 'add_event'] import networkx as nx from solar.core.log import log -from solar.interfaces import orm from solar.events.controls import Dep, React, StateChange from solar.dblayer.solar_models import Resource diff --git a/solar/solar/interfaces/__init__.py b/solar/solar/interfaces/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/solar/solar/interfaces/db/__init__.py b/solar/solar/interfaces/db/__init__.py deleted file mode 100644 index a4c23d08..00000000 --- a/solar/solar/interfaces/db/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import importlib - -db_backends = { - 'neo4j_db': ('solar.interfaces.db.neo4j', 'Neo4jDB'), - 'redis_db': ('solar.interfaces.db.redis_db', 'RedisDB'), - 'fakeredis_db': ('solar.interfaces.db.redis_db', 'FakeRedisDB'), - 'redis_graph_db': ('solar.interfaces.db.redis_graph_db', 'RedisGraphDB'), - 'fakeredis_graph_db': ('solar.interfaces.db.redis_graph_db', 'FakeRedisGraphDB'), -} - -CURRENT_DB = 'redis_graph_db' -#CURRENT_DB = 'neo4j_db' - -DB = None - - -def get_db(backend=CURRENT_DB): - # Should be retrieved from config - global DB - if DB is None: - import_path, klass = db_backends[backend] - module = importlib.import_module(import_path) - DB = getattr(module, klass)() - return DB diff --git a/solar/solar/interfaces/db/base.py b/solar/solar/interfaces/db/base.py deleted file mode 100644 index ddb448ac..00000000 --- a/solar/solar/interfaces/db/base.py +++ /dev/null @@ -1,236 +0,0 @@ -import abc -from enum import Enum -from functools import partial - - -class Node(object): - def __init__(self, db, uid, labels, properties): - self.db = db - self.uid = uid - self.labels = labels - self.properties = properties - - @property - def collection(self): - return getattr( - BaseGraphDB.COLLECTIONS, - list(self.labels)[0] - ) - - -class Relation(object): - def __init__(self, db, start_node, end_node, properties): - self.db = db - self.start_node = start_node - self.end_node = end_node - self.properties = properties - - -class DBObjectMeta(abc.ABCMeta): - # Tuples of: function name, is-multi (i.e. returns a list) - node_db_read_methods = [ - ('all', True), - ('create', False), - ('get', False), - ('get_or_create', False), - ] - relation_db_read_methods = [ - ('all_relations', True), - ('create_relation', False), - ('get_relations', True), - ('get_relation', False), - ('get_or_create_relation', False), - ] - - def __new__(cls, name, parents, dct): - def from_db_list_decorator(converting_func, method): - def wrapper(self, *args, **kwargs): - db_convert = kwargs.pop('db_convert', True) - - result = method(self, *args, **kwargs) - - if db_convert: - return map(partial(converting_func, self), result) - - return result - - return wrapper - - def from_db_decorator(converting_func, method): - def wrapper(self, *args, **kwargs): - db_convert = kwargs.pop('db_convert', True) - - result = method(self, *args, **kwargs) - - if result is None: - return - - if db_convert: - return converting_func(self, result) - - return result - - return wrapper - - node_db_to_object = cls.find_method( - 'node_db_to_object', name, parents, dct - ) - relation_db_to_object = cls.find_method( - 'relation_db_to_object', name, parents, dct - ) - - # Node conversions - for method_name, is_list in cls.node_db_read_methods: - method = cls.find_method(method_name, name, parents, dct) - if is_list: - func = from_db_list_decorator - else: - func = from_db_decorator - # Handle subclasses - if not getattr(method, '_wrapped', None): - dct[method_name] = func(node_db_to_object, method) - setattr(dct[method_name], '_wrapped', True) - - # Relation conversions - for method_name, is_list in cls.relation_db_read_methods: - method = cls.find_method(method_name, name, parents, dct) - if is_list: - func = from_db_list_decorator - else: - func = from_db_decorator - # Handle subclasses - if not getattr(method, '_wrapped', None): - dct[method_name] = func(relation_db_to_object, method) - setattr(dct[method_name], '_wrapped', True) - - return super(DBObjectMeta, cls).__new__(cls, name, parents, dct) - - @classmethod - def find_method(cls, method_name, class_name, parents, dict): - if method_name in dict: - return dict[method_name] - - for parent in parents: - method = getattr(parent, method_name) - if method: - return method - - raise NotImplementedError( - '"{}" method not implemented in class {}'.format( - method_name, class_name - ) - ) - - -class BaseGraphDB(object): - __metaclass__ = DBObjectMeta - - COLLECTIONS = Enum( - 'Collections', - 'input resource state_data state_log plan_node plan_graph events stage_log commit_log resource_events' - ) - DEFAULT_COLLECTION=COLLECTIONS.resource - RELATION_TYPES = Enum( - 'RelationTypes', - 'input_to_input resource_input plan_edge graph_to_node resource_event commited' - ) - DEFAULT_RELATION=RELATION_TYPES.resource_input - - @staticmethod - def node_db_to_object(node_db): - """Convert node DB object to Node object.""" - - @staticmethod - def object_to_node_db(node_obj): - """Convert Node object to node DB object.""" - - @staticmethod - def relation_db_to_object(relation_db): - """Convert relation DB object to Relation object.""" - - @staticmethod - def object_to_relation_db(relation_obj): - """Convert Relation object to relation DB object.""" - - @abc.abstractmethod - def all(self, collection=DEFAULT_COLLECTION): - """Return all elements (nodes) of type `collection`.""" - - @abc.abstractmethod - def all_relations(self, type_=DEFAULT_RELATION): - """Return all relations of type `type_`.""" - - @abc.abstractmethod - def clear(self): - """Clear the whole DB.""" - - @abc.abstractmethod - def clear_collection(self, collection=DEFAULT_COLLECTION): - """Clear all elements (nodes) of type `collection`.""" - - @abc.abstractmethod - def create(self, name, properties={}, collection=DEFAULT_COLLECTION): - """Create element (node) with given name, args, of type `collection`.""" - - @abc.abstractmethod - def delete(self, name, collection=DEFAULT_COLLECTION): - """Delete element with given name. of type `collection`.""" - - @abc.abstractmethod - def create_relation(self, - source, - dest, - properties={}, - type_=DEFAULT_RELATION): - """ - Create relation (connection) of type `type_` from source to dest with - given args. - """ - - @abc.abstractmethod - def get(self, name, collection=DEFAULT_COLLECTION): - """Fetch element with given name and collection type.""" - - @abc.abstractmethod - def get_or_create(self, - name, - properties={}, - collection=DEFAULT_COLLECTION): - """ - Fetch or create element (if not exists) with given name, args of type - `collection`. - """ - - @abc.abstractmethod - def delete_relations(self, - source=None, - dest=None, - type_=DEFAULT_RELATION, - has_properties=None): - """Delete all relations of type `type_` from source to dest.""" - - @abc.abstractmethod - def get_relations(self, - source=None, - dest=None, - type_=DEFAULT_RELATION, - has_properties=None): - """Fetch all relations of type `type_` from source to dest. - - NOTE that this function must return only direct relations (edges) - between vertices `source` and `dest` of type `type_`. - - If you want all PATHS between `source` and `dest`, write another - method for this (`get_paths`).""" - - @abc.abstractmethod - def get_relation(self, source, dest, type_=DEFAULT_RELATION): - """Fetch relations with given source, dest and type_.""" - - @abc.abstractmethod - def get_or_create_relation(self, - source, - dest, - properties={}, - type_=DEFAULT_RELATION): - """Fetch or create relation with given args.""" diff --git a/solar/solar/interfaces/db/neo4j.py b/solar/solar/interfaces/db/neo4j.py deleted file mode 100644 index 5425a434..00000000 --- a/solar/solar/interfaces/db/neo4j.py +++ /dev/null @@ -1,205 +0,0 @@ -import json -from copy import deepcopy -import py2neo - -from solar.core import log - -from .base import BaseGraphDB, Node, Relation - - -class Neo4jDB(BaseGraphDB): - DB = { - 'host': 'localhost', - 'port': 7474, - } - NEO4J_CLIENT = py2neo.Graph - - def __init__(self): - self._r = self.NEO4J_CLIENT('http://{host}:{port}/db/data/'.format( - **self.DB - )) - - def node_db_to_object(self, node_db): - return Node( - self, - node_db.properties['name'], - node_db.labels, - # Neo4j Node.properties is some strange PropertySet, use dict instead - dict(**node_db.properties) - ) - - def relation_db_to_object(self, relation_db): - return Relation( - self, - self.node_db_to_object(relation_db.start_node), - self.node_db_to_object(relation_db.end_node), - relation_db.properties - ) - - def all(self, collection=BaseGraphDB.DEFAULT_COLLECTION): - return [ - r.n for r in self._r.cypher.execute( - 'MATCH (n:%(collection)s) RETURN n' % { - 'collection': collection.name, - } - ) - ] - - def all_relations(self, type_=BaseGraphDB.DEFAULT_RELATION): - return [ - r.r for r in self._r.cypher.execute( - *self._relations_query( - source=None, dest=None, type_=type_ - ) - ) - ] - - def clear(self): - log.log.debug('Clearing whole DB') - - self._r.delete_all() - - def clear_collection(self, collection=BaseGraphDB.DEFAULT_COLLECTION): - log.log.debug('Clearing collection %s', collection.name) - - # TODO: make single DELETE query - self._r.delete([r.n for r in self.all(collection=collection)]) - - def create(self, name, properties={}, collection=BaseGraphDB.DEFAULT_COLLECTION): - log.log.debug( - 'Creating %s, name %s with properties %s', - collection.name, - name, - properties - ) - - properties = deepcopy(properties) - properties['name'] = name - - n = py2neo.Node(collection.name, **properties) - return self._r.create(n)[0] - - def create_relation(self, - source, - dest, - properties={}, - type_=BaseGraphDB.DEFAULT_RELATION): - log.log.debug( - 'Creating %s from %s to %s with properties %s', - type_.name, - source.properties['name'], - dest.properties['name'], - properties - ) - s = self.get( - source.properties['name'], - collection=source.collection, - db_convert=False - ) - d = self.get( - dest.properties['name'], - collection=dest.collection, - db_convert=False - ) - r = py2neo.Relationship(s, type_.name, d, **properties) - self._r.create(r) - - return r - - def _get_query(self, name, collection=BaseGraphDB.DEFAULT_COLLECTION): - return 'MATCH (n:%(collection)s {name:{name}}) RETURN n' % { - 'collection': collection.name, - }, { - 'name': name, - } - - def get(self, name, collection=BaseGraphDB.DEFAULT_COLLECTION): - query, kwargs = self._get_query(name, collection=collection) - res = self._r.cypher.execute(query, kwargs) - - if res: - return res[0].n - - def get_or_create(self, - name, - properties={}, - collection=BaseGraphDB.DEFAULT_COLLECTION): - n = self.get(name, collection=collection, db_convert=False) - - if n: - if properties != n.properties: - n.properties.update(properties) - n.push() - return n - - return self.create(name, properties=properties, collection=collection) - - def _relations_query(self, - source=None, - dest=None, - type_=BaseGraphDB.DEFAULT_RELATION, - query_type='RETURN'): - kwargs = {} - source_query = '(n)' - if source: - source_query = '(n {name:{source_name}})' - kwargs['source_name'] = source.properties['name'] - dest_query = '(m)' - if dest: - dest_query = '(m {name:{dest_name}})' - kwargs['dest_name'] = dest.properties['name'] - rel_query = '[r:%(type_)s]' % {'type_': type_.name} - - query = ('MATCH %(source_query)s-%(rel_query)s->' - '%(dest_query)s %(query_type)s r' % { - 'dest_query': dest_query, - 'query_type': query_type, - 'rel_query': rel_query, - 'source_query': source_query, - }) - - return query, kwargs - - def delete_relations(self, - source=None, - dest=None, - type_=BaseGraphDB.DEFAULT_RELATION): - query, kwargs = self._relations_query( - source=source, dest=dest, type_=type_, query_type='DELETE' - ) - - self._r.cypher.execute(query, kwargs) - - def get_relations(self, - source=None, - dest=None, - type_=BaseGraphDB.DEFAULT_RELATION): - query, kwargs = self._relations_query( - source=source, dest=dest, type_=type_ - ) - - res = self._r.cypher.execute(query, kwargs) - - return [r.r for r in res] - - def get_relation(self, source, dest, type_=BaseGraphDB.DEFAULT_RELATION): - rel = self.get_relations(source=source, dest=dest, type_=type_) - - if rel: - return rel[0] - - def get_or_create_relation(self, - source, - dest, - properties={}, - type_=BaseGraphDB.DEFAULT_RELATION): - rel = self.get_relations(source=source, dest=dest, type_=type_) - - if rel: - r = rel[0] - if properties != r.properties: - r.properties.update(properties) - r.push() - return r - - return self.create_relation(source, dest, properties=properties, type_=type_) diff --git a/solar/solar/interfaces/db/redis_db.py b/solar/solar/interfaces/db/redis_db.py deleted file mode 100644 index a177f1fb..00000000 --- a/solar/solar/interfaces/db/redis_db.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from enum import Enum -try: - import ujson as json -except ImportError: - import json - -import redis -import fakeredis - - -class RedisDB(object): - COLLECTIONS = Enum( - 'Collections', - 'connection resource state_data state_log events' - ) - DB = { - 'host': 'localhost', - 'port': 6379, - } - REDIS_CLIENT = redis.StrictRedis - - def __init__(self): - self._r = self.REDIS_CLIENT(**self.DB) - self.entities = {} - - def read(self, uid, collection=COLLECTIONS.resource): - try: - return json.loads( - self._r.get(self._make_key(collection, uid)) - ) - except TypeError: - return None - - def get_list(self, collection=COLLECTIONS.resource): - key_glob = self._make_key(collection, '*') - - keys = self._r.keys(key_glob) - - with self._r.pipeline() as pipe: - pipe.multi() - - values = [self._r.get(key) for key in keys] - - pipe.execute() - - for value in values: - yield json.loads(value) - - def save(self, uid, data, collection=COLLECTIONS.resource): - ret = self._r.set( - self._make_key(collection, uid), - json.dumps(data) - ) - - return ret - - def save_list(self, lst, collection=COLLECTIONS.resource): - with self._r.pipeline() as pipe: - pipe.multi() - - for uid, data in lst: - key = self._make_key(collection, uid) - pipe.set(key, json.dumps(data)) - - pipe.execute() - - def clear(self): - self._r.flushdb() - - def get_ordered_hash(self, collection): - return OrderedHash(self._r, collection) - - def clear_collection(self, collection=COLLECTIONS.resource): - key_glob = self._make_key(collection, '*') - - self._r.delete(self._r.keys(key_glob)) - - def delete(self, uid, collection=COLLECTIONS.resource): - self._r.delete(self._make_key(collection, uid)) - - def _make_key(self, collection, _id): - if isinstance(collection, self.COLLECTIONS): - collection = collection.name - - # NOTE: hiera-redis backend depends on this! - return '{0}:{1}'.format(collection, _id) - - -class OrderedHash(object): - - def __init__(self, client, collection): - self.r = client - self.collection = collection - self.order_counter = '{}:incr'.format(collection) - self.order = '{}:order'.format(collection) - - def add(self, items): - pipe = self.r.pipeline() - for key, value in items: - count = self.r.incr(self.order_counter) - pipe.zadd(self.order, count, key) - pipe.hset(self.collection, key, json.dumps(value)) - pipe.execute() - - def rem(self, keys): - pipe = self.r.pipeline() - for key in keys: - pipe.zrem(self.order, key) - pipe.hdel(self.collection, key) - pipe.execute() - - def get(self, key): - value = self.r.hget(self.collection, key) - if value: - return json.loads(value) - return None - - def update(self, key, value): - self.r.hset(self.collection, key, json.dumps(value)) - - def clean(self): - self.rem(self.r.zrange(self.order, 0, -1)) - - def rem_left(self, n=1): - self.rem(self.r.zrevrange(self.order, 0, n-1)) - - def reverse(self, n=1): - result = [] - for key in self.r.zrevrange(self.order, 0, n-1): - result.append(self.get(key)) - return result - - def list(self, n=0): - result = [] - for key in self.r.zrange(self.order, 0, n-1): - result.append(self.get(key)) - return result - - -class FakeRedisDB(RedisDB): - - REDIS_CLIENT = fakeredis.FakeStrictRedis diff --git a/solar/solar/interfaces/db/redis_graph_db.py b/solar/solar/interfaces/db/redis_graph_db.py deleted file mode 100644 index a254b099..00000000 --- a/solar/solar/interfaces/db/redis_graph_db.py +++ /dev/null @@ -1,300 +0,0 @@ -try: - import ujson as json -except ImportError: - import json -import redis -import fakeredis - -from .base import BaseGraphDB, Node, Relation -from .redis_db import OrderedHash - - -class RedisGraphDB(BaseGraphDB): - DB = { - 'host': 'localhost', - 'port': 6379, - } - REDIS_CLIENT = redis.StrictRedis - - def __init__(self): - self._r = self.REDIS_CLIENT(**self.DB) - self.entities = {} - - def node_db_to_object(self, node_db): - if isinstance(node_db, Node): - return node_db - - return Node( - self, - node_db['name'], - [node_db['collection']], - node_db['properties'] - ) - - def relation_db_to_object(self, relation_db): - if isinstance(relation_db, Relation): - return relation_db - - if relation_db['type_'] == BaseGraphDB.RELATION_TYPES.input_to_input.name: - source_collection = BaseGraphDB.COLLECTIONS.input - dest_collection = BaseGraphDB.COLLECTIONS.input - elif relation_db['type_'] == BaseGraphDB.RELATION_TYPES.resource_input.name: - source_collection = BaseGraphDB.COLLECTIONS.resource - dest_collection = BaseGraphDB.COLLECTIONS.input - elif relation_db['type_'] == BaseGraphDB.RELATION_TYPES.resource_event.name: - source_collection = BaseGraphDB.COLLECTIONS.resource_events - dest_collection = BaseGraphDB.COLLECTIONS.events - - source = self.get(relation_db['source'], collection=source_collection) - dest = self.get(relation_db['dest'], collection=dest_collection) - - return Relation( - self, - source, - dest, - relation_db['properties'] - ) - - def all(self, collection=BaseGraphDB.DEFAULT_COLLECTION): - """Return all elements (nodes) of type `collection`.""" - - key_glob = self._make_collection_key(collection, '*') - - for result in self._all(key_glob): - yield result - - def all_relations(self, type_=BaseGraphDB.DEFAULT_RELATION): - """Return all relations of type `type_`.""" - key_glob = self._make_relation_key(type_, '*') - for result in self._all(key_glob): - yield result - - def _all(self, key_glob): - keys = self._r.keys(key_glob) - - with self._r.pipeline() as pipe: - pipe.multi() - - values = [self._r.get(key) for key in keys] - - pipe.execute() - - for value in values: - yield json.loads(value) - - def clear(self): - """Clear the whole DB.""" - - self._r.flushdb() - - def clear_collection(self, collection=BaseGraphDB.DEFAULT_COLLECTION): - """Clear all elements (nodes) of type `collection`.""" - - key_glob = self._make_collection_key(collection, '*') - - self._r.delete(self._r.keys(key_glob)) - - def create(self, name, properties={}, collection=BaseGraphDB.DEFAULT_COLLECTION): - """Create element (node) with given name, properties, of type `collection`.""" - - if isinstance(collection, self.COLLECTIONS): - collection = collection.name - - properties = { - 'name': name, - 'properties': properties, - 'collection': collection, - } - - self._r.set( - self._make_collection_key(collection, name), - json.dumps(properties) - ) - - return properties - - def create_relation(self, - source, - dest, - properties={}, - type_=BaseGraphDB.DEFAULT_RELATION): - """ - Create relation (connection) of type `type_` from source to dest with - given properties. - """ - return self.create_relation_str( - source.uid, dest.uid, properties, type_=type_) - - def create_relation_str(self, source, dest, - properties={}, type_=BaseGraphDB.DEFAULT_RELATION): - if isinstance(type_, self.RELATION_TYPES): - type_ = type_.name - - uid = self._make_relation_uid(source, dest) - - properties = { - 'source': source, - 'dest': dest, - 'properties': properties, - 'type_': type_, - } - - self._r.set( - self._make_relation_key(type_, uid), - json.dumps(properties) - ) - - return properties - - def get(self, name, collection=BaseGraphDB.DEFAULT_COLLECTION, - return_empty=False): - """Fetch element with given name and collection type.""" - try: - collection_key = self._make_collection_key(collection, name) - item = self._r.get(collection_key) - if not item and return_empty: - return item - return json.loads(item) - except TypeError: - raise KeyError(collection_key) - - def delete(self, name, collection=BaseGraphDB.DEFAULT_COLLECTION): - keys = self._r.keys(self._make_collection_key(collection, name)) - if keys: - self._r.delete(*keys) - - def get_or_create(self, - name, - properties={}, - collection=BaseGraphDB.DEFAULT_COLLECTION): - """ - Fetch or create element (if not exists) with given name, properties of - type `collection`. - """ - - try: - return self.get(name, collection=collection) - except KeyError: - return self.create(name, properties=properties, collection=collection) - - def _relations_glob(self, - source=None, - dest=None, - type_=BaseGraphDB.DEFAULT_RELATION): - if source is None: - source = '*' - else: - source = source.uid - if dest is None: - dest = '*' - else: - dest = dest.uid - - return self._make_relation_key(type_, self._make_relation_uid(source, dest)) - - def delete_relations(self, - source=None, - dest=None, - type_=BaseGraphDB.DEFAULT_RELATION, - has_properties=None): - """Delete all relations of type `type_` from source to dest.""" - - glob = self._relations_glob(source=source, dest=dest, type_=type_) - keys = self._r.keys(glob) - - if not keys: - return - - if not has_properties: - self._r.delete(*keys) - - rels = self.get_relations( - source=source, dest=dest, type_=type_, has_properties=has_properties - ) - for r in rels: - self.delete_relations( - source=r.start_node, - dest=r.end_node, - type_=type_ - ) - - def get_relations(self, - source=None, - dest=None, - type_=BaseGraphDB.DEFAULT_RELATION, - has_properties=None): - """Fetch all relations of type `type_` from source to dest.""" - - glob = self._relations_glob(source=source, dest=dest, type_=type_) - - def check_has_properties(r): - if has_properties: - for k, v in has_properties.items(): - if not r['properties'].get(k) == v: - return False - - return True - - for r in self._all(glob): - # Glob is primitive, we must filter stuff correctly here - if source and r['source'] != source.uid: - continue - if dest and r['dest'] != dest.uid: - continue - if not check_has_properties(r): - continue - yield r - - def get_relation(self, source, dest, type_=BaseGraphDB.DEFAULT_RELATION): - """Fetch relations with given source, dest and type_.""" - - uid = self._make_relation_key(source.uid, dest.uid) - try: - return json.loads( - self._r.get(self._make_relation_key(type_, uid)) - ) - except TypeError: - raise KeyError - - def get_or_create_relation(self, - source, - dest, - properties=None, - type_=BaseGraphDB.DEFAULT_RELATION): - """Fetch or create relation with given properties.""" - properties = properties or {} - - try: - return self.get_relation(source, dest, type_=type_) - except KeyError: - return self.create_relation(source, dest, properties=properties, type_=type_) - - def _make_collection_key(self, collection, _id): - if isinstance(collection, self.COLLECTIONS): - collection = collection.name - - # NOTE: hiera-redis backend depends on this! - return '{0}:{1}'.format(collection, _id) - - def _make_relation_uid(self, source, dest): - """ - There can be only one relation from source to dest, that's why - this function works. - """ - - return '{0}-{1}'.format(source, dest) - - def _make_relation_key(self, type_, _id): - if isinstance(type_, self.RELATION_TYPES): - type_ = type_.name - - # NOTE: hiera-redis backend depends on this! - return '{0}:{1}'.format(type_, _id) - - def get_ordered_hash(self, collection): - return OrderedHash(self._r, collection) - - -class FakeRedisGraphDB(RedisGraphDB): - - REDIS_CLIENT = fakeredis.FakeStrictRedis diff --git a/solar/solar/interfaces/orm.py b/solar/solar/interfaces/orm.py deleted file mode 100644 index 6444baa0..00000000 --- a/solar/solar/interfaces/orm.py +++ /dev/null @@ -1,735 +0,0 @@ -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import inspect -import networkx -import uuid - -from solar import errors -from solar.core import validation -from solar.interfaces.db import base -from solar.interfaces.db import get_db - -import os - -# USE_CACHE could be set only from CLI -USE_CACHE = int(os.getenv("USE_CACHE", 0)) - - -db = get_db() - - -from functools import wraps - -def _delete_from(store): - def _wrp(key): - try: - del store[key] - except KeyError: - pass - return _wrp - - -def cache_me(store): - def _inner(f): - # attaching to functions even when no cache enabled for consistency - f._cache_store = store - f._cache_del = _delete_from(store) - @wraps(f) - def _inner2(obj, *args, **kwargs): - try: - return store[obj.id] - except KeyError: - pass - val = f(obj, *args, **kwargs) - if obj.id.startswith('location_id'): - if not val.value: - return val - if obj.id.startswith('transports_id'): - if not val.value: - return val - if isinstance(val, list): - return val - else: - if not val.value: - return val - store[obj.id] = val - return val - if USE_CACHE: - return _inner2 - else: - return f - return _inner - - -class DBField(object): - is_primary = False - schema = None - schema_in_field = None - default_value = None - - def __init__(self, name, value=None): - self.name = name - self.value = value - if value is None: - self.value = self.default_value - - def __eq__(self, inst): - return self.name == inst.name and self.value == inst.value - - def __ne__(self, inst): - return not self.__eq__(inst) - - def __hash__(self): - return hash('{}:{}'.format(self.name, self.value)) - - def validate(self): - if self.schema is None: - return - - es = validation.validate_input(self.value, schema=self.schema) - if es: - raise errors.ValidationError('"{}": {}'.format(self.name, es[0])) - - -def db_field(schema=None, - schema_in_field=None, - default_value=None, - is_primary=False): - """Definition for the DB field. - - schema - simple schema according to the one in solar.core.validation - schema_in_field - if you don't want to fix schema, you can specify - another field in DBObject that will represent the schema used - for validation of this field - is_primary - only one field in db object can be primary. This key is used - for creating key in the DB - """ - - class DBFieldX(DBField): - pass - - DBFieldX.is_primary = is_primary - DBFieldX.schema = schema - DBFieldX.schema_in_field = schema_in_field - if default_value is not None: - DBFieldX.default_value = default_value - - return DBFieldX - - -class DBRelatedField(object): - source_db_class = None - destination_db_class = None - relation_type = None - - def __init__(self, name, source_db_object): - self.name = name - self.source_db_object = source_db_object - - @classmethod - def graph(self): - relations = db.get_relations(type_=self.relation_type) - - g = networkx.MultiDiGraph() - - for r in relations: - source = self.source_db_class(**r.start_node.properties) - dest = self.destination_db_class(**r.end_node.properties) - properties = r.properties.copy() - g.add_edge( - source, - dest, - attr_dict=properties - ) - - return g - - def all(self): - source_db_node = self.source_db_object._db_node - - if source_db_node is None: - return [] - - return db.get_relations(source=source_db_node, - type_=self.relation_type) - - def all_by_dest(self, destination_db_object): - destination_db_node = destination_db_object._db_node - - if destination_db_node is None: - return set() - - return db.get_relations(dest=destination_db_node, - type_=self.relation_type) - - def add(self, *destination_db_objects): - for dest in destination_db_objects: - if not isinstance(dest, self.destination_db_class): - raise errors.SolarError( - 'Object {} is of incompatible type {}.'.format( - dest, self.destination_db_class - ) - ) - - db.get_or_create_relation( - self.source_db_object._db_node, - dest._db_node, - properties={}, - type_=self.relation_type - ) - - def add_hash(self, destination_db_object, destination_key, tag=None): - if not isinstance(destination_db_object, self.destination_db_class): - raise errors.SolarError( - 'Object {} is of incompatible type {}.'.format( - destination_db_object, self.destination_db_class - ) - ) - - db.get_or_create_relation( - self.source_db_object._db_node, - destination_db_object._db_node, - properties={'destination_key': destination_key, 'tag': tag}, - type_=self.relation_type - ) - - def remove(self, *destination_db_objects): - for dest in destination_db_objects: - db.delete_relations( - source=self.source_db_object._db_node, - dest=dest._db_node, - type_=self.relation_type - ) - - def as_set(self): - """ - Return DB objects that are destinations for self.source_db_object. - """ - - relations = self.all() - - ret = set() - - for rel in relations: - ret.add( - self.destination_db_class(**rel.end_node.properties) - ) - - return ret - - def as_list(self): - relations = self.all() - - ret = [] - - for rel in relations: - ret.append( - self.destination_db_class(**rel.end_node.properties) - ) - - return ret - - def sources(self, destination_db_object): - """ - Reverse of self.as_set, i.e. for given destination_db_object, - return source DB objects. - """ - - relations = self.all_by_dest(destination_db_object) - - ret = set() - - for rel in relations: - ret.add( - self.source_db_class(**rel.start_node.properties) - ) - - return ret - - def delete_all_incoming(self, - destination_db_object, - destination_key=None, - tag=None): - """ - Delete all relations for which destination_db_object is an end node. - - If object is a hash, you can additionally specify the dst_key argument. - Then only connections that are destinations of dst_key will be deleted. - - Same with tag. - """ - properties = {} - if destination_key is not None: - properties['destination_key'] = destination_key - if tag is not None: - properties['tag'] = tag - - db.delete_relations( - dest=destination_db_object._db_node, - type_=self.relation_type, - has_properties=properties or None - ) - - -def db_related_field(relation_type, destination_db_class): - class DBRelatedFieldX(DBRelatedField): - pass - - DBRelatedFieldX.relation_type = relation_type - DBRelatedFieldX.destination_db_class = destination_db_class - - return DBRelatedFieldX - - -class DBObjectMeta(type): - def __new__(cls, name, parents, dct): - collection = dct.get('_collection') - if not collection: - raise NotImplementedError('Collection is required.') - - dct['_meta'] = {} - dct['_meta']['fields'] = {} - dct['_meta']['related_to'] = {} - - has_primary = False - - for field_name, field_klass in dct.items(): - if not inspect.isclass(field_klass): - continue - if issubclass(field_klass, DBField): - dct['_meta']['fields'][field_name] = field_klass - - if field_klass.is_primary: - if has_primary: - raise errors.SolarError('Object cannot have 2 primary fields.') - - has_primary = True - - dct['_meta']['primary'] = field_name - elif issubclass(field_klass, DBRelatedField): - dct['_meta']['related_to'][field_name] = field_klass - - if not has_primary: - raise errors.SolarError('Object needs to have a primary field.') - - klass = super(DBObjectMeta, cls).__new__(cls, name, parents, dct) - - # Support for self-references in relations - for field_name, field_klass in klass._meta['related_to'].items(): - field_klass.source_db_class = klass - if field_klass.destination_db_class == klass.__name__: - field_klass.destination_db_class = klass - - return klass - - -class DBObject(object): - # Enum from BaseGraphDB.COLLECTIONS - _collection = None - - def __init__(self, **kwargs): - wrong_fields = set(kwargs) - set(self._meta['fields']) - if wrong_fields: - raise errors.SolarError( - 'Unknown fields {}'.format(wrong_fields) - ) - - self._fields = {} - - for field_name, field_klass in self._meta['fields'].items(): - value = kwargs.get(field_name, field_klass.default_value) - - self._fields[field_name] = field_klass(field_name, value=value) - - self._related_to = {} - - for field_name, field_klass in self._meta['related_to'].items(): - inst = field_klass(field_name, self) - self._related_to[field_name] = inst - - self._update_values() - - def __eq__(self, inst): - # NOTE: don't compare related fields - self._update_fields_values() - return self._fields == inst._fields - - def __ne__(self, inst): - return not self.__eq__(inst) - - def __hash__(self): - return hash(self._db_key) - - def _update_fields_values(self): - """Copy values from self to self._fields.""" - - for field in self._fields.values(): - field.value = getattr(self, field.name) - - def _update_values(self): - """ - Reverse of _update_fields_values, i.e. copy values from self._fields to - self.""" - - for field in self._fields.values(): - setattr(self, field.name, field.value) - - for field in self._related_to.values(): - setattr(self, field.name, field) - - @property - def _db_key(self): - """Key for the DB document (in KV-store). - - You can overwrite this with custom keys.""" - if not self._primary_field.value: - setattr(self, self._primary_field.name, unicode(uuid.uuid4())) - self._update_fields_values() - return self._primary_field.value - - @property - def _primary_field(self): - return self._fields[self._meta['primary']] - - @property - def _db_node(self): - try: - return db.get(self._db_key, collection=self._collection) - except KeyError: - return - - def validate(self): - self._update_fields_values() - for field in self._fields.values(): - if field.schema_in_field is not None: - field.schema = self._fields[field.schema_in_field].value - field.validate() - - def to_dict(self): - self._update_fields_values() - return { - f.name: f.value for f in self._fields.values() - } - - @classmethod - def load(cls, key): - r = db.get(key, collection=cls._collection) - return cls(**r.properties) - - @classmethod - def load_all(cls): - rs = db.all(collection=cls._collection) - - return [cls(**r.properties) for r in rs] - - def save(self): - db.create( - self._db_key, - properties=self.to_dict(), - collection=self._collection - ) - - def delete(self): - db.delete( - self._db_key, - collection=self._collection - ) - - -class DBResourceInput(DBObject): - __metaclass__ = DBObjectMeta - - _collection = base.BaseGraphDB.COLLECTIONS.input - - id = db_field(schema='str!', is_primary=True) - name = db_field(schema='str!') - schema = db_field() - value = db_field(schema_in_field='schema') - is_list = db_field(schema='bool!', default_value=False) - is_hash = db_field(schema='bool!', default_value=False) - - receivers = db_related_field(base.BaseGraphDB.RELATION_TYPES.input_to_input, - 'DBResourceInput') - - @property - def resource(self): - return DBResource( - **db.get_relations( - dest=self._db_node, - type_=base.BaseGraphDB.RELATION_TYPES.resource_input - )[0].start_node.properties - ) - - def save(self): - self.backtrack_value_emitter._cache_del(self.id) - return super(DBResourceInput, self).save() - - def delete(self): - db.delete_relations( - source=self._db_node, - type_=base.BaseGraphDB.RELATION_TYPES.input_to_input - ) - db.delete_relations( - dest=self._db_node, - type_=base.BaseGraphDB.RELATION_TYPES.input_to_input - ) - self.backtrack_value_emitter._cache_del(self.id) - super(DBResourceInput, self).delete() - - def edges(self): - - out = db.get_relations( - source=self._db_node, - type_=base.BaseGraphDB.RELATION_TYPES.input_to_input) - incoming = db.get_relations( - dest=self._db_node, - type_=base.BaseGraphDB.RELATION_TYPES.input_to_input) - for relation in out + incoming: - meta = relation.properties - source = DBResourceInput(**relation.start_node.properties) - dest = DBResourceInput(**relation.end_node.properties) - yield source, dest, meta - - def check_other_val(self, other_val=None): - if not other_val: - return self - res = self.resource - # TODO: needs to be refactored a lot to be more effective. - # We don't have way of getting single input / value for given resource. - inps = {i.name: i for i in res.inputs.as_set()} - correct_input = inps[other_val] - return correct_input.backtrack_value() - - @cache_me({}) - def backtrack_value_emitter(self, level=None, other_val=None): - # TODO: this is actually just fetching head element in linked list - # so this whole algorithm can be moved to the db backend probably - # TODO: cycle detection? - # TODO: write this as a Cypher query? Move to DB? - if level is not None and other_val is not None: - raise Exception("Not supported yet") - - if level == 0: - return self - - def backtrack_func(i): - if level is None: - return i.backtrack_value_emitter(other_val=other_val) - - return i.backtrack_value_emitter(level=level - 1, other_val=other_val) - - inputs = self.receivers.sources(self) - relations = self.receivers.all_by_dest(self) - source_class = self.receivers.source_db_class - - if not inputs: - return self.check_other_val(other_val) - - # if lazy_val is None: - # return self.value - # print self.resource.name - # print [x.name for x in self.resource.inputs.as_set()] - # _input = next(x for x in self.resource.inputs.as_set() if x.name == lazy_val) - # return _input.backtrack_value() - # # return self.value - if self.is_list: - if not self.is_hash: - return [backtrack_func(i) for i in inputs] - - # NOTE: we return a list of values, but we need to group them - # hence this dict here - # NOTE: grouping is done by resource.name by default, but this - # can be overwritten by the 'tag' property in relation - ret = {} - - for r in relations: - source = source_class(**r.start_node.properties) - tag = r.properties['tag'] - ret.setdefault(tag, {}) - key = r.properties['destination_key'] - value = backtrack_func(source) - - ret[tag].update({key: value}) - - return ret.values() - elif self.is_hash: - ret = self.value or {} - for r in relations: - source = source_class( - **r.start_node.properties - ) - # NOTE: hard way to do this, what if there are more relations - # and some of them do have destination_key while others - # don't? - if 'destination_key' not in r.properties: - return backtrack_func(source) - key = r.properties['destination_key'] - ret[key] = backtrack_func(source) - return ret - - return backtrack_func(inputs.pop()) - - def parse_backtracked_value(self, v): - if isinstance(v, DBResourceInput): - return v.value - - if isinstance(v, list): - return [self.parse_backtracked_value(vv) for vv in v] - - if isinstance(v, dict): - return { - k: self.parse_backtracked_value(vv) for k, vv in v.items() - } - - return v - - def backtrack_value(self, other_val=None): - return self.parse_backtracked_value(self.backtrack_value_emitter(other_val=other_val)) - - -class DBEvent(DBObject): - - __metaclass__ = DBObjectMeta - - _collection = base.BaseGraphDB.COLLECTIONS.events - - id = db_field(is_primary=True) - parent = db_field(schema='str!') - parent_action = db_field(schema='str!') - etype = db_field('str!') - state = db_field('str') - child = db_field('str') - child_action = db_field('str') - - def delete(self): - db.delete_relations( - dest=self._db_node, - type_=base.BaseGraphDB.RELATION_TYPES.resource_event - ) - super(DBEvent, self).delete() - - -class DBResourceEvents(DBObject): - - __metaclass__ = DBObjectMeta - - _collection = base.BaseGraphDB.COLLECTIONS.resource_events - - id = db_field(schema='str!', is_primary=True) - events = db_related_field(base.BaseGraphDB.RELATION_TYPES.resource_event, - DBEvent) - - @classmethod - def get_or_create(cls, name): - r = db.get_or_create( - name, - properties={'id': name}, - collection=cls._collection) - return cls(**r.properties) - - -class DBCommitedState(DBObject): - - __metaclass__ = DBObjectMeta - - _collection = base.BaseGraphDB.COLLECTIONS.state_data - - id = db_field(schema='str!', is_primary=True) - inputs = db_field(schema={}, default_value={}) - connections = db_field(schema=[], default_value=[]) - base_path = db_field(schema='str') - tags = db_field(schema=[], default_value=[]) - state = db_field(schema='str', default_value='removed') - - @classmethod - def get_or_create(cls, name): - r = db.get_or_create( - name, - properties={'id': name}, - collection=cls._collection) - return cls(**r.properties) - - -class DBResource(DBObject): - __metaclass__ = DBObjectMeta - - _collection = base.BaseGraphDB.COLLECTIONS.resource - - id = db_field(schema='str', is_primary=True) - name = db_field(schema='str!') - actions_path = db_field(schema='str') - actions = db_field(schema={}, default_value={}) - base_name = db_field(schema='str') - base_path = db_field(schema='str') - handler = db_field(schema='str') # one of: {'ansible_playbook', 'ansible_template', 'puppet', etc} - puppet_module = db_field(schema='str') - version = db_field(schema='str') - tags = db_field(schema=[], default_value=[]) - meta_inputs = db_field(schema={}, default_value={}) - state = db_field(schema='str') - - inputs = db_related_field(base.BaseGraphDB.RELATION_TYPES.resource_input, - DBResourceInput) - - def add_input(self, name, schema, value): - # NOTE: Inputs need to have uuid added because there can be many - # inputs with the same name - uid = '{}-{}'.format(name, uuid.uuid4()) - input = DBResourceInput(id=uid, - name=name, - schema=schema, - value=value, - is_list=isinstance(schema, list), - is_hash=isinstance(schema, dict) or (isinstance(schema, list) and len(schema) > 0 and isinstance(schema[0], dict))) - input.save() - - self.inputs.add(input) - - def add_event(self, action, state, etype, child, child_action): - event = DBEvent( - parent=self.name, - parent_action=action, - state=state, - etype=etype, - child=child, - child_action=child_action - ) - event.save() - self.events.add(event) - - def delete(self): - for input in self.inputs.as_set(): - self.inputs.remove(input) - input.delete() - super(DBResource, self).delete() - - def graph(self): - mdg = networkx.MultiDiGraph() - for input in self.inputs.as_list(): - mdg.add_edges_from(input.edges()) - return mdg - - def add_tags(self, *tags): - self.tags = list(set(self.tags) | set(tags)) - self.save() - - def remove_tags(self, *tags): - self.tags = list(set(self.tags) - set(tags)) - self.save() - -# TODO: remove this -if __name__ == '__main__': - r = DBResource(name=1) - r.validate() diff --git a/solar/solar/system_log/operations.py b/solar/solar/system_log/operations.py index 1ce131f4..2eb9f42f 100644 --- a/solar/solar/system_log/operations.py +++ b/solar/solar/system_log/operations.py @@ -15,7 +15,6 @@ from solar.system_log import data from solar.dblayer.solar_models import CommitedResource from dictdiffer import patch -from solar.interfaces import orm from solar.core.resource import resource from .consts import CHANGES diff --git a/solar/solar/test/test_orm.py b/solar/solar/test/test_orm.py deleted file mode 100644 index 03cbc517..00000000 --- a/solar/solar/test/test_orm.py +++ /dev/null @@ -1,488 +0,0 @@ -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from .base import BaseResourceTest - -from solar.core import resource -from solar.core import signals -from solar import errors -from solar.interfaces import orm -from solar.interfaces.db import base - - -class TestORM(BaseResourceTest): - def test_no_collection_defined(self): - with self.assertRaisesRegexp(NotImplementedError, 'Collection is required.'): - class TestDBObject(orm.DBObject): - __metaclass__ = orm.DBObjectMeta - - def test_has_primary(self): - with self.assertRaisesRegexp(errors.SolarError, 'Object needs to have a primary field.'): - class TestDBObject(orm.DBObject): - _collection = base.BaseGraphDB.COLLECTIONS.resource - __metaclass__ = orm.DBObjectMeta - - test1 = orm.db_field(schema='str') - - def test_no_multiple_primaries(self): - with self.assertRaisesRegexp(errors.SolarError, 'Object cannot have 2 primary fields.'): - class TestDBObject(orm.DBObject): - _collection = base.BaseGraphDB.COLLECTIONS.resource - __metaclass__ = orm.DBObjectMeta - - test1 = orm.db_field(schema='str', is_primary=True) - test2 = orm.db_field(schema='str', is_primary=True) - - def test_primary_field(self): - class TestDBObject(orm.DBObject): - _collection = base.BaseGraphDB.COLLECTIONS.resource - __metaclass__ = orm.DBObjectMeta - - test1 = orm.db_field(schema='str', is_primary=True) - - t = TestDBObject(test1='abc') - - self.assertEqual('test1', t._primary_field.name) - self.assertEqual('abc', t._db_key) - - t = TestDBObject() - self.assertIsNotNone(t._db_key) - self.assertIsNotNone(t.test1) - - def test_default_value(self): - class TestDBObject(orm.DBObject): - _collection = base.BaseGraphDB.COLLECTIONS.resource - __metaclass__ = orm.DBObjectMeta - - test1 = orm.db_field(schema='str', - is_primary=True, - default_value='1') - - t = TestDBObject() - - self.assertEqual('1', t.test1) - - def test_field_validation(self): - class TestDBObject(orm.DBObject): - _collection = base.BaseGraphDB.COLLECTIONS.resource - __metaclass__ = orm.DBObjectMeta - - id = orm.db_field(schema='str', is_primary=True) - - t = TestDBObject(id=1) - - with self.assertRaises(errors.ValidationError): - t.validate() - - t = TestDBObject(id='1') - t.validate() - - def test_dynamic_schema_validation(self): - class TestDBObject(orm.DBObject): - _collection = base.BaseGraphDB.COLLECTIONS.resource - __metaclass__ = orm.DBObjectMeta - - id = orm.db_field(schema='str', is_primary=True) - schema = orm.db_field() - value = orm.db_field(schema_in_field='schema') - - t = TestDBObject(id='1', schema='str', value=1) - - with self.assertRaises(errors.ValidationError): - t.validate() - - self.assertEqual(t._fields['value'].schema, t._fields['schema'].value) - - t = TestDBObject(id='1', schema='int', value=1) - t.validate() - self.assertEqual(t._fields['value'].schema, t._fields['schema'].value) - - def test_unknown_fields(self): - class TestDBObject(orm.DBObject): - _collection = base.BaseGraphDB.COLLECTIONS.resource - __metaclass__ = orm.DBObjectMeta - - id = orm.db_field(schema='str', is_primary=True) - - with self.assertRaisesRegexp(errors.SolarError, 'Unknown fields .*iid'): - TestDBObject(iid=1) - - def test_equality(self): - class TestDBObject(orm.DBObject): - _collection = base.BaseGraphDB.COLLECTIONS.resource - __metaclass__ = orm.DBObjectMeta - - id = orm.db_field(schema='str', is_primary=True) - test = orm.db_field(schema='str') - - t1 = TestDBObject(id='1', test='test') - - t2 = TestDBObject(id='2', test='test') - self.assertNotEqual(t1, t2) - - t2 = TestDBObject(id='1', test='test2') - self.assertNotEqual(t1, t2) - - t2 = TestDBObject(id='1', test='test') - self.assertEqual(t1, t2) - - -class TestORMRelation(BaseResourceTest): - def test_children_value(self): - class TestDBRelatedObject(orm.DBObject): - _collection = base.BaseGraphDB.COLLECTIONS.input - __metaclass__ = orm.DBObjectMeta - - id = orm.db_field(schema='str', is_primary=True) - - class TestDBObject(orm.DBObject): - _collection = base.BaseGraphDB.COLLECTIONS.resource - __metaclass__ = orm.DBObjectMeta - - id = orm.db_field(schema='str', is_primary=True) - related = orm.db_related_field( - base.BaseGraphDB.RELATION_TYPES.resource_input, - TestDBRelatedObject - ) - - r1 = TestDBRelatedObject(id='1') - r1.save() - r2 = TestDBRelatedObject(id='2') - r2.save() - - o = TestDBObject(id='a') - o.save() - - self.assertSetEqual(o.related.as_set(), set()) - - o.related.add(r1) - self.assertSetEqual(o.related.as_set(), {r1}) - - o.related.add(r2) - self.assertSetEqual(o.related.as_set(), {r1, r2}) - - o.related.remove(r2) - self.assertSetEqual(o.related.as_set(), {r1}) - - o.related.add(r2) - self.assertSetEqual(o.related.as_set(), {r1, r2}) - - o.related.remove(r1, r2) - self.assertSetEqual(o.related.as_set(), set()) - - o.related.add(r1, r2) - self.assertSetEqual(o.related.as_set(), {r1, r2}) - - with self.assertRaisesRegexp(errors.SolarError, '.*incompatible type.*'): - o.related.add(o) - - def test_relation_to_self(self): - class TestDBObject(orm.DBObject): - _collection = base.BaseGraphDB.COLLECTIONS.input - __metaclass__ = orm.DBObjectMeta - - id = orm.db_field(schema='str', is_primary=True) - related = orm.db_related_field( - base.BaseGraphDB.RELATION_TYPES.input_to_input, - 'TestDBObject' - ) - - o1 = TestDBObject(id='1') - o1.save() - o2 = TestDBObject(id='2') - o2.save() - o3 = TestDBObject(id='2') - o3.save() - - o1.related.add(o2) - o2.related.add(o3) - - self.assertEqual(o1.related.as_set(), {o2}) - self.assertEqual(o2.related.as_set(), {o3}) - - -class TestResourceORM(BaseResourceTest): - def test_save(self): - r = orm.DBResource(id='test1', name='test1', base_path='x') - r.save() - - rr = resource.load(r.id) - - self.assertEqual(r, rr.db_obj) - - def test_add_input(self): - r = orm.DBResource(id='test1', name='test1', base_path='x') - r.save() - - r.add_input('ip', 'str!', '10.0.0.2') - self.assertEqual(len(r.inputs.as_set()), 1) - - def test_delete_resource(self): - r = orm.DBResource(id='test1', name='test1', base_path='x') - r.save() - - r.add_input('ip', 'str!', '10.0.0.2') - - -class TestResourceInputORM(BaseResourceTest): - def test_backtrack_simple(self): - sample_meta_dir = self.make_resource_meta(""" -id: sample -handler: ansible -version: 1.0.0 -input: - value: - schema: str! - value: - """) - - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'value': 'x'} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, {'value': 'y'} - ) - sample3 = self.create_resource( - 'sample3', sample_meta_dir, {'value': 'z'} - ) - vi = sample2.resource_inputs()['value'] - self.assertEqual(vi.backtrack_value_emitter(), vi) - - # sample1 -> sample2 - signals.connect(sample1, sample2) - self.assertEqual(vi.backtrack_value_emitter(), - sample1.resource_inputs()['value']) - - # sample3 -> sample1 -> sample2 - signals.connect(sample3, sample1) - self.assertEqual(vi.backtrack_value_emitter(), - sample3.resource_inputs()['value']) - - # sample2 disconnected - signals.disconnect(sample1, sample2) - self.assertEqual(vi.backtrack_value_emitter(), vi) - - def test_backtrack_list(self): - sample_meta_dir = self.make_resource_meta(""" -id: sample -handler: ansible -version: 1.0.0 -input: - value: - schema: str! - value: - """) - sample_list_meta_dir = self.make_resource_meta(""" -id: sample_list -handler: ansible -version: 1.0.0 -input: - values: - schema: [str!] - value: - """) - - sample_list = self.create_resource( - 'sample_list', sample_list_meta_dir - ) - vi = sample_list.resource_inputs()['values'] - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'value': 'x'} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, {'value': 'y'} - ) - sample3 = self.create_resource( - 'sample3', sample_meta_dir, {'value': 'z'} - ) - self.assertEqual(vi.backtrack_value_emitter(), vi) - - # [sample1] -> sample_list - signals.connect(sample1, sample_list, {'value': 'values'}) - self.assertEqual(vi.backtrack_value_emitter(), - [sample1.resource_inputs()['value']]) - - # [sample3, sample1] -> sample_list - signals.connect(sample3, sample_list, {'value': 'values'}) - self.assertSetEqual(set(vi.backtrack_value_emitter()), - set([sample1.resource_inputs()['value'], - sample3.resource_inputs()['value']])) - - # sample2 disconnected - signals.disconnect(sample1, sample_list) - self.assertEqual(vi.backtrack_value_emitter(), - [sample3.resource_inputs()['value']]) - - def test_backtrack_dict(self): - sample_meta_dir = self.make_resource_meta(""" -id: sample -handler: ansible -version: 1.0.0 -input: - value: - schema: str! - value: - """) - sample_dict_meta_dir = self.make_resource_meta(""" -id: sample_dict -handler: ansible -version: 1.0.0 -input: - value: - schema: {a: str!} - value: - """) - - sample_dict = self.create_resource( - 'sample_dict', sample_dict_meta_dir - ) - vi = sample_dict.resource_inputs()['value'] - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'value': 'x'} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, {'value': 'z'} - ) - self.assertEqual(vi.backtrack_value_emitter(), vi) - - # {a: sample1} -> sample_dict - signals.connect(sample1, sample_dict, {'value': 'value:a'}) - self.assertDictEqual(vi.backtrack_value_emitter(), - {'a': sample1.resource_inputs()['value']}) - - # {a: sample2} -> sample_dict - signals.connect(sample2, sample_dict, {'value': 'value:a'}) - self.assertDictEqual(vi.backtrack_value_emitter(), - {'a': sample2.resource_inputs()['value']}) - - # sample2 disconnected - signals.disconnect(sample2, sample_dict) - self.assertEqual(vi.backtrack_value_emitter(), vi) - - def test_backtrack_dict_list(self): - sample_meta_dir = self.make_resource_meta(""" -id: sample -handler: ansible -version: 1.0.0 -input: - value: - schema: str! - value: - """) - sample_dict_list_meta_dir = self.make_resource_meta(""" -id: sample_dict_list -handler: ansible -version: 1.0.0 -input: - value: - schema: [{a: str!}] - value: - """) - - sample_dict_list = self.create_resource( - 'sample_dict', sample_dict_list_meta_dir - ) - vi = sample_dict_list.resource_inputs()['value'] - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'value': 'x'} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, {'value': 'y'} - ) - sample3 = self.create_resource( - 'sample3', sample_meta_dir, {'value': 'z'} - ) - self.assertEqual(vi.backtrack_value_emitter(), vi) - - # [{a: sample1}] -> sample_dict_list - signals.connect(sample1, sample_dict_list, {'value': 'value:a'}) - self.assertListEqual(vi.backtrack_value_emitter(), - [{'a': sample1.resource_inputs()['value']}]) - - # [{a: sample1}, {a: sample3}] -> sample_dict_list - signals.connect(sample3, sample_dict_list, {'value': 'value:a'}) - self.assertItemsEqual(vi.backtrack_value_emitter(), - [{'a': sample1.resource_inputs()['value']}, - {'a': sample3.resource_inputs()['value']}]) - - # [{a: sample1}, {a: sample2}] -> sample_dict_list - signals.connect(sample2, sample_dict_list, {'value': 'value:a|sample3'}) - self.assertItemsEqual(vi.backtrack_value_emitter(), - [{'a': sample1.resource_inputs()['value']}, - {'a': sample2.resource_inputs()['value']}]) - - # sample2 disconnected - signals.disconnect(sample2, sample_dict_list) - self.assertEqual(vi.backtrack_value_emitter(), - [{'a': sample1.resource_inputs()['value']}]) - - -class TestEventORM(BaseResourceTest): - - def test_return_emtpy_set(self): - r = orm.DBResourceEvents(id='test1') - r.save() - self.assertEqual(r.events.as_set(), set()) - - def test_save_and_load_by_parent(self): - ev = orm.DBEvent( - parent='n1', - parent_action='run', - state='success', - child_action='run', - child='n2', - etype='dependency') - ev.save() - - rst = orm.DBEvent.load(ev.id) - self.assertEqual(rst, ev) - - def test_save_several(self): - ev = orm.DBEvent( - parent='n1', - parent_action='run', - state='success', - child_action='run', - child='n2', - etype='dependency') - ev.save() - ev1 = orm.DBEvent( - parent='n1', - parent_action='run', - state='success', - child_action='run', - child='n3', - etype='dependency') - ev1.save() - self.assertEqual(len(orm.DBEvent.load_all()), 2) - - def test_removal_of_event(self): - r = orm.DBResourceEvents(id='test1') - r.save() - - ev = orm.DBEvent( - parent='test1', - parent_action='run', - state='success', - child_action='run', - child='n2', - etype='dependency') - ev.save() - r.events.add(ev) - - self.assertEqual(r.events.as_set(), {ev}) - ev.delete() - - r = orm.DBResourceEvents.load('test1') - self.assertEqual(r.events.as_set(), set()) From db86b75d353e86bee6fbd666b30f0080302fdf85 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 6 Nov 2015 15:55:23 +0200 Subject: [PATCH 153/191] Dockerfile and run.sh for solar-celery image --- Dockerfile | 25 +++++++++++++++++++++++++ run.sh | 12 ++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 Dockerfile create mode 100755 run.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e2c17e28 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM ubuntu:14.04 + +WORKDIR / + +RUN apt-get update +# Install pip's dependency: setuptools: +RUN apt-get install -y python python-dev python-distribute python-pip +RUN pip install ansible + +ADD bootstrap/playbooks/celery.yaml /celery.yaml +ADD solar /solar +ADD solard /solard +ADD resources /resources +ADD templates /templates +ADD run.sh /run.sh + + +RUN apt-get install -y libffi-dev libssl-dev +RUN pip install riak peewee +RUN pip install -U setuptools>=17.1 +RUN cd /solar && python setup.py install +RUN cd /solard && python setup.py install +RUN ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags slave + +CMD ["/run.sh"] diff --git a/run.sh b/run.sh new file mode 100755 index 00000000..ef5ce6a7 --- /dev/null +++ b/run.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# required for ease of development +pushd /solar +python setup.py develop +popd + +pushd /solard +python setup.py develop +popd + +tail -f /var/run/celery/*.log From 4a3ef360a21e9c35d9dbb61f8a1e26e5ee6ac093 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 6 Nov 2015 15:55:44 +0200 Subject: [PATCH 154/191] Basic config to provide dynamic path for resources --- solar/solar/config.py | 84 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 solar/solar/config.py diff --git a/solar/solar/config.py b/solar/solar/config.py new file mode 100644 index 00000000..81e0a33f --- /dev/null +++ b/solar/solar/config.py @@ -0,0 +1,84 @@ + +import os + +class DictWrp(object): + + def __init__(self, store): + self.store = store + + def __getitem__(self, item): + return self.store[item] + + __getattr__ = __getitem__ + + +class Conf(object): + + def __init__(self): + self.store = {} + self.types = {} + + def add(self, name, _type=None, default=None): + if default: + if hasattr(default, '__call__'): + val = default() + else: + val = default + _type = type(val) + self.types[name] = _type + if '.' in name: + parent, child = name.split('.') + if parent not in self.store: + self.store[parent] = {} + self.types[parent] = dict + self.store[parent][child] = val + else: + self.store[name] = val + + def __getitem__(self, item): + val = self.store[item] + if isinstance(val, dict): + return DictWrp(val) + return val + + def __setitem__(self, item, val): + stack = item.split('.') + while stack[:-1]: + nxt = stack.pop(0) + store = self.store[nxt] + store[stack[-1]] = val + + def init_env(self): + for var, _type in self.types.iteritems(): + if '.' in var: + variable = '_'.join(var.split('.')) + else: + variable = var + env_var = variable.upper() + val = os.getenv(env_var) + if not val: continue + + if _type == list: + val_lst = val.split('|') + self.store[var].extend(val_lst) + elif _type == dict: + pass + else: + self.store[var] = val + + + + __getattr__ = __getitem__ + + +C = Conf() +C.add('redis.port', default='6379') +C.add('redis.host', default='10.0.0.2') +C.add('riak.host', default='10.0.0.2') +C.add('riak.port', default='8087') +C.add('riak.protocol', default='pbc') +C.add('directories', default=[os.path.dirname(os.path.realpath(__file__))]) +C.init_env() + +if __name__ == '__main__': + print C.store From 866ca840b060ed127fd6321af13fdfa9f16d0331 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 6 Nov 2015 15:56:42 +0200 Subject: [PATCH 155/191] Docker compose for development --- devenv_compose.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 devenv_compose.yaml diff --git a/devenv_compose.yaml b/devenv_compose.yaml new file mode 100644 index 00000000..f17c792b --- /dev/null +++ b/devenv_compose.yaml @@ -0,0 +1,13 @@ +solar-celery: + build: . + volumes: + - .solar:/solar + - .solard:/solard + - .templates:/templates + - .resources:/resources + environment: + - REDIS_HOST=10.0.0.2 + - REDIS_PORT=6379 + - RIAK_HOST=10.0.0.2 + - RIAK_PORT=8087 + - DIRECTORIES=/resources/templates From b5be49e33515abdcdf6d03f0d5193fa31c6e50ec Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Mon, 9 Nov 2015 10:32:36 +0200 Subject: [PATCH 156/191] Use docker compose --- devenv_compose.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/devenv_compose.yaml b/devenv_compose.yaml index f17c792b..0d4329f3 100644 --- a/devenv_compose.yaml +++ b/devenv_compose.yaml @@ -11,3 +11,12 @@ solar-celery: - RIAK_HOST=10.0.0.2 - RIAK_PORT=8087 - DIRECTORIES=/resources/templates +riak: + image: tutum/riak + ports: + - 8087:8087 + - 8098:8098 +redis: + image: tutum/redis + ports: + - 6379:6379 From c598ed539967d9a332e8bdbda20f308f6fb73cbc Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Mon, 9 Nov 2015 11:31:30 +0200 Subject: [PATCH 157/191] Use config.py:C in solar and fix compose file --- devenv_compose.yaml | 22 ---------------------- docker-compose.yml | 29 +++++++++++++++++++++++++++++ solar/solar/config.py | 1 - solar/solar/dblayer/__init__.py | 4 +++- solar/solar/orchestration/runner.py | 8 ++++++-- solar/solar/orchestration/tasks.py | 3 --- 6 files changed, 38 insertions(+), 29 deletions(-) delete mode 100644 devenv_compose.yaml create mode 100644 docker-compose.yml diff --git a/devenv_compose.yaml b/devenv_compose.yaml deleted file mode 100644 index 0d4329f3..00000000 --- a/devenv_compose.yaml +++ /dev/null @@ -1,22 +0,0 @@ -solar-celery: - build: . - volumes: - - .solar:/solar - - .solard:/solard - - .templates:/templates - - .resources:/resources - environment: - - REDIS_HOST=10.0.0.2 - - REDIS_PORT=6379 - - RIAK_HOST=10.0.0.2 - - RIAK_PORT=8087 - - DIRECTORIES=/resources/templates -riak: - image: tutum/riak - ports: - - 8087:8087 - - 8098:8098 -redis: - image: tutum/redis - ports: - - 6379:6379 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..57070c27 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,29 @@ +solar-celery: + image: solar/solar-celery + # path inside of the container should be exactly the same as outside + # because solar uses absolute path to find resoruce actions files + volumes: + - /vagrant/.vagrant:/vagrant/.vagrant + - /vagrant/solar:/solar + - /vagrant/solard:/solard + - /vagrant/templates:/vagrant/templates + - /vagrant/resources:/vagrant/resources + environment: + - REDIS_HOST=10.0.0.2 + - REDIS_PORT=6379 + - RIAK_HOST=10.0.0.2 + - RIAK_PORT=8087 + # links are not used for configuration because we can rely on non-container + # based datastores + links: + - riak + - redis +riak: + image: tutum/riak + ports: + - 8087:8087 + - 8098:8098 +redis: + image: tutum/redis + ports: + - 6379:6379 diff --git a/solar/solar/config.py b/solar/solar/config.py index 81e0a33f..eaebdd04 100644 --- a/solar/solar/config.py +++ b/solar/solar/config.py @@ -77,7 +77,6 @@ C.add('redis.host', default='10.0.0.2') C.add('riak.host', default='10.0.0.2') C.add('riak.port', default='8087') C.add('riak.protocol', default='pbc') -C.add('directories', default=[os.path.dirname(os.path.realpath(__file__))]) C.init_env() if __name__ == '__main__': diff --git a/solar/solar/dblayer/__init__.py b/solar/solar/dblayer/__init__.py index 7ed377c1..1a3f54c4 100644 --- a/solar/solar/dblayer/__init__.py +++ b/solar/solar/dblayer/__init__.py @@ -1,7 +1,9 @@ from solar.dblayer.model import ModelMeta from solar.dblayer.riak_client import RiakClient +from solar.config import C -client = RiakClient(protocol='pbc', host='10.0.0.2', pb_port=8087) +client = RiakClient( + protocol=C.riak.protcol, host=C.riak.host, pb_port=C.riak.port) # client = RiakClient(protocol='http', host='10.0.0.2', http_port=8098) ModelMeta.setup(client) diff --git a/solar/solar/orchestration/runner.py b/solar/solar/orchestration/runner.py index af4ba9c9..448a30f1 100644 --- a/solar/solar/orchestration/runner.py +++ b/solar/solar/orchestration/runner.py @@ -14,9 +14,13 @@ from celery import Celery +from solar.config import C + +_url = 'redis://{}:{}/1'.format(C.redis.host, C.redis.port) + app = Celery( include=['solar.system_log.tasks', 'solar.orchestration.tasks'], - backend='redis://10.0.0.2:6379/1', - broker='redis://10.0.0.2:6379/1') + backend=_url, + broker=_url) app.conf.update(CELERY_ACCEPT_CONTENT = ['json']) app.conf.update(CELERY_TASK_SERIALIZER = 'json') diff --git a/solar/solar/orchestration/tasks.py b/solar/solar/orchestration/tasks.py index 7c558a2a..85fa960f 100644 --- a/solar/solar/orchestration/tasks.py +++ b/solar/solar/orchestration/tasks.py @@ -17,7 +17,6 @@ import subprocess import time from celery.app import task -import redis from solar.orchestration import graph from solar.core import actions @@ -28,8 +27,6 @@ from solar.orchestration.traversal import traverse from solar.orchestration import limits from solar.orchestration import executor -r = redis.StrictRedis(host='10.0.0.2', port=6379, db=1) - __all__ = ['solar_resource', 'cmd', 'sleep', 'error', 'fault_tolerance', 'schedule_start', 'schedule_next'] From b4e8b119b934cb5ec367db2d8120a7a1415276a3 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Mon, 9 Nov 2015 12:00:17 +0200 Subject: [PATCH 158/191] Fix run.sh to start celery --- run.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run.sh b/run.sh index ef5ce6a7..ac857907 100755 --- a/run.sh +++ b/run.sh @@ -9,4 +9,7 @@ pushd /solard python setup.py develop popd +#used only to start celery on docker +ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags slave + tail -f /var/run/celery/*.log From 1835ac5b382b052a3cc4806e3fa2be0a76005fb7 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Mon, 9 Nov 2015 12:35:08 +0200 Subject: [PATCH 159/191] Make redis run without password --- docker-compose.yml | 2 ++ solar/solar/dblayer/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 57070c27..886dd7d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,3 +27,5 @@ redis: image: tutum/redis ports: - 6379:6379 + environment: + - REDIS_PASS=**None** diff --git a/solar/solar/dblayer/__init__.py b/solar/solar/dblayer/__init__.py index 1a3f54c4..bf4c7e00 100644 --- a/solar/solar/dblayer/__init__.py +++ b/solar/solar/dblayer/__init__.py @@ -3,7 +3,7 @@ from solar.dblayer.riak_client import RiakClient from solar.config import C client = RiakClient( - protocol=C.riak.protcol, host=C.riak.host, pb_port=C.riak.port) + protocol=C.riak.protocol, host=C.riak.host, pb_port=C.riak.port) # client = RiakClient(protocol='http', host='10.0.0.2', http_port=8098) ModelMeta.setup(client) From 77c4dee5f939dd97a5e0ef57f1c275334dd85b07 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Mon, 9 Nov 2015 12:53:51 +0200 Subject: [PATCH 160/191] Use solarproject repository --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 886dd7d8..ad5f87d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ solar-celery: - image: solar/solar-celery + image: solarproject/solar-celery # path inside of the container should be exactly the same as outside # because solar uses absolute path to find resoruce actions files volumes: From db0e7881975100d6b6fdcf9e94a84c070e6ea79d Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 17 Nov 2015 15:44:42 +0200 Subject: [PATCH 161/191] Rewrite config using dotted dict object and add loading from files --- .config | 11 ++++ .gitignore | 1 + solar/requirements.txt | 1 + solar/solar/config.py | 111 ++++++++++++-------------------- solar/solar/dblayer/__init__.py | 26 +++++++- solar/solar/test/conftest.py | 15 ----- 6 files changed, 78 insertions(+), 87 deletions(-) create mode 100644 .config diff --git a/.config b/.config new file mode 100644 index 00000000..1aaf8b4d --- /dev/null +++ b/.config @@ -0,0 +1,11 @@ +dblayer: riak +redis: + host: localhost + port: '6379' +riak: + host: localhost + port: '8087' + protocol: pbc +sqlite: + backend: memory + location: ':memory:' diff --git a/.gitignore b/.gitignore index 6187e89f..038b03ca 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,4 @@ solar/.coverage # pytest cache solar/.cache +.config.override diff --git a/solar/requirements.txt b/solar/requirements.txt index 955086fd..c245b255 100644 --- a/solar/requirements.txt +++ b/solar/requirements.txt @@ -19,3 +19,4 @@ celery mock multipledispatch==0.4.8 pydot +bunch diff --git a/solar/solar/config.py b/solar/solar/config.py index eaebdd04..6624675d 100644 --- a/solar/solar/config.py +++ b/solar/solar/config.py @@ -1,83 +1,56 @@ import os +import yaml +from bunch import Bunch -class DictWrp(object): +CWD = os.getcwd() - def __init__(self, store): - self.store = store - - def __getitem__(self, item): - return self.store[item] - - __getattr__ = __getitem__ +C = Bunch() +C.redis = Bunch(port='6379', host='10.0.0.2') +C.riak = Bunch(port='8087', host='10.0.0.2', protocol='pbc') +C.sqlite = Bunch(backend='memory', location=':memory:') +C.dblayer = 'riak' -class Conf(object): - - def __init__(self): - self.store = {} - self.types = {} - - def add(self, name, _type=None, default=None): - if default: - if hasattr(default, '__call__'): - val = default() +def _lookup_vals(setter, config, prefix=None): + for key, val in config.iteritems(): + if prefix is None: + sub = [key] else: - val = default - _type = type(val) - self.types[name] = _type - if '.' in name: - parent, child = name.split('.') - if parent not in self.store: - self.store[parent] = {} - self.types[parent] = dict - self.store[parent][child] = val - else: - self.store[name] = val - - def __getitem__(self, item): - val = self.store[item] - if isinstance(val, dict): - return DictWrp(val) - return val - - def __setitem__(self, item, val): - stack = item.split('.') - while stack[:-1]: - nxt = stack.pop(0) - store = self.store[nxt] - store[stack[-1]] = val - - def init_env(self): - for var, _type in self.types.iteritems(): - if '.' in var: - variable = '_'.join(var.split('.')) + sub = prefix + [key] + if isinstance(val, Bunch): + _lookup_vals(setter, val, sub) else: - variable = var - env_var = variable.upper() - val = os.getenv(env_var) - if not val: continue + setter(config, sub) - if _type == list: - val_lst = val.split('|') - self.store[var].extend(val_lst) - elif _type == dict: - pass - else: - self.store[var] = val +def from_configs(): + paths = [ + os.path.join(CWD, '.config'), + os.path.join(CWD, '.config.override') + ] + data = {} + for path in paths: + with open(path) as f: + loaded = yaml.load(f) + if loaded: + data.update(loaded) + def _setter(config, path): + vals = data + for key in path: + vals = vals[key] + config[path[-1]] = vals + _lookup_vals(_setter, C) +def from_env(): + def _setter(config, path): + env_key = '_'.join(path).upper() + if env_key in os.environ: + config[path[-1]] = os.environ[env_key] + _lookup_vals(_setter, C) - __getattr__ = __getitem__ - - -C = Conf() -C.add('redis.port', default='6379') -C.add('redis.host', default='10.0.0.2') -C.add('riak.host', default='10.0.0.2') -C.add('riak.port', default='8087') -C.add('riak.protocol', default='pbc') -C.init_env() +from_configs() +from_env() if __name__ == '__main__': - print C.store + print C diff --git a/solar/solar/dblayer/__init__.py b/solar/solar/dblayer/__init__.py index bf4c7e00..09e0e7d6 100644 --- a/solar/solar/dblayer/__init__.py +++ b/solar/solar/dblayer/__init__.py @@ -2,9 +2,29 @@ from solar.dblayer.model import ModelMeta from solar.dblayer.riak_client import RiakClient from solar.config import C -client = RiakClient( - protocol=C.riak.protocol, host=C.riak.host, pb_port=C.riak.port) -# client = RiakClient(protocol='http', host='10.0.0.2', http_port=8098) +if C.dblayer == 'sqlite': + from solar.dblayer.sql_client import SqlClient + if C.sqlite.backend == 'memory': + client = SqlClient(C.sqlite.location, threadlocals=False, autocommit=False) + elif C.sqlite.backend == 'file': + client = SqlClient(C.sqlite.location, threadlocals=True, + autocommit=False, pragmas=(('journal_mode', 'WAL'), + ('synchronous', 'NORMAL'))) + else: + raise Exception('Unknown sqlite backend %s', C.sqlite.backend) + +elif C.dblayer == 'riak': + from solar.dblayer.riak_client import RiakClient + if C.riak.protocol == 'pbc': + client = RiakClient( + protocol=C.riak.protocol, host=C.riak.host, pb_port=C.riak.port) + elif C.riak.protocol == 'http': + client = RiakClient( + protocol=C.riak.protocol, host=C.riak.host, http_port=C.riak.port) + else: + raise Exception('Unknown riak protocol %s', C.riak.protocol) +else: + raise Exception('Unknown dblayer backend %s', C.dblayer) ModelMeta.setup(client) diff --git a/solar/solar/test/conftest.py b/solar/solar/test/conftest.py index b8d62bbf..be3a5582 100644 --- a/solar/solar/test/conftest.py +++ b/solar/solar/test/conftest.py @@ -19,8 +19,6 @@ import time def patched_get_bucket_name(cls): return cls.__name__ + str(time.time()) - - @pytest.fixture(autouse=True) def setup(request): @@ -49,16 +47,3 @@ def pytest_runtest_call(item): Model.get_bucket_name = classmethod(patched_get_bucket_name) - -# from solar.dblayer.sql_client import SqlClient -# client = SqlClient(':memory:', threadlocals=False, autocommit=False) -# client = SqlClient('/tmp/blah.db', threadlocals=True, -# autocommit=False, pragmas=(('journal_mode', 'WAL'), -# ('synchronous', 'NORMAL'))) - -from solar.dblayer.riak_client import RiakClient -client = RiakClient(protocol='pbc', host='10.0.0.2', pb_port=8087) -# client = RiakClient(protocol='http', host='10.0.0.3', http_port=18098) - - -ModelMeta.setup(client) From 2b063520a75e1e56df597f90ab7768a165647286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 17 Nov 2015 18:39:05 +0100 Subject: [PATCH 162/191] py.test handles sessions --- solar/solar/test/base.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/solar/solar/test/base.py b/solar/solar/test/base.py index 0586d901..597fb14c 100644 --- a/solar/solar/test/base.py +++ b/solar/solar/test/base.py @@ -33,14 +33,9 @@ class BaseResourceTest(unittest.TestCase): def setUp(self): self.storage_dir = tempfile.mkdtemp() - for model in ModelMeta._defined_models: - model.bucket = get_bucket(None, model, ModelMeta) - ModelMeta.session_start() def tearDown(self): shutil.rmtree(self.storage_dir) - ModelMeta.session_end() - def make_resource_meta(self, meta_yaml): meta = yaml.load(meta_yaml) From e108b78f8bbe348efa461e2d48df6f8207846401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 17 Nov 2015 18:39:32 +0100 Subject: [PATCH 163/191] Fix test_signals --- solar/solar/dblayer/solar_models.py | 9 ++++--- solar/solar/test/test_signals.py | 41 ++++++++++++++++++----------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index ea5217e1..5c75509a 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from solar.dblayer.model import (Model, Field, IndexField, IndexFieldWrp, DBLayerException, @@ -7,7 +8,6 @@ from solar.dblayer.model import (Model, Field, IndexField, from types import NoneType from operator import itemgetter from enum import Enum -from itertools import groupby from uuid import uuid4 from collections import defaultdict @@ -595,15 +595,18 @@ class Resource(Model): other_inputs = other.inputs if mapping is None: return + if self == other: + raise Exception('Trying to connect value-.* to itself') solar_map(lambda (my_name, other_name): self._connect_single(other_inputs, other_name, my_name), mapping.iteritems(), concurrency=2) def disconnect(self, other, inputs): other_key = other.key edges = other.inputs._edges() - edges = filter(lambda (receiver, emitter, _): emitter[0] == other_key and receiver[1] in inputs, + edges = filter(lambda (emitter, receiver, _): receiver[0] == other_key + and emitter[1] in inputs and emitter[0] == self.name, edges) - inputs = [x[1][1] for x in edges] + inputs = ['{}|{}'.format(x[1][1], self.name) for x in edges] solar_map(other.inputs.disconnect, inputs, concurrency=2) def save(self, *args, **kwargs): diff --git a/solar/solar/test/test_signals.py b/solar/solar/test/test_signals.py index eb2bd5b1..b6bc0f26 100644 --- a/solar/solar/test/test_signals.py +++ b/solar/solar/test/test_signals.py @@ -16,6 +16,8 @@ import base from solar.core import signals as xs +import pytest + class TestBaseInput(base.BaseResourceTest): def test_no_self_connection(self): @@ -38,6 +40,7 @@ input: 'Trying to connect value-.* to itself'): xs.connect(sample, sample, {'value'}) + @pytest.mark.xfail(reason="No cycle detection in new_db") def test_no_cycles(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -102,7 +105,7 @@ input: # Check disconnect # TODO: should sample2.value be reverted to original value? - xs.disconnect(sample1, sample2) + sample1.disconnect(sample2) sample1.update({'values': {'a': 3}}) self.assertEqual( sample1.args['values'], @@ -210,6 +213,7 @@ input: sample2.update({'ip': '10.0.0.3'}) self.assertEqual(sample2.args['ip'], sample.args['ip']) + @pytest.mark.xfail(reason="No cycle detection in new_db") def test_circular_connection_prevention(self): # TODO: more complex cases sample_meta_dir = self.make_resource_meta(""" @@ -265,7 +269,7 @@ input: 'list-input-single', list_input_single_meta_dir, {'ips': []} ) - xs.connect(sample1, list_input_single, mapping={'ip': 'ips'}) + sample1.connect(list_input_single, mapping={'ip': 'ips'}) self.assertItemsEqual( #[ip['value'] for ip in list_input_single.args['ips']], list_input_single.args['ips'], @@ -278,7 +282,7 @@ input: # [(sample1.args['ip'].attached_to.name, 'ip')] #) - xs.connect(sample2, list_input_single, mapping={'ip': 'ips'}) + sample2.connect(list_input_single, mapping={'ip': 'ips'}) self.assertItemsEqual( #[ip['value'] for ip in list_input_single.args['ips']], list_input_single.args['ips'], @@ -305,7 +309,7 @@ input: ) # Test disconnect - xs.disconnect(sample2, list_input_single) + sample2.disconnect(list_input_single) self.assertItemsEqual( #[ip['value'] for ip in list_input_single.args['ips']], list_input_single.args['ips'], @@ -395,7 +399,7 @@ input: #) # Test disconnect - xs.disconnect(sample2, list_input_multi) + sample2.disconnect(list_input_multi) self.assertItemsEqual( #[ip['value'] for ip in list_input_multi.args['ips']], list_input_multi.args['ips'], @@ -407,6 +411,7 @@ input: [sample1.args['port']] ) + @pytest.mark.xfail(reason="Nested lists are not supported in new_db") def test_nested_list_input(self): """ Make sure that single input change is propagated along the chain of @@ -463,9 +468,9 @@ input: 'list-input-nested', list_input_nested_meta_dir, ) - xs.connect(sample1, list_input, mapping={'ip': 'ips', 'port': 'ports'}) - xs.connect(sample2, list_input, mapping={'ip': 'ips', 'port': 'ports'}) - xs.connect(list_input, list_input_nested, mapping={'ips': 'ipss', 'ports': 'portss'}) + sample1.connect(list_input, mapping={'ip': 'ips', 'port': 'ports'}) + sample2.connect(list_input, mapping={'ip': 'ips', 'port': 'ports'}) + list_input.connect(list_input_nested, mapping={'ips': 'ipss', 'ports': 'portss'}) self.assertListEqual( #[ips['value'] for ips in list_input_nested.args['ipss']], list_input_nested.args['ipss'], @@ -491,6 +496,7 @@ input: class TestHashInput(base.BaseResourceTest): + @pytest.mark.xfail(reason="Connect should raise an error if already connected") def test_hash_input_basic(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -535,6 +541,8 @@ input: {'ip': sample1.args['ip'], 'port': sample1.args['port']}, receiver.args['server'], ) + # XXX: We need to disconnect first + # XXX: it should raise error when connecting already connected inputs xs.connect(sample2, receiver, mapping={'ip': 'server:ip'}) self.assertDictEqual( {'ip': sample2.args['ip'], 'port': sample1.args['port']}, @@ -546,6 +554,7 @@ input: receiver.args['server'], ) + @pytest.mark.xfail(reason="Not supported case when some elements of dict are connected and some not") def test_hash_input_mixed(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -574,7 +583,7 @@ input: receiver = self.create_resource( 'receiver', receiver_meta_dir, args={'server': {'port': 5001}} ) - xs.connect(sample, receiver, mapping={'ip': 'server:ip'}) + sample.connect(receiver, mapping={'ip': 'server:ip'}) self.assertDictEqual( {'ip': sample.args['ip'], 'port': 5001}, receiver.args['server'], @@ -627,12 +636,13 @@ input: {'ip': sample2.args['ip'], 'port': sample2.args['port']}], receiver.args['server'], ) - xs.disconnect(sample1, receiver) + sample1.disconnect(receiver) self.assertItemsEqual( [{'ip': sample2.args['ip'], 'port': sample2.args['port']}], receiver.args['server'], ) + @pytest.mark.xfail(reason='no idea') def test_hash_input_with_multiple_connections(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -671,6 +681,7 @@ input: receiver.args['server'], ) + @pytest.mark.xfail(reason='no idea') def test_hash_input_multiple_resources_with_tag_connect(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -702,8 +713,8 @@ input: receiver = self.create_resource( 'receiver', receiver_meta_dir ) - xs.connect(sample1, receiver, mapping={'ip': 'server:ip'}) - xs.connect(sample2, receiver, mapping={'port': 'server:port|sample1'}) + sample1.connect(receiver, mapping={'ip': 'server:ip'}) + sample2.connect(receiver, mapping={'port': 'server:port|sample1'}) self.assertItemsEqual( [{'ip': sample1.args['ip'], 'port': sample2.args['port']}], receiver.args['server'], @@ -711,7 +722,7 @@ input: sample3 = self.create_resource( 'sample3', sample_meta_dir, args={'ip': '10.0.0.3', 'port': 5002} ) - xs.connect(sample3, receiver, mapping={'ip': 'server:ip', 'port': 'server:port'}) + sample3.connect(receiver, mapping={'ip': 'server:ip', 'port': 'server:port'}) self.assertItemsEqual( [{'ip': sample1.args['ip'], 'port': sample2.args['port']}, {'ip': sample3.args['ip'], 'port': sample3.args['port']}], @@ -720,14 +731,14 @@ input: sample4 = self.create_resource( 'sample4', sample_meta_dir, args={'ip': '10.0.0.4', 'port': 5003} ) - xs.connect(sample4, receiver, mapping={'port': 'server:port|sample3'}) + sample4.connect(receiver, mapping={'port': 'server:port|sample3'}) self.assertItemsEqual( [{'ip': sample1.args['ip'], 'port': sample2.args['port']}, {'ip': sample3.args['ip'], 'port': sample4.args['port']}], receiver.args['server'], ) # There can be no sample3 connections left now - xs.connect(sample4, receiver, mapping={'ip': 'server:ip|sample3'}) + sample4.connect(receiver, mapping={'ip': 'server:ip|sample3'}) self.assertItemsEqual( [{'ip': sample1.args['ip'], 'port': sample2.args['port']}, {'ip': sample4.args['ip'], 'port': sample4.args['port']}], From 438f430f415a3078df23ec24121de18ec0514f05 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 17 Nov 2015 18:39:53 +0100 Subject: [PATCH 164/191] Make other config files optional --- solar/solar/config.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/solar/solar/config.py b/solar/solar/config.py index 6624675d..39d129e1 100644 --- a/solar/solar/config.py +++ b/solar/solar/config.py @@ -1,4 +1,3 @@ - import os import yaml from bunch import Bunch @@ -13,23 +12,36 @@ C.dblayer = 'riak' def _lookup_vals(setter, config, prefix=None): - for key, val in config.iteritems(): - if prefix is None: - sub = [key] - else: - sub = prefix + [key] - if isinstance(val, Bunch): - _lookup_vals(setter, val, sub) - else: - setter(config, sub) + for key, val in config.iteritems(): + if prefix is None: + sub = [key] + else: + sub = prefix + [key] + if isinstance(val, Bunch): + _lookup_vals(setter, val, sub) + else: + setter(config, sub) + def from_configs(): + paths = [ os.path.join(CWD, '.config'), os.path.join(CWD, '.config.override') ] data = {} - for path in paths: + + def _load_from_path(data, path): + with open(path) as f: + loaded = yaml.load(f) + if loaded: + data.update(loaded) + + _load_from_path(data, paths[0]) + + for path in paths[1:]: + if not os.path.exists(path): + continue with open(path) as f: loaded = yaml.load(f) if loaded: @@ -51,6 +63,3 @@ def from_env(): from_configs() from_env() - -if __name__ == '__main__': - print C From ee2fdba3f9b1134d2df6197e5976f44de6e79a56 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 17 Nov 2015 18:46:32 +0100 Subject: [PATCH 165/191] Allow to set major config in SOLAR_CONFIG env variable --- solar/solar/config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/solar/solar/config.py b/solar/solar/config.py index 39d129e1..87284557 100644 --- a/solar/solar/config.py +++ b/solar/solar/config.py @@ -26,7 +26,7 @@ def _lookup_vals(setter, config, prefix=None): def from_configs(): paths = [ - os.path.join(CWD, '.config'), + os.getenv('SOLAR_CONFIG', os.path.join(CWD, '.config')), os.path.join(CWD, '.config.override') ] data = {} @@ -37,9 +37,7 @@ def from_configs(): if loaded: data.update(loaded) - _load_from_path(data, paths[0]) - - for path in paths[1:]: + for path in paths: if not os.path.exists(path): continue with open(path) as f: From 8b3685f6af728ac252fc542c41956cd1e53fe99f Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 17 Nov 2015 19:05:45 +0100 Subject: [PATCH 166/191] Add riak to requirements --- solar/requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/solar/requirements.txt b/solar/requirements.txt index c245b255..2cc1ed95 100644 --- a/solar/requirements.txt +++ b/solar/requirements.txt @@ -20,3 +20,7 @@ mock multipledispatch==0.4.8 pydot bunch +# if you want to use riak backend then +riak +# if you want to use sql backend then +# peewee From 11336fd7ddd1e5202e88ffec2b81d37fc269ace1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 17 Nov 2015 19:41:36 +0100 Subject: [PATCH 167/191] Restore resources fixture --- solar/solar/test/conftest.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/solar/solar/test/conftest.py b/solar/solar/test/conftest.py index be3a5582..9240c582 100644 --- a/solar/solar/test/conftest.py +++ b/solar/solar/test/conftest.py @@ -11,14 +11,35 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -from solar.dblayer.model import Model, ModelMeta, get_bucket -import pytest +import os import time +from solar.core.resource import Resource +from solar.dblayer.model import Model, ModelMeta, get_bucket + +import pytest + def patched_get_bucket_name(cls): return cls.__name__ + str(time.time()) +@pytest.fixture +def resources(): + base_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'resource_fixtures') + + node_path = os.path.join(base_path, 'node') + node1 = Resource('node1', node_path, args={'ip':'10.0.0.1'}) + node2 = Resource('node2', node_path, args={'ip':'10.0.0.2'}) + + base_service_path = os.path.join(base_path, 'base_service') + service1 = Resource('service1', base_service_path) + return {'node1' : node1, + 'node2' : node2, + 'service1': service1 + } + @pytest.fixture(autouse=True) def setup(request): From e974855ce3c9827060046d736c4e4254552e9e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 17 Nov 2015 19:57:32 +0100 Subject: [PATCH 168/191] Set config path. Run only tests from test dir --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 7fd73f86..d12ede1d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -34,4 +34,4 @@ pip-accel install -r solar/test-requirements.txt pushd solar -PYTHONPATH=$WORKSPACE/solar CONFIG_FILE=$CONFIG_FILE py.test --cov=solar -s solar +SOLAR_CONFIG=../.config PYTHONPATH=$WORKSPACE/solar CONFIG_FILE=$CONFIG_FILE py.test --cov=solar -s solar/test From bfccbd9e6237cec134b176e035271e9c876dc069 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 18 Nov 2015 13:29:30 +0100 Subject: [PATCH 169/191] More fixes in test_signals --- solar/solar/dblayer/solar_models.py | 88 +++++++++++++++++++++++------ solar/solar/test/test_signals.py | 3 - 2 files changed, 72 insertions(+), 19 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 5c75509a..477c880a 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -130,20 +130,52 @@ class InputsFieldWrp(IndexFieldWrp): def _connect_other_simple(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): other_ind_name = '{}_emit_bin'.format(self.fname) - other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, - other_inp_name, - my_resource.key, - my_inp_name) real_my_type = self._input_type(my_resource, my_inp_name) - if real_my_type == InputTypes.simple: + if real_my_type == InputTypes.simple or ':' not in my_inp_name: + other_ind_val = '{}|{}|{}|{}'.format(other_resource.key, + other_inp_name, + my_resource.key, + my_inp_name) for ind_name, ind_value in my_resource._riak_object.indexes: if ind_name == other_ind_name: - mr, mn = ind_value.rsplit('|')[2:] + try: + mr, mn = ind_value.rsplit('|')[2:] + except ValueError: + if len(ind_value.split('|')) == 6: + continue + else: + raise if mr == my_resource.key and mn == my_inp_name: my_resource._remove_index(ind_name, ind_value) break + elif real_my_type in (InputTypes.list_hash, InputTypes.hash, InputTypes.list): + my_key, my_val = my_inp_name.split(':', 1) + if '|' in my_val: + my_val, my_tag = my_val.split('|', 1) + else: + my_tag = other_resource.name + my_inp_name = my_key + other_ind_val = '{}|{}|{}|{}|{}|{}'.format(other_resource.key, + other_inp_name, + my_resource.key, + my_inp_name, + my_tag, + my_val) + for ind_name, ind_value in my_resource._riak_object.indexes: + if ind_name == other_ind_name: + try: + mr, mn, mt, mv = ind_value.rsplit('|')[2:] + except ValueError: + if len(ind_value.split('|')) == 4: + continue + else: + raise + if mr == my_resource.key and mn == my_inp_name \ + and mt == my_tag and mv == my_val: + my_resource._remove_index(ind_name, ind_value) + break my_resource._add_index(other_ind_name, other_ind_val) return other_inp_name @@ -184,6 +216,7 @@ class InputsFieldWrp(IndexFieldWrp): other_type = self._input_type(other_resource, other_inp_name) my_type = self._input_type(my_resource, my_inp_name) + # import ipdb; ipdb.set_trace() if my_type == other_type: # if the type is the same map 1:1 my_type = InputTypes.simple @@ -290,10 +323,10 @@ class InputsFieldWrp(IndexFieldWrp): return other_res return _res my_meth = getattr(self, '_map_field_val_{}'.format(my_type.name)) - return my_meth(recvs, my_name, other=other) + return my_meth(recvs, name, my_name, other=other) - def _map_field_val_simple(self, recvs, name, other=None): + def _map_field_val_simple(self, recvs, input_name, name, other=None): recvs = recvs[0] index_val, obj_key = recvs _, inp, emitter_key, emitter_inp, _mapping_type = index_val.split('|', 4) @@ -301,7 +334,7 @@ class InputsFieldWrp(IndexFieldWrp): self._cache[name] = res return res - def _map_field_val_list(self, recvs, name, other=None): + def _map_field_val_list(self, recvs, input_name, name, other=None): if len(recvs) == 1: recv = recvs[0] index_val, obj_key = recv @@ -319,7 +352,7 @@ class InputsFieldWrp(IndexFieldWrp): self._cache[name] = res return res - def _map_field_val_hash_single(self, recvs, other): + def _map_field_val_hash_single(self, recvs, input_name, other): items = [] tags = set() for recv in recvs: @@ -330,16 +363,39 @@ class InputsFieldWrp(IndexFieldWrp): tags.add(my_tag) return items, tags - def _map_field_val_hash(self, recvs, name, other=None): + def _map_field_val_hash(self, recvs, input_name, name, other=None): if len(recvs) == 1: + # just one connected recv = recvs[0] index_val, obj_key = recv - _, inp, emitter_key, emitter_inp, mapping_type = index_val.split('|', 4) - res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) - if mapping_type != "{}_{}".format(InputTypes.simple.value, InputTypes.simple.value): - raise NotImplementedError() + splitted = index_val.split('|') + splen = len(splitted) + if splen == 5: + # 1:1 + _, inp, emitter_key, emitter_inp, mapping_type = splitted + if mapping_type != "{}_{}".format(InputTypes.simple.value, InputTypes.simple.value): + raise NotImplementedError() + res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) + elif splen == 7: + # partial + _, _, emitter_key, emitter_inp, my_tag, my_val, mapping_type = splitted + cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) + res = {my_val: cres} + my_resource = self._instance + my_resource_value = my_resource.inputs._get_raw_field_val(input_name) + if my_resource_value: + for my_val, cres in my_resource_value.iteritems(): + res[my_val] = cres + else: + raise Exception("Not supported splen %s", splen) else: items, tags = self._map_field_val_hash_single(recvs, other) + my_resource = self._instance + my_resource_value = my_resource.inputs._get_raw_field_val(input_name) + if my_resource_value: + for my_val, cres in my_resource_value.iteritems(): + items.append((my_resource.name, my_val, cres)) + tags.add(my_resource.name) if len(tags) != 1: # TODO: add it also for during connecting raise Exception("Detected dict with different tags") @@ -349,7 +405,7 @@ class InputsFieldWrp(IndexFieldWrp): self._cache[name] = res return res - def _map_field_val_list_hash(self, recvs, name, other=None): + def _map_field_val_list_hash(self, recvs, input_name, name, other=None): items = [] tags = set() for recv in recvs: diff --git a/solar/solar/test/test_signals.py b/solar/solar/test/test_signals.py index b6bc0f26..78ee3741 100644 --- a/solar/solar/test/test_signals.py +++ b/solar/solar/test/test_signals.py @@ -554,7 +554,6 @@ input: receiver.args['server'], ) - @pytest.mark.xfail(reason="Not supported case when some elements of dict are connected and some not") def test_hash_input_mixed(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -642,7 +641,6 @@ input: receiver.args['server'], ) - @pytest.mark.xfail(reason='no idea') def test_hash_input_with_multiple_connections(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -681,7 +679,6 @@ input: receiver.args['server'], ) - @pytest.mark.xfail(reason='no idea') def test_hash_input_multiple_resources_with_tag_connect(self): sample_meta_dir = self.make_resource_meta(""" id: sample From bcf8dbe25d6db401e38fe1ca0294c80243451319 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 18 Nov 2015 13:30:04 +0100 Subject: [PATCH 170/191] Removed duplicated test --- solar/solar/test/test_signals.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/solar/solar/test/test_signals.py b/solar/solar/test/test_signals.py index 78ee3741..1f37f396 100644 --- a/solar/solar/test/test_signals.py +++ b/solar/solar/test/test_signals.py @@ -40,34 +40,6 @@ input: 'Trying to connect value-.* to itself'): xs.connect(sample, sample, {'value'}) - @pytest.mark.xfail(reason="No cycle detection in new_db") - def test_no_cycles(self): - sample_meta_dir = self.make_resource_meta(""" -id: sample -handler: ansible -version: 1.0.0 -input: - value: - schema: str! - value: - """) - - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'value': 'x'} - ) - - sample2 = self.create_resource( - 'sample2', sample_meta_dir, {'value': 'y'} - ) - - xs.connect(sample1, sample2) - - with self.assertRaisesRegexp( - Exception, - 'Prevented creating a cycle'): - xs.connect(sample2, sample1) - - # TODO: more complex cycles def test_input_dict_type(self): sample_meta_dir = self.make_resource_meta(""" From 3b68111b7d3229cd6371eea2bc29acfea4c4c213 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 18 Nov 2015 13:30:25 +0100 Subject: [PATCH 171/191] Commented test for unused functionality (nested lists) --- solar/solar/test/test_signals.py | 159 ++++++++++++++++--------------- 1 file changed, 80 insertions(+), 79 deletions(-) diff --git a/solar/solar/test/test_signals.py b/solar/solar/test/test_signals.py index 1f37f396..e2504b78 100644 --- a/solar/solar/test/test_signals.py +++ b/solar/solar/test/test_signals.py @@ -383,88 +383,89 @@ input: [sample1.args['port']] ) - @pytest.mark.xfail(reason="Nested lists are not supported in new_db") - def test_nested_list_input(self): - """ - Make sure that single input change is propagated along the chain of - lists. - """ +# XXX: not used for now, not implemented in new db (jnowak) +# @pytest.mark.xfail(reason="Nested lists are not supported in new_db") +# def test_nested_list_input(self): +# """ +# Make sure that single input change is propagated along the chain of +# lists. +# """ - sample_meta_dir = self.make_resource_meta(""" -id: sample -handler: ansible -version: 1.0.0 -input: - ip: - schema: str - value: - port: - schema: int - value: - """) - list_input_meta_dir = self.make_resource_meta(""" -id: list-input -handler: ansible -version: 1.0.0 -input: - ips: - schema: [str] - value: [] - ports: - schema: [int] - value: [] - """) - list_input_nested_meta_dir = self.make_resource_meta(""" -id: list-input-nested -handler: ansible -version: 1.0.0 -input: - ipss: - schema: [[str]] - value: [] - portss: - schema: [[int]] - value: [] - """) +# sample_meta_dir = self.make_resource_meta(""" +# id: sample +# handler: ansible +# version: 1.0.0 +# input: +# ip: +# schema: str +# value: +# port: +# schema: int +# value: +# """) +# list_input_meta_dir = self.make_resource_meta(""" +# id: list-input +# handler: ansible +# version: 1.0.0 +# input: +# ips: +# schema: [str] +# value: [] +# ports: +# schema: [int] +# value: [] +# """) +# list_input_nested_meta_dir = self.make_resource_meta(""" +# id: list-input-nested +# handler: ansible +# version: 1.0.0 +# input: +# ipss: +# schema: [[str]] +# value: [] +# portss: +# schema: [[int]] +# value: [] +# """) - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'ip': '10.0.0.1', 'port': 1000} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, {'ip': '10.0.0.2', 'port': 1001} - ) - list_input = self.create_resource( - 'list-input', list_input_meta_dir, - ) - list_input_nested = self.create_resource( - 'list-input-nested', list_input_nested_meta_dir, - ) +# sample1 = self.create_resource( +# 'sample1', sample_meta_dir, {'ip': '10.0.0.1', 'port': 1000} +# ) +# sample2 = self.create_resource( +# 'sample2', sample_meta_dir, {'ip': '10.0.0.2', 'port': 1001} +# ) +# list_input = self.create_resource( +# 'list-input', list_input_meta_dir, +# ) +# list_input_nested = self.create_resource( +# 'list-input-nested', list_input_nested_meta_dir, +# ) - sample1.connect(list_input, mapping={'ip': 'ips', 'port': 'ports'}) - sample2.connect(list_input, mapping={'ip': 'ips', 'port': 'ports'}) - list_input.connect(list_input_nested, mapping={'ips': 'ipss', 'ports': 'portss'}) - self.assertListEqual( - #[ips['value'] for ips in list_input_nested.args['ipss']], - list_input_nested.args['ipss'], - [list_input.args['ips']] - ) - self.assertListEqual( - #[ps['value'] for ps in list_input_nested.args['portss']], - list_input_nested.args['portss'], - [list_input.args['ports']] - ) +# sample1.connect(list_input, mapping={'ip': 'ips', 'port': 'ports'}) +# sample2.connect(list_input, mapping={'ip': 'ips', 'port': 'ports'}) +# list_input.connect(list_input_nested, mapping={'ips': 'ipss', 'ports': 'portss'}) +# self.assertListEqual( +# #[ips['value'] for ips in list_input_nested.args['ipss']], +# list_input_nested.args['ipss'], +# [list_input.args['ips']] +# ) +# self.assertListEqual( +# #[ps['value'] for ps in list_input_nested.args['portss']], +# list_input_nested.args['portss'], +# [list_input.args['ports']] +# ) - # Test disconnect - xs.disconnect(sample1, list_input) - self.assertListEqual( - #[[ip['value'] for ip in ips['value']] for ips in list_input_nested.args['ipss']], - list_input_nested.args['ipss'], - [[sample2.args['ip']]] - ) - self.assertListEqual( - list_input_nested.args['portss'], - [[sample2.args['port']]] - ) +# # Test disconnect +# xs.disconnect(sample1, list_input) +# self.assertListEqual( +# #[[ip['value'] for ip in ips['value']] for ips in list_input_nested.args['ipss']], +# list_input_nested.args['ipss'], +# [[sample2.args['ip']]] +# ) +# self.assertListEqual( +# list_input_nested.args['portss'], +# [[sample2.args['port']]] +# ) class TestHashInput(base.BaseResourceTest): @@ -513,7 +514,7 @@ input: {'ip': sample1.args['ip'], 'port': sample1.args['port']}, receiver.args['server'], ) - # XXX: We need to disconnect first + # XXX: We need to disconnect first # XXX: it should raise error when connecting already connected inputs xs.connect(sample2, receiver, mapping={'ip': 'server:ip'}) self.assertDictEqual( From 668282dfe8a10f463a3aaeea899f4175340a59e0 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 18 Nov 2015 13:34:28 +0100 Subject: [PATCH 172/191] Added exception when unsupported connection type --- solar/solar/dblayer/solar_models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 477c880a..3ee63be4 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -176,6 +176,8 @@ class InputsFieldWrp(IndexFieldWrp): and mt == my_tag and mv == my_val: my_resource._remove_index(ind_name, ind_value) break + else: + raise Exception("Unsupported connection type") my_resource._add_index(other_ind_name, other_ind_val) return other_inp_name From 1b1874786982af0b50fa705538b4d4bce3ea4e04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Wed, 18 Nov 2015 13:44:59 +0100 Subject: [PATCH 173/191] Adjust travis conf --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 063d847e..83d7c68a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: 2.7 env: - - PIP_ACCEL_CACHE=$HOME/.pip-accel-cache + - PIP_ACCEL_CACHE=$HOME/.pip-accel-cache SOLAR_CONFIG=$TRAVIS_BUILD_DIR/.config cache: directories: - $HOME/.pip-accel-cache @@ -10,8 +10,8 @@ install: - pip-accel install coveralls - pip-accel install -r solar/test-requirements.txt script: - - cd solar && py.test --cov=solar -s solar && cd .. + - cd solar && py.test --cov=solar -s solar/test && cd .. services: - - redis-server + - riak after_success: cd solar && coveralls From 0dedcd867d90b7cf6603c632393c8f0ba005cc01 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 18 Nov 2015 14:25:37 +0100 Subject: [PATCH 174/191] Changed config keys --- .config | 6 ++---- solar/solar/config.py | 4 ++-- solar/solar/dblayer/__init__.py | 25 +++++++++++++------------ 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.config b/.config index 1aaf8b4d..424b2169 100644 --- a/.config +++ b/.config @@ -2,10 +2,8 @@ dblayer: riak redis: host: localhost port: '6379' -riak: +solar_db: + mode: riak host: localhost port: '8087' protocol: pbc -sqlite: - backend: memory - location: ':memory:' diff --git a/solar/solar/config.py b/solar/solar/config.py index 87284557..ab40b990 100644 --- a/solar/solar/config.py +++ b/solar/solar/config.py @@ -6,8 +6,7 @@ CWD = os.getcwd() C = Bunch() C.redis = Bunch(port='6379', host='10.0.0.2') -C.riak = Bunch(port='8087', host='10.0.0.2', protocol='pbc') -C.sqlite = Bunch(backend='memory', location=':memory:') +C.solar_db = Bunch(mode='riak', port='8087', host='10.0.0.2', protocol='pbc') C.dblayer = 'riak' @@ -52,6 +51,7 @@ def from_configs(): config[path[-1]] = vals _lookup_vals(_setter, C) + def from_env(): def _setter(config, path): env_key = '_'.join(path).upper() diff --git a/solar/solar/dblayer/__init__.py b/solar/solar/dblayer/__init__.py index 09e0e7d6..a7827b0d 100644 --- a/solar/solar/dblayer/__init__.py +++ b/solar/solar/dblayer/__init__.py @@ -2,27 +2,28 @@ from solar.dblayer.model import ModelMeta from solar.dblayer.riak_client import RiakClient from solar.config import C -if C.dblayer == 'sqlite': + +if C.solar_db.mode == 'sqlite': from solar.dblayer.sql_client import SqlClient - if C.sqlite.backend == 'memory': - client = SqlClient(C.sqlite.location, threadlocals=False, autocommit=False) - elif C.sqlite.backend == 'file': - client = SqlClient(C.sqlite.location, threadlocals=True, + if C.solar_db.backend == 'memory': + client = SqlClient(C.solar_db.location, threadlocals=False, autocommit=False) + elif C.solar_db.backend == 'file': + client = SqlClient(C.solar_db.location, threadlocals=True, autocommit=False, pragmas=(('journal_mode', 'WAL'), ('synchronous', 'NORMAL'))) else: - raise Exception('Unknown sqlite backend %s', C.sqlite.backend) + raise Exception('Unknown sqlite backend %s', C.solar_db.backend) -elif C.dblayer == 'riak': +elif C.solar_db.mode == 'riak': from solar.dblayer.riak_client import RiakClient - if C.riak.protocol == 'pbc': + if C.solar_db.protocol == 'pbc': client = RiakClient( - protocol=C.riak.protocol, host=C.riak.host, pb_port=C.riak.port) - elif C.riak.protocol == 'http': + protocol=C.solar_db.protocol, host=C.solar_db.host, pb_port=C.solar_db.port) + elif C.solar_db.protocol == 'http': client = RiakClient( - protocol=C.riak.protocol, host=C.riak.host, http_port=C.riak.port) + protocol=C.solar_db.protocol, host=C.solar_db.host, http_port=C.solar_db.port) else: - raise Exception('Unknown riak protocol %s', C.riak.protocol) + raise Exception('Unknown riak protocol %s', C.solar_db.protocol) else: raise Exception('Unknown dblayer backend %s', C.dblayer) From a7480424a93a0758112e53e43330ebe1e241f170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Nowak?= Date: Wed, 18 Nov 2015 14:31:17 +0100 Subject: [PATCH 175/191] removed dblayer option from config --- solar/solar/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/solar/solar/config.py b/solar/solar/config.py index ab40b990..3b724e05 100644 --- a/solar/solar/config.py +++ b/solar/solar/config.py @@ -7,7 +7,6 @@ CWD = os.getcwd() C = Bunch() C.redis = Bunch(port='6379', host='10.0.0.2') C.solar_db = Bunch(mode='riak', port='8087', host='10.0.0.2', protocol='pbc') -C.dblayer = 'riak' def _lookup_vals(setter, config, prefix=None): From 4be29698fd9cf22153a0cccd4eb793f5e599c07b Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 18 Nov 2015 14:32:15 +0100 Subject: [PATCH 176/191] Don't add nested models into defined_models, and don't create buckets for them --- solar/solar/dblayer/model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 600f7e72..072e4cdf 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -549,6 +549,9 @@ class ModelMeta(type): if bases == (object, ): return cls + if issubclass(cls, NestedModel): + return cls + cls.bucket = Replacer('bucket', get_bucket, mcs) mcs._defined_models.add(cls) return cls From cb24435771e3a1dca3bc4504850df48f3e2db80e Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 18 Nov 2015 14:38:38 +0100 Subject: [PATCH 177/191] Fixed dblayer tests --- solar/solar/dblayer/test/test_basic.py | 8 ++++---- solar/solar/dblayer/test/test_real.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/solar/solar/dblayer/test/test_basic.py b/solar/solar/dblayer/test/test_basic.py index 8a285550..54ef5b0a 100644 --- a/solar/solar/dblayer/test/test_basic.py +++ b/solar/solar/dblayer/test/test_basic.py @@ -84,7 +84,7 @@ def test_cache_logic(rk): k = next(rk) M1.session_start() assert M1._c.obj_cache == {} - + m1 = M1.from_dict(k, {'f1': 'blah', 'f2': 150}) m1.save() M1.session_end() @@ -115,9 +115,9 @@ def test_normal_index(rk): m2 = M1.from_dict(key2, {'f1': 'blah', 'f2': 150, 'ind': {'blah': 'something2'}}) m2.save() - assert M1.ind.filter('blah=somethi*') == set([key, key2]) - assert M1.ind.filter('blah=something') == set([key]) - assert M1.ind.filter('blah=something2') == set([key2]) + assert set(M1.ind.filter('blah=somethi*')) == set([key, key2]) + assert set(M1.ind.filter('blah=something')) == set([key]) + assert set(M1.ind.filter('blah=something2')) == set([key2]) def test_update(rk): diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index a1db5e53..4a26f6c0 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -394,7 +394,7 @@ def test_dict_to_list_inputs(rk): r2.save() r3.save() - assert r1.inputs['modules'] == [{'name': 'blah2'}, {'name': 'blah3'}] + assert sorted(r1.inputs['modules']) == sorted([{'name': 'blah2'}, {'name': 'blah3'}]) @@ -502,7 +502,7 @@ def test_events(rk): r1.save() assert r1.events == ['event1', 'event2'] r1.events.pop() - r1.save() + assert r1.events == ['event1'] From c0ba7276a44b9be893d9eacb81399a7d84ed147b Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 19 Nov 2015 15:54:00 +0100 Subject: [PATCH 178/191] Removed db configuration from dblayer conftest --- solar/solar/dblayer/model.py | 2 ++ solar/solar/dblayer/test/conftest.py | 13 ------------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/solar/solar/dblayer/model.py b/solar/solar/dblayer/model.py index 072e4cdf..f10211c1 100644 --- a/solar/solar/dblayer/model.py +++ b/solar/solar/dblayer/model.py @@ -559,6 +559,8 @@ class ModelMeta(type): @classmethod def setup(mcs, riak_client): + if hasattr(mcs, 'riak_client'): + raise DBLayerException("Setup already done") mcs.riak_client = riak_client diff --git a/solar/solar/dblayer/test/conftest.py b/solar/solar/dblayer/test/conftest.py index 2ba01da2..ad36c689 100644 --- a/solar/solar/dblayer/test/conftest.py +++ b/solar/solar/dblayer/test/conftest.py @@ -56,16 +56,3 @@ def pytest_runtest_call(item): Model.get_bucket_name = classmethod(patched_get_bucket_name) - -# from solar.dblayer.sql_client import SqlClient -# client = SqlClient(':memory:', threadlocals=False, autocommit=False) -# client = SqlClient('/tmp/blah.db', threadlocals=True, -# autocommit=False, pragmas=(('journal_mode', 'WAL'), -# ('synchronous', 'NORMAL'))) - -from solar.dblayer.riak_client import RiakClient -client = RiakClient(protocol='pbc', host='10.0.0.2', pb_port=8087) -# client = RiakClient(protocol='http', host='10.0.0.3', http_port=18098) - - -ModelMeta.setup(client) From 3e1c92523259acbad2d475fabb7d8c42d0114e19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Thu, 19 Nov 2015 16:20:06 +0100 Subject: [PATCH 179/191] Run all tests from solar dir --- .travis.yml | 4 ++-- solar/test-requirements.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 83d7c68a..232e9775 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: 2.7 env: - - PIP_ACCEL_CACHE=$HOME/.pip-accel-cache SOLAR_CONFIG=$TRAVIS_BUILD_DIR/.config + - PIP_ACCEL_CACHE=$HOME/.pip-accel-cache SOLAR_CONFIG=$TRAVIS_BUILD_DIR/.config SOLAR_SOLAR_DB_HOST=localhost cache: directories: - $HOME/.pip-accel-cache @@ -10,7 +10,7 @@ install: - pip-accel install coveralls - pip-accel install -r solar/test-requirements.txt script: - - cd solar && py.test --cov=solar -s solar/test && cd .. + - cd solar && py.test --cov=solar -s solar && cd .. services: - riak after_success: diff --git a/solar/test-requirements.txt b/solar/test-requirements.txt index 04572781..7e25f773 100644 --- a/solar/test-requirements.txt +++ b/solar/test-requirements.txt @@ -3,3 +3,4 @@ hacking==0.7 pytest-cov pytest-mock tox +peewee From 776f3596ca0cdc87695da6c0c8323e338c3f3eda Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 20 Nov 2015 10:40:45 +0100 Subject: [PATCH 180/191] Remove peewee from test requirements --- solar/test-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/solar/test-requirements.txt b/solar/test-requirements.txt index 7e25f773..04572781 100644 --- a/solar/test-requirements.txt +++ b/solar/test-requirements.txt @@ -3,4 +3,3 @@ hacking==0.7 pytest-cov pytest-mock tox -peewee From 1529c2c96d2f98058bad7fb4b0e1163b97118d42 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 19 Nov 2015 23:53:20 +0100 Subject: [PATCH 181/191] Fixed complex disconnects scenarios with new_db --- solar/solar/dblayer/solar_models.py | 119 +++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 21 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 3ee63be4..329d8d01 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -218,7 +218,6 @@ class InputsFieldWrp(IndexFieldWrp): other_type = self._input_type(other_resource, other_inp_name) my_type = self._input_type(my_resource, my_inp_name) - # import ipdb; ipdb.set_trace() if my_type == other_type: # if the type is the same map 1:1 my_type = InputTypes.simple @@ -248,21 +247,49 @@ class InputsFieldWrp(IndexFieldWrp): def disconnect(self, name): # ind_name = '{}_recv_bin'.format(self.fname) + if ':' in name: + # disconnect from hash with tag + normalized_name, tag_and_target = name.split(':', 1) + my_val, my_tag = tag_and_target.split('|', 1) + emit_name = None + # emit_name = '{}|{}'.format(my_tag, my_val) + full_name = '{}|{}|{}'.format(normalized_name, my_tag, my_val) + name = normalized_name + elif '|'in name: + # disconnect everything from given input|resource + my_input, other_resource, other_input = name.split('|', 2) + full_name = my_input + emit_name = '{}|{}'.format(other_resource, other_input) + normalized_name = "{}|{}".format(my_input, other_resource) + name = name.split('|', 1)[0] + my_val, my_tag = None, None + else: + # disconnect everything from given input + full_name = name + emit_name = None + normalized_name = name + my_val, my_tag = None, None indexes = self._instance._riak_object.indexes to_dels = [] recvs = filter(lambda x: x[0] == '{}_recv_bin'.format(self.fname), indexes) for recv in recvs: _, ind_value = recv - recv_name = name - if ':' in recv_name: - recv_name = recv_name.split(':')[0] - if ind_value.startswith('{}|{}|'.format(self._instance.key, recv_name)): - to_dels.append(recv) + if ind_value.startswith('{}|{}|'.format(self._instance.key, normalized_name)): + spl = ind_value.split('|') + if len(spl) == 7 and my_tag and my_val: + if spl[-3] == my_tag and spl[-2] == my_val: + to_dels.append(recv) + else: + to_dels.append(recv) emits = filter(lambda x: x[0] == '{}_emit_bin'.format(self.fname), indexes) for emit in emits: _, ind_value = emit - if ind_value.endswith('|{}|{}'.format(self._instance.key, name)): - to_dels.append(emit) + if ind_value.endswith('|{}|{}'.format(self._instance.key, full_name)): + if emit_name: + if ind_value.startswith(emit_name): + to_dels.append(emit) + else: + to_dels.append(emit) for to_del in to_dels: self._instance._remove_index(*to_del) @@ -391,17 +418,16 @@ class InputsFieldWrp(IndexFieldWrp): else: raise Exception("Not supported splen %s", splen) else: - items, tags = self._map_field_val_hash_single(recvs, other) + items, tags = self._map_field_val_hash_single(recvs, input_name, other) my_resource = self._instance my_resource_value = my_resource.inputs._get_raw_field_val(input_name) if my_resource_value: - for my_val, cres in my_resource_value.iteritems(): - items.append((my_resource.name, my_val, cres)) - tags.add(my_resource.name) + res = my_resource_value + else: + res = {} if len(tags) != 1: # TODO: add it also for during connecting raise Exception("Detected dict with different tags") - res = {} for _, my_val, value in items: res[my_val] = value self._cache[name] = res @@ -659,12 +685,54 @@ class Resource(Model): mapping.iteritems(), concurrency=2) def disconnect(self, other, inputs): + def _to_disconnect((emitter, receiver, meta)): + if not receiver[0] == other_key: + return False + # name there? + if not emitter[0] == self.key: + return False + # TODO: `delete_hash` test works with receiver[1], while lists works with emitter[1] + key = emitter[1] + if not key in converted: + return False + convs = converted[key] + for conv in convs: + if conv: + if meta['tag'] == conv['tag'] \ + and meta['destination_key'] == conv['destination_key']: + return True + else: + return True + return False + + def _convert_input(input): + spl = input.split('|') + spl_len = len(spl) + if spl_len == 1: + # normal input + return input, None + elif spl_len == 3: + return spl[0], {'tag': spl[1], + 'destination_key': spl[2]} + else: + raise Exception("Cannot convert input %r" % input) + + def _format_for_disconnect((emitter, receiver, meta)): + input = receiver[1] + if not meta: + return "{}|{}|{}".format(receiver[1], emitter[0], emitter[1]) + dest_key = meta['destination_key'] + tag = meta.get('tag', other.name) + return '{}:{}|{}'.format(input, dest_key, tag) + + + converted = defaultdict(list) + for k, v in map(_convert_input, inputs): + converted[k].append(v) other_key = other.key edges = other.inputs._edges() - edges = filter(lambda (emitter, receiver, _): receiver[0] == other_key - and emitter[1] in inputs and emitter[0] == self.name, - edges) - inputs = ['{}|{}'.format(x[1][1], self.name) for x in edges] + edges = filter(_to_disconnect, edges) + inputs = map(_format_for_disconnect, edges) solar_map(other.inputs.disconnect, inputs, concurrency=2) def save(self, *args, **kwargs): @@ -707,12 +775,21 @@ class Resource(Model): return_terms=True, max_results=999999) + to_disconnect_all = defaultdict(list) for emit_bin in inputs_index.results: index_vals = emit_bin[0].split('|') - - my_res, my_key, other_res, other_key = index_vals[:4] - emit_obj = Resource.get(other_res) - emit_obj.inputs.disconnect(other_key) + index_vals_len = len(index_vals) + if index_vals_len == 6: # hash + _, my_input, other_res, other_input, my_tag, my_val = index_vals + to_disconnect_all[other_res].append("{}|{}|{}".format(my_input, my_tag, my_val)) + elif index_vals_len == 4: + _, my_input, other_res, other_input = index_vals + to_disconnect_all[other_res].append(other_input) + else: + raise Exception("Unknown input %r" % index_vals) + for other_obj_key, to_disconnect in to_disconnect_all.items(): + other_obj = Resource.get(other_obj_key) + self.disconnect(other_obj, to_disconnect) super(Resource, self).delete() From eaad0d4dd54a8067c0a8500b21234615347888d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Fri, 20 Nov 2015 12:04:06 +0100 Subject: [PATCH 182/191] Lookup vals only if config was found --- solar/solar/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solar/solar/config.py b/solar/solar/config.py index 3b724e05..9a92c501 100644 --- a/solar/solar/config.py +++ b/solar/solar/config.py @@ -48,7 +48,8 @@ def from_configs(): for key in path: vals = vals[key] config[path[-1]] = vals - _lookup_vals(_setter, C) + if data: + _lookup_vals(_setter, C) def from_env(): From 707dfb3070855ad2e0259f8106622d2cf14e9b92 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 20 Nov 2015 12:21:32 +0100 Subject: [PATCH 183/191] Install libffi-dev and libssl-dev in bootstrap --- bootstrap/playbooks/tasks/base.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap/playbooks/tasks/base.yaml b/bootstrap/playbooks/tasks/base.yaml index 9b7a33a4..d79b25ce 100644 --- a/bootstrap/playbooks/tasks/base.yaml +++ b/bootstrap/playbooks/tasks/base.yaml @@ -32,6 +32,9 @@ - build-essential # for torrent transport - python-libtorrent + # for riak python package + - libffi-dev + - libssl-dev # PIP #- apt: name=python-pip state=absent From 2c385c173a8b80fe5ecf147c03c0dcc5ef84ad19 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 20 Nov 2015 13:15:33 +0100 Subject: [PATCH 184/191] Added session start / end to celery using hooks --- solar/solar/orchestration/tasks.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/solar/solar/orchestration/tasks.py b/solar/solar/orchestration/tasks.py index 85fa960f..3f2d005e 100644 --- a/solar/solar/orchestration/tasks.py +++ b/solar/solar/orchestration/tasks.py @@ -17,6 +17,7 @@ import subprocess import time from celery.app import task +from celery.signals import task_prerun, task_postrun from solar.orchestration import graph from solar.core import actions @@ -26,6 +27,7 @@ from solar.orchestration.runner import app from solar.orchestration.traversal import traverse from solar.orchestration import limits from solar.orchestration import executor +from solar.dblayer import ModelMeta __all__ = ['solar_resource', 'cmd', 'sleep', @@ -52,6 +54,14 @@ class ReportTask(task.Task): report_task = partial(app.task, base=ReportTask, bind=True) +@task_prerun.connect +def start_solar_session(task_id, task, *args, **kwargs): + ModelMeta.session_start() + +@task_postrun.connect +def end_solar_session(task_id, task, *args, **kwargs): + ModelMeta.session_end() + @report_task(name='solar_resource') def solar_resource(ctxt, resource_name, action): From 3369cf898b920ec519619e062f0bc1bf73b7a908 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 20 Nov 2015 17:02:02 +0100 Subject: [PATCH 185/191] Test for failing case --- solar/solar/dblayer/test/test_real.py | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 4a26f6c0..69d3195e 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -554,3 +554,40 @@ def test_delete_hash(rk): if 'recv' in index[0] or 'emit' in index[0]: recv_emit_bin.append(index) assert recv_emit_bin == [] + + +def test_nested_simple_listdict(rk): + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + k4 = next(rk) + k5 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'config': [{}]}}) + r2 = create_resource(k2, {'name': 'second', + 'inputs': {'backend': {}}}) + r3 = create_resource(k3, {'name': 'second', + 'inputs': {'backend': {}}}) + r5 = create_resource(k5, {'name': 'fifth', + 'inputs': {"port": 5, + "host": "fifth_host"}}) + r4 = create_resource(k4, {'name': 'fourth', + 'inputs': {"port": 4, + "host": "fourth_host"}}) + + r4.connect(r2, {'port': "backend:port", + 'host': 'backend:host'}) + r5.connect(r3, {'port': "backend:port", + 'host': 'backend:host'}) + + + assert r2.inputs['backend'] == {'host': 'fourth_host', 'port': 4} + assert r3.inputs['backend'] == {'host': 'fifth_host', 'port': 5} + + r2.connect(r1, {'backend': 'config'}) + r3.connect(r1, {'backend': 'config'}) + + print r1.inputs['config'] + + From a7a05a281a23be5638f1f1a824eb5b3444faa3e7 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 20 Nov 2015 18:27:33 +0100 Subject: [PATCH 186/191] Implementation + test for failing case --- solar/solar/dblayer/solar_models.py | 32 +++++++++++++++++++-------- solar/solar/dblayer/test/test_real.py | 14 +++++++----- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index 329d8d01..d8d70345 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -225,7 +225,10 @@ class InputsFieldWrp(IndexFieldWrp): elif my_type == InputTypes.list_hash and other_type == InputTypes.hash: # whole dict to list with dicts # TODO: solve this problem - my_type = InputTypes.list + if ':' in my_inp_name: + my_type = InputTypes.hash + else: + my_type = InputTypes.list # set my side my_meth = getattr(self, '_connect_my_{}'.format(my_type.name)) @@ -436,6 +439,7 @@ class InputsFieldWrp(IndexFieldWrp): def _map_field_val_list_hash(self, recvs, input_name, name, other=None): items = [] tags = set() + maybe_list = set() for recv in recvs: index_val, obj_key = recv splitted_val = index_val.split('|', 6) @@ -447,16 +451,27 @@ class InputsFieldWrp(IndexFieldWrp): else: _, _, emitter_key, emitter_inp, my_tag, my_val, mapping_type = splitted_val cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) - items.append((my_tag, my_val, cres)) + mapping_type = splitted_val[-1] + if mapping_type == '{}_{}'.format(InputTypes.hash.value, InputTypes.hash.value): + items.append((mapping_type, my_val, cres)) + maybe_list.add((mapping_type, my_val)) + else: + items.append((my_tag, my_val, cres)) tmp_res = {} - for my_tag, my_val, value in items: + for first, my_val, value in items: if my_val is None: - tmp_res[my_tag] = value + tmp_res[first] = value else: - try: - tmp_res[my_tag][my_val] = value - except KeyError: - tmp_res[my_tag] = {my_val: value} + if (first, my_val) in maybe_list: + try: + tmp_res[first][my_val].append(value) + except KeyError: + tmp_res[first] = {my_val: [value]} + else: + try: + tmp_res[first][my_val] = value + except KeyError: + tmp_res[first] = {my_val: value} res = tmp_res.values() self._cache[name] = res return res @@ -691,7 +706,6 @@ class Resource(Model): # name there? if not emitter[0] == self.key: return False - # TODO: `delete_hash` test works with receiver[1], while lists works with emitter[1] key = emitter[1] if not key in converted: return False diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 69d3195e..cc078c4a 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -564,10 +564,11 @@ def test_nested_simple_listdict(rk): k5 = next(rk) r1 = create_resource(k1, {'name': 'first', - 'inputs': {'config': [{}]}}) + 'inputs': {'config': [{"backends": [{}], + 'listen_port': 1}]}}) r2 = create_resource(k2, {'name': 'second', 'inputs': {'backend': {}}}) - r3 = create_resource(k3, {'name': 'second', + r3 = create_resource(k3, {'name': 'third', 'inputs': {'backend': {}}}) r5 = create_resource(k5, {'name': 'fifth', 'inputs': {"port": 5, @@ -585,9 +586,12 @@ def test_nested_simple_listdict(rk): assert r2.inputs['backend'] == {'host': 'fourth_host', 'port': 4} assert r3.inputs['backend'] == {'host': 'fifth_host', 'port': 5} - r2.connect(r1, {'backend': 'config'}) - r3.connect(r1, {'backend': 'config'}) + r2.connect(r1, {'backend': 'config:backends'}) + r3.connect(r1, {'backend': 'config:backends'}) - print r1.inputs['config'] + Resource.save_all_lazy() + + backends = next(x['backends'] for x in r1.inputs['config'] if 'backends' in x) + assert len(backends) == 2 From 2f91879f7606d7b7f0cdc0e77fbf379ddf46b422 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 23 Nov 2015 10:21:05 +0100 Subject: [PATCH 187/191] One more test (haproxy config problem) --- solar/solar/dblayer/test/test_real.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index cc078c4a..1d06e7c3 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -595,3 +595,28 @@ def test_nested_simple_listdict(rk): assert len(backends) == 2 +def test_nested_two_listdict(rk): + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + + r1 = create_resource(k1, {'name': 'first', + 'inputs': {'config': [{"backends": [{}], + 'something': 0}]}}) + r2 = create_resource(k2, {'name': 'second', + 'inputs': {"backends": [{"host": "second_host", "port": 2}], + 'something': 1}}) + r3 = create_resource(k3, {'name': 'third', + 'inputs': {"backends": [{"host": "third_host", "port": 3}], + 'something': 2}}) + + r2.connect(r1, {'backends': 'config:backends', + 'something': 'config:something'}) + r3.connect(r1, {'backends': 'config:backends', + 'something': 'config:something'}) + + Resource.save_all_lazy() + + for sc in r1.inputs['config']: + assert 'something' in sc + assert 'backends' in sc From b1c1a911ec867346c89ff3c4bfa768141bf1ef27 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 23 Nov 2015 13:13:50 +0100 Subject: [PATCH 188/191] Nested keys are working properly right now --- solar/solar/dblayer/solar_models.py | 27 ++++++++++----------------- solar/solar/dblayer/test/test_real.py | 2 ++ 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index d8d70345..a52df166 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -185,6 +185,9 @@ class InputsFieldWrp(IndexFieldWrp): def _connect_other_hash(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): return self._connect_other_simple(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) + def _connect_other_list_hash(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): + return self._connect_other_simple(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) + def _connect_my_list(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): ret = self._connect_my_simple(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) return ret @@ -218,8 +221,8 @@ class InputsFieldWrp(IndexFieldWrp): other_type = self._input_type(other_resource, other_inp_name) my_type = self._input_type(my_resource, my_inp_name) - if my_type == other_type: - # if the type is the same map 1:1 + if my_type == other_type and not ':' in my_inp_name: + # if the type is the same map 1:1, and flat my_type = InputTypes.simple other_type = InputTypes.simple elif my_type == InputTypes.list_hash and other_type == InputTypes.hash: @@ -452,26 +455,16 @@ class InputsFieldWrp(IndexFieldWrp): _, _, emitter_key, emitter_inp, my_tag, my_val, mapping_type = splitted_val cres = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) mapping_type = splitted_val[-1] - if mapping_type == '{}_{}'.format(InputTypes.hash.value, InputTypes.hash.value): - items.append((mapping_type, my_val, cres)) - maybe_list.add((mapping_type, my_val)) - else: - items.append((my_tag, my_val, cres)) + items.append((my_tag, my_val, cres)) tmp_res = {} for first, my_val, value in items: if my_val is None: tmp_res[first] = value else: - if (first, my_val) in maybe_list: - try: - tmp_res[first][my_val].append(value) - except KeyError: - tmp_res[first] = {my_val: [value]} - else: - try: - tmp_res[first][my_val] = value - except KeyError: - tmp_res[first] = {my_val: value} + try: + tmp_res[first][my_val] = value + except KeyError: + tmp_res[first] = {my_val: value} res = tmp_res.values() self._cache[name] = res return res diff --git a/solar/solar/dblayer/test/test_real.py b/solar/solar/dblayer/test/test_real.py index 1d06e7c3..bdabb036 100644 --- a/solar/solar/dblayer/test/test_real.py +++ b/solar/solar/dblayer/test/test_real.py @@ -620,3 +620,5 @@ def test_nested_two_listdict(rk): for sc in r1.inputs['config']: assert 'something' in sc assert 'backends' in sc + assert isinstance(sc['backends'], list) + assert isinstance(sc['something'], int) From 33a25e10a6d687c9205884a1c3a7f6800ad3c7c2 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 23 Nov 2015 13:14:04 +0100 Subject: [PATCH 189/191] Added -rv (raw value) option to backtrack_inputs --- solar/solar/cli/resource.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/solar/solar/cli/resource.py b/solar/solar/cli/resource.py index 0207aaa6..20a2aea2 100644 --- a/solar/solar/cli/resource.py +++ b/solar/solar/cli/resource.py @@ -82,9 +82,10 @@ def backtrack_single(i): @resource.command() @click.option('-v', '--values', default=False, is_flag=True) +@click.option('-r', '--real_values', default=False, is_flag=True) @click.option('-i', '--input', default=None) @click.argument('resource') -def backtrack_inputs(resource, input, values): +def backtrack_inputs(resource, input, values, real_values): r = sresource.load(resource) db_obj = r.db_obj @@ -101,7 +102,7 @@ def backtrack_inputs(resource, input, values): for (rname, rinput), _, meta in se: l.append(dict(resource=resource, name=name)) val = single(rname, rinput, get_val) - if meta: + if meta and isinstance(val, dict): val['meta'] = meta l.append(val) return l @@ -115,6 +116,8 @@ def backtrack_inputs(resource, input, values): for name, values in inps.iteritems(): click.echo(yaml.safe_dump({name: values}, default_flow_style=False)) + if real_values: + click.echo('! Real value: %r' % sresource.load(resource).db_obj.inputs[name] , nl=True) @resource.command() def compile_all(): From 4ce8abf42eee9aa9440e916d0aecf732b7f506ce Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 23 Nov 2015 13:14:42 +0100 Subject: [PATCH 190/191] Fixed haproxy meta.yaml input type --- resources/haproxy_config/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/haproxy_config/meta.yaml b/resources/haproxy_config/meta.yaml index 10c8d44a..45eeb53b 100644 --- a/resources/haproxy_config/meta.yaml +++ b/resources/haproxy_config/meta.yaml @@ -10,7 +10,7 @@ input: value: {src: /etc/solar/haproxy, dst: /etc/haproxy} config: schema: [{backends: [{server: str!, port: int!}], listen_port: int!, protocol: str!, name: str!}] - value: [] + value: [{}] # ssh_user: # schema: str! # value: From f6498d0ad66073da4eaa6da9c771d1007ae78e70 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 23 Nov 2015 13:19:34 +0100 Subject: [PATCH 191/191] Removed unused variable --- solar/solar/dblayer/solar_models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/solar/solar/dblayer/solar_models.py b/solar/solar/dblayer/solar_models.py index a52df166..4524e1db 100644 --- a/solar/solar/dblayer/solar_models.py +++ b/solar/solar/dblayer/solar_models.py @@ -442,7 +442,6 @@ class InputsFieldWrp(IndexFieldWrp): def _map_field_val_list_hash(self, recvs, input_name, name, other=None): items = [] tags = set() - maybe_list = set() for recv in recvs: index_val, obj_key = recv splitted_val = index_val.split('|', 6)