Merge pull request #413 from pigmej/computable_inputs

Computable inputs

.travis.yml

@@ -1,5 +1,6 @@
 language: python
 python: 2.7
+sudo: false
 env:
   - PIP_ACCEL_CACHE=$HOME/.pip-accel-cache SOLAR_CONFIG=$TRAVIS_BUILD_DIR/.config SOLAR_SOLAR_DB_HOST=localhost
 cache:
@@ -15,3 +16,7 @@ services:
     - riak
 after_success:
     coveralls
+addons:
+  apt:
+    packages:
+    - libluajit-5.1-dev

bootstrap/playbooks/tasks/base.yaml

@@ -34,6 +34,9 @@
   - libffi-dev
   - libssl-dev
 
+  # computable inputs lua
+  - libluajit-5.1-dev
+
 
 # PIP
 #- apt: name=python-pip state=absent

examples/riak/riaks.py

@@ -45,8 +45,9 @@ def setup_riak():
                       'resources/riak_node',
                       {'riak_self_name': 'riak%d' % num,
                        'storage_backend': 'leveldb',
-                       'riak_hostname': 'riak_server%d.solar' % num,
+                       'riak_hostname': 'riak_server%d.solar' % num})[0]
-                       'riak_name': 'riak%d@riak_server%d.solar' % (num, num)})[0]
+        r.connect(r, {'riak_self_name': 'riak_name',
+                      'riak_hostname': 'riak_name'})
         riak_services.append(r)
 
     for i, riak in enumerate(riak_services):

requirements.txt

@@ -24,3 +24,7 @@ bunch
 riak
 # if you want to use sql backend then
 # peewee
+
+
+# if you want to use lua computable inputs
+# lupa

resources/riak_node/meta.yaml

@@ -9,12 +9,6 @@ input:
     ip:
       schema: str!
       value:
-    # ssh_key:
-    #   schema: str!
-    #   value:
-    # ssh_user:
-    #   schema: str!
-    #   value:
     riak_self_name:
       schema: str!
       value:
@@ -23,8 +17,11 @@ input:
       value:
     riak_name:
       schema: str!
-      # value: "{{riak_self_name}}@{{riak_hostname}}"
+      value: null
-      value: "{{riak_self_name}}@{{ip}}"
+      computable:
+        lang: jinja2
+        type: full
+        func: "{{riak_self_name}}@{{riak_hostname}}"
     riak_port_http:
       schema: int!
       value: 18098

solar/computable_inputs/__init__.py

@@ -0,0 +1,32 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from enum import Enum
+import os
+
+ComputablePassedTypes = Enum('ComputablePassedTypes', 'values full')
+
+HELPERS_PATH = os.path.normpath(
+    os.path.join(os.path.realpath(__file__), '..', 'helpers'))
+
+
+class ComputableInputProcessor(object):
+
+    def __init__(self):
+        pass
+
+    def process(self, resource_name, computable_type, funct, data):
+        if funct is None or funct == 'noop':
+            return data
+        return self.run(resource_name, computable_type, funct, data)

solar/computable_inputs/ci_jinja.py

@@ -0,0 +1,47 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from jinja2.sandbox import SandboxedEnvironment
+from solar.computable_inputs import ComputableInputProcessor
+from solar.computable_inputs import ComputablePassedTypes
+
+
+def make_arr(data):
+    t = {}
+    for ov in data:
+        if t.get(ov['resource']) is None:
+            t[ov['resource']] = {}
+        t[ov['resource']][ov['other_input']] = ov['value']
+    return t
+
+
+class JinjaProcessor(ComputableInputProcessor):
+
+    def __init__(self):
+        self.env = SandboxedEnvironment(trim_blocks=True,
+                                        lstrip_blocks=True)
+        self._globals = {'make_arr': make_arr}
+
+    def run(self, resource_name, computable_type, funct, data):
+        t = self.env.from_string(funct, globals=self._globals)
+        if computable_type == ComputablePassedTypes.full.name:
+            arr = make_arr(data)
+            my_inputs = arr[resource_name]
+        else:
+            my_inputs = {}
+            arr = {}
+        return t.render(resource_name=resource_name,
+                        D=data,
+                        R=arr,
+                        **my_inputs).strip()

solar/computable_inputs/ci_lua.py

@@ -0,0 +1,58 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import os
+
+from lupa import LuaRuntime
+from solar.computable_inputs import ComputableInputProcessor
+from solar.computable_inputs import HELPERS_PATH
+from solar.dblayer.solar_models import ComputablePassedTypes
+
+
+_LUA_HELPERS = open(os.path.join(HELPERS_PATH, 'lua_helpers.lua')).read()
+
+
+# TODO: (jnowak) add sandboxing (http://lua-users.org/wiki/SandBoxes)
+
+
+class LuaProcessor(ComputableInputProcessor):
+
+    def __init__(self):
+        self.lua = LuaRuntime()
+        self.lua.execute(_LUA_HELPERS)
+
+    def check_funct(self, funct, computable_type):
+        # dummy insert function start / end
+        if not funct.startswith('function') \
+           and not funct.endswith('end'):
+            if computable_type == ComputablePassedTypes.full.name:
+                make_arr = 'local R = make_arr(D)'
+                funct = "%s\n%s" % (make_arr, funct)
+            return 'function (D, resource_name) %s end' % funct
+        return funct
+
+    def run(self, resource_name, computable_type, funct, data):
+        # when computable_type == full then raw python object is passed
+        # to lua (counts from 0 etc)
+
+        if isinstance(data, list) \
+           and computable_type == ComputablePassedTypes.values.name:
+            lua_data = self.lua.table_from(data)
+        else:
+            lua_data = data
+
+        funct = self.check_funct(funct, computable_type)
+        funct_lua = self.lua.eval(funct)
+        return funct_lua(lua_data, resource_name)

solar/computable_inputs/ci_python.py

@@ -0,0 +1,112 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import json
+import os
+import struct
+import subprocess
+
+from solar.computable_inputs import ComputableInputProcessor
+from solar.computable_inputs import ComputablePassedTypes
+from solar.computable_inputs import HELPERS_PATH
+
+
+_PYTHON_WORKER = os.path.join(HELPERS_PATH, 'python_loop.py')
+_PYTHON_HELPERS = open(os.path.join(HELPERS_PATH, 'python_helpers.py')).read()
+
+
+class Mgr(object):
+
+    def __init__(self):
+        self.child = None
+
+    def run(self):
+        self.child = subprocess.Popen(['/usr/bin/env', 'python',
+                                       _PYTHON_WORKER],
+                                      stdin=subprocess.PIPE,
+                                      stdout=subprocess.PIPE)
+        self.prepare()
+
+    def prepare(self):
+        self.run_code(fname=None,
+                      code=_PYTHON_HELPERS,
+                      kwargs={},
+                      copy_env=False)
+
+    def kill_child(self):
+        self.child.kill()
+
+    def ensure_running(self):
+        if self.child is None:
+            self.run()
+        try:
+            os.waitpid(self.child.pid, os.WNOHANG)
+        except Exception:
+            running = False
+        else:
+            running = True
+        if not running:
+            self.run()
+
+    def send(self, data):
+        c_data = json.dumps(data)
+        dlen = len(c_data)
+        self.ensure_running()
+        self.child.stdin.write(struct.pack('<L', dlen))
+        self.child.stdin.write(c_data)
+        self.child.stdin.flush()
+
+    def read(self):
+        # TODO (jnowak) this int may be unsafe
+        hdr = self.child.stdout.read(struct.calcsize('<L'))
+        if not hdr:
+            raise Exception("Loop crashed, probably invalid code")
+        dlen = int(struct.unpack('<L', hdr)[0])
+        data = self.child.stdout.read(dlen)
+        return json.loads(data)
+
+    def run_code(self, fname, code, kwargs, copy_env=True):
+        self.send({'fname': fname,
+                   'code': code,
+                   'kwargs': kwargs,
+                   'copy_env': copy_env})
+        result = self.read()
+        if 'error' in result:
+            raise Exception("Loop error: %r" % result['error'])
+        return result['result']
+
+
+class PyProcessor(ComputableInputProcessor):
+
+    def __init__(self):
+        self.mgr = Mgr()
+        self.mgr.run()
+
+    def check_funct(self, funct, computable_type):
+        if not funct.startswith('def calculate_input('):
+            code = funct.splitlines()
+            if computable_type == ComputablePassedTypes.full.name:
+                code.insert(0, 'R = make_arr(D)')
+            code = '\n    '.join(code)
+            return 'def calculate_input(D, resource_name):\n    %s' % code
+        return funct
+
+    def run(self, resource_name, computable_type, funct, data):
+        funct = self.check_funct(funct, computable_type)
+        value = self.mgr.run_code(code=funct,
+                                  fname='calculate_input',
+                                  kwargs={'D': data,
+                                          'resource_name': resource_name})
+        return value

solar/computable_inputs/helpers/lua_helpers.lua

@@ -0,0 +1,10 @@
+function make_arr(data)
+   local t = {}
+   for orig_value in python.iter(data) do
+      if t[orig_value["resource"]] == nil then
+         t[orig_value["resource"]] = {}
+      end
+      t[orig_value["resource"]][orig_value['other_input']] = orig_value['value']
+   end
+   return t
+end

solar/computable_inputs/helpers/python_helpers.py

@@ -0,0 +1,7 @@
+def make_arr(data):
+    t = {}
+    for ov in data:
+        if t.get(ov['resource']) is None:
+            t[ov['resource']] = {}
+        t[ov['resource']][ov['other_input']] = ov['value']
+    return t

solar/computable_inputs/helpers/python_loop.py

@@ -0,0 +1,76 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# Consider this part as POC
+
+import json
+import struct
+import sys
+
+_hdr_size = struct.calcsize('<L')
+
+try:
+    from seccomp import *  # noqa
+except ImportError:
+    # TODO: (jnowak) unsafe fallback for now
+    pass
+else:
+    RULES = SyscallFilter(defaction=KILL)
+
+    RULES.add_rule(ALLOW, 'read', Arg(0, EQ, sys.stdin.fileno()))
+    RULES.add_rule(ALLOW, 'fstat')
+    RULES.add_rule(ALLOW, 'mmap')
+    RULES.add_rule(ALLOW, 'write', Arg(0, EQ, sys.stdout.fileno()))
+    RULES.add_rule(ALLOW, "exit_group")
+    RULES.add_rule(ALLOW, "rt_sigaction")
+    RULES.add_rule(ALLOW, "brk")
+
+    RULES.load()
+
+
+_env = {}
+
+
+def exec_remote(fname, code, kwargs, copy_env=True):
+    if copy_env:
+        local_env = _env.copy()
+    else:
+        local_env = _env
+    local_env.update(**kwargs)
+    exec code in _env
+    if fname is not None:
+        return _env[fname](**kwargs)
+    return True
+
+
+while True:
+    read = sys.stdin.read(_hdr_size)
+    if not read:
+        break
+    d_size = int(struct.unpack('<L', read)[0])
+    data = sys.stdin.read(d_size)
+    cmd = json.loads(data)
+    result, error = None, None
+    try:
+        result = exec_remote(**cmd)
+    except Exception as ex:
+        error = str(ex)
+    if result:
+        resp = {'result': result}
+    else:
+        resp = {'error': error}
+    resp_json = json.dumps(resp)
+    sys.stdout.write(struct.pack("<L", len(resp_json)))
+    sys.stdout.write(resp_json)
+    sys.stdout.flush()

solar/computable_inputs/processor.py

@@ -0,0 +1,53 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+_av_processors = {}
+
+try:
+    from solar.computable_inputs.ci_lua import LuaProcessor
+except ImportError:
+    pass
+else:
+    _av_processors['lua'] = LuaProcessor
+
+try:
+    from solar.computable_inputs.ci_python import PyProcessor
+except ImportError:
+    pass
+else:
+    _av_processors['py'] = PyProcessor
+    _av_processors['python'] = PyProcessor
+
+try:
+    from solar.computable_inputs.ci_jinja import JinjaProcessor
+except ImportError:
+    pass
+else:
+    _av_processors['j2'] = JinjaProcessor
+    _av_processors['jinja'] = JinjaProcessor
+    _av_processors['jinja2'] = JinjaProcessor
+
+
+_processors = {}
+
+
+def get_processor(resource, input_name, computable_type, data, other=None):
+    computable = resource.meta_inputs[input_name]['computable']
+    lang = computable['lang']
+    funct = computable['func']
+    if lang not in _processors:
+        _processors[lang] = processor = _av_processors[lang]()
+    else:
+        processor = _processors[lang]
+    return processor.process(resource.name, computable_type, funct, data)

solar/core/resource/resource.py

@@ -273,7 +273,8 @@ class Resource(object):
         # signals.connect(self, receiver, mapping=mapping)
         # TODO: implement events
         if use_defaults:
-            api.add_default_events(self, receiver)
+            if self != receiver:
+                api.add_default_events(self, receiver)
         if events:
             api.add_events(self.name, events)
 

solar/core/signals.py

@@ -141,6 +141,8 @@ def location_and_transports(emitter, receiver, orig_mapping):
 
 
 def get_mapping(emitter, receiver, mapping=None):
+    if emitter == receiver:
+        return mapping
     if mapping is None:
         mapping = guess_mapping(emitter, receiver)
     location_and_transports(emitter, receiver, mapping)

solar/dblayer/solar_models.py

@@ -20,6 +20,8 @@ from types import NoneType
 from uuid import uuid4
 
 from enum import Enum
+from solar.computable_inputs import ComputablePassedTypes
+from solar.computable_inputs.processor import get_processor
 from solar.dblayer.model import check_state_for
 from solar.dblayer.model import CompositeIndexField
 from solar.dblayer.model import DBLayerException
@@ -32,7 +34,8 @@ from solar.dblayer.model import SingleIndexCache
 from solar.dblayer.model import StrInt
 from solar.utils import solar_map
 
-InputTypes = Enum('InputTypes', 'simple list hash list_hash')
+
+InputTypes = Enum('InputTypes', 'simple list hash list_hash computable')
 
 
 class DBLayerSolarException(DBLayerException):
@@ -53,7 +56,11 @@ class InputsFieldWrp(IndexFieldWrp):
         # XXX: it could be worth to precalculate it
         if ':' in name:
             name = name.split(":", 1)[0]
-        schema = resource.meta_inputs[name].get('schema', None)
+        mi = resource.meta_inputs[name]
+        schema = mi.get('schema', None)
+        is_computable = mi.get('computable', None) is not None
+        if is_computable:
+            return InputTypes.computable
         if isinstance(schema, self._simple_types):
             return InputTypes.simple
         if isinstance(schema, list):
@@ -216,6 +223,13 @@ class InputsFieldWrp(IndexFieldWrp):
             my_resource, my_inp_name, other_resource, other_inp_name, my_type,
             other_type)
 
+    def _connect_other_computable(self, my_resource, my_inp_name,
+                                  other_resource, other_inp_name, my_type,
+                                  other_type):
+        return self._connect_other_simple(
+            my_resource, my_inp_name, other_resource, other_inp_name, my_type,
+            other_type)
+
     def _connect_my_list(self, my_resource, my_inp_name, other_resource,
                          other_inp_name, my_type, other_type):
         ret = self._connect_my_simple(my_resource, my_inp_name, other_resource,
@@ -250,6 +264,12 @@ class InputsFieldWrp(IndexFieldWrp):
         return self._connect_my_hash(my_resource, my_inp_name, other_resource,
                                      other_inp_name, my_type, other_type)
 
+    def _connect_my_computable(self, my_resource, my_inp_name, other_resource,
+                               other_inp_name, my_type, other_type):
+        return self._connect_my_simple(my_resource, my_inp_name,
+                                       other_resource, other_inp_name,
+                                       my_type, other_type)
+
     def connect(self, my_inp_name, other_resource, other_inp_name):
         my_resource = self._instance
         other_type = self._input_type(other_resource, other_inp_name)
@@ -526,6 +546,27 @@ class InputsFieldWrp(IndexFieldWrp):
         self._cache[name] = res
         return res
 
+    def _map_field_val_computable(self, recvs, input_name, name, other=None):
+        to_calc = []
+        computable = self._instance.meta_inputs[input_name]['computable']
+        computable_type = computable.get('type',
+                                         ComputablePassedTypes.values.name)
+        for recv in recvs:
+            index_val, obj_key = recv
+            splitted = index_val.split('|', 4)
+            _, inp, emitter_key, emitter_inp, _ = splitted
+            res = Resource.get(emitter_key)
+            inp_value = res.inputs._get_field_val(emitter_inp,
+                                                  other)
+            if computable_type == ComputablePassedTypes.values.name:
+                to_calc.append(inp_value)
+            else:
+                to_calc.append({'value': inp_value,
+                                'resource': res.name,
+                                'other_input': emitter_inp})
+        return get_processor(self._instance, input_name,
+                             computable_type, to_calc, other)
+
     def _get_raw_field_val(self, name):
         return self._instance._data_container[self.fname][name]
 
@@ -747,7 +788,9 @@ class Resource(Model):
         if mapping is None:
             return
         if self == other:
-            raise Exception('Trying to connect value-.* to itself')
+            for k, v in mapping.items():
+                if k == v:
+                    raise Exception('Trying to connect value-.* to itself')
         solar_map(
             lambda (my_name, other_name): self._connect_single(other_inputs,
                                                                other_name,

solar/dblayer/test/test_computable_inputs.py

@@ -0,0 +1,350 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from __future__ import print_function
+
+import pytest
+
+from solar.computable_inputs import ComputablePassedTypes as CPT
+from solar.dblayer.test.test_real import create_resource
+
+pytest.importorskip('lupa')
+
+dth = pytest.dicts_to_hashable
+
+
+def test_lua_simple_noop(rk):
+    k1 = next(rk)
+    k2 = next(rk)
+
+    r1 = create_resource(k1, {'name': 'source1',
+                              'inputs': {'input1': 10}})
+    r2 = create_resource(k2, {'name': 'target1',
+                              'inputs': {'input1': None}})
+
+    r2.meta_inputs['input1']['computable'] = {'func': None,
+                                              'lang': 'lua'}
+    r1.connect(r2, {'input1': 'input1'})
+    r1.save()
+    r2.save()
+
+    assert r2.inputs['input1'] == [10]
+
+
+def test_lua_full_noop(rk):
+    k1 = next(rk)
+    k2 = next(rk)
+
+    r1 = create_resource(k1, {'name': 'source1',
+                              'inputs': {'input1': 10}})
+    r2 = create_resource(k2, {'name': 'target1',
+                              'inputs': {'input1': None}})
+
+    r2.meta_inputs['input1']['computable'] = {'func': None,
+                                              'type': CPT.full.name,
+                                              'lang': 'lua'}
+    r1.connect(r2, {'input1': 'input1'})
+    r1.save()
+    r2.save()
+
+    assert r2.inputs['input1'] == [{'value': 10, 'resource': r1.name,
+                                    'other_input': 'input1'}]
+
+
+def test_lua_simple_lua_simple_max(rk):
+    k1 = next(rk)
+    k2 = next(rk)
+    k3 = next(rk)
+
+    r1 = create_resource(k1, {'name': 'source1',
+                              'inputs': {'input1': 10}})
+    r3 = create_resource(k3, {'name': 'source1',
+                              'inputs': {'input1': 11}})
+    r2 = create_resource(k2, {'name': 'target1',
+                              'inputs': {'input1': None}})
+
+    lua_funct = 'return math.max(unpack(D))'
+    r2.meta_inputs['input1']['computable'] = {'func': lua_funct,
+                                              'lang': 'lua'}
+    r1.connect(r2, {'input1': 'input1'})
+    r3.connect(r2, {'input1': 'input1'})
+
+    r1.save()
+    r2.save()
+    r3.save()
+
+    assert r2.inputs['input1'] == 11
+
+
+def test_lua_full_lua_array(rk):
+    k1 = next(rk)
+    k2 = next(rk)
+    k3 = next(rk)
+
+    r1 = create_resource(k1, {'name': 'source1',
+                              'inputs': {'input1': 10}})
+    r3 = create_resource(k3, {'name': 'source1',
+                              'inputs': {'input1': 11}})
+    r2 = create_resource(k2, {'name': 'target1',
+                              'inputs': {'input1': None}})
+
+    # raw python object, counts from 0
+    lua_funct = 'return D'
+    r2.meta_inputs['input1']['computable'] = {'func': lua_funct,
+                                              'type': CPT.full.name,
+                                              'lang': 'lua'}
+    r1.connect(r2, {'input1': 'input1'})
+    r3.connect(r2, {'input1': 'input1'})
+
+    r1.save()
+    r2.save()
+    r3.save()
+
+    res_inputs = set(dth(r2.inputs['input1']))
+    comparsion = set(dth([{'value': 11, 'resource': r3.name,
+                           'other_input': 'input1'},
+                          {'value': 10, 'resource': r1.name,
+                           'other_input': 'input1'}]))
+    assert res_inputs == comparsion
+
+
+def test_lua_connect_to_computed(rk):
+    k1 = next(rk)
+    k2 = next(rk)
+    k3 = next(rk)
+    k4 = next(rk)
+
+    r1 = create_resource(k1, {'name': 'source1',
+                              'inputs': {'input1': 10}})
+    r3 = create_resource(k3, {'name': 'source1',
+                              'inputs': {'input1': 11}})
+    r2 = create_resource(k2, {'name': 'target1',
+                              'inputs': {'input1': None}})
+    r4 = create_resource(k4, {'name': 'target1',
+                              'inputs': {'input1': None}})
+
+    lua_funct = 'return math.max(unpack(D))'
+    r2.meta_inputs['input1']['computable'] = {'func': lua_funct,
+                                              'lang': 'lua'}
+    r1.connect(r2, {'input1': 'input1'})
+    r3.connect(r2, {'input1': 'input1'})
+
+    r2.connect(r4, {'input1': 'input1'})
+
+    r1.save()
+    r2.save()
+    r3.save()
+    r4.save()
+
+    assert r4.inputs['input1'] == 11
+
+
+def test_lua_join_different_values(rk):
+    k1 = next(rk)
+    k2 = next(rk)
+    k3 = next(rk)
+    k4 = next(rk)
+
+    r1 = create_resource(k1, {'name': 'r1',
+                              'inputs': {'input1': "blah"}})
+    r2 = create_resource(k2, {'name': 'r2',
+                              'inputs': {'input2': "blub"}})
+    r3 = create_resource(k3, {'name': 'r3',
+                              'inputs': {'input': None}})
+    r4 = create_resource(k4, {'name': 'r4',
+                              'inputs': {'input': None}})
+
+    lua_funct = """
+return R["r1"]["input1"] .. "@" .. R["r2"]["input2"]"""
+
+    r3.meta_inputs['input']['computable'] = {"func": lua_funct,
+                                             'lang': 'lua',
+                                             'type': CPT.full.name}
+
+    r1.connect(r3, {'input1': 'input'})
+    r2.connect(r3, {'input2': 'input'})
+
+    r1.save()
+    r2.save()
+    r3.save()
+
+    assert r3.inputs['input'] == 'blah@blub'
+
+    r3.connect(r4, {'input': 'input'})
+    r4.save()
+
+    assert r4.inputs['input'] == 'blah@blub'
+
+
+def test_lua_join_replace_in_lua(rk):
+    k1 = next(rk)
+    k2 = next(rk)
+    k3 = next(rk)
+    k4 = next(rk)
+
+    r1 = create_resource(k1, {'name': 'r1',
+                              'inputs': {'input1': "blah"}})
+    r2 = create_resource(k2, {'name': 'r2',
+                              'inputs': {'input2': "blub"}})
+    r3 = create_resource(k3, {'name': 'r3',
+                              'inputs': {'input': None}})
+    r4 = create_resource(k4, {'name': 'r4',
+                              'inputs': {'input': None}})
+
+    lua_funct = """
+return R["r1"]["input1"] .. "@" .. R["r2"]["input2"]
+"""
+
+    r3.meta_inputs['input']['computable'] = {"func": lua_funct,
+                                             'lang': 'lua',
+                                             'type': CPT.full.name}
+
+    lua_funct2 = """local v = D[1]
+v = v:gsub("@", "-", 1)
+return v
+"""
+
+    r4.meta_inputs['input']['computable'] = {"func": lua_funct2,
+                                             'lang': 'lua',
+                                             'type': CPT.values.name}
+
+    r1.connect(r3, {'input1': 'input'})
+    r2.connect(r3, {'input2': 'input'})
+
+    r1.save()
+    r2.save()
+    r3.save()
+
+    assert r3.inputs['input'] == 'blah@blub'
+
+    r3.connect(r4, {'input': 'input'})
+    r4.save()
+
+    assert r4.inputs['input'] == 'blah-blub'
+
+
+def test_lua_join_self_computable(rk):
+    k1 = next(rk)
+
+    r1 = create_resource(k1, {'name': "r1",
+                              'inputs': {'input1': 'bar',
+                                         'input2': 'foo',
+                                         'input3': None}})
+
+    lua_funct = """
+return resource_name .. R["r1"]["input2"] .. R["r1"]["input1"]
+"""
+
+    r1.meta_inputs['input3']['computable'] = {'func': lua_funct,
+                                              'lang': 'lua',
+                                              'type': CPT.full.name}
+
+    r1.connect(r1, {'input1': 'input3'})
+    r1.connect(r1, {'input2': 'input3'})
+
+    r1.save()
+
+    assert r1.inputs['input3'] == 'r1foobar'
+
+
+def test_python_join_self_computable(rk):
+    k1 = next(rk)
+
+    r1 = create_resource(k1, {'name': "r1",
+                              'inputs': {'input1': 'bar',
+                                         'input2': 'foo',
+                                         'input3': None}})
+    py_funct = """
+return resource_name + R["r1"]["input2"] + R["r1"]["input1"]
+"""
+
+    r1.meta_inputs['input3']['computable'] = {'func': py_funct,
+                                              'lang': 'py',
+                                              'type': CPT.full.name}
+
+    r1.connect(r1, {'input1': 'input3'})
+    r1.connect(r1, {'input2': 'input3'})
+
+    r1.save()
+
+    assert r1.inputs['input3'] == 'r1foobar'
+
+
+def test_jinja_join_self_computable(rk):
+    k1 = next(rk)
+
+    r1 = create_resource(k1, {'name': "r1",
+                              'inputs': {'input1': 'bar',
+                                         'input2': 'foo',
+                                         'input3': None}})
+    jinja_funct = """
+{{resource_name}}{{input2}}{{input1}}
+"""
+
+    r1.meta_inputs['input3']['computable'] = {'func': jinja_funct,
+                                              'lang': 'jinja2',
+                                              'type': CPT.full.name}
+
+    r1.connect(r1, {'input1': 'input3'})
+    r1.connect(r1, {'input2': 'input3'})
+
+    r1.save()
+
+    assert r1.inputs['input3'] == 'r1foobar'
+
+
+def test_jinja_join_self_sum(rk):
+    k1 = next(rk)
+
+    r1 = create_resource(k1, {'name': "r1",
+                              'inputs': {'input1': 3,
+                                         'input2': 2,
+                                         'input3': None}})
+    jinja_funct = """
+{{[input1, input2]|sum}}
+"""
+
+    r1.meta_inputs['input3']['computable'] = {'func': jinja_funct,
+                                              'lang': 'jinja2',
+                                              'type': CPT.full.name}
+
+    r1.connect(r1, {'input1': 'input3'})
+    r1.connect(r1, {'input2': 'input3'})
+
+    r1.save()
+
+    assert r1.inputs['input3'] == '5'
+
+
+def test_jinja_join_self_sum_simple(rk):
+    k1 = next(rk)
+
+    r1 = create_resource(k1, {'name': "r1",
+                              'inputs': {'input1': 3,
+                                         'input2': 2,
+                                         'input3': None}})
+    jinja_funct = """
+{{D|sum}}
+"""
+
+    r1.meta_inputs['input3']['computable'] = {'func': jinja_funct,
+                                              'lang': 'jinja2',
+                                              'type': CPT.values.name}
+
+    r1.connect(r1, {'input1': 'input3'})
+    r1.connect(r1, {'input2': 'input3'})
+
+    r1.save()
+
+    assert r1.inputs['input3'] == '5'

solar/dblayer/test/test_real.py

@@ -32,6 +32,9 @@ def create_resource(key, data):
         elif isinstance(inp_value, dict):
             schema = {}
         else:
+            if inp_value is None:
+                mi.setdefault(inp_name, {})
+                continue
             schema = '%s!' % type(inp_value).__name__
         mi.setdefault(inp_name, {"schema": schema})
     data['meta_inputs'] = mi

test-requirements.txt

@@ -6,3 +6,6 @@ pytest-mock
 tox
 pytest-subunit
 os-testr
+
+# for computable inputs
+lupa