From 582b8c17e14dd4bc73a166c8ae5837e78dd299ec Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 2 Dec 2015 18:27:31 +0100 Subject: [PATCH 01/24] First skeleton of Computable Inputs --- solar/computable_inputs/__init__.py | 24 +++++++++ solar/computable_inputs/ci_lua.py | 31 +++++++++++ solar/computable_inputs/processor.py | 36 +++++++++++++ solar/core/computable_inputs.py | 14 +++++ solar/dblayer/solar_models.py | 24 ++++++++- solar/dblayer/test/__init__.py | 0 solar/dblayer/test/test_computable_inputs.py | 55 ++++++++++++++++++++ solar/dblayer/test/test_real.py | 3 ++ 8 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 solar/computable_inputs/__init__.py create mode 100644 solar/computable_inputs/ci_lua.py create mode 100644 solar/computable_inputs/processor.py create mode 100644 solar/core/computable_inputs.py create mode 100644 solar/dblayer/test/__init__.py create mode 100644 solar/dblayer/test/test_computable_inputs.py diff --git a/solar/computable_inputs/__init__.py b/solar/computable_inputs/__init__.py new file mode 100644 index 00000000..fa1d3b0c --- /dev/null +++ b/solar/computable_inputs/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class ComputableInputProcessor(object): + + def __init__(self): + pass + + def process(self, funct, data): + if funct is None or funct == 'noop': + return data + return self.run(funct, data) diff --git a/solar/computable_inputs/ci_lua.py b/solar/computable_inputs/ci_lua.py new file mode 100644 index 00000000..80fdded8 --- /dev/null +++ b/solar/computable_inputs/ci_lua.py @@ -0,0 +1,31 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import lupa +from lupa import LuaRuntime +from solar.computable_inputs import ComputableInputProcessor + + +class LuaProcessor(ComputableInputProcessor): + + def __init__(self): + self.lua = LuaRuntime() + + def run(self, funct, data): + if isinstance(data, list): + lua_data = self.lua.table_from(data) + else: + lua_data = data + funct_lua = self.lua.eval(funct) + return funct_lua(lua_data) diff --git a/solar/computable_inputs/processor.py b/solar/computable_inputs/processor.py new file mode 100644 index 00000000..89a86cfb --- /dev/null +++ b/solar/computable_inputs/processor.py @@ -0,0 +1,36 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +_av_processors = {} + +try: + from solar.computable_inputs.ci_lua import LuaProcessor +except ImportError: + pass +else: + _av_processors['lua'] = LuaProcessor + + +_processors = {} + + +def get_processor(resource, input_name, data, other=None): + computable = resource.meta_inputs[input_name]['computable'] + lang = computable['lang'] + funct = computable['func'] + if lang not in _processors: + _processors[lang] = processor = _av_processors[lang]() + else: + processor = _processors[lang] + return processor.process(funct, data) diff --git a/solar/core/computable_inputs.py b/solar/core/computable_inputs.py new file mode 100644 index 00000000..aee8d92e --- /dev/null +++ b/solar/core/computable_inputs.py @@ -0,0 +1,14 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + diff --git a/solar/dblayer/solar_models.py b/solar/dblayer/solar_models.py index 724c940b..994d10f7 100644 --- a/solar/dblayer/solar_models.py +++ b/solar/dblayer/solar_models.py @@ -20,6 +20,7 @@ from types import NoneType from uuid import uuid4 from enum import Enum +from solar.computable_inputs.processor import get_processor from solar.dblayer.model import check_state_for from solar.dblayer.model import CompositeIndexField from solar.dblayer.model import DBLayerException @@ -32,7 +33,8 @@ from solar.dblayer.model import SingleIndexCache from solar.dblayer.model import StrInt from solar.utils import solar_map -InputTypes = Enum('InputTypes', 'simple list hash list_hash') + +InputTypes = Enum('InputTypes', 'simple list hash list_hash computable') class DBLayerSolarException(DBLayerException): @@ -54,6 +56,9 @@ class InputsFieldWrp(IndexFieldWrp): if ':' in name: name = name.split(":", 1)[0] schema = resource.meta_inputs[name].get('schema', None) + is_computable = resource.meta_inputs[name].get('computable', None) is not None + if is_computable: + return InputTypes.computable if isinstance(schema, self._simple_types): return InputTypes.simple if isinstance(schema, list): @@ -239,6 +244,12 @@ class InputsFieldWrp(IndexFieldWrp): return self._connect_my_hash(my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) + def _connect_my_computable(self, my_resource, my_inp_name, other_resource, + other_inp_name, my_type, other_type): + return self._connect_my_simple(my_resource, my_inp_name, + other_resource, other_inp_name, + my_type, other_type) + def connect(self, my_inp_name, other_resource, other_inp_name): my_resource = self._instance other_type = self._input_type(other_resource, other_inp_name) @@ -514,6 +525,17 @@ class InputsFieldWrp(IndexFieldWrp): self._cache[name] = res return res + def _map_field_val_computable(self, recvs, input_name, name, other=None): + to_calc = [] + for recv in recvs: + index_val, obj_key = recv + splitted = index_val.split('|', 4) + _, inp, emitter_key, emitter_inp, _mapping_type = splitted + res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, + other) + to_calc.append(res) + return get_processor(self._instance, input_name, to_calc, other) + def _get_raw_field_val(self, name): return self._instance._data_container[self.fname][name] diff --git a/solar/dblayer/test/__init__.py b/solar/dblayer/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/solar/dblayer/test/test_computable_inputs.py b/solar/dblayer/test/test_computable_inputs.py new file mode 100644 index 00000000..c08788b1 --- /dev/null +++ b/solar/dblayer/test/test_computable_inputs.py @@ -0,0 +1,55 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function +import pytest + +from solar.dblayer.test.test_real import create_resource +from solar.dblayer.solar_models import InputTypes + + +def test_simple_noop(rk): + k1 = next(rk) + k2 = next(rk) + + r1 = create_resource(k1, {'name': 'source1', + 'inputs': {'input1': 10}}) + r2 = create_resource(k2, {'name': 'target1', + 'inputs': {'input1': None}}) + + r2.meta_inputs['input1']['computable'] = {'func': None, + 'lang': 'lua'} + r1.connect(r2, {'input1': 'input1'}) + r1.save() + r2.save() + + assert r2.inputs['input1'] == [10] + + +def test_simple_lua_simple_max(rk): + k1 = next(rk) + k2 = next(rk) + + r1 = create_resource(k1, {'name': 'source1', + 'inputs': {'input1': 10}}) + r2 = create_resource(k2, {'name': 'target1', + 'inputs': {'input1': None}}) + + r2.meta_inputs['input1']['computable'] = {'func': 'function(arr) return math.max(unpack(arr)) end', + 'lang': 'lua'} + r1.connect(r2, {'input1': 'input1'}) + r1.save() + r2.save() + + assert r2.inputs['input1'] == 10 diff --git a/solar/dblayer/test/test_real.py b/solar/dblayer/test/test_real.py index 7cf58f0c..cefcd4fe 100644 --- a/solar/dblayer/test/test_real.py +++ b/solar/dblayer/test/test_real.py @@ -32,6 +32,9 @@ def create_resource(key, data): elif isinstance(inp_value, dict): schema = {} else: + if inp_value is None: + mi.setdefault(inp_name, {}) + continue schema = '%s!' % type(inp_value).__name__ mi.setdefault(inp_name, {"schema": schema}) data['meta_inputs'] = mi From 7289352f7577683c2fbfb343c00276529428f565 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Wed, 2 Dec 2015 18:50:46 +0100 Subject: [PATCH 02/24] Extended test for max --- solar/dblayer/test/test_computable_inputs.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/solar/dblayer/test/test_computable_inputs.py b/solar/dblayer/test/test_computable_inputs.py index c08788b1..423322b0 100644 --- a/solar/dblayer/test/test_computable_inputs.py +++ b/solar/dblayer/test/test_computable_inputs.py @@ -40,16 +40,21 @@ def test_simple_noop(rk): def test_simple_lua_simple_max(rk): k1 = next(rk) k2 = next(rk) + k3 = next(rk) r1 = create_resource(k1, {'name': 'source1', 'inputs': {'input1': 10}}) + r3 = create_resource(k3, {'name': 'source1', + 'inputs': {'input1': 11}}) r2 = create_resource(k2, {'name': 'target1', 'inputs': {'input1': None}}) r2.meta_inputs['input1']['computable'] = {'func': 'function(arr) return math.max(unpack(arr)) end', 'lang': 'lua'} r1.connect(r2, {'input1': 'input1'}) + r3.connect(r2, {'input1': 'input1'}) + r1.save() r2.save() - assert r2.inputs['input1'] == 10 + assert r2.inputs['input1'] == 11 From 7f6a965370b4eec0837fb7bcf72c49b1ad164cc8 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 3 Dec 2015 14:57:24 +0100 Subject: [PATCH 03/24] Basic ops are working --- solar/computable_inputs/__init__.py | 8 +- solar/computable_inputs/ci_lua.py | 9 +- solar/computable_inputs/processor.py | 6 +- solar/dblayer/solar_models.py | 22 ++++- solar/dblayer/test/test_computable_inputs.py | 94 +++++++++++++++++++- 5 files changed, 127 insertions(+), 12 deletions(-) diff --git a/solar/computable_inputs/__init__.py b/solar/computable_inputs/__init__.py index fa1d3b0c..13bc0f85 100644 --- a/solar/computable_inputs/__init__.py +++ b/solar/computable_inputs/__init__.py @@ -12,13 +12,17 @@ # License for the specific language governing permissions and limitations # under the License. +from enum import Enum + +ComputablePassedTypes = Enum('ComputablePassedTypes', 'values full') + class ComputableInputProcessor(object): def __init__(self): pass - def process(self, funct, data): + def process(self, computable_type, funct, data): if funct is None or funct == 'noop': return data - return self.run(funct, data) + return self.run(computable_type, funct, data) diff --git a/solar/computable_inputs/ci_lua.py b/solar/computable_inputs/ci_lua.py index 80fdded8..40303148 100644 --- a/solar/computable_inputs/ci_lua.py +++ b/solar/computable_inputs/ci_lua.py @@ -15,6 +15,7 @@ import lupa from lupa import LuaRuntime from solar.computable_inputs import ComputableInputProcessor +from solar.dblayer.solar_models import ComputablePassedTypes class LuaProcessor(ComputableInputProcessor): @@ -22,10 +23,14 @@ class LuaProcessor(ComputableInputProcessor): def __init__(self): self.lua = LuaRuntime() - def run(self, funct, data): - if isinstance(data, list): + def run(self, computable_type, funct, data): + # when computable_type == full then raw python object is passed + # to lua (counts from 0 etc) + + if isinstance(data, list) and computable_type == ComputablePassedTypes.values: lua_data = self.lua.table_from(data) else: lua_data = data + funct_lua = self.lua.eval(funct) return funct_lua(lua_data) diff --git a/solar/computable_inputs/processor.py b/solar/computable_inputs/processor.py index 89a86cfb..e008ec8c 100644 --- a/solar/computable_inputs/processor.py +++ b/solar/computable_inputs/processor.py @@ -17,7 +17,7 @@ _av_processors = {} try: from solar.computable_inputs.ci_lua import LuaProcessor except ImportError: - pass + raise else: _av_processors['lua'] = LuaProcessor @@ -25,7 +25,7 @@ else: _processors = {} -def get_processor(resource, input_name, data, other=None): +def get_processor(resource, input_name, computable_type, data, other=None): computable = resource.meta_inputs[input_name]['computable'] lang = computable['lang'] funct = computable['func'] @@ -33,4 +33,4 @@ def get_processor(resource, input_name, data, other=None): _processors[lang] = processor = _av_processors[lang]() else: processor = _processors[lang] - return processor.process(funct, data) + return processor.process(computable_type, funct, data) diff --git a/solar/dblayer/solar_models.py b/solar/dblayer/solar_models.py index 994d10f7..488bdf0f 100644 --- a/solar/dblayer/solar_models.py +++ b/solar/dblayer/solar_models.py @@ -20,6 +20,7 @@ from types import NoneType from uuid import uuid4 from enum import Enum +from solar.computable_inputs import ComputablePassedTypes from solar.computable_inputs.processor import get_processor from solar.dblayer.model import check_state_for from solar.dblayer.model import CompositeIndexField @@ -215,6 +216,13 @@ class InputsFieldWrp(IndexFieldWrp): my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) + def _connect_other_computable(self, my_resource, my_inp_name, + other_resource, other_inp_name, my_type, + other_type): + return self._connect_other_simple( + my_resource, my_inp_name, other_resource, other_inp_name, my_type, + other_type) + def _connect_my_list(self, my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type): ret = self._connect_my_simple(my_resource, my_inp_name, other_resource, @@ -527,14 +535,22 @@ class InputsFieldWrp(IndexFieldWrp): def _map_field_val_computable(self, recvs, input_name, name, other=None): to_calc = [] + computable = self._instance.meta_inputs[input_name]['computable'] + computable_type = computable.get('type', ComputablePassedTypes.values) for recv in recvs: index_val, obj_key = recv splitted = index_val.split('|', 4) - _, inp, emitter_key, emitter_inp, _mapping_type = splitted + _, inp, emitter_key, emitter_inp, _ = splitted res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, other) - to_calc.append(res) - return get_processor(self._instance, input_name, to_calc, other) + if computable_type == ComputablePassedTypes.values: + to_calc.append(res) + else: + to_calc.append({'value': res, + 'resource': emitter_key, + 'other_input': emitter_inp}) + return get_processor(self._instance, input_name, + computable_type, to_calc, other) def _get_raw_field_val(self, name): return self._instance._data_container[self.fname][name] diff --git a/solar/dblayer/test/test_computable_inputs.py b/solar/dblayer/test/test_computable_inputs.py index 423322b0..345c2dc6 100644 --- a/solar/dblayer/test/test_computable_inputs.py +++ b/solar/dblayer/test/test_computable_inputs.py @@ -13,10 +13,14 @@ # under the License. from __future__ import print_function + import pytest from solar.dblayer.test.test_real import create_resource -from solar.dblayer.solar_models import InputTypes +from solar.computable_inputs import ComputablePassedTypes as CPT + + +dth = pytest.dicts_to_hashable def test_simple_noop(rk): @@ -37,6 +41,26 @@ def test_simple_noop(rk): assert r2.inputs['input1'] == [10] +def test_full_noop(rk): + k1 = next(rk) + k2 = next(rk) + + r1 = create_resource(k1, {'name': 'source1', + 'inputs': {'input1': 10}}) + r2 = create_resource(k2, {'name': 'target1', + 'inputs': {'input1': None}}) + + r2.meta_inputs['input1']['computable'] = {'func': None, + 'type': CPT.full.name, + 'lang': 'lua'} + r1.connect(r2, {'input1': 'input1'}) + r1.save() + r2.save() + + assert r2.inputs['input1'] == [{'value': 10, 'resource': r1.key, + 'other_input': 'input1'}] + + def test_simple_lua_simple_max(rk): k1 = next(rk) k2 = next(rk) @@ -49,12 +73,78 @@ def test_simple_lua_simple_max(rk): r2 = create_resource(k2, {'name': 'target1', 'inputs': {'input1': None}}) - r2.meta_inputs['input1']['computable'] = {'func': 'function(arr) return math.max(unpack(arr)) end', + lua_funct = 'function(arr) return math.max(unpack(arr)) end' + r2.meta_inputs['input1']['computable'] = {'func': lua_funct, 'lang': 'lua'} r1.connect(r2, {'input1': 'input1'}) r3.connect(r2, {'input1': 'input1'}) r1.save() r2.save() + r3.save() assert r2.inputs['input1'] == 11 + + +def test_full_lua_array(rk): + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + + r1 = create_resource(k1, {'name': 'source1', + 'inputs': {'input1': 10}}) + r3 = create_resource(k3, {'name': 'source1', + 'inputs': {'input1': 11}}) + r2 = create_resource(k2, {'name': 'target1', + 'inputs': {'input1': None}}) + + # raw python object, counts from 0 + lua_funct = 'function(arr) return arr end' + r2.meta_inputs['input1']['computable'] = {'func': lua_funct, + 'type': CPT.full.name, + 'lang': 'lua'} + r1.connect(r2, {'input1': 'input1'}) + r3.connect(r2, {'input1': 'input1'}) + + r1.save() + r2.save() + r3.save() + + res_inputs = set(dth(r2.inputs['input1'])) + comparsion = set(dth([{'value': 11, 'resource': r3.key, + 'other_input': 'input1'}, + {'value': 10, 'resource': r1.key, + 'other_input': 'input1'}])) + assert res_inputs == comparsion + + +def test_connect_to_computed(rk): + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + k4 = next(rk) + + r1 = create_resource(k1, {'name': 'source1', + 'inputs': {'input1': 10}}) + r3 = create_resource(k3, {'name': 'source1', + 'inputs': {'input1': 11}}) + r2 = create_resource(k2, {'name': 'target1', + 'inputs': {'input1': None}}) + r4 = create_resource(k4, {'name': 'target1', + 'inputs': {'input1': None}}) + + lua_funct = 'function(arr) return math.max(unpack(arr)) end' + r2.meta_inputs['input1']['computable'] = {'func': lua_funct, + 'lang': 'lua'} + r1.connect(r2, {'input1': 'input1'}) + r3.connect(r2, {'input1': 'input1'}) + + r2.connect(r4, {'input1': 'input1'}) + + r1.save() + r2.save() + r3.save() + r4.save() + + assert r4.inputs['input1'] == 11 + From 307707ce3d351c153d029b67cc000436d1257831 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 3 Dec 2015 15:25:27 +0100 Subject: [PATCH 04/24] Added more complex case + added helpers to lua + code pretify --- solar/computable_inputs/__init__.py | 4 +++ solar/computable_inputs/ci_lua.py | 15 +++++++-- .../computable_inputs/helpers/lua_helpers.lua | 10 ++++++ solar/core/computable_inputs.py | 14 -------- solar/dblayer/solar_models.py | 22 +++++++------ solar/dblayer/test/test_computable_inputs.py | 32 ++++++++++++++++++- 6 files changed, 70 insertions(+), 27 deletions(-) create mode 100644 solar/computable_inputs/helpers/lua_helpers.lua delete mode 100644 solar/core/computable_inputs.py diff --git a/solar/computable_inputs/__init__.py b/solar/computable_inputs/__init__.py index 13bc0f85..4838d5f2 100644 --- a/solar/computable_inputs/__init__.py +++ b/solar/computable_inputs/__init__.py @@ -13,9 +13,13 @@ # under the License. from enum import Enum +import os ComputablePassedTypes = Enum('ComputablePassedTypes', 'values full') +HELPERS_PATH = os.path.normpath( + os.path.join(os.path.realpath(__file__), '..', 'helpers')) + class ComputableInputProcessor(object): diff --git a/solar/computable_inputs/ci_lua.py b/solar/computable_inputs/ci_lua.py index 40303148..204b0735 100644 --- a/solar/computable_inputs/ci_lua.py +++ b/solar/computable_inputs/ci_lua.py @@ -12,22 +12,33 @@ # License for the specific language governing permissions and limitations # under the License. -import lupa + +import os + from lupa import LuaRuntime from solar.computable_inputs import ComputableInputProcessor +from solar.computable_inputs import HELPERS_PATH from solar.dblayer.solar_models import ComputablePassedTypes +_LUA_HELPERS = open(os.path.join(HELPERS_PATH, 'lua_helpers.lua')).read() + + +# TODO: (jnowak) add sandoxing + + class LuaProcessor(ComputableInputProcessor): def __init__(self): self.lua = LuaRuntime() + self.lua.execute(_LUA_HELPERS) def run(self, computable_type, funct, data): # when computable_type == full then raw python object is passed # to lua (counts from 0 etc) - if isinstance(data, list) and computable_type == ComputablePassedTypes.values: + if isinstance(data, list) \ + and computable_type == ComputablePassedTypes.values: lua_data = self.lua.table_from(data) else: lua_data = data diff --git a/solar/computable_inputs/helpers/lua_helpers.lua b/solar/computable_inputs/helpers/lua_helpers.lua new file mode 100644 index 00000000..ee863a38 --- /dev/null +++ b/solar/computable_inputs/helpers/lua_helpers.lua @@ -0,0 +1,10 @@ +function make_arr(data) + local t = {} + for orig_value in python.iter(data) do + if t[orig_value["resource"]] == nil then + t[orig_value["resource"]] = {} + end + t[orig_value["resource"]][orig_value['other_input']] = orig_value['value'] + end + return t +end diff --git a/solar/core/computable_inputs.py b/solar/core/computable_inputs.py deleted file mode 100644 index aee8d92e..00000000 --- a/solar/core/computable_inputs.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - diff --git a/solar/dblayer/solar_models.py b/solar/dblayer/solar_models.py index 488bdf0f..0b3c60d7 100644 --- a/solar/dblayer/solar_models.py +++ b/solar/dblayer/solar_models.py @@ -56,8 +56,9 @@ class InputsFieldWrp(IndexFieldWrp): # XXX: it could be worth to precalculate it if ':' in name: name = name.split(":", 1)[0] - schema = resource.meta_inputs[name].get('schema', None) - is_computable = resource.meta_inputs[name].get('computable', None) is not None + mi = resource.meta_inputs[name] + schema = mi.get('schema', None) + is_computable = mi.get('computable', None) is not None if is_computable: return InputTypes.computable if isinstance(schema, self._simple_types): @@ -217,8 +218,8 @@ class InputsFieldWrp(IndexFieldWrp): other_type) def _connect_other_computable(self, my_resource, my_inp_name, - other_resource, other_inp_name, my_type, - other_type): + other_resource, other_inp_name, my_type, + other_type): return self._connect_other_simple( my_resource, my_inp_name, other_resource, other_inp_name, my_type, other_type) @@ -540,14 +541,15 @@ class InputsFieldWrp(IndexFieldWrp): for recv in recvs: index_val, obj_key = recv splitted = index_val.split('|', 4) - _, inp, emitter_key, emitter_inp, _ = splitted - res = Resource.get(emitter_key).inputs._get_field_val(emitter_inp, - other) + _, inp, emitter_key, emitter_inp, _ = splitted + res = Resource.get(emitter_key) + inp_value = res.inputs._get_field_val(emitter_inp, + other) if computable_type == ComputablePassedTypes.values: - to_calc.append(res) + to_calc.append(inp_value) else: - to_calc.append({'value': res, - 'resource': emitter_key, + to_calc.append({'value': inp_value, + 'resource': res.name, 'other_input': emitter_inp}) return get_processor(self._instance, input_name, computable_type, to_calc, other) diff --git a/solar/dblayer/test/test_computable_inputs.py b/solar/dblayer/test/test_computable_inputs.py index 345c2dc6..11087b7f 100644 --- a/solar/dblayer/test/test_computable_inputs.py +++ b/solar/dblayer/test/test_computable_inputs.py @@ -16,8 +16,8 @@ from __future__ import print_function import pytest -from solar.dblayer.test.test_real import create_resource from solar.computable_inputs import ComputablePassedTypes as CPT +from solar.dblayer.test.test_real import create_resource dth = pytest.dicts_to_hashable @@ -148,3 +148,33 @@ def test_connect_to_computed(rk): assert r4.inputs['input1'] == 11 + +def test_join_different_values(rk): + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + + r1 = create_resource(k1, {'name': 'r1', + 'inputs': {'input1': "blah"}}) + r2 = create_resource(k2, {'name': 'r2', + 'inputs': {'input2': "blub"}}) + r3 = create_resource(k3, {'name': 'r3', + 'inputs': {'input': None}}) + + lua_funct = """function (data) +local l = make_arr(data) +return l["r1"]["input1"] .. "@" .. l["r2"]["input2"] +end""" + + r3.meta_inputs['input']['computable'] = {"func": lua_funct, + 'lang': 'lua', + 'type': CPT.full.name} + + r1.connect(r3, {'input1': 'input'}) + r2.connect(r3, {'input2': 'input'}) + + r1.save() + r2.save() + r3.save() + + assert r3.inputs['input'] == 'blah@blub' From cdc9a0f5f3781a082dba39999fe89c1453cbd199 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 3 Dec 2015 16:36:34 +0100 Subject: [PATCH 05/24] Naming adjustments --- solar/computable_inputs/ci_lua.py | 2 +- solar/dblayer/solar_models.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/solar/computable_inputs/ci_lua.py b/solar/computable_inputs/ci_lua.py index 204b0735..b7690e15 100644 --- a/solar/computable_inputs/ci_lua.py +++ b/solar/computable_inputs/ci_lua.py @@ -38,7 +38,7 @@ class LuaProcessor(ComputableInputProcessor): # to lua (counts from 0 etc) if isinstance(data, list) \ - and computable_type == ComputablePassedTypes.values: + and computable_type == ComputablePassedTypes.values.name: lua_data = self.lua.table_from(data) else: lua_data = data diff --git a/solar/dblayer/solar_models.py b/solar/dblayer/solar_models.py index 0b3c60d7..032ea2b9 100644 --- a/solar/dblayer/solar_models.py +++ b/solar/dblayer/solar_models.py @@ -537,7 +537,8 @@ class InputsFieldWrp(IndexFieldWrp): def _map_field_val_computable(self, recvs, input_name, name, other=None): to_calc = [] computable = self._instance.meta_inputs[input_name]['computable'] - computable_type = computable.get('type', ComputablePassedTypes.values) + computable_type = computable.get('type', + ComputablePassedTypes.values.name) for recv in recvs: index_val, obj_key = recv splitted = index_val.split('|', 4) @@ -545,7 +546,7 @@ class InputsFieldWrp(IndexFieldWrp): res = Resource.get(emitter_key) inp_value = res.inputs._get_field_val(emitter_inp, other) - if computable_type == ComputablePassedTypes.values: + if computable_type == ComputablePassedTypes.values.name: to_calc.append(inp_value) else: to_calc.append({'value': inp_value, From 0bb3dfd9cfe475021e2d835855f2a5f1781af66d Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 3 Dec 2015 16:36:45 +0100 Subject: [PATCH 06/24] Test improvements for computable inputs --- solar/dblayer/test/test_computable_inputs.py | 64 +++++++++++++++++++- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/solar/dblayer/test/test_computable_inputs.py b/solar/dblayer/test/test_computable_inputs.py index 11087b7f..28865b88 100644 --- a/solar/dblayer/test/test_computable_inputs.py +++ b/solar/dblayer/test/test_computable_inputs.py @@ -57,7 +57,7 @@ def test_full_noop(rk): r1.save() r2.save() - assert r2.inputs['input1'] == [{'value': 10, 'resource': r1.key, + assert r2.inputs['input1'] == [{'value': 10, 'resource': r1.name, 'other_input': 'input1'}] @@ -111,9 +111,9 @@ def test_full_lua_array(rk): r3.save() res_inputs = set(dth(r2.inputs['input1'])) - comparsion = set(dth([{'value': 11, 'resource': r3.key, + comparsion = set(dth([{'value': 11, 'resource': r3.name, 'other_input': 'input1'}, - {'value': 10, 'resource': r1.key, + {'value': 10, 'resource': r1.name, 'other_input': 'input1'}])) assert res_inputs == comparsion @@ -153,6 +153,7 @@ def test_join_different_values(rk): k1 = next(rk) k2 = next(rk) k3 = next(rk) + k4 = next(rk) r1 = create_resource(k1, {'name': 'r1', 'inputs': {'input1': "blah"}}) @@ -160,6 +161,8 @@ def test_join_different_values(rk): 'inputs': {'input2': "blub"}}) r3 = create_resource(k3, {'name': 'r3', 'inputs': {'input': None}}) + r4 = create_resource(k4, {'name': 'r4', + 'inputs': {'input': None}}) lua_funct = """function (data) local l = make_arr(data) @@ -178,3 +181,58 @@ end""" r3.save() assert r3.inputs['input'] == 'blah@blub' + + r3.connect(r4, {'input': 'input'}) + r4.save() + + assert r4.inputs['input'] == 'blah@blub' + + +def test_join_replace_in_lua(rk): + k1 = next(rk) + k2 = next(rk) + k3 = next(rk) + k4 = next(rk) + + r1 = create_resource(k1, {'name': 'r1', + 'inputs': {'input1': "blah"}}) + r2 = create_resource(k2, {'name': 'r2', + 'inputs': {'input2': "blub"}}) + r3 = create_resource(k3, {'name': 'r3', + 'inputs': {'input': None}}) + r4 = create_resource(k4, {'name': 'r4', + 'inputs': {'input': None}}) + + lua_funct = """function (data) +local l = make_arr(data) +return l["r1"]["input1"] .. "@" .. l["r2"]["input2"] +end""" + + r3.meta_inputs['input']['computable'] = {"func": lua_funct, + 'lang': 'lua', + 'type': CPT.full.name} + + lua_funct2 = """function (data) +local v = data[1] +v = v:gsub("@", "-", 1) +return v +end + """ + + r4.meta_inputs['input']['computable'] = {"func": lua_funct2, + 'lang': 'lua', + 'type': CPT.values.name} + + r1.connect(r3, {'input1': 'input'}) + r2.connect(r3, {'input2': 'input'}) + + r1.save() + r2.save() + r3.save() + + assert r3.inputs['input'] == 'blah@blub' + + r3.connect(r4, {'input': 'input'}) + r4.save() + + assert r4.inputs['input'] == 'blah-blub' From d46fd775565372005a52925adbaba6c8959645e6 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 3 Dec 2015 17:15:58 +0100 Subject: [PATCH 07/24] Self referenced computable inputs --- solar/dblayer/solar_models.py | 4 +++- solar/dblayer/test/test_computable_inputs.py | 25 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/solar/dblayer/solar_models.py b/solar/dblayer/solar_models.py index 032ea2b9..d095ae87 100644 --- a/solar/dblayer/solar_models.py +++ b/solar/dblayer/solar_models.py @@ -776,7 +776,9 @@ class Resource(Model): if mapping is None: return if self == other: - raise Exception('Trying to connect value-.* to itself') + for k, v in mapping.items(): + if k == v: + raise Exception('Trying to connect value-.* to itself') solar_map( lambda (my_name, other_name): self._connect_single(other_inputs, other_name, diff --git a/solar/dblayer/test/test_computable_inputs.py b/solar/dblayer/test/test_computable_inputs.py index 28865b88..9e7db39e 100644 --- a/solar/dblayer/test/test_computable_inputs.py +++ b/solar/dblayer/test/test_computable_inputs.py @@ -236,3 +236,28 @@ end r4.save() assert r4.inputs['input'] == 'blah-blub' + + +def test_join_self_computable(rk): + k1 = next(rk) + + r1 = create_resource(k1, {'name': "r1", + 'inputs': {'input1': 'bar', + 'input2': 'foo', + 'input3': None}}) + + lua_funct = """function (data) +local l = make_arr(data) +return l["r1"]["input2"] .. l["r1"]["input1"] +end""" + + r1.meta_inputs['input3']['computable'] = {'func': lua_funct, + 'lang': 'lua', + 'type': CPT.full.name} + + r1.connect(r1, {'input1': 'input3'}) + r1.connect(r1, {'input2': 'input3'}) + + r1.save() + + assert r1.inputs['input3'] == 'foobar' From f508c33e39a2c3c9813ce519f7b9223991a7ec8f Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 3 Dec 2015 17:16:29 +0100 Subject: [PATCH 08/24] Skip tests when lupa is not installed --- solar/computable_inputs/processor.py | 2 +- solar/dblayer/test/test_computable_inputs.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/solar/computable_inputs/processor.py b/solar/computable_inputs/processor.py index e008ec8c..b59dbe36 100644 --- a/solar/computable_inputs/processor.py +++ b/solar/computable_inputs/processor.py @@ -17,7 +17,7 @@ _av_processors = {} try: from solar.computable_inputs.ci_lua import LuaProcessor except ImportError: - raise + pass else: _av_processors['lua'] = LuaProcessor diff --git a/solar/dblayer/test/test_computable_inputs.py b/solar/dblayer/test/test_computable_inputs.py index 9e7db39e..085ecc90 100644 --- a/solar/dblayer/test/test_computable_inputs.py +++ b/solar/dblayer/test/test_computable_inputs.py @@ -19,6 +19,7 @@ import pytest from solar.computable_inputs import ComputablePassedTypes as CPT from solar.dblayer.test.test_real import create_resource +pytest.importorskip('lupa') dth = pytest.dicts_to_hashable From 2f0d636fea598a816416bc4e3e458a16eb749d4c Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 3 Dec 2015 17:16:54 +0100 Subject: [PATCH 09/24] Added lupa to test-requirements, and added note in requirements --- requirements.txt | 4 ++++ test-requirements.txt | 3 +++ 2 files changed, 7 insertions(+) diff --git a/requirements.txt b/requirements.txt index 2ef88140..cc41aa8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,7 @@ bunch riak # if you want to use sql backend then # peewee + + +# if you want to use lua computable inputs +# lupa diff --git a/test-requirements.txt b/test-requirements.txt index f84eb8ad..500eec70 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,3 +6,6 @@ pytest-mock tox pytest-subunit os-testr + +# for computable inputs +lupa From 9b56660abe2292f5eefd6dd408c89a4dbdd9fbae Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 3 Dec 2015 17:40:48 +0100 Subject: [PATCH 10/24] Auto wrap code with lua function declaration --- solar/computable_inputs/__init__.py | 4 +-- solar/computable_inputs/ci_lua.py | 12 ++++++-- solar/computable_inputs/processor.py | 2 +- solar/dblayer/test/test_computable_inputs.py | 30 +++++++++----------- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/solar/computable_inputs/__init__.py b/solar/computable_inputs/__init__.py index 4838d5f2..64b3a899 100644 --- a/solar/computable_inputs/__init__.py +++ b/solar/computable_inputs/__init__.py @@ -26,7 +26,7 @@ class ComputableInputProcessor(object): def __init__(self): pass - def process(self, computable_type, funct, data): + def process(self, resource_name, computable_type, funct, data): if funct is None or funct == 'noop': return data - return self.run(computable_type, funct, data) + return self.run(resource_name, computable_type, funct, data) diff --git a/solar/computable_inputs/ci_lua.py b/solar/computable_inputs/ci_lua.py index b7690e15..fd143fad 100644 --- a/solar/computable_inputs/ci_lua.py +++ b/solar/computable_inputs/ci_lua.py @@ -33,7 +33,14 @@ class LuaProcessor(ComputableInputProcessor): self.lua = LuaRuntime() self.lua.execute(_LUA_HELPERS) - def run(self, computable_type, funct, data): + def check_funct(self, funct): + # dummy insert function start / end + if not funct.startswith('function') \ + and not funct.endswith('end'): + return 'function (data, resource_name) %s end' % funct + return funct + + def run(self, resource_name, computable_type, funct, data): # when computable_type == full then raw python object is passed # to lua (counts from 0 etc) @@ -43,5 +50,6 @@ class LuaProcessor(ComputableInputProcessor): else: lua_data = data + funct = self.check_funct(funct) funct_lua = self.lua.eval(funct) - return funct_lua(lua_data) + return funct_lua(lua_data, resource_name) diff --git a/solar/computable_inputs/processor.py b/solar/computable_inputs/processor.py index b59dbe36..47634088 100644 --- a/solar/computable_inputs/processor.py +++ b/solar/computable_inputs/processor.py @@ -33,4 +33,4 @@ def get_processor(resource, input_name, computable_type, data, other=None): _processors[lang] = processor = _av_processors[lang]() else: processor = _processors[lang] - return processor.process(computable_type, funct, data) + return processor.process(resource.name, computable_type, funct, data) diff --git a/solar/dblayer/test/test_computable_inputs.py b/solar/dblayer/test/test_computable_inputs.py index 085ecc90..8e6d6634 100644 --- a/solar/dblayer/test/test_computable_inputs.py +++ b/solar/dblayer/test/test_computable_inputs.py @@ -74,7 +74,7 @@ def test_simple_lua_simple_max(rk): r2 = create_resource(k2, {'name': 'target1', 'inputs': {'input1': None}}) - lua_funct = 'function(arr) return math.max(unpack(arr)) end' + lua_funct = 'return math.max(unpack(data))' r2.meta_inputs['input1']['computable'] = {'func': lua_funct, 'lang': 'lua'} r1.connect(r2, {'input1': 'input1'}) @@ -100,7 +100,7 @@ def test_full_lua_array(rk): 'inputs': {'input1': None}}) # raw python object, counts from 0 - lua_funct = 'function(arr) return arr end' + lua_funct = 'return data' r2.meta_inputs['input1']['computable'] = {'func': lua_funct, 'type': CPT.full.name, 'lang': 'lua'} @@ -134,7 +134,7 @@ def test_connect_to_computed(rk): r4 = create_resource(k4, {'name': 'target1', 'inputs': {'input1': None}}) - lua_funct = 'function(arr) return math.max(unpack(arr)) end' + lua_funct = 'return math.max(unpack(data))' r2.meta_inputs['input1']['computable'] = {'func': lua_funct, 'lang': 'lua'} r1.connect(r2, {'input1': 'input1'}) @@ -165,10 +165,9 @@ def test_join_different_values(rk): r4 = create_resource(k4, {'name': 'r4', 'inputs': {'input': None}}) - lua_funct = """function (data) + lua_funct = """ local l = make_arr(data) -return l["r1"]["input1"] .. "@" .. l["r2"]["input2"] -end""" +return l["r1"]["input1"] .. "@" .. l["r2"]["input2"]""" r3.meta_inputs['input']['computable'] = {"func": lua_funct, 'lang': 'lua', @@ -204,21 +203,19 @@ def test_join_replace_in_lua(rk): r4 = create_resource(k4, {'name': 'r4', 'inputs': {'input': None}}) - lua_funct = """function (data) + lua_funct = """ local l = make_arr(data) return l["r1"]["input1"] .. "@" .. l["r2"]["input2"] -end""" +""" r3.meta_inputs['input']['computable'] = {"func": lua_funct, 'lang': 'lua', 'type': CPT.full.name} - lua_funct2 = """function (data) -local v = data[1] + lua_funct2 = """local v = data[1] v = v:gsub("@", "-", 1) return v -end - """ +""" r4.meta_inputs['input']['computable'] = {"func": lua_funct2, 'lang': 'lua', @@ -247,10 +244,9 @@ def test_join_self_computable(rk): 'input2': 'foo', 'input3': None}}) - lua_funct = """function (data) -local l = make_arr(data) -return l["r1"]["input2"] .. l["r1"]["input1"] -end""" + lua_funct = """local l = make_arr(data) +return resource_name .. l["r1"]["input2"] .. l["r1"]["input1"] +""" r1.meta_inputs['input3']['computable'] = {'func': lua_funct, 'lang': 'lua', @@ -261,4 +257,4 @@ end""" r1.save() - assert r1.inputs['input3'] == 'foobar' + assert r1.inputs['input3'] == 'r1foobar' From fd0616f64b5da325100622837ecd69291160215c Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 3 Dec 2015 17:49:15 +0100 Subject: [PATCH 11/24] Dont guess mapping when emitter == receiver --- solar/core/signals.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solar/core/signals.py b/solar/core/signals.py index c40c6290..573b8832 100644 --- a/solar/core/signals.py +++ b/solar/core/signals.py @@ -141,6 +141,8 @@ def location_and_transports(emitter, receiver, orig_mapping): def get_mapping(emitter, receiver, mapping=None): + if emitter == receiver: + return mapping if mapping is None: mapping = guess_mapping(emitter, receiver) location_and_transports(emitter, receiver, mapping) From fb64bfcdc8373ce5e7887e78772d77cd048231d6 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 4 Dec 2015 13:03:35 +0100 Subject: [PATCH 12/24] Added computable inputs riak example --- resources/riak_node_comp/actions/commit.yaml | 6 + resources/riak_node_comp/actions/join.yaml | 15 + resources/riak_node_comp/actions/remove.yaml | 6 + resources/riak_node_comp/actions/run.yaml | 28 + .../actions/scripts/add_riak_repo.sh | 143 +++++ resources/riak_node_comp/actions/update.yaml | 12 + resources/riak_node_comp/meta.yaml | 41 ++ .../riak_node_comp/templates/riak.conf.jinja | 494 ++++++++++++++++++ 8 files changed, 745 insertions(+) create mode 100644 resources/riak_node_comp/actions/commit.yaml create mode 100644 resources/riak_node_comp/actions/join.yaml create mode 100644 resources/riak_node_comp/actions/remove.yaml create mode 100644 resources/riak_node_comp/actions/run.yaml create mode 100755 resources/riak_node_comp/actions/scripts/add_riak_repo.sh create mode 100644 resources/riak_node_comp/actions/update.yaml create mode 100644 resources/riak_node_comp/meta.yaml create mode 100644 resources/riak_node_comp/templates/riak.conf.jinja diff --git a/resources/riak_node_comp/actions/commit.yaml b/resources/riak_node_comp/actions/commit.yaml new file mode 100644 index 00000000..52bc4712 --- /dev/null +++ b/resources/riak_node_comp/actions/commit.yaml @@ -0,0 +1,6 @@ +- hosts: [{{host}}] + sudo: yes + tasks: + # - shell: sleep 30 + - shell: riak-admin cluster plan + - shell: riak-admin cluster commit diff --git a/resources/riak_node_comp/actions/join.yaml b/resources/riak_node_comp/actions/join.yaml new file mode 100644 index 00000000..01d202f7 --- /dev/null +++ b/resources/riak_node_comp/actions/join.yaml @@ -0,0 +1,15 @@ +- hosts: [{{host}}] + sudo: yes + tasks: + - shell: riak-admin cluster join {{join_to}} + ignore_errors: true + register: join_output + # those below are hacky solution for "this node is already member of a cluster + # solar for now lacks logic that would allow to avoid it + - shell: /bin/true + when: join_output|failed and join_output.stdout.find("This node is already a member of a cluster") != -1 + - shell: /bin/false + when: join_output|failed and join_output.stdout.find("This node is already a member of a cluster") == -1 + - shell: /bin/true + when: join_output|success + diff --git a/resources/riak_node_comp/actions/remove.yaml b/resources/riak_node_comp/actions/remove.yaml new file mode 100644 index 00000000..9af84785 --- /dev/null +++ b/resources/riak_node_comp/actions/remove.yaml @@ -0,0 +1,6 @@ +- hosts: [{{host}}] + sudo: yes + tasks: + - apt: + name: riak + state: absent diff --git a/resources/riak_node_comp/actions/run.yaml b/resources/riak_node_comp/actions/run.yaml new file mode 100644 index 00000000..cb553d47 --- /dev/null +++ b/resources/riak_node_comp/actions/run.yaml @@ -0,0 +1,28 @@ +- hosts: [{{host}}] + sudo: yes + tasks: + # those below are mostly for tests + - shell: killall -u riak + ignore_errors: yes + # remove above when non tests + + # we install ubuntu repo there, + # NOT recommended on production + - shell: curl -s https://packagecloud.io/install/repositories/basho/riak/script.deb.sh | sudo bash + + - apt: + name: riak + state: present + - service: + name: riak + state: stopped + - file: path=/etc/riak/riak.conf state=touch + - template: + src: {{templates_dir}}/riak.conf + dest: /etc/riak/riak.conf + - shell: rm -fr /var/lib/riak/kv_vnode/* + - shell: rm -fr /var/lib/riak/ring/* + + - service: + name: riak + state: reloaded diff --git a/resources/riak_node_comp/actions/scripts/add_riak_repo.sh b/resources/riak_node_comp/actions/scripts/add_riak_repo.sh new file mode 100755 index 00000000..b8b451a2 --- /dev/null +++ b/resources/riak_node_comp/actions/scripts/add_riak_repo.sh @@ -0,0 +1,143 @@ +#!/bin/bash + +unknown_os () +{ + echo "Unfortunately, your operating system distribution and version are not supported by this script." + echo "Please email support@packagecloud.io and we will be happy to help." + exit 1 +} + +curl_check () +{ + echo "Checking for curl..." + if command -v curl > /dev/null; then + echo "Detected curl..." + else + echo "Installing curl..." + apt-get install -q -y curl + fi +} + +os= +dist= +host= + +get_hostname () +{ + echo "Getting the hostname of this machine..." + + host=`hostname -f 2>/dev/null` + if [ "$host" = "" ]; then + host=`hostname 2>/dev/null` + if [ "$host" = "" ]; then + host=$HOSTNAME + fi + fi + + if [ "$host" = "" -o "$host" = "(none)" ]; then + echo "Unable to determine the hostname of your system!" + echo + echo "Please consult the documentation for your system. The files you need " + echo "to modify to do this vary between Linux distribution and version." + echo + exit 1 + fi + + echo "Found hostname: ${host}" +} + + +# some systems dont have lsb-release yet have the lsb_release binary and +# vice-versa +if [ -e /etc/lsb-release ]; then + . /etc/lsb-release + os=${DISTRIB_ID} + dist=${DISTRIB_CODENAME} + + if [ -z "$dist" ]; then + dist=${DISTRIB_RELEASE} + fi + +elif [ `which lsb_release 2>/dev/null` ]; then + dist=`lsb_release -c | cut -f2` + os=`lsb_release -i | cut -f2 | awk '{ print tolower($1) }'` + +elif [ -e /etc/debian_version ]; then + # some Debians have jessie/sid in their /etc/debian_version + # while others have '6.0.7' + os=`cat /etc/issue | head -1 | awk '{ print tolower($1) }'` + if grep -q '/' /etc/debian_version; then + dist=`cut --delimiter='/' -f1 /etc/debian_version` + else + dist=`cut --delimiter='.' -f1 /etc/debian_version` + fi + +else + unknown_os +fi + +if [ -z "$dist" ]; then + unknown_os +fi + +echo "Detected operating system as $os/$dist." + +curl_check + +echo -n "Installing apt-transport-https... " + +# install apt-https-transport +apt-get install -y apt-transport-https &> /dev/null + +echo "done." + +get_hostname + +apt_source_path="/etc/apt/sources.list.d/basho_riak.list" +apt_config_url="https://packagecloud.io/install/repositories/basho/riak/config_file.list?os=${os}&dist=${dist}&name=${host}&source=script" + +echo -n "Installing $apt_source_path..." + +# create an apt config file for this repository +curl -f "${apt_config_url}" > $apt_source_path +curl_exit_code=$? + +if [ "$curl_exit_code" = "22" ]; then + echo -n "Unable to download repo config from: " + echo "${apt_config_url}" + echo + echo "Please contact support@packagecloud.io and report this." + [ -e $apt_source_path ] && rm $apt_source_path + exit 1 +elif [ "$curl_exit_code" = "35" ]; then + echo "curl is unable to connect to packagecloud.io over TLS when running: " + echo " curl ${apt_config_url}" + echo "This is usually due to one of two things:" + echo + echo " 1.) Missing CA root certificates (make sure the ca-certificates package is installed)" + echo " 2.) An old version of libssl. Try upgrading libssl on your system to a more recent version" + echo + echo "Contact support@packagecloud.io with information about your system for help." + [ -e $apt_source_path ] && rm $apt_source_path + exit 1 +elif [ "$curl_exit_code" -gt "0" ]; then + echo + echo "Unable to run: " + echo " curl ${apt_config_url}" + echo + echo "Double check your curl installation and try again." + [ -e $apt_source_path ] && rm $apt_source_path + exit 1 +else + echo "done." +fi + +echo -n "Importing packagecloud gpg key... " +# import the gpg key +curl https://packagecloud.io/gpg.key 2> /dev/null | apt-key add - &>/dev/null +echo "done." + +echo -n "Running apt-get update... " +# update apt on this system +apt-get update &> /dev/null +echo "done." diff --git a/resources/riak_node_comp/actions/update.yaml b/resources/riak_node_comp/actions/update.yaml new file mode 100644 index 00000000..a4a5af70 --- /dev/null +++ b/resources/riak_node_comp/actions/update.yaml @@ -0,0 +1,12 @@ +- hosts: [{{host}}] + sudo: yes + tasks: + - service: + name: riak + state: stopped + - template: + src: {{templates_dir}}/riak.conf + dest: /etc/riak/riak.conf + - service: + name: riak + state: reloaded diff --git a/resources/riak_node_comp/meta.yaml b/resources/riak_node_comp/meta.yaml new file mode 100644 index 00000000..e836faa4 --- /dev/null +++ b/resources/riak_node_comp/meta.yaml @@ -0,0 +1,41 @@ +id: riak_node +handler: ansible +version: 1.0.0 +actions: + commit: commit.yaml + run: run.yaml + join: join.yaml +input: + ip: + schema: str! + value: + riak_self_name: + schema: str! + value: + riak_hostname: + schema: str! + value: + riak_name: + schema: str! + value: null + computable: + lang: lua + type: full + func: | + local l = make_arr(data) + return l[resource_name]["riak_self_name"] .. "@" .. l[resource_name]["riak_hostname"] + riak_port_http: + schema: int! + value: 18098 + riak_port_pb: + schema: int! + value: 18087 + riak_port_solr: + schema: int! + value: 8985 + join_to: + schema: str + value: + storage_backend: + schema: str! + value: bitcask diff --git a/resources/riak_node_comp/templates/riak.conf.jinja b/resources/riak_node_comp/templates/riak.conf.jinja new file mode 100644 index 00000000..a9f5d977 --- /dev/null +++ b/resources/riak_node_comp/templates/riak.conf.jinja @@ -0,0 +1,494 @@ +## Where to emit the default log messages (typically at 'info' +## severity): +## off: disabled +## file: the file specified by log.console.file +## console: to standard output (seen when using `riak attach-direct`) +## both: log.console.file and standard out. +## +## Default: file +## +## Acceptable values: +## - one of: off, file, console, both +log.console = file + +## The severity level of the console log, default is 'info'. +## +## Default: info +## +## Acceptable values: +## - one of: debug, info, notice, warning, error, critical, alert, emergency, none +log.console.level = info + +## When 'log.console' is set to 'file' or 'both', the file where +## console messages will be logged. +## +## Default: $(platform_log_dir)/console.log +## +## Acceptable values: +## - the path to a file +log.console.file = $(platform_log_dir)/console.log + +## The file where error messages will be logged. +## +## Default: $(platform_log_dir)/error.log +## +## Acceptable values: +## - the path to a file +log.error.file = $(platform_log_dir)/error.log + +## When set to 'on', enables log output to syslog. +## +## Default: off +## +## Acceptable values: +## - on or off +log.syslog = off + +## Whether to enable the crash log. +## +## Default: on +## +## Acceptable values: +## - on or off +log.crash = on + +## If the crash log is enabled, the file where its messages will +## be written. +## +## Default: $(platform_log_dir)/crash.log +## +## Acceptable values: +## - the path to a file +log.crash.file = $(platform_log_dir)/crash.log + +## Maximum size in bytes of individual messages in the crash log +## +## Default: 64KB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +log.crash.maximum_message_size = 64KB + +## Maximum size of the crash log in bytes, before it is rotated +## +## Default: 10MB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +log.crash.size = 10MB + +## The schedule on which to rotate the crash log. For more +## information see: +## https://github.com/basho/lager/blob/master/README.md#internal-log-rotation +## +## Default: $D0 +## +## Acceptable values: +## - text +log.crash.rotation = $D0 + +## The number of rotated crash logs to keep. When set to +## 'current', only the current open log file is kept. +## +## Default: 5 +## +## Acceptable values: +## - an integer +## - the text "current" +log.crash.rotation.keep = 5 + +## Name of the Erlang node +## +## Default: riak@127.0.0.1 +## +## Acceptable values: +## - text +nodename = {{riak_name}} + +## Cookie for distributed node communication. All nodes in the +## same cluster should use the same cookie or they will not be able to +## communicate. +## +## Default: riak +## +## Acceptable values: +## - text +distributed_cookie = riak + +## Sets the number of threads in async thread pool, valid range +## is 0-1024. If thread support is available, the default is 64. +## More information at: http://erlang.org/doc/man/erl.html +## +## Default: 64 +## +## Acceptable values: +## - an integer +erlang.async_threads = 64 + +## The number of concurrent ports/sockets +## Valid range is 1024-134217727 +## +## Default: 65536 +## +## Acceptable values: +## - an integer +erlang.max_ports = 65536 + +## Set scheduler forced wakeup interval. All run queues will be +## scanned each Interval milliseconds. While there are sleeping +## schedulers in the system, one scheduler will be woken for each +## non-empty run queue found. An Interval of zero disables this +## feature, which also is the default. +## This feature is a workaround for lengthy executing native code, and +## native code that do not bump reductions properly. +## More information: http://www.erlang.org/doc/man/erl.html#+sfwi +## +## Default: 500 +## +## Acceptable values: +## - an integer +## erlang.schedulers.force_wakeup_interval = 500 + +## Enable or disable scheduler compaction of load. By default +## scheduler compaction of load is enabled. When enabled, load +## balancing will strive for a load distribution which causes as many +## scheduler threads as possible to be fully loaded (i.e., not run out +## of work). This is accomplished by migrating load (e.g. runnable +## processes) into a smaller set of schedulers when schedulers +## frequently run out of work. When disabled, the frequency with which +## schedulers run out of work will not be taken into account by the +## load balancing logic. +## More information: http://www.erlang.org/doc/man/erl.html#+scl +## +## Default: false +## +## Acceptable values: +## - one of: true, false +## erlang.schedulers.compaction_of_load = false + +## Enable or disable scheduler utilization balancing of load. By +## default scheduler utilization balancing is disabled and instead +## scheduler compaction of load is enabled which will strive for a +## load distribution which causes as many scheduler threads as +## possible to be fully loaded (i.e., not run out of work). When +## scheduler utilization balancing is enabled the system will instead +## try to balance scheduler utilization between schedulers. That is, +## strive for equal scheduler utilization on all schedulers. +## More information: http://www.erlang.org/doc/man/erl.html#+sub +## +## Acceptable values: +## - one of: true, false +## erlang.schedulers.utilization_balancing = true + +## Number of partitions in the cluster (only valid when first +## creating the cluster). Must be a power of 2, minimum 8 and maximum +## 1024. +## +## Default: 64 +## +## Acceptable values: +## - an integer +ring_size = 8 + +## Number of concurrent node-to-node transfers allowed. +## +## Default: 2 +## +## Acceptable values: +## - an integer +## transfer_limit = 2 + +## Default cert location for https can be overridden +## with the ssl config variable, for example: +## +## Acceptable values: +## - the path to a file +## ssl.certfile = $(platform_etc_dir)/cert.pem + +## Default key location for https can be overridden with the ssl +## config variable, for example: +## +## Acceptable values: +## - the path to a file +## ssl.keyfile = $(platform_etc_dir)/key.pem + +## Default signing authority location for https can be overridden +## with the ssl config variable, for example: +## +## Acceptable values: +## - the path to a file +## ssl.cacertfile = $(platform_etc_dir)/cacertfile.pem + +## DTrace support Do not enable 'dtrace' unless your Erlang/OTP +## runtime is compiled to support DTrace. DTrace is available in +## R15B01 (supported by the Erlang/OTP official source package) and in +## R14B04 via a custom source repository & branch. +## +## Default: off +## +## Acceptable values: +## - on or off +dtrace = off + +## Platform-specific installation paths (substituted by rebar) +## +## Default: ./bin +## +## Acceptable values: +## - the path to a directory +platform_bin_dir = ./bin + +## +## Default: ./data +## +## Acceptable values: +## - the path to a directory +platform_data_dir = ./data + +## +## Default: ./etc +## +## Acceptable values: +## - the path to a directory +platform_etc_dir = ./etc + +## +## Default: ./lib +## +## Acceptable values: +## - the path to a directory +platform_lib_dir = ./lib + +## +## Default: ./log +## +## Acceptable values: +## - the path to a directory +platform_log_dir = ./log + +## Enable consensus subsystem. Set to 'on' to enable the +## consensus subsystem used for strongly consistent Riak operations. +## +## Default: off +## +## Acceptable values: +## - on or off +## strong_consistency = on + +## listener.http. is an IP address and TCP port that the Riak +## HTTP interface will bind. +## +## Default: 127.0.0.1:8098 +## +## Acceptable values: +## - an IP/port pair, e.g. 127.0.0.1:10011 +listener.http.internal = 0.0.0.0:{{riak_port_http}} + +## listener.protobuf. is an IP address and TCP port that the Riak +## Protocol Buffers interface will bind. +## +## Default: 127.0.0.1:8087 +## +## Acceptable values: +## - an IP/port pair, e.g. 127.0.0.1:10011 +listener.protobuf.internal = 0.0.0.0:{{riak_port_pb}} + +## The maximum length to which the queue of pending connections +## may grow. If set, it must be an integer > 0. If you anticipate a +## huge number of connections being initialized *simultaneously*, set +## this number higher. +## +## Default: 128 +## +## Acceptable values: +## - an integer +## protobuf.backlog = 128 + +## listener.https. is an IP address and TCP port that the Riak +## HTTPS interface will bind. +## +## Acceptable values: +## - an IP/port pair, e.g. 127.0.0.1:10011 +## listener.https.internal = 127.0.0.1:8098 + +## How Riak will repair out-of-sync keys. Some features require +## this to be set to 'active', including search. +## * active: out-of-sync keys will be repaired in the background +## * passive: out-of-sync keys are only repaired on read +## * active-debug: like active, but outputs verbose debugging +## information +## +## Default: active +## +## Acceptable values: +## - one of: active, passive, active-debug +anti_entropy = active + +## Specifies the storage engine used for Riak's key-value data +## and secondary indexes (if supported). +## +## Default: bitcask +## +## Acceptable values: +## - one of: bitcask, leveldb, memory, multi +storage_backend = {{storage_backend}} + +## Controls which binary representation of a riak value is stored +## on disk. +## * 0: Original erlang:term_to_binary format. Higher space overhead. +## * 1: New format for more compact storage of small values. +## +## Default: 1 +## +## Acceptable values: +## - the integer 1 +## - the integer 0 +object.format = 1 + +## Reading or writing objects bigger than this size will write a +## warning in the logs. +## +## Default: 5MB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +object.size.warning_threshold = 5MB + +## Writing an object bigger than this will send a failure to the +## client. +## +## Default: 50MB +## +## Acceptable values: +## - a byte size with units, e.g. 10GB +object.size.maximum = 50MB + +## Writing an object with more than this number of siblings will +## generate a warning in the logs. +## +## Default: 25 +## +## Acceptable values: +## - an integer +object.siblings.warning_threshold = 25 + +## Writing an object with more than this number of siblings will +## send a failure to the client. +## +## Default: 100 +## +## Acceptable values: +## - an integer +object.siblings.maximum = 100 + +## A path under which bitcask data files will be stored. +## +## Default: $(platform_data_dir)/bitcask +## +## Acceptable values: +## - the path to a directory +bitcask.data_root = $(platform_data_dir)/bitcask + +## Configure how Bitcask writes data to disk. +## erlang: Erlang's built-in file API +## nif: Direct calls to the POSIX C API +## The NIF mode provides higher throughput for certain +## workloads, but has the potential to negatively impact +## the Erlang VM, leading to higher worst-case latencies +## and possible throughput collapse. +## +## Default: erlang +## +## Acceptable values: +## - one of: erlang, nif +bitcask.io_mode = erlang + +## Set to 'off' to disable the admin panel. +## +## Default: off +## +## Acceptable values: +## - on or off +riak_control = on + +## Authentication mode used for access to the admin panel. +## +## Default: off +## +## Acceptable values: +## - one of: off, userlist +riak_control.auth.mode = off + +## If riak control's authentication mode (riak_control.auth.mode) +## is set to 'userlist' then this is the list of usernames and +## passwords for access to the admin panel. +## To create users with given names, add entries of the format: +## riak_control.auth.user.USERNAME.password = PASSWORD +## replacing USERNAME with the desired username and PASSWORD with the +## desired password for that user. +## +## Acceptable values: +## - text +## riak_control.auth.user.admin.password = pass + +## This parameter defines the percentage of total server memory +## to assign to LevelDB. LevelDB will dynamically adjust its internal +## cache sizes to stay within this size. The memory size can +## alternately be assigned as a byte count via leveldb.maximum_memory +## instead. +## +## Default: 70 +## +## Acceptable values: +## - an integer +leveldb.maximum_memory.percent = 70 + +## To enable Search set this 'on'. +## +## Default: off +## +## Acceptable values: +## - on or off +search = off + +## How long Riak will wait for Solr to start. The start sequence +## will be tried twice. If both attempts timeout, then the Riak node +## will be shutdown. This may need to be increased as more data is +## indexed and Solr takes longer to start. Values lower than 1s will +## be rounded up to the minimum 1s. +## +## Default: 30s +## +## Acceptable values: +## - a time duration with units, e.g. '10s' for 10 seconds +search.solr.start_timeout = 30s + +## The port number which Solr binds to. +## NOTE: Binds on every interface. +## +## Default: 8093 +## +## Acceptable values: +## - an integer +search.solr.port = 8093 + +## The port number which Solr JMX binds to. +## NOTE: Binds on every interface. +## +## Default: 8985 +## +## Acceptable values: +## - an integer +search.solr.jmx_port = 8985 + +## The options to pass to the Solr JVM. Non-standard options, +## i.e. -XX, may not be portable across JVM implementations. +## E.g. -XX:+UseCompressedStrings +## +## Default: -d64 -Xms1g -Xmx1g -XX:+UseStringCache -XX:+UseCompressedOops +## +## Acceptable values: +## - text +search.solr.jvm_options = -d64 -Xms1g -Xmx1g -XX:+UseStringCache -XX:+UseCompressedOops + +# new From 8a3a99996774d48b5143bd34b4940ccb3e80a972 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 4 Dec 2015 17:29:53 +0100 Subject: [PATCH 13/24] Install libluajit-5.1-dev on travis --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9951f676..560b35bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,9 @@ env: cache: directories: - $HOME/.pip-accel-cache +before_install: + - sudo apt-get update -qq + - sudo apt-get install -y libluajit-5.1-dev install: - pip install pip-accel - pip-accel install coveralls From 7728b6eaac465fd205c149138748d2c104201484 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Fri, 4 Dec 2015 17:36:53 +0100 Subject: [PATCH 14/24] Python computable inputs Simple repl like server with simple sandbox based on seccomp --- solar/computable_inputs/ci_python.py | 109 ++++++++++++++++++ solar/computable_inputs/helpers/__init__.py | 0 .../helpers/python_helpers.py | 7 ++ .../computable_inputs/helpers/python_loop.py | 74 ++++++++++++ solar/computable_inputs/processor.py | 8 ++ solar/dblayer/test/test_computable_inputs.py | 39 +++++-- 6 files changed, 229 insertions(+), 8 deletions(-) create mode 100644 solar/computable_inputs/ci_python.py create mode 100644 solar/computable_inputs/helpers/__init__.py create mode 100644 solar/computable_inputs/helpers/python_helpers.py create mode 100644 solar/computable_inputs/helpers/python_loop.py diff --git a/solar/computable_inputs/ci_python.py b/solar/computable_inputs/ci_python.py new file mode 100644 index 00000000..8d598135 --- /dev/null +++ b/solar/computable_inputs/ci_python.py @@ -0,0 +1,109 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import json +import os +import struct +import subprocess + +from solar.computable_inputs import ComputableInputProcessor +from solar.computable_inputs import HELPERS_PATH + + +_PYTHON_WORKER = os.path.join(HELPERS_PATH, 'python_loop.py') +_PYTHON_HELPERS = open(os.path.join(HELPERS_PATH, 'python_helpers.py')).read() + + +class Mgr(object): + + def __init__(self): + self.child = None + + def run(self): + self.child = subprocess.Popen(['/usr/bin/env', 'python', + _PYTHON_WORKER], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + self.prepare() + + def prepare(self): + self.run_code(fname=None, + code=_PYTHON_HELPERS, + kwargs={}, + copy_env=False) + + def kill_child(self): + self.child.kill() + + def ensure_running(self): + if self.child is None: + self.run() + try: + os.waitpid(self.child.pid, os.WNOHANG) + except Exception: + running = False + else: + running = True + if not running: + self.run() + + def send(self, data): + c_data = json.dumps(data) + dlen = len(c_data) + self.ensure_running() + self.child.stdin.write(struct.pack(' Date: Fri, 4 Dec 2015 18:52:25 +0100 Subject: [PATCH 15/24] Added jinja2 computable input types --- solar/computable_inputs/ci_jinja.py | 37 ++++++++++++++++++++ solar/computable_inputs/processor.py | 11 +++++- solar/dblayer/test/test_computable_inputs.py | 23 ++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 solar/computable_inputs/ci_jinja.py diff --git a/solar/computable_inputs/ci_jinja.py b/solar/computable_inputs/ci_jinja.py new file mode 100644 index 00000000..374caad6 --- /dev/null +++ b/solar/computable_inputs/ci_jinja.py @@ -0,0 +1,37 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from jinja2.sandbox import SandboxedEnvironment +from solar.computable_inputs import ComputableInputProcessor + + +def make_arr(data): + t = {} + for ov in data: + if t.get(ov['resource']) is None: + t[ov['resource']] = {} + t[ov['resource']][ov['other_input']] = ov['value'] + return t + + +class JinjaProcessor(ComputableInputProcessor): + + def __init__(self): + self.env = SandboxedEnvironment(trim_blocks=True, + lstrip_blocks=True) + self._globals = {'make_arr': make_arr} + + def run(self, resource_name, computable_type, funct, data): + t = self.env.from_string(funct, globals=self._globals) + return t.render(resource_name=resource_name, data=data).strip() diff --git a/solar/computable_inputs/processor.py b/solar/computable_inputs/processor.py index b6baeb61..2bfe3c90 100644 --- a/solar/computable_inputs/processor.py +++ b/solar/computable_inputs/processor.py @@ -24,11 +24,20 @@ else: try: from solar.computable_inputs.ci_python import PyProcessor except ImportError: - raise + pass else: _av_processors['py'] = PyProcessor _av_processors['python'] = PyProcessor +try: + from solar.computable_inputs.ci_jinja import JinjaProcessor +except ImportError: + pass +else: + _av_processors['j2'] = JinjaProcessor + _av_processors['jinja'] = JinjaProcessor + _av_processors['jinja2'] = JinjaProcessor + _processors = {} diff --git a/solar/dblayer/test/test_computable_inputs.py b/solar/dblayer/test/test_computable_inputs.py index d765e0f6..bcf426e1 100644 --- a/solar/dblayer/test/test_computable_inputs.py +++ b/solar/dblayer/test/test_computable_inputs.py @@ -281,3 +281,26 @@ return resource_name + l["r1"]["input2"] + l["r1"]["input1"] r1.save() assert r1.inputs['input3'] == 'r1foobar' + + +def test_jinja_join_self_computable(rk): + k1 = next(rk) + + r1 = create_resource(k1, {'name': "r1", + 'inputs': {'input1': 'bar', + 'input2': 'foo', + 'input3': None}}) + jinja_funct = """{% set l = make_arr(data) %} +{{resource_name}}{{l['r1']['input2']}}{{l['r1']['input1']}} +""" + + r1.meta_inputs['input3']['computable'] = {'func': jinja_funct, + 'lang': 'jinja2', + 'type': CPT.full.name} + + r1.connect(r1, {'input1': 'input3'}) + r1.connect(r1, {'input2': 'input3'}) + + r1.save() + + assert r1.inputs['input3'] == 'r1foobar' From 7d791f3460f072faf816760220b7d811882c702a Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 7 Dec 2015 13:45:09 +0100 Subject: [PATCH 16/24] Automaticaly do make_arr with `full` mode --- solar/computable_inputs/ci_lua.py | 7 +++++-- solar/dblayer/test/test_computable_inputs.py | 10 ++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/solar/computable_inputs/ci_lua.py b/solar/computable_inputs/ci_lua.py index fd143fad..b34e16fb 100644 --- a/solar/computable_inputs/ci_lua.py +++ b/solar/computable_inputs/ci_lua.py @@ -33,10 +33,13 @@ class LuaProcessor(ComputableInputProcessor): self.lua = LuaRuntime() self.lua.execute(_LUA_HELPERS) - def check_funct(self, funct): + def check_funct(self, funct, computable_type): # dummy insert function start / end if not funct.startswith('function') \ and not funct.endswith('end'): + if computable_type == ComputablePassedTypes.full.name: + make_arr = 'local res = make_arr(data)' + funct = "%s\n%s" % (make_arr, funct) return 'function (data, resource_name) %s end' % funct return funct @@ -50,6 +53,6 @@ class LuaProcessor(ComputableInputProcessor): else: lua_data = data - funct = self.check_funct(funct) + funct = self.check_funct(funct, computable_type) funct_lua = self.lua.eval(funct) return funct_lua(lua_data, resource_name) diff --git a/solar/dblayer/test/test_computable_inputs.py b/solar/dblayer/test/test_computable_inputs.py index 8e6d6634..6bc61764 100644 --- a/solar/dblayer/test/test_computable_inputs.py +++ b/solar/dblayer/test/test_computable_inputs.py @@ -166,8 +166,7 @@ def test_join_different_values(rk): 'inputs': {'input': None}}) lua_funct = """ -local l = make_arr(data) -return l["r1"]["input1"] .. "@" .. l["r2"]["input2"]""" +return res["r1"]["input1"] .. "@" .. res["r2"]["input2"]""" r3.meta_inputs['input']['computable'] = {"func": lua_funct, 'lang': 'lua', @@ -204,8 +203,7 @@ def test_join_replace_in_lua(rk): 'inputs': {'input': None}}) lua_funct = """ -local l = make_arr(data) -return l["r1"]["input1"] .. "@" .. l["r2"]["input2"] +return res["r1"]["input1"] .. "@" .. res["r2"]["input2"] """ r3.meta_inputs['input']['computable'] = {"func": lua_funct, @@ -244,8 +242,8 @@ def test_join_self_computable(rk): 'input2': 'foo', 'input3': None}}) - lua_funct = """local l = make_arr(data) -return resource_name .. l["r1"]["input2"] .. l["r1"]["input1"] + lua_funct = """ +return resource_name .. res["r1"]["input2"] .. res["r1"]["input1"] """ r1.meta_inputs['input3']['computable'] = {'func': lua_funct, From 72762979dac3c14bbf8492432f746f5c5e574acb Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 7 Dec 2015 15:18:20 +0100 Subject: [PATCH 17/24] Standarize API for inputs D is data with raw passed values R is make_arr result for jinja we also pass flat input as kwargs --- solar/computable_inputs/ci_jinja.py | 12 +++- solar/computable_inputs/ci_lua.py | 4 +- solar/computable_inputs/ci_python.py | 11 ++-- solar/dblayer/test/test_computable_inputs.py | 68 ++++++++++++++++---- 4 files changed, 77 insertions(+), 18 deletions(-) diff --git a/solar/computable_inputs/ci_jinja.py b/solar/computable_inputs/ci_jinja.py index 374caad6..6423477a 100644 --- a/solar/computable_inputs/ci_jinja.py +++ b/solar/computable_inputs/ci_jinja.py @@ -14,6 +14,7 @@ from jinja2.sandbox import SandboxedEnvironment from solar.computable_inputs import ComputableInputProcessor +from solar.computable_inputs import ComputablePassedTypes def make_arr(data): @@ -34,4 +35,13 @@ class JinjaProcessor(ComputableInputProcessor): def run(self, resource_name, computable_type, funct, data): t = self.env.from_string(funct, globals=self._globals) - return t.render(resource_name=resource_name, data=data).strip() + if computable_type == ComputablePassedTypes.full.name: + arr = make_arr(data) + my_inputs = arr[resource_name] + else: + my_inputs = {} + arr = {} + return t.render(resource_name=resource_name, + D=data, + R=arr, + **my_inputs).strip() diff --git a/solar/computable_inputs/ci_lua.py b/solar/computable_inputs/ci_lua.py index b34e16fb..2e163c3e 100644 --- a/solar/computable_inputs/ci_lua.py +++ b/solar/computable_inputs/ci_lua.py @@ -38,9 +38,9 @@ class LuaProcessor(ComputableInputProcessor): if not funct.startswith('function') \ and not funct.endswith('end'): if computable_type == ComputablePassedTypes.full.name: - make_arr = 'local res = make_arr(data)' + make_arr = 'local R = make_arr(D)' funct = "%s\n%s" % (make_arr, funct) - return 'function (data, resource_name) %s end' % funct + return 'function (D, resource_name) %s end' % funct return funct def run(self, resource_name, computable_type, funct, data): diff --git a/solar/computable_inputs/ci_python.py b/solar/computable_inputs/ci_python.py index 8d598135..412fbb97 100644 --- a/solar/computable_inputs/ci_python.py +++ b/solar/computable_inputs/ci_python.py @@ -19,6 +19,7 @@ import struct import subprocess from solar.computable_inputs import ComputableInputProcessor +from solar.computable_inputs import ComputablePassedTypes from solar.computable_inputs import HELPERS_PATH @@ -93,17 +94,19 @@ class PyProcessor(ComputableInputProcessor): self.mgr = Mgr() self.mgr.run() - def check_funct(self, funct): + def check_funct(self, funct, computable_type): if not funct.startswith('def calculate_input('): code = funct.splitlines() + if computable_type == ComputablePassedTypes.full.name: + code.insert(0, 'R = make_arr(D)') code = '\n '.join(code) - return 'def calculate_input(data, resource_name):\n %s' % code + return 'def calculate_input(D, resource_name):\n %s' % code return funct def run(self, resource_name, computable_type, funct, data): - funct = self.check_funct(funct) + funct = self.check_funct(funct, computable_type) value = self.mgr.run_code(code=funct, fname='calculate_input', - kwargs={'data': data, + kwargs={'D': data, 'resource_name': resource_name}) return value diff --git a/solar/dblayer/test/test_computable_inputs.py b/solar/dblayer/test/test_computable_inputs.py index 60e19ea0..80e548e2 100644 --- a/solar/dblayer/test/test_computable_inputs.py +++ b/solar/dblayer/test/test_computable_inputs.py @@ -74,7 +74,7 @@ def test_lua_simple_lua_simple_max(rk): r2 = create_resource(k2, {'name': 'target1', 'inputs': {'input1': None}}) - lua_funct = 'return math.max(unpack(data))' + lua_funct = 'return math.max(unpack(D))' r2.meta_inputs['input1']['computable'] = {'func': lua_funct, 'lang': 'lua'} r1.connect(r2, {'input1': 'input1'}) @@ -100,7 +100,7 @@ def test_lua_full_lua_array(rk): 'inputs': {'input1': None}}) # raw python object, counts from 0 - lua_funct = 'return data' + lua_funct = 'return D' r2.meta_inputs['input1']['computable'] = {'func': lua_funct, 'type': CPT.full.name, 'lang': 'lua'} @@ -134,7 +134,7 @@ def test_lua_connect_to_computed(rk): r4 = create_resource(k4, {'name': 'target1', 'inputs': {'input1': None}}) - lua_funct = 'return math.max(unpack(data))' + lua_funct = 'return math.max(unpack(D))' r2.meta_inputs['input1']['computable'] = {'func': lua_funct, 'lang': 'lua'} r1.connect(r2, {'input1': 'input1'}) @@ -166,7 +166,7 @@ def test_lua_join_different_values(rk): 'inputs': {'input': None}}) lua_funct = """ -return res["r1"]["input1"] .. "@" .. res["r2"]["input2"]""" +return R["r1"]["input1"] .. "@" .. R["r2"]["input2"]""" r3.meta_inputs['input']['computable'] = {"func": lua_funct, 'lang': 'lua', @@ -203,14 +203,14 @@ def test_lua_join_replace_in_lua(rk): 'inputs': {'input': None}}) lua_funct = """ -return res["r1"]["input1"] .. "@" .. res["r2"]["input2"] +return R["r1"]["input1"] .. "@" .. R["r2"]["input2"] """ r3.meta_inputs['input']['computable'] = {"func": lua_funct, 'lang': 'lua', 'type': CPT.full.name} - lua_funct2 = """local v = data[1] + lua_funct2 = """local v = D[1] v = v:gsub("@", "-", 1) return v """ @@ -243,7 +243,7 @@ def test_lua_join_self_computable(rk): 'input3': None}}) lua_funct = """ -return resource_name .. res["r1"]["input2"] .. res["r1"]["input1"] +return resource_name .. R["r1"]["input2"] .. R["r1"]["input1"] """ r1.meta_inputs['input3']['computable'] = {'func': lua_funct, @@ -265,8 +265,8 @@ def test_python_join_self_computable(rk): 'inputs': {'input1': 'bar', 'input2': 'foo', 'input3': None}}) - py_funct = """l = make_arr(data) -return resource_name + l["r1"]["input2"] + l["r1"]["input1"] + py_funct = """ +return resource_name + R["r1"]["input2"] + R["r1"]["input1"] """ r1.meta_inputs['input3']['computable'] = {'func': py_funct, @@ -288,8 +288,8 @@ def test_jinja_join_self_computable(rk): 'inputs': {'input1': 'bar', 'input2': 'foo', 'input3': None}}) - jinja_funct = """{% set l = make_arr(data) %} -{{resource_name}}{{l['r1']['input2']}}{{l['r1']['input1']}} + jinja_funct = """ +{{resource_name}}{{input2}}{{input1}} """ r1.meta_inputs['input3']['computable'] = {'func': jinja_funct, @@ -302,3 +302,49 @@ def test_jinja_join_self_computable(rk): r1.save() assert r1.inputs['input3'] == 'r1foobar' + + +def test_jinja_join_self_sum(rk): + k1 = next(rk) + + r1 = create_resource(k1, {'name': "r1", + 'inputs': {'input1': 3, + 'input2': 2, + 'input3': None}}) + jinja_funct = """ +{{[input1, input2]|sum}} +""" + + r1.meta_inputs['input3']['computable'] = {'func': jinja_funct, + 'lang': 'jinja2', + 'type': CPT.full.name} + + r1.connect(r1, {'input1': 'input3'}) + r1.connect(r1, {'input2': 'input3'}) + + r1.save() + + assert r1.inputs['input3'] == '5' + + +def test_jinja_join_self_sum_simple(rk): + k1 = next(rk) + + r1 = create_resource(k1, {'name': "r1", + 'inputs': {'input1': 3, + 'input2': 2, + 'input3': None}}) + jinja_funct = """ +{{D|sum}} +""" + + r1.meta_inputs['input3']['computable'] = {'func': jinja_funct, + 'lang': 'jinja2', + 'type': CPT.values.name} + + r1.connect(r1, {'input1': 'input3'}) + r1.connect(r1, {'input2': 'input3'}) + + r1.save() + + assert r1.inputs['input3'] == '5' From 729d3b023f160f9354d3954b1dfd15337c45a1c9 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 7 Dec 2015 17:21:10 +0100 Subject: [PATCH 18/24] Moved riak_node_comp to riak_node --- examples/riak/riaks.py | 5 +- resources/riak_node/meta.yaml | 15 +- resources/riak_node_comp/actions/commit.yaml | 6 - resources/riak_node_comp/actions/join.yaml | 15 - resources/riak_node_comp/actions/remove.yaml | 6 - resources/riak_node_comp/actions/run.yaml | 28 - .../actions/scripts/add_riak_repo.sh | 143 ----- resources/riak_node_comp/actions/update.yaml | 12 - resources/riak_node_comp/meta.yaml | 41 -- .../riak_node_comp/templates/riak.conf.jinja | 494 ------------------ 10 files changed, 9 insertions(+), 756 deletions(-) delete mode 100644 resources/riak_node_comp/actions/commit.yaml delete mode 100644 resources/riak_node_comp/actions/join.yaml delete mode 100644 resources/riak_node_comp/actions/remove.yaml delete mode 100644 resources/riak_node_comp/actions/run.yaml delete mode 100755 resources/riak_node_comp/actions/scripts/add_riak_repo.sh delete mode 100644 resources/riak_node_comp/actions/update.yaml delete mode 100644 resources/riak_node_comp/meta.yaml delete mode 100644 resources/riak_node_comp/templates/riak.conf.jinja diff --git a/examples/riak/riaks.py b/examples/riak/riaks.py index c5e8c42e..4c43d1eb 100755 --- a/examples/riak/riaks.py +++ b/examples/riak/riaks.py @@ -45,8 +45,9 @@ def setup_riak(): 'resources/riak_node', {'riak_self_name': 'riak%d' % num, 'storage_backend': 'leveldb', - 'riak_hostname': 'riak_server%d.solar' % num, - 'riak_name': 'riak%d@riak_server%d.solar' % (num, num)})[0] + 'riak_hostname': 'riak_server%d.solar' % num})[0] + r.connect(r, {'riak_self_name': 'riak_name', + 'riak_hostname': 'riak_name'}) riak_services.append(r) for i, riak in enumerate(riak_services): diff --git a/resources/riak_node/meta.yaml b/resources/riak_node/meta.yaml index 12022aca..fb2c584d 100644 --- a/resources/riak_node/meta.yaml +++ b/resources/riak_node/meta.yaml @@ -9,22 +9,19 @@ input: ip: schema: str! value: - # ssh_key: - # schema: str! - # value: - # ssh_user: - # schema: str! - # value: riak_self_name: schema: str! value: riak_hostname: schema: str! - value: + value: riak_name: schema: str! - # value: "{{riak_self_name}}@{{riak_hostname}}" - value: "{{riak_self_name}}@{{ip}}" + value: null + computable: + lang: jinja2 + type: full + func: "{{riak_self_name}}@{{riak_hostname}}" riak_port_http: schema: int! value: 18098 diff --git a/resources/riak_node_comp/actions/commit.yaml b/resources/riak_node_comp/actions/commit.yaml deleted file mode 100644 index 52bc4712..00000000 --- a/resources/riak_node_comp/actions/commit.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- hosts: [{{host}}] - sudo: yes - tasks: - # - shell: sleep 30 - - shell: riak-admin cluster plan - - shell: riak-admin cluster commit diff --git a/resources/riak_node_comp/actions/join.yaml b/resources/riak_node_comp/actions/join.yaml deleted file mode 100644 index 01d202f7..00000000 --- a/resources/riak_node_comp/actions/join.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- hosts: [{{host}}] - sudo: yes - tasks: - - shell: riak-admin cluster join {{join_to}} - ignore_errors: true - register: join_output - # those below are hacky solution for "this node is already member of a cluster - # solar for now lacks logic that would allow to avoid it - - shell: /bin/true - when: join_output|failed and join_output.stdout.find("This node is already a member of a cluster") != -1 - - shell: /bin/false - when: join_output|failed and join_output.stdout.find("This node is already a member of a cluster") == -1 - - shell: /bin/true - when: join_output|success - diff --git a/resources/riak_node_comp/actions/remove.yaml b/resources/riak_node_comp/actions/remove.yaml deleted file mode 100644 index 9af84785..00000000 --- a/resources/riak_node_comp/actions/remove.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- hosts: [{{host}}] - sudo: yes - tasks: - - apt: - name: riak - state: absent diff --git a/resources/riak_node_comp/actions/run.yaml b/resources/riak_node_comp/actions/run.yaml deleted file mode 100644 index cb553d47..00000000 --- a/resources/riak_node_comp/actions/run.yaml +++ /dev/null @@ -1,28 +0,0 @@ -- hosts: [{{host}}] - sudo: yes - tasks: - # those below are mostly for tests - - shell: killall -u riak - ignore_errors: yes - # remove above when non tests - - # we install ubuntu repo there, - # NOT recommended on production - - shell: curl -s https://packagecloud.io/install/repositories/basho/riak/script.deb.sh | sudo bash - - - apt: - name: riak - state: present - - service: - name: riak - state: stopped - - file: path=/etc/riak/riak.conf state=touch - - template: - src: {{templates_dir}}/riak.conf - dest: /etc/riak/riak.conf - - shell: rm -fr /var/lib/riak/kv_vnode/* - - shell: rm -fr /var/lib/riak/ring/* - - - service: - name: riak - state: reloaded diff --git a/resources/riak_node_comp/actions/scripts/add_riak_repo.sh b/resources/riak_node_comp/actions/scripts/add_riak_repo.sh deleted file mode 100755 index b8b451a2..00000000 --- a/resources/riak_node_comp/actions/scripts/add_riak_repo.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash - -unknown_os () -{ - echo "Unfortunately, your operating system distribution and version are not supported by this script." - echo "Please email support@packagecloud.io and we will be happy to help." - exit 1 -} - -curl_check () -{ - echo "Checking for curl..." - if command -v curl > /dev/null; then - echo "Detected curl..." - else - echo "Installing curl..." - apt-get install -q -y curl - fi -} - -os= -dist= -host= - -get_hostname () -{ - echo "Getting the hostname of this machine..." - - host=`hostname -f 2>/dev/null` - if [ "$host" = "" ]; then - host=`hostname 2>/dev/null` - if [ "$host" = "" ]; then - host=$HOSTNAME - fi - fi - - if [ "$host" = "" -o "$host" = "(none)" ]; then - echo "Unable to determine the hostname of your system!" - echo - echo "Please consult the documentation for your system. The files you need " - echo "to modify to do this vary between Linux distribution and version." - echo - exit 1 - fi - - echo "Found hostname: ${host}" -} - - -# some systems dont have lsb-release yet have the lsb_release binary and -# vice-versa -if [ -e /etc/lsb-release ]; then - . /etc/lsb-release - os=${DISTRIB_ID} - dist=${DISTRIB_CODENAME} - - if [ -z "$dist" ]; then - dist=${DISTRIB_RELEASE} - fi - -elif [ `which lsb_release 2>/dev/null` ]; then - dist=`lsb_release -c | cut -f2` - os=`lsb_release -i | cut -f2 | awk '{ print tolower($1) }'` - -elif [ -e /etc/debian_version ]; then - # some Debians have jessie/sid in their /etc/debian_version - # while others have '6.0.7' - os=`cat /etc/issue | head -1 | awk '{ print tolower($1) }'` - if grep -q '/' /etc/debian_version; then - dist=`cut --delimiter='/' -f1 /etc/debian_version` - else - dist=`cut --delimiter='.' -f1 /etc/debian_version` - fi - -else - unknown_os -fi - -if [ -z "$dist" ]; then - unknown_os -fi - -echo "Detected operating system as $os/$dist." - -curl_check - -echo -n "Installing apt-transport-https... " - -# install apt-https-transport -apt-get install -y apt-transport-https &> /dev/null - -echo "done." - -get_hostname - -apt_source_path="/etc/apt/sources.list.d/basho_riak.list" -apt_config_url="https://packagecloud.io/install/repositories/basho/riak/config_file.list?os=${os}&dist=${dist}&name=${host}&source=script" - -echo -n "Installing $apt_source_path..." - -# create an apt config file for this repository -curl -f "${apt_config_url}" > $apt_source_path -curl_exit_code=$? - -if [ "$curl_exit_code" = "22" ]; then - echo -n "Unable to download repo config from: " - echo "${apt_config_url}" - echo - echo "Please contact support@packagecloud.io and report this." - [ -e $apt_source_path ] && rm $apt_source_path - exit 1 -elif [ "$curl_exit_code" = "35" ]; then - echo "curl is unable to connect to packagecloud.io over TLS when running: " - echo " curl ${apt_config_url}" - echo "This is usually due to one of two things:" - echo - echo " 1.) Missing CA root certificates (make sure the ca-certificates package is installed)" - echo " 2.) An old version of libssl. Try upgrading libssl on your system to a more recent version" - echo - echo "Contact support@packagecloud.io with information about your system for help." - [ -e $apt_source_path ] && rm $apt_source_path - exit 1 -elif [ "$curl_exit_code" -gt "0" ]; then - echo - echo "Unable to run: " - echo " curl ${apt_config_url}" - echo - echo "Double check your curl installation and try again." - [ -e $apt_source_path ] && rm $apt_source_path - exit 1 -else - echo "done." -fi - -echo -n "Importing packagecloud gpg key... " -# import the gpg key -curl https://packagecloud.io/gpg.key 2> /dev/null | apt-key add - &>/dev/null -echo "done." - -echo -n "Running apt-get update... " -# update apt on this system -apt-get update &> /dev/null -echo "done." diff --git a/resources/riak_node_comp/actions/update.yaml b/resources/riak_node_comp/actions/update.yaml deleted file mode 100644 index a4a5af70..00000000 --- a/resources/riak_node_comp/actions/update.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- hosts: [{{host}}] - sudo: yes - tasks: - - service: - name: riak - state: stopped - - template: - src: {{templates_dir}}/riak.conf - dest: /etc/riak/riak.conf - - service: - name: riak - state: reloaded diff --git a/resources/riak_node_comp/meta.yaml b/resources/riak_node_comp/meta.yaml deleted file mode 100644 index e836faa4..00000000 --- a/resources/riak_node_comp/meta.yaml +++ /dev/null @@ -1,41 +0,0 @@ -id: riak_node -handler: ansible -version: 1.0.0 -actions: - commit: commit.yaml - run: run.yaml - join: join.yaml -input: - ip: - schema: str! - value: - riak_self_name: - schema: str! - value: - riak_hostname: - schema: str! - value: - riak_name: - schema: str! - value: null - computable: - lang: lua - type: full - func: | - local l = make_arr(data) - return l[resource_name]["riak_self_name"] .. "@" .. l[resource_name]["riak_hostname"] - riak_port_http: - schema: int! - value: 18098 - riak_port_pb: - schema: int! - value: 18087 - riak_port_solr: - schema: int! - value: 8985 - join_to: - schema: str - value: - storage_backend: - schema: str! - value: bitcask diff --git a/resources/riak_node_comp/templates/riak.conf.jinja b/resources/riak_node_comp/templates/riak.conf.jinja deleted file mode 100644 index a9f5d977..00000000 --- a/resources/riak_node_comp/templates/riak.conf.jinja +++ /dev/null @@ -1,494 +0,0 @@ -## Where to emit the default log messages (typically at 'info' -## severity): -## off: disabled -## file: the file specified by log.console.file -## console: to standard output (seen when using `riak attach-direct`) -## both: log.console.file and standard out. -## -## Default: file -## -## Acceptable values: -## - one of: off, file, console, both -log.console = file - -## The severity level of the console log, default is 'info'. -## -## Default: info -## -## Acceptable values: -## - one of: debug, info, notice, warning, error, critical, alert, emergency, none -log.console.level = info - -## When 'log.console' is set to 'file' or 'both', the file where -## console messages will be logged. -## -## Default: $(platform_log_dir)/console.log -## -## Acceptable values: -## - the path to a file -log.console.file = $(platform_log_dir)/console.log - -## The file where error messages will be logged. -## -## Default: $(platform_log_dir)/error.log -## -## Acceptable values: -## - the path to a file -log.error.file = $(platform_log_dir)/error.log - -## When set to 'on', enables log output to syslog. -## -## Default: off -## -## Acceptable values: -## - on or off -log.syslog = off - -## Whether to enable the crash log. -## -## Default: on -## -## Acceptable values: -## - on or off -log.crash = on - -## If the crash log is enabled, the file where its messages will -## be written. -## -## Default: $(platform_log_dir)/crash.log -## -## Acceptable values: -## - the path to a file -log.crash.file = $(platform_log_dir)/crash.log - -## Maximum size in bytes of individual messages in the crash log -## -## Default: 64KB -## -## Acceptable values: -## - a byte size with units, e.g. 10GB -log.crash.maximum_message_size = 64KB - -## Maximum size of the crash log in bytes, before it is rotated -## -## Default: 10MB -## -## Acceptable values: -## - a byte size with units, e.g. 10GB -log.crash.size = 10MB - -## The schedule on which to rotate the crash log. For more -## information see: -## https://github.com/basho/lager/blob/master/README.md#internal-log-rotation -## -## Default: $D0 -## -## Acceptable values: -## - text -log.crash.rotation = $D0 - -## The number of rotated crash logs to keep. When set to -## 'current', only the current open log file is kept. -## -## Default: 5 -## -## Acceptable values: -## - an integer -## - the text "current" -log.crash.rotation.keep = 5 - -## Name of the Erlang node -## -## Default: riak@127.0.0.1 -## -## Acceptable values: -## - text -nodename = {{riak_name}} - -## Cookie for distributed node communication. All nodes in the -## same cluster should use the same cookie or they will not be able to -## communicate. -## -## Default: riak -## -## Acceptable values: -## - text -distributed_cookie = riak - -## Sets the number of threads in async thread pool, valid range -## is 0-1024. If thread support is available, the default is 64. -## More information at: http://erlang.org/doc/man/erl.html -## -## Default: 64 -## -## Acceptable values: -## - an integer -erlang.async_threads = 64 - -## The number of concurrent ports/sockets -## Valid range is 1024-134217727 -## -## Default: 65536 -## -## Acceptable values: -## - an integer -erlang.max_ports = 65536 - -## Set scheduler forced wakeup interval. All run queues will be -## scanned each Interval milliseconds. While there are sleeping -## schedulers in the system, one scheduler will be woken for each -## non-empty run queue found. An Interval of zero disables this -## feature, which also is the default. -## This feature is a workaround for lengthy executing native code, and -## native code that do not bump reductions properly. -## More information: http://www.erlang.org/doc/man/erl.html#+sfwi -## -## Default: 500 -## -## Acceptable values: -## - an integer -## erlang.schedulers.force_wakeup_interval = 500 - -## Enable or disable scheduler compaction of load. By default -## scheduler compaction of load is enabled. When enabled, load -## balancing will strive for a load distribution which causes as many -## scheduler threads as possible to be fully loaded (i.e., not run out -## of work). This is accomplished by migrating load (e.g. runnable -## processes) into a smaller set of schedulers when schedulers -## frequently run out of work. When disabled, the frequency with which -## schedulers run out of work will not be taken into account by the -## load balancing logic. -## More information: http://www.erlang.org/doc/man/erl.html#+scl -## -## Default: false -## -## Acceptable values: -## - one of: true, false -## erlang.schedulers.compaction_of_load = false - -## Enable or disable scheduler utilization balancing of load. By -## default scheduler utilization balancing is disabled and instead -## scheduler compaction of load is enabled which will strive for a -## load distribution which causes as many scheduler threads as -## possible to be fully loaded (i.e., not run out of work). When -## scheduler utilization balancing is enabled the system will instead -## try to balance scheduler utilization between schedulers. That is, -## strive for equal scheduler utilization on all schedulers. -## More information: http://www.erlang.org/doc/man/erl.html#+sub -## -## Acceptable values: -## - one of: true, false -## erlang.schedulers.utilization_balancing = true - -## Number of partitions in the cluster (only valid when first -## creating the cluster). Must be a power of 2, minimum 8 and maximum -## 1024. -## -## Default: 64 -## -## Acceptable values: -## - an integer -ring_size = 8 - -## Number of concurrent node-to-node transfers allowed. -## -## Default: 2 -## -## Acceptable values: -## - an integer -## transfer_limit = 2 - -## Default cert location for https can be overridden -## with the ssl config variable, for example: -## -## Acceptable values: -## - the path to a file -## ssl.certfile = $(platform_etc_dir)/cert.pem - -## Default key location for https can be overridden with the ssl -## config variable, for example: -## -## Acceptable values: -## - the path to a file -## ssl.keyfile = $(platform_etc_dir)/key.pem - -## Default signing authority location for https can be overridden -## with the ssl config variable, for example: -## -## Acceptable values: -## - the path to a file -## ssl.cacertfile = $(platform_etc_dir)/cacertfile.pem - -## DTrace support Do not enable 'dtrace' unless your Erlang/OTP -## runtime is compiled to support DTrace. DTrace is available in -## R15B01 (supported by the Erlang/OTP official source package) and in -## R14B04 via a custom source repository & branch. -## -## Default: off -## -## Acceptable values: -## - on or off -dtrace = off - -## Platform-specific installation paths (substituted by rebar) -## -## Default: ./bin -## -## Acceptable values: -## - the path to a directory -platform_bin_dir = ./bin - -## -## Default: ./data -## -## Acceptable values: -## - the path to a directory -platform_data_dir = ./data - -## -## Default: ./etc -## -## Acceptable values: -## - the path to a directory -platform_etc_dir = ./etc - -## -## Default: ./lib -## -## Acceptable values: -## - the path to a directory -platform_lib_dir = ./lib - -## -## Default: ./log -## -## Acceptable values: -## - the path to a directory -platform_log_dir = ./log - -## Enable consensus subsystem. Set to 'on' to enable the -## consensus subsystem used for strongly consistent Riak operations. -## -## Default: off -## -## Acceptable values: -## - on or off -## strong_consistency = on - -## listener.http. is an IP address and TCP port that the Riak -## HTTP interface will bind. -## -## Default: 127.0.0.1:8098 -## -## Acceptable values: -## - an IP/port pair, e.g. 127.0.0.1:10011 -listener.http.internal = 0.0.0.0:{{riak_port_http}} - -## listener.protobuf. is an IP address and TCP port that the Riak -## Protocol Buffers interface will bind. -## -## Default: 127.0.0.1:8087 -## -## Acceptable values: -## - an IP/port pair, e.g. 127.0.0.1:10011 -listener.protobuf.internal = 0.0.0.0:{{riak_port_pb}} - -## The maximum length to which the queue of pending connections -## may grow. If set, it must be an integer > 0. If you anticipate a -## huge number of connections being initialized *simultaneously*, set -## this number higher. -## -## Default: 128 -## -## Acceptable values: -## - an integer -## protobuf.backlog = 128 - -## listener.https. is an IP address and TCP port that the Riak -## HTTPS interface will bind. -## -## Acceptable values: -## - an IP/port pair, e.g. 127.0.0.1:10011 -## listener.https.internal = 127.0.0.1:8098 - -## How Riak will repair out-of-sync keys. Some features require -## this to be set to 'active', including search. -## * active: out-of-sync keys will be repaired in the background -## * passive: out-of-sync keys are only repaired on read -## * active-debug: like active, but outputs verbose debugging -## information -## -## Default: active -## -## Acceptable values: -## - one of: active, passive, active-debug -anti_entropy = active - -## Specifies the storage engine used for Riak's key-value data -## and secondary indexes (if supported). -## -## Default: bitcask -## -## Acceptable values: -## - one of: bitcask, leveldb, memory, multi -storage_backend = {{storage_backend}} - -## Controls which binary representation of a riak value is stored -## on disk. -## * 0: Original erlang:term_to_binary format. Higher space overhead. -## * 1: New format for more compact storage of small values. -## -## Default: 1 -## -## Acceptable values: -## - the integer 1 -## - the integer 0 -object.format = 1 - -## Reading or writing objects bigger than this size will write a -## warning in the logs. -## -## Default: 5MB -## -## Acceptable values: -## - a byte size with units, e.g. 10GB -object.size.warning_threshold = 5MB - -## Writing an object bigger than this will send a failure to the -## client. -## -## Default: 50MB -## -## Acceptable values: -## - a byte size with units, e.g. 10GB -object.size.maximum = 50MB - -## Writing an object with more than this number of siblings will -## generate a warning in the logs. -## -## Default: 25 -## -## Acceptable values: -## - an integer -object.siblings.warning_threshold = 25 - -## Writing an object with more than this number of siblings will -## send a failure to the client. -## -## Default: 100 -## -## Acceptable values: -## - an integer -object.siblings.maximum = 100 - -## A path under which bitcask data files will be stored. -## -## Default: $(platform_data_dir)/bitcask -## -## Acceptable values: -## - the path to a directory -bitcask.data_root = $(platform_data_dir)/bitcask - -## Configure how Bitcask writes data to disk. -## erlang: Erlang's built-in file API -## nif: Direct calls to the POSIX C API -## The NIF mode provides higher throughput for certain -## workloads, but has the potential to negatively impact -## the Erlang VM, leading to higher worst-case latencies -## and possible throughput collapse. -## -## Default: erlang -## -## Acceptable values: -## - one of: erlang, nif -bitcask.io_mode = erlang - -## Set to 'off' to disable the admin panel. -## -## Default: off -## -## Acceptable values: -## - on or off -riak_control = on - -## Authentication mode used for access to the admin panel. -## -## Default: off -## -## Acceptable values: -## - one of: off, userlist -riak_control.auth.mode = off - -## If riak control's authentication mode (riak_control.auth.mode) -## is set to 'userlist' then this is the list of usernames and -## passwords for access to the admin panel. -## To create users with given names, add entries of the format: -## riak_control.auth.user.USERNAME.password = PASSWORD -## replacing USERNAME with the desired username and PASSWORD with the -## desired password for that user. -## -## Acceptable values: -## - text -## riak_control.auth.user.admin.password = pass - -## This parameter defines the percentage of total server memory -## to assign to LevelDB. LevelDB will dynamically adjust its internal -## cache sizes to stay within this size. The memory size can -## alternately be assigned as a byte count via leveldb.maximum_memory -## instead. -## -## Default: 70 -## -## Acceptable values: -## - an integer -leveldb.maximum_memory.percent = 70 - -## To enable Search set this 'on'. -## -## Default: off -## -## Acceptable values: -## - on or off -search = off - -## How long Riak will wait for Solr to start. The start sequence -## will be tried twice. If both attempts timeout, then the Riak node -## will be shutdown. This may need to be increased as more data is -## indexed and Solr takes longer to start. Values lower than 1s will -## be rounded up to the minimum 1s. -## -## Default: 30s -## -## Acceptable values: -## - a time duration with units, e.g. '10s' for 10 seconds -search.solr.start_timeout = 30s - -## The port number which Solr binds to. -## NOTE: Binds on every interface. -## -## Default: 8093 -## -## Acceptable values: -## - an integer -search.solr.port = 8093 - -## The port number which Solr JMX binds to. -## NOTE: Binds on every interface. -## -## Default: 8985 -## -## Acceptable values: -## - an integer -search.solr.jmx_port = 8985 - -## The options to pass to the Solr JVM. Non-standard options, -## i.e. -XX, may not be portable across JVM implementations. -## E.g. -XX:+UseCompressedStrings -## -## Default: -d64 -Xms1g -Xmx1g -XX:+UseStringCache -XX:+UseCompressedOops -## -## Acceptable values: -## - text -search.solr.jvm_options = -d64 -Xms1g -Xmx1g -XX:+UseStringCache -XX:+UseCompressedOops - -# new From 4e15f1475dfe3f74dbdb7c33507434e721dee3e4 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 7 Dec 2015 17:23:32 +0100 Subject: [PATCH 19/24] Fixed events cycles with self referenced inputs --- solar/core/resource/resource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solar/core/resource/resource.py b/solar/core/resource/resource.py index 6f24a88d..0c7d6152 100644 --- a/solar/core/resource/resource.py +++ b/solar/core/resource/resource.py @@ -273,7 +273,8 @@ class Resource(object): # signals.connect(self, receiver, mapping=mapping) # TODO: implement events if use_defaults: - api.add_default_events(self, receiver) + if self != receiver: + api.add_default_events(self, receiver) if events: api.add_events(self.name, events) From 7e12ac573653a74a42ba05c5421344a72416ec73 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 8 Dec 2015 10:15:57 +0100 Subject: [PATCH 20/24] Add note about python_loop --- solar/computable_inputs/helpers/python_loop.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solar/computable_inputs/helpers/python_loop.py b/solar/computable_inputs/helpers/python_loop.py index af95d65a..f27cc425 100644 --- a/solar/computable_inputs/helpers/python_loop.py +++ b/solar/computable_inputs/helpers/python_loop.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +# Consider this part as POC + import json import struct import sys From ec28d22c69204593f32bb2f37499d3d22bd40d1d Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 8 Dec 2015 10:34:14 +0100 Subject: [PATCH 21/24] Use apt addon in travis --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 560b35bb..e91c058d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,9 +5,6 @@ env: cache: directories: - $HOME/.pip-accel-cache -before_install: - - sudo apt-get update -qq - - sudo apt-get install -y libluajit-5.1-dev install: - pip install pip-accel - pip-accel install coveralls @@ -18,3 +15,7 @@ services: - riak after_success: coveralls +addons: + apt: + packages: + - libluajit-5.1-dev From 1d6cb28b69067159b04705a898a925a570d701cd Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 8 Dec 2015 10:39:57 +0100 Subject: [PATCH 22/24] Set `sudo: false` in travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e91c058d..f22aacff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ language: python python: 2.7 +sudo: false env: - PIP_ACCEL_CACHE=$HOME/.pip-accel-cache SOLAR_CONFIG=$TRAVIS_BUILD_DIR/.config SOLAR_SOLAR_DB_HOST=localhost cache: From 22f337109d6c763de329ee4f5cefa4169fbd3db9 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 8 Dec 2015 10:55:53 +0100 Subject: [PATCH 23/24] Corrected comment about sandboxing --- solar/computable_inputs/ci_lua.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/computable_inputs/ci_lua.py b/solar/computable_inputs/ci_lua.py index 2e163c3e..a60a170b 100644 --- a/solar/computable_inputs/ci_lua.py +++ b/solar/computable_inputs/ci_lua.py @@ -24,7 +24,7 @@ from solar.dblayer.solar_models import ComputablePassedTypes _LUA_HELPERS = open(os.path.join(HELPERS_PATH, 'lua_helpers.lua')).read() -# TODO: (jnowak) add sandoxing +# TODO: (jnowak) add sandboxing (http://lua-users.org/wiki/SandBoxes) class LuaProcessor(ComputableInputProcessor): From c25efa2da6e6e6dbf341dafd5dfb29e96121c5b2 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 8 Dec 2015 11:35:47 +0100 Subject: [PATCH 24/24] Added libluajit-5.1-dev to base.yaml --- bootstrap/playbooks/tasks/base.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap/playbooks/tasks/base.yaml b/bootstrap/playbooks/tasks/base.yaml index 18806769..96b27692 100644 --- a/bootstrap/playbooks/tasks/base.yaml +++ b/bootstrap/playbooks/tasks/base.yaml @@ -34,6 +34,9 @@ - libffi-dev - libssl-dev + # computable inputs lua + - libluajit-5.1-dev + # PIP #- apt: name=python-pip state=absent