openstack/yaql
Code Issues Proposed changes

yaql 1.0

Browse Source 
* Support for kwargs and keyword-only args (Py3)
* Optional function arguments
* Smart algorithm to find matching function overload without side effects
* Ability to organize functions into layers
* Configurable list of operators (left/right associative binary, prefix/suffix unary with precedence)
* No global variables. There can be  more than one parser with different set of operators simultaneously
* List literals ([a, b])
* Dictionary literals ({ a => b})
* Handling of escape characters in string literals
* Verbatim strings (`...`) and double-quotes ("...")
* =~ and !~ operators in default configuration (similar to Perl)
* -> operator to pass context
* Alternate operator names (for example '*equal' instead of '#operator_=')
   so that it will be possible to have different symbol for particular operator
   without breaking standard library that expects operator to have well known names
* Set operations
* Support for lists and dictionaries as a dictionary keys and set elements
* New framework to decorate functions
* Ability to distinguish between functions and methods
* Switchable naming conventions
* Unicode support
* Execution options available to all invoked functions
* Iterators limitation
* Ability to limit memory consumption
* Can work with custom context classes
* It is possible to extend both parser and set of expression classes on user-side
* It is possible to create user-defined types (also can be used for dependency injection)
* Legacy yaql 0.2.x backward compatibility mode
* Comprehensive standard library of functions
* High unit test coverage

Change-Id: Ie31b7c3cbadbff5b7728f55b3ba7bcb78a39d156
This commit is contained in:
Stan Lagun 2015-02-02 21:08:12 +03:00
parent aedc4d90d9
commit 152a88140a
60 changed files with 7004 additions and 2455 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/source/conf.py
View File

@ -22,7 +22,7 @@ sys.path.insert(0, os.path.abspath('../..'))
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.autodoc',
#'sphinx.ext.intersphinx',
# 'sphinx.ext.intersphinx',
'oslosphinx'
]
@ -72,4 +72,4 @@ latex_documents = [
]
# Example configuration for intersphinx: refer to the Python standard library.
#intersphinx_mapping = {'http://docs.python.org/': None}
# intersphinx_mapping = {'http://docs.python.org/': None}

56
examples/mydata.json
View File

@ -1,56 +0,0 @@
{
"test" : {
"Name" : "root entity",
"Child" : {
"Name" : "child entity level 1",
"Child" : {
"Name" : "child entity level 2",
"Child": {
"Name" : "Last child",
"Child" : null
}
}
}
},
"name" : "Env",
"helper":{
"num" : 2
},
"services" :
[{
"ServiceName" : "service1",
"MajorVersion" : 1,
"MinorVersion" : 0,
"units" : [{
"Name" : "Unit1",
"Number" : 1
},
{
"Name" : "Unit2",
"Number" : 2
},
{
"Name" : "Unit3",
"Number" : 3
}
]
},
{
"ServiceName" : "service2",
"MajorVersion" : 1,
"MinorVersion" : 1,
"units" : [{
"Name" : "Unit10",
"Number" : 10
},
{
"Name" : "Unit20",
"Number" : 20
},
{
"Name" : "Unit30",
"Number" : 30
}
]
}]
}

10
setup.cfg
View File

@ -1,12 +1,12 @@
[metadata]
name = yaql
version = 0.3.0
version = 1.0.0
summary = YAQL - Yet Another Query Language
description-file =
README.rst
author = OpenStack
author-email = openstack-dev@lists.openstack.org
home-page = http://www.openstack.org/
author = Stan Lagun
author-email = slagun@mirantis.com
home-page = https://launchpad.net/yaql
classifier =
Environment :: OpenStack
Intended Audience :: Information Technology
@ -18,7 +18,7 @@ classifier =
Programming Language :: Python :: 2.7
Programming Language :: Python :: 2.6
Programming Language :: Python :: 3
Programming Language :: Python :: 3.3
Programming Language :: Python :: 3.4
[files]
packages =

106
yaql/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2013 Mirantis, Inc.
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -12,19 +12,101 @@
# License for the specific language governing permissions and limitations
# under the License.
from yaql import functions
from yaql.language import parser, context
from yaql.language import conventions
from yaql.language import context as yaqlcontext
from yaql.language import factory
from yaql.language import specs
from yaql.language import utils
from yaql.language import yaqltypes
from yaql.standard_library import boolean as std_boolean
from yaql.standard_library import branching as std_branching
from yaql.standard_library import collections as std_collections
from yaql.standard_library import common as std_common
from yaql.standard_library import math as std_math
from yaql.standard_library import queries as std_queries
from yaql.standard_library import regex as std_regex
from yaql.standard_library import strings as std_strings
from yaql.standard_library import system as std_system
__versioninfo__ = (0, 3, 0)
__versioninfo__ = (1, 0, 0)
__version__ = '.'.join(map(str, __versioninfo__))
def parse(expression):
return parser.parse(expression)
_cached_expressions = {}
_cached_engine = None
_default_context = None
def create_context(register_functions=True):
cont = context.Context()
if register_functions:
functions.register(cont)
return context.Context(cont)
def _setup_context(data, context, finalizer):
if context is None:
context = yaqlcontext.Context(
convention=conventions.CamelCaseConvention())
if finalizer is None:
@specs.parameter('iterator', yaqltypes.Iterable())
@specs.name('#iter')
def limit(iterator):
return iterator
@specs.inject('limiter', yaqltypes.Delegate('#iter'))
@specs.inject('engine', yaqltypes.Engine())
@specs.name('#finalize')
def finalize(obj, limiter, engine):
return utils.convert_output_data(obj, limiter, engine)
context.register_function(limit)
context.register_function(finalize)
else:
context.register_function(finalizer)
if data is not utils.NO_VALUE:
context['$'] = utils.convert_input_data(data)
return context
def create_context(data=utils.NO_VALUE, context=None, system=True,
common=True, boolean=True, strings=True,
math=True, collections=True, queries=True,
regex=True, branching=True,
no_sets=False, finalizer=None):
context = _setup_context(data, context, finalizer)
if system:
std_system.register(context)
if common:
std_common.register(context)
if boolean:
std_boolean.register(context)
if strings:
std_strings.register(context)
if math:
std_math.register(context)
if collections:
std_collections.register(context, no_sets)
if queries:
std_queries.register(context)
if regex:
std_regex.register(context)
if branching:
std_branching.register(context)
return context
YaqlFactory = factory.YaqlFactory
def eval(expression, data=None):
global _cached_engine, _cached_expressions, _default_context
if _cached_engine is None:
_cached_engine = YaqlFactory().create()
engine = _cached_expressions.get(expression)
if engine is None:
engine = _cached_engine(expression)
_cached_expressions[expression] = engine
if _default_context is None:
_default_context = create_context()
return engine.evaluate(
data=data, context=_default_context.create_child_context())

62
yaql/cli/cli_functions.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2013 Mirantis, Inc.
# Copyright (c) 2013-2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -12,34 +12,28 @@
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import json
import os
import re
import readline
import types
from json import JSONDecoder
import yaql
from yaql.language.context import Context
from yaql.language.exceptions import YaqlParsingException
from yaql import __version__ as version
from yaql.language import lexer
from yaql.language.engine import context_aware
from yaql.language.utils import limit
from yaql.language import utils
PROMPT = "yaql> "
LIMIT = 100
@context_aware
def main(context, show_tokens):
def main(context, show_tokens, parser):
print("Yet Another Query Language - command-line query tool")
print("Version {0}".format(version))
print("Copyright (c) 2014 Mirantis, Inc")
print("Copyright (c) 2013-2015 Mirantis, Inc")
print("")
if not context.get_data():
if not context['']:
print("No data loaded into context ")
print("Type '@load data-file.json' to load data")
print("")
@ -57,21 +51,24 @@ def main(context, show_tokens):
if comm[0] == '@':
func_name, args = parse_service_command(comm)
if func_name not in SERVICE_FUNCTIONS:
print("Unknown command " + func_name)
print('Unknown command ' + func_name)
else:
SERVICE_FUNCTIONS[func_name](args, context)
continue
try:
if show_tokens:
lexer.lexer.input(comm)
parser.lexer.input(comm)
tokens = []
while True:
tok = lexer.lexer.token()
tok = parser.lexer.token()
if not tok:
break
tokens.append(tok)
print("Tokens: " + str(tokens))
expr = yaql.parse(comm)
print('Tokens: ' + str(tokens))
expr = parser(comm)
if show_tokens:
print('Expression: ' + str(expr))
except YaqlParsingException as ex:
if ex.position:
pointer_string = (" " * (ex.position + len(PROMPT))) + '^'
@ -79,16 +76,16 @@ def main(context, show_tokens):
print(ex.message)
continue
try:
res = expr.evaluate(context=Context(context))
if isinstance(res, types.GeneratorType):
res = limit(res)
res = expr.evaluate(context=context)
if utils.is_iterator(res):
res = list(itertools.islice(res, LIMIT))
print(json.dumps(res, indent=4))
except Exception as ex:
print("Execution exception:")
print('Execution exception:')
if hasattr(ex, 'message'):
print(ex.message)
else:
print("Unknown")
print('Unknown')
def load_data(data_file, context):
@ -99,26 +96,27 @@ def load_data(data_file, context):
e.strerror))
return
try:
decoder = JSONDecoder()
data = decoder.decode(json_str)
data = json.loads(json_str)
except Exception as e:
print("Unable to parse data: " + e.message)
print('Unable to parse data: ' + e.message)
return
context.set_data(data)
print("Data from file '{0}' loaded into context".format(data_file))
context['$'] = utils.convert_input_data(data)
print('Data from file {0} loaded into context'.format(data_file))
def regexp(self, pattern):
match = re.match(pattern(), self())
match = re.match(pattern, self)
if match:
return match.groups()
else:
return None
def register_in_context(context):
context.register_function(main, '__main')
context.register_function(regexp, 'regexp')
def register_in_context(context, parser):
context.register_function(
lambda context, show_tokens: main(context, show_tokens, parser),
name='__main')
context.register_function(regexp)
def parse_service_command(comm):

25
yaql/cli/run.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
# Copyright (c) 2013 Mirantis, Inc.
# Copyright (c) 2013-2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -14,8 +14,10 @@
# License for the specific language governing permissions and limitations
# under the License.
import json
import optparse
from json import JSONDecoder
import yaql
from yaql.cli import cli_functions
@ -28,21 +30,26 @@ def main():
options, arguments = p.parse_args()
if options.data:
try:
json_str = open(options.data).read()
decoder = JSONDecoder()
data = decoder.decode(json_str)
with open(options.data) as f:
data = json.load(f)
except Exception:
print("Unable to load data from " + options.data)
print('Unable to load data from ' + options.data)
return
else:
data = None
context = yaql.create_context()
cli_functions.register_in_context(context)
engine_options = {
'yaql.limitIterators': 100,
'yaql.treatSetsAsLists': True,
'yaql.memoryQuota': 10000
}
parser = yaql.YaqlFactory().create(options=engine_options)
cli_functions.register_in_context(context, parser)
if options.tokens:
yaql.parse('__main(true)').evaluate(data, context)
parser('__main(true)').evaluate(data, context)
else:
yaql.parse('__main(false)').evaluate(data, context)
parser('__main(false)').evaluate(data, context)
if __name__ == "__main__":

23
yaql/functions/__init__.py
View File

@ -1,23 +0,0 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from yaql.functions import system, strings, containers, arithmetic, boolean
def register(context):
system.add_to_context(context)
strings.add_to_context(context)
containers.add_to_context(context)
arithmetic.add_to_context(context)
boolean.add_to_context(context)

128
yaql/functions/arithmetic.py
View File

@ -1,128 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import random
import six
from yaql.language.engine import parameter
from yaql.language.exceptions import YaqlExecutionException
def _is_a_number(value):
return isinstance(value, six.integer_types + (float, complex))
@parameter('value', custom_validator=_is_a_number)
def unary_minus(value):
return -1 * value
@parameter('value', custom_validator=_is_a_number)
def unary_plus(value):
return value
@parameter('a', custom_validator=_is_a_number)
@parameter('b', custom_validator=_is_a_number)
def plus(a, b):
return a + b
@parameter('a', custom_validator=_is_a_number)
@parameter('b', custom_validator=_is_a_number)
def minus(a, b):
return a - b
@parameter('a', custom_validator=_is_a_number)
@parameter('b', custom_validator=_is_a_number)
def multiply(a, b):
return a * b
@parameter('a', custom_validator=_is_a_number)
@parameter('b', custom_validator=_is_a_number)
def divide(a, b):
if isinstance(a, six.integer_types) and isinstance(b, six.integer_types):
return a // b
return a / b
# comparison
def less_then(a, b):
return a < b
def greater_or_equals(a, b):
return a >= b
def less_or_equals(a, b):
return a <= b
def greater_then(a, b):
return a > b
def equals(a, b):
return a == b
def not_equals(a, b):
return a != b
def to_int(value):
try:
return int(value)
except Exception as e:
raise YaqlExecutionException("Unable to convert to integer", e)
def to_float(value):
try:
return float(value)
except Exception as e:
raise YaqlExecutionException("Unable to convert to float", e)
def rand():
return random.random()
def add_to_context(context):
# prefix unary
context.register_function(unary_minus, 'unary_-')
context.register_function(unary_plus, 'unary_+')
# arithmetic actions
context.register_function(plus, 'operator_+')
context.register_function(minus, 'operator_-')
context.register_function(multiply, 'operator_*')
context.register_function(divide, 'operator_/')
# comparison
context.register_function(greater_then, 'operator_>')
context.register_function(less_then, 'operator_<')
context.register_function(greater_or_equals, 'operator_>=')
context.register_function(less_or_equals, 'operator_<=')
context.register_function(equals, 'operator_=')
context.register_function(not_equals, 'operator_!=')
#conversion
context.register_function(to_int, 'int')
context.register_function(to_float, 'float')
#random
context.register_function(rand, 'random')

47
yaql/functions/boolean.py
View File

@ -1,47 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from yaql.language.engine import parameter
from yaql.language.exceptions import YaqlExecutionException
@parameter('a', arg_type=bool)
@parameter('b', arg_type=bool)
def _and(a, b):
return a and b
@parameter('a', arg_type=bool)
@parameter('b', arg_type=bool)
def _or(a, b):
return a or b
@parameter('data', arg_type=bool)
def _not(data):
return not data
def to_bool(value):
try:
return bool(value)
except Exception as e:
raise YaqlExecutionException("Unable to convert to boolean", e)
def add_to_context(context):
context.register_function(_and, 'operator_and')
context.register_function(_or, 'operator_or')
context.register_function(_not, 'unary_not')
context.register_function(_not, 'unary_!')
context.register_function(to_bool, 'bool')

188
yaql/functions/containers.py
View File

@ -1,188 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import itertools
import six
import types
from yaql.language.exceptions import YaqlExecutionException
from yaql.language.engine import parameter
from yaql.language.utils import limit
def collection_parameter(name):
return parameter(name, arg_type=collections.Iterable,
custom_validator=
lambda v: not isinstance(v, six.string_types))
@parameter("index", arg_type=int)
def get_by_index(data, index):
if isinstance(data, types.GeneratorType):
data = list(data)
return data[index]
@collection_parameter('self')
@parameter("predicate", function_only=True, lazy=True)
def filter_by_predicate(self, predicate):
for item in self:
r = predicate(item)
if not isinstance(r, bool):
raise YaqlExecutionException("Not a predicate")
if r is True:
yield item
def build_list(*args):
res = []
for arg in args:
if isinstance(arg, types.GeneratorType):
arg = limit(arg)
res.append(arg)
return res
@collection_parameter('b')
def is_in(a, b):
return a in b
@collection_parameter('self')
@parameter('att_name', constant_only=True)
def collection_attribution(self, att_name):
def get_att_or_key(col_item):
value = att_name
if isinstance(col_item, dict):
return col_item.get(value)
else:
return getattr(col_item, value)
for item in self:
val = get_att_or_key(item)
yield val
@parameter('arg1', lazy=True,
custom_validator=lambda v: v.key != 'operator_=>')
def build_new_tuple(arg1, arg2):
return arg1(), arg2
@parameter('arg1', lazy=True,
custom_validator=lambda v: v.key == 'operator_=>')
def append_tuple(arg1, arg2):
res = []
for tup in arg1():
res.append(tup)
res.append(arg2)
return tuple(res)
def build_dict(*tuples):
try:
d = {}
for key, value in tuples:
d[key] = value
return d
except ValueError as e:
raise YaqlExecutionException("Not a valid dictionary", e)
@collection_parameter('self')
@collection_parameter('others')
@parameter('join_predicate', lazy=True, function_only=True)
@parameter('composer', lazy=True, function_only=True)
def join(self, others, join_predicate, composer):
for self_item in self:
for other_item in others:
res = join_predicate(self_item, other_item)
if not isinstance(res, bool):
raise YaqlExecutionException("Not a predicate")
if res:
yield composer(self_item, other_item)
@collection_parameter('self')
@parameter('composer', lazy=True, function_only=True)
def select(self, composer):
for item in self:
yield composer(item)
@collection_parameter('self')
def _sum(self):
try:
return sum(self)
except TypeError as e:
raise YaqlExecutionException("Not a collection of numbers", e)
@parameter('start', arg_type=int)
@parameter('end', arg_type=int)
def _range_limited(start, end):
for i in six.moves.range(int(start), int(end)):
yield i
@parameter('start', arg_type=int)
def _range_infinite(start):
for i in itertools.count(start):
yield i
@collection_parameter('self')
@parameter('predicate', lazy=True, function_only=True)
def take_while(self, predicate):
for item in self:
res = predicate(item)
if not isinstance(res, bool):
raise YaqlExecutionException("Not a predicate")
if res:
yield item
else:
return
@parameter('self', arg_type=types.GeneratorType)
def _list(self):
return limit(self)
@collection_parameter('self')
@parameter('function', lazy=True, function_only=True)
def for_each(self, function):
for item in self:
yield function(sender=item)
def add_to_context(context):
context.register_function(get_by_index, 'where')
context.register_function(filter_by_predicate, 'where')
context.register_function(build_list, 'list')
context.register_function(build_dict, 'dict')
context.register_function(is_in, 'operator_in')
context.register_function(collection_attribution, 'operator_.')
context.register_function(build_new_tuple, 'operator_=>')
context.register_function(append_tuple, 'operator_=>')
context.register_function(join)
context.register_function(select)
context.register_function(_sum, 'sum')
context.register_function(_range_limited, 'range')
context.register_function(_range_infinite, 'range')
context.register_function(take_while)
context.register_function(_list, 'list')
context.register_function(for_each)

80
yaql/functions/ns.py
View File

@ -1,80 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from yaql.language.engine import context_aware, parameter
from yaql.language.exceptions import YaqlException
class NamespaceResolutionException(YaqlException):
def __init__(self, alias):
super(NamespaceResolutionException, self).__init__(
"Unable to resolve namespace: %s" % alias)
class NamespaceValidationException(YaqlException):
def __init__(self, name, symbol):
super(NamespaceValidationException, self).__init__(
"Namespace %s does not define %s" % (name, symbol))
class Namespace(object):
def __init__(self, name, *symbols):
self.name = name
self.symbols = symbols
def validate(self, symbol):
if symbol not in self.symbols:
raise NamespaceValidationException(self.name, symbol)
class NamespaceResolver(object):
def __init__(self):
self._ns = {}
def register(self, alias, namespace):
self._ns[alias] = namespace
def resolve(self, alias):
if alias in self._ns:
return self._ns[alias]
else:
raise NamespaceResolutionException(alias)
@context_aware
@parameter('symbol', constant_only=True)
def resolve_prop(alias, symbol, context):
resolver = get_resolver(context)
namespace = resolver.resolve(alias)
namespace.validate(symbol)
return namespace.name + '.' + symbol
@context_aware
@parameter('symbol', function_only=True, lazy=True)
def resolve_function(self, alias, symbol, context):
resolver = get_resolver(context)
namespace = resolver.resolve(alias)
namespace.validate(symbol.function_name)
symbol.function_name = namespace.name + '.' + symbol.function_name
return symbol(sender=self)
def add_to_context(context, resolver=None):
context.set_data(resolver or NamespaceResolver(), '$__ns_resolver')
context.register_function(resolve_prop, 'operator_:')
context.register_function(resolve_function, 'operator_:')
def get_resolver(context):
return context.get_data('$__ns_resolver')

42
yaql/functions/strings.py
View File

@ -1,42 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
from yaql.language.engine import parameter
@parameter('a', arg_type=six.string_types)
@parameter('b', arg_type=six.string_types)
def string_concatenation(a, b):
return a + b
@parameter('self', arg_type=six.string_types, is_self=True)
def as_list(self):
return list(self)
def to_string(self):
return str(self)
def _to_string_func(data):
return to_string(data)
def add_to_context(context):
context.register_function(string_concatenation, 'operator_+')
context.register_function(as_list, 'asList')
context.register_function(to_string)
context.register_function(_to_string_func, 'string')

90
yaql/functions/system.py
View File

@ -1,90 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import six
from yaql.language.exceptions import YaqlExecutionException
from yaql.language.engine import parameter, context_aware, inverse_context
# This unit defines basic YAQL functions, such as
# context retrieval, object property retrieval, method calls etc
def _is_object(value):
return not isinstance(value, (dict, collections.Iterable)) \
or isinstance(value, six.string_types)
@context_aware
def get_context_data(context, path):
return context.get_data(path)
@parameter('att_name', constant_only=True)
@parameter('self', custom_validator=_is_object)
def obj_attribution(self, att_name):
try:
return getattr(self, att_name)
except AttributeError:
raise YaqlExecutionException("Unable to retrieve object attribute")
@parameter('self', arg_type=dict)
@parameter('att_name', constant_only=True)
def dict_attribution(self, att_name):
return self.get(att_name)
@parameter('method', lazy=True, function_only=True)
@parameter('self')
@inverse_context
def method_call(self, method):
return method(sender=self)
@context_aware
@parameter('tuple_preds', lazy=True)
def _as(self, context, *tuple_preds):
self = self
for t in tuple_preds:
tup = t(self)
val = tup[0]
key_name = tup[1]
context.set_data(val, key_name)
return self
@parameter('conditions', lazy=True)
def switch(self, *conditions):
for cond in conditions:
res = cond(self)
if not isinstance(res, tuple):
raise YaqlExecutionException("Switch must have tuple parameters")
if len(res) != 2:
raise YaqlExecutionException("Switch tuples must be of size 2")
if res[0]:
return res[1]
return None
def add_to_context(context):
context.register_function(get_context_data)
context.register_function(obj_attribution, 'operator_.')
context.register_function(dict_attribution, 'operator_.')
context.register_function(method_call, 'operator_.')
context.register_function(_as, 'as')
context.register_function(switch)
context.register_function(lambda val: val, 'wrap')

36
yaql/language/__init__.py
View File

@ -0,0 +1,36 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
def _python_2_unicode_compatible(class_):
"""
A decorator that defines __unicode__ and __str__ methods under Python 2.
Under Python 3 it does nothing.
To support Python 2 and 3 with a single code base, define a __str__ method
returning text and apply this decorator to the class.
Copyright (c) 2010-2015 Benjamin Peterson
"""
if six.PY2:
if '__str__' not in class_.__dict__:
raise ValueError("@python_2_unicode_compatible cannot be applied "
"to %s because it doesn't define __str__()." %
class_.__name__)
class_.__unicode__ = class_.__str__
class_.__str__ = lambda self: self.__unicode__().encode('utf-8')
return class_
if not hasattr(six, 'python_2_unicode_compatible'):
six.python_2_unicode_compatible = _python_2_unicode_compatible

176
yaql/language/context.py
View File

@ -11,72 +11,134 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import types
import yaql.language
import six
from yaql.language import specs
from yaql.language import exceptions
from yaql.language import runner
from yaql.language import utils
class Context():
def __init__(self, parent_context=None, data=None):
self.parent_context = parent_context
self.functions = {}
self.data = {}
class ContextBase(object):
def __init__(self, parent_context):
self._parent_context = parent_context
if data:
self.data['$'] = data
if parent_context:
self.depth = parent_context.depth + 1
else:
self.depth = 0
@property
def parent(self):
return self._parent_context
def take_snapshot(self):
return {
'functions': self.functions.copy(),
'data': self.data.copy()
}
def register_function(self, spec, *args, **kwargs):
pass
def restore(self, snapshot):
self.data = snapshot['data'].copy()
self.functions = snapshot['functions'].copy()
def __getitem__(self, name):
return None
def register_function(self, function, name=None):
func_def = yaql.language.engine.yaql_function(function)
func_def.build()
if isinstance(function, types.MethodType):
func_def.restrict_to_class(function.im_class)
num_params = func_def.get_num_params()
if not name:
name = func_def.function.__name__
def __setitem__(self, name, value):
pass
if name not in self.functions:
self.functions[name] = {}
if num_params not in self.functions[name]:
self.functions[name][num_params] = [func_def]
else:
self.functions[name][num_params].append(func_def)
def __delitem__(self, name):
pass
def get_functions(self, function_name, num_params):
result = []
if function_name in self.functions:
if num_params in self.functions[function_name]:
result += self.functions[function_name][num_params]
if -1 in self.functions[function_name]:
result += self.functions[function_name][-1]
def __call__(self, name, engine, sender=utils.NO_VALUE,
data_context=None, return_context=False,
use_convention=False):
raise exceptions.NoFunctionRegisteredException(name)
if self.parent_context:
result += self.parent_context.get_functions(function_name,
num_params)
return result
def get_functions(self, name, predicate=None, use_convention=False):
return []
def set_data(self, data, path='$'):
if not path.startswith('$'):
path = '$' + path
self.data[path] = data
if path == '$':
self.data['$1'] = data
def collect_functions(self, name, predicate=None, use_convention=False):
return [[]]
def get_data(self, path='$'):
if path in self.data:
return self.data[path]
if self.parent_context:
return self.parent_context.get_data(path)
def create_child_context(self):
return type(self)(self)
class Context(ContextBase):
def __init__(self, parent_context=None, data=utils.NO_VALUE,
convention=None):
super(Context, self).__init__(parent_context)
self._functions = {}
self._data = {}
self._convention = convention
if data is not utils.NO_VALUE:
self['$'] = data
@staticmethod
def _import_function_definition(fd):
return fd
def register_function(self, spec, *args, **kwargs):
exclusive = kwargs.pop('exclusive', False)
if not isinstance(spec, specs.FunctionDefinition) \
and six.callable(spec):
spec = specs.get_function_definition(
spec, *args, convention=self._convention, **kwargs)
spec = self._import_function_definition(spec)
if spec.is_method:
if not spec.is_valid_method():
raise exceptions.InvalidMethodException(spec.name)
self._functions.setdefault(spec.name, list()).append((spec, exclusive))
def get_functions(self, name, predicate=None, use_convention=False):
if use_convention and self._convention is not None:
name = self._convention.convert_function_name(name)
if predicate is None:
predicate = lambda x: True
return six.moves.filter(predicate, list(six.moves.map(
lambda x: x[0],
self._functions.get(name, list()))))
def collect_functions(self, name, predicate=None, use_convention=False):
if use_convention and self._convention is not None:
name = self._convention.convert_function_name(name)
overloads = []
p = self
while p is not None:
layer_overloads = p._functions.get(name)
p = p.parent
if layer_overloads:
layer = []
for spec, exclusive in layer_overloads:
if exclusive:
p = None
if predicate and not predicate(spec):
continue
layer.append(spec)
if layer:
overloads.append(layer)
return overloads
def __call__(self, name, engine, sender=utils.NO_VALUE,
data_context=None, return_context=False,
use_convention=False):
return lambda *args, **kwargs: runner.call(
name, self, args, kwargs, engine, sender,
data_context, return_context, use_convention)
@staticmethod
def _normalize_name(name):
if not name.startswith('$'):
name = ('$' + name)
if name == '$':
name = '$1'
return name
def __setitem__(self, name, value):
self._data[self._normalize_name(name)] = value
def __getitem__(self, name):
name = self._normalize_name(name)
if name in self._data:
return self._data[name]
if self.parent:
return self.parent[name]
def __delitem__(self, name):
self._data.pop(self._normalize_name(name))
def create_child_context(self):
return Context(self, convention=self._convention)

56
yaql/language/conventions.py Normal file
View File

@ -0,0 +1,56 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
class Convention(object):
def convert_function_name(self, name):
return name
def convert_parameter_name(self, name):
return name
class PythonConvention(Convention):
def convert_function_name(self, name):
if not name or not name[0].isalpha():
return name
return name.rstrip('_')
def convert_parameter_name(self, name):
if not name or not name[0].isalpha():
return name
return name.rstrip('_')
class CamelCaseConvention(Convention):
def __init__(self):
self.regex = re.compile(r'(?!^)_(\w)', flags=re.UNICODE)
def convert_function_name(self, name):
if not name or not name[0].isalpha():
return name
return self._to_camel_case(name.strip('_'))
def convert_parameter_name(self, name):
if not name or not name[0].isalpha():
return name
return self._to_camel_case(name.rstrip('_', ))
def _to_camel_case(self, name):
return self.regex.sub(lambda m: m.group(1).upper(), name)

269
yaql/language/engine.py
View File

@ -1,269 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import inspect
import sys
import types
import yaql.language.context
from yaql.language import exceptions
import yaql.language.expressions
def yaql_function(function):
if not hasattr(function, "__yaql__"):
if isinstance(function, types.MethodType):
function = function.im_func
function.__yaql__ = YaqlFunctionDefinition(function)
return function.__yaql__
class YaqlFunctionDefinition(object):
def __init__(self, function):
self.function = function
self.is_context_aware = False
self.context_param_name = None
self.self_param_name = None
self.context_owner_param_name = None
self.param_definitions = {}
self._arg_spec = inspect.getargspec(function)
self._inverse_context = False
if self._arg_spec.keywords:
raise exceptions.YaqlException(
"Keyword parameters are not supported")
def register_param_constraint(self, param):
if param.name not in self._arg_spec.args \
and self._arg_spec.varargs != param.name:
raise exceptions.NoParameterFoundException(
function_name=self.function.func_name,
param_name=param.name)
if param.name in self.param_definitions:
raise exceptions.DuplicateParameterDecoratorException(
function_name=self.function.func_name,
param_name=param.name)
if self.is_context_aware and param.is_context:
raise exceptions.DuplicateContextDecoratorException(
function_name=self.function.func_name)
if self.context_owner_param_name and param.own_context:
raise exceptions.DuplicateContextOwnerDecoratorException(
function_name=self.function.func_name)
self.param_definitions[param.name] = param
if param.is_context:
self.is_context_aware = True
self.context_param_name = param.name
if param.is_self is None:
if param.name in self._arg_spec.args:
param.is_self = self._arg_spec.args.index(
param.name) == 0 and param.name == 'self'
if param.is_self:
self.self_param_name = param.name
if param.lazy:
raise exceptions.YaqlException("Self parameter cannot be lazy")
def get_num_params(self):
if self._arg_spec.varargs or self._arg_spec.keywords:
return -1
if self.is_context_aware:
return len(self._arg_spec.args) - 1
else:
return len(self._arg_spec.args)
def get_context_owner_index(self):
for param in self.param_definitions.values():
if param.inverse_context:
return param.name
return None
def build(self):
for arg_name in self._arg_spec.args:
if arg_name not in self.param_definitions:
self.register_param_constraint(ParameterDefinition(arg_name))
if self._arg_spec.varargs and\
self._arg_spec.varargs not in self.param_definitions:
self.register_param_constraint(
ParameterDefinition(self._arg_spec.varargs))
def inverse_context(self):
self._inverse_context = True
def __repr__(self):
return self.function.func_name + "_" + str(self.get_num_params())
def __call__(self, context, sender, *args):
if sender and not self.self_param_name:
raise exceptions.YaqlExecutionException(
"The function cannot be run as a method")
num_args = len(args) + 1 if sender else len(args)
if 0 <= self.get_num_params() != num_args:
raise exceptions.YaqlExecutionException(
"Expected {0} args, got {1}".format(self.get_num_params(),
len(args)))
input_position = 0
prepared_list = []
if self._inverse_context:
context_to_pass = context
else:
context_to_pass = yaql.language.context.Context(context)
for arg_name in self._arg_spec.args:
definition = self.param_definitions[arg_name]
if sender and definition.is_self:
definition.validate_value(sender)
prepared_list.append(sender)
elif definition.is_context:
prepared_list.append(context)
else:
arg = args[input_position]
input_position += 1
value, base_context = definition.validate(
arg.create_callable(context_to_pass))
prepared_list.append(value)
if self._inverse_context:
context_to_pass = yaql.language.context.Context(
base_context)
else:
context_to_pass = yaql.language.context.Context(
context)
if self._arg_spec.varargs:
while input_position < len(args):
definition = self.param_definitions[self._arg_spec.varargs]
arg = args[input_position]
input_position += 1
c = arg.create_callable(context_to_pass)
val = definition.validate(c)[0]
base_context = c.yaql_context
prepared_list.append(val)
if self._inverse_context:
context_to_pass = yaql.language.context.Context(
base_context)
else:
context_to_pass = yaql.language.context.Context(context)
if self._inverse_context:
final_context = context_to_pass
else:
final_context = context
return self.function(*prepared_list), final_context
def restrict_to_class(self, class_type):
if self.self_param_name:
definition = self.param_definitions.get(self.self_param_name)
if not definition.arg_type:
definition.arg_type = class_type
class ParameterDefinition(object):
def __init__(self,
name,
lazy=False,
arg_type=None,
custom_validator=None,
constant_only=False,
function_only=False,
is_context=False,
is_self=None):
self.arg_type = arg_type
self.name = name
self.lazy = lazy
self.arg_type = arg_type
self.custom_validator = custom_validator
self.constant_only = constant_only
self.function_only = function_only
self.is_context = is_context
self.is_self = is_self
def validate(self, value):
if self.constant_only:
if not isinstance(value,
yaql.language.expressions.Constant.Callable):
raise exceptions.YaqlExecutionException(
"Parameter {0} has to be a constant".format(self.name))
if self.function_only:
if not isinstance(value,
yaql.language.expressions.Function.Callable):
raise exceptions.YaqlExecutionException(
"Parameter {0} has to be a function".format(self.name))
if not self.lazy:
try:
res = value()
except Exception:
raise exceptions.YaqlExecutionException(
"Unable to evaluate parameter {0}".format(self.name),
sys.exc_info())
else:
res = value
context = value.yaql_context
self.validate_value(res)
return res, context
def validate_value(self, value):
if self.arg_type:
# we need a special handling for booleans, as
# isinstance(boolean_value, integer_type)
# will return true, which is not what we expect
if type(value) is bool:
if self.arg_type is not bool:
raise exceptions.YaqlExecutionException(
"Type of the parameter is not boolean")
elif not isinstance(value, self.arg_type):
raise exceptions.YaqlExecutionException(
"Type of the parameter is not {0}".format(
str(self.arg_type)))
if self.custom_validator:
if not self.custom_validator(value):
raise exceptions.YaqlExecutionException(
"Parameter didn't pass the custom validation")
def parameter(name,
lazy=False,
arg_type=None,
custom_validator=None,
constant_only=False,
function_only=False,
is_context=False,
is_self=None):
def get_wrapper(func):
param = ParameterDefinition(name,
lazy,
arg_type,
custom_validator,
constant_only,
function_only,
is_context,
is_self)
yaql_function(func).register_param_constraint(param)
return func
return get_wrapper
def inverse_context(func):
yaql_function(func).inverse_context()
return func
def context_aware(arg):
if callable(arg): # no-arg decorator case, arg is a decorated function
yaql_function(arg).register_param_constraint(
ParameterDefinition('context', is_context=True))
return arg
else: # decorator is called with args, arg is the name of parameter
return parameter(arg, is_context=True)

139
yaql/language/exceptions.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2013 Mirantis, Inc.
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -14,52 +14,94 @@
class YaqlException(Exception):
def __init__(self, message):
super(YaqlException, self).__init__()
self.message = message
pass
class NoFunctionRegisteredException(YaqlException):
def __init__(self, func_name, arg_num=None):
self.func_name = func_name
self.arg_num = arg_num
msg = "No function called '{0}' is registered".format(self.func_name)
if self.arg_num:
msg += " which has {0} arguments".format(self.arg_num)
super(NoFunctionRegisteredException, self).__init__(msg)
class ResolutionError(YaqlException):
pass
class YaqlExecutionException(YaqlException):
def __init__(self, message, inner=None):
super(YaqlExecutionException, self).__init__(message)
self.inner_exception = inner
class FunctionResolutionError(ResolutionError):
pass
class MethodResolutionError(ResolutionError):
pass
class NoFunctionRegisteredException(FunctionResolutionError):
def __init__(self, name):
super(NoFunctionRegisteredException, self).__init__(
u'Unknown function "{0}"'.format(name))
class NoMethodRegisteredException(MethodResolutionError):
def __init__(self, name, sender):
super(NoMethodRegisteredException, self).__init__(
u'Unknown method "{0}" for type {1}'.format(name, type(sender)))
class NoMatchingFunctionException(FunctionResolutionError):
def __init__(self, name):
super(NoMatchingFunctionException, self).__init__(
u'No function "{0}" matches supplied arguments'.format(name))
class NoMatchingMethodException(MethodResolutionError):
def __init__(self, name, sender):
super(NoMatchingMethodException, self).__init__(
u'No method "{0}" for type {1} matches supplied arguments'.format(
name, type(sender)))
class AmbiguousFunctionException(FunctionResolutionError):
def __init__(self, name):
super(AmbiguousFunctionException, self).__init__(
u'Ambiguous function "{0}"'.format(name))
class AmbiguousMethodException(MethodResolutionError):
def __init__(self, name, sender):
super(AmbiguousMethodException, self).__init__(
u'Ambiguous method "{0}" for type {1}'.format(name, type(sender)))
class ArgumentException(YaqlException):
def __init__(self, argument_name):
self.parameter_name = argument_name
super(ArgumentException, self).__init__(
u'Invalid argument {0}'.format(argument_name))
class MappingTranslationException(YaqlException):
def __init__(self):
super(MappingTranslationException, self).__init__(
u'Cannot convert mapping to keyword argument')
class ArgumentValueException(YaqlException):
def __init__(self):
super(ArgumentValueException, self).__init__()
class DuplicateParameterDecoratorException(YaqlException):
def __init__(self, function_name, param_name):
message = "Function '{0}' has multiple " \
"decorators for parameter '{1}'". \
message = u"Function '{0}' has multiple " \
u"decorators for parameter '{1}'". \
format(function_name, param_name)
super(DuplicateParameterDecoratorException, self).__init__(message)
class DuplicateContextDecoratorException(YaqlException):
class InvalidMethodException(YaqlException):
def __init__(self, function_name):
message = "Function '{0}' has multiple context-param decorators". \
message = u"Function '{0}' cannot be called as a method". \
format(function_name)
super(DuplicateContextDecoratorException, self).__init__(message)
class DuplicateContextOwnerDecoratorException(YaqlException):
def __init__(self, function_name):
message = "Function '{0}' has multiple context-owner decorators". \
format(function_name)
super(DuplicateContextOwnerDecoratorException, self).__init__(message)
super(InvalidMethodException, self).__init__(message)
class NoParameterFoundException(YaqlException):
def __init__(self, function_name, param_name):
message = "Function '{0}' has no parameter called '{1}'". \
message = u"Function '{0}' has no parameter called '{1}'". \
format(function_name, param_name)
super(NoParameterFoundException, self).__init__(message)
@ -73,21 +115,42 @@ class YaqlParsingException(YaqlException):
class YaqlGrammarException(YaqlParsingException):
def __init__(self, value, position):
msg = "Parse error: unexpected '{0}' at position {1}" \
.format(value, position)
def __init__(self, expr, value, position):
if position is None:
msg = u'Parse error: unexpected end of statement'
else:
msg = u"Parse error: unexpected '{0}' at position {1} of " \
u"expression '{2}'".format(value, position, expr)
super(YaqlGrammarException, self).__init__(value, position, msg)
class YaqlLexicalException(YaqlParsingException):
def __init__(self, value, position):
msg = "Lexical error: illegal character '{0}' at position {1}" \
msg = u"Lexical error: illegal character '{0}' at position {1}" \
.format(value, position)
super(YaqlLexicalException, self).__init__(value, position, msg)
class YaqlSequenceException(YaqlException):
def __init__(self, size):
self.size = size
super(YaqlSequenceException, self). \
__init__("Generator sequence too long ({0})".format(self.size))
class InvalidOperatorTableException(YaqlException):
def __init__(self, op):
super(InvalidOperatorTableException, self). \
__init__(u"Invalid records in operator table for operator "
u"'{0}".format(op))
class WrappedException(YaqlException):
def __init__(self, exception):
self.wrapped = exception
super(WrappedException, self).__init__(str(exception))
class CollectionTooLargeException(YaqlException):
def __init__(self, count):
super(CollectionTooLargeException, self).__init__(
'Collection length exceeds {0} elements'.format(count))
class MemoryQuotaExceededException(YaqlException):
def __init__(self):
super(MemoryQuotaExceededException, self).__init__(
'Expression consumed too much memory')

222
yaql/language/expressions.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2013 Mirantis, Inc.
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -11,145 +11,159 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from yaql.language.context import Context
from yaql.language import exceptions
import sys
import six
import yaql
from yaql.language import exceptions
from yaql.language import utils
class Expression(object):
class Callable(object):
def __init__(self, wrapped_object, context, key=None):
self.wrapped_object = wrapped_object
self.yaql_context = context
self.key = key
def __str__(self):
return str(self.key)
def evaluate(self, data=None, context=None):
if not context:
context = Context(yaql.create_context())
if data:
context.set_data(data)
f = self.create_callable(context)
# noinspection PyCallingNonCallable
return f()
def create_callable(self, context):
def __call__(self, sender, context, engine):
pass
@six.python_2_unicode_compatible
class Function(Expression):
def __init__(self, name, *args):
self.name = name
self.args = args
self.uses_sender = True
class Callable(Expression.Callable):
def __init__(self, wrapped, context, function_name, args):
super(Function.Callable, self).__init__(wrapped, None,
key=function_name)
self.function_name = function_name
self.args = args
self.yaql_context = context
def __call__(self, sender, context, engine):
return context(self.name, engine, sender, context,
return_context=True)(*self.args)
def __call__(self, *context_args, **context_kwargs):
sender = context_kwargs.pop('sender', None)
if context_args: # passed args have to be placed in the context
self.yaql_context.set_data(context_args[0])
for i, param in enumerate(context_args):
self.yaql_context.set_data(param, '$' + str(i + 1))
for arg_name, arg_value in context_kwargs.items():
self.yaql_context.set_data(arg_value, '$' + arg_name)
num_args = len(self.args) + 1 if sender else len(self.args)
fs = self.yaql_context.get_functions(self.function_name, num_args)
if not fs:
raise exceptions.NoFunctionRegisteredException(
self.function_name,
num_args)
snapshot = self.yaql_context.take_snapshot()
errors = []
for func in fs:
try:
result, res_context = func(self.yaql_context, sender,
*self.args)
self.yaql_context = res_context
return result
except exceptions.YaqlExecutionException as e:
self.yaql_context.restore(snapshot)
errors.append(e)
continue
raise exceptions.YaqlExecutionException(
"Registered function(s) matched but none"
" could run successfully", errors)
def create_callable(self, context):
return Function.Callable(self, context, self.name, self.args)
def __str__(self):
return u'{0}({1})'.format(self.name, ', '.join(
map(six.text_type, self.args)))
class BinaryOperator(Function):
def __init__(self, op, obj1, obj2):
super(BinaryOperator, self).__init__("operator_" + op, obj1,
obj2)
def __init__(self, op, obj1, obj2, alias):
if alias is None:
func_name = '#operator_' + op
else:
func_name = '*' + alias
self.operator = op
super(BinaryOperator, self).__init__(func_name, obj1, obj2)
self.uses_sender = False
class UnaryOperator(Function):
def __init__(self, op, obj):
super(UnaryOperator, self).__init__("unary_" + op, obj)
class Filter(Function):
def __init__(self, value, expression):
super(Filter, self).__init__("where", value, expression)
class Tuple(Function):
def __init__(self, left, right):
super(Tuple, self).__init__('tuple', left, right)
@staticmethod
def create_tuple(left, right):
if isinstance(left, Tuple):
new_args = list(left.args)
new_args.append(right)
left.args = tuple(new_args)
return left
def __init__(self, op, obj, alias):
if alias is None:
func_name = '#unary_operator_' + op
else:
return Tuple(left, right)
func_name = '*' + alias
self.operator = op
super(UnaryOperator, self).__init__(func_name, obj)
self.uses_sender = False
class Wrap(Function):
def __init__(self, content):
super(Wrap, self).__init__('wrap', content)
class IndexExpression(Function):
def __init__(self, value, *args):
super(IndexExpression, self).__init__('#indexer', value, *args)
self.uses_sender = False
class ListExpression(Function):
def __init__(self, *args):
super(ListExpression, self).__init__('#list', *args)
self.uses_sender = False
class MapExpression(Function):
def __init__(self, *args):
super(MapExpression, self).__init__('#map', *args)
self.uses_sender = False
@six.python_2_unicode_compatible
class GetContextValue(Function):
def __init__(self, path):
super(GetContextValue, self).__init__("get_context_data", path)
super(GetContextValue, self).__init__('#get_context_data', path)
self.path = path
self.uses_sender = False
def __str__(self):
return self.path
return self.path.value
@six.python_2_unicode_compatible
class Constant(Expression):
def __init__(self, value):
self.value = value
self.uses_sender = False
def __str__(self):
return str(self.value)
if isinstance(self.value, six.text_type):
return u"'{0}'".format(self.value)
return six.text_type(self.value)
class Callable(Expression.Callable):
def __init__(self, wrapped, value, context):
super(Constant.Callable, self).__init__(wrapped, context,
key=value)
self.value = value
def __call__(self, sender, context, engine):
return self.value, context
# noinspection PyUnusedLocal
def __call__(self, *args):
return self.value
def create_callable(self, context):
return Constant.Callable(self, self.value, context)
class KeywordConstant(Constant):
pass
@six.python_2_unicode_compatible
class Wrap(Expression):
def __init__(self, expression):
self.expr = expression
self.uses_sender = False
def __str__(self):
return str(self.expr)
def __call__(self, sender, context, engine):
return self.expr(sender, context, engine)
@six.python_2_unicode_compatible
class MappingRuleExpression(Expression):
def __init__(self, source, destination):
self.source = source
self.destination = destination
self.uses_sender = False
def __str__(self):
return u'{0} => {1}'.format(self.source, self.destination)
def __call__(self, sender, context, engine):
return utils.MappingRule(
self.source(sender, context, engine)[0],
self.destination(sender, context, engine)[0]), context
@six.python_2_unicode_compatible
class Statement(Function):
def __init__(self, expression, engine):
self.expression = expression
self.uses_sender = False
self.engine = engine
super(Statement, self).__init__('#finalize', expression)
def __call__(self, sender, context, engine):
if not context.collect_functions('#finalize'):
context = context.create_child_context()
context.register_function(lambda x: x, name='#finalize')
try:
return super(Statement, self).__call__(sender, context, engine)
except exceptions.WrappedException as e:
six.reraise(type(e.wrapped), e.wrapped, sys.exc_info()[2])
def evaluate(self, data=utils.NO_VALUE, context=utils.NO_VALUE):
if context is utils.NO_VALUE:
context = yaql.create_context()
if data is not utils.NO_VALUE:
context['$'] = utils.convert_input_data(data)
return self(utils.NO_VALUE, context, self.engine)[0]
def __str__(self):
return str(self.expression)

231
yaql/language/factory.py Normal file
View File

@ -0,0 +1,231 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import re
from ply import lex
from ply import yacc
from yaql.language import exceptions
from yaql.language import expressions
from yaql.language import lexer
from yaql.language import parser
from yaql.language import utils
OperatorType = collections.namedtuple('OperatorType', [
'PREFIX_UNARY', 'SUFFIX_UNARY',
'BINARY_LEFT_ASSOCIATIVE', 'BINARY_RIGHT_ASSOCIATIVE',
'NAME_VALUE_PAIR'
])(
PREFIX_UNARY='PREFIX_UNARY',
SUFFIX_UNARY='SUFFIX_UNARY',
BINARY_LEFT_ASSOCIATIVE='BINARY_LEFT_ASSOCIATIVE',
BINARY_RIGHT_ASSOCIATIVE='BINARY_RIGHT_ASSOCIATIVE',
NAME_VALUE_PAIR='NAME_VALUE_PAIR'
)
class YaqlOperators(object):
def __init__(self, operators, name_value_op=None):
self.operators = operators
self.name_value_op = name_value_op
class YaqlEngine(object):
def __init__(self, ply_lexer, ply_parser, options, factory):
self._lexer = ply_lexer
self._parser = ply_parser
self._options = utils.FrozenDict(options or {})
self._factory = factory
@property
def lexer(self):
return self._lexer
@property
def parser(self):
return self._parser
@property
def options(self):
return self._options
@property
def factory(self):
return self._factory
def __call__(self, expression, options=None):
if options:
return self.copy(options)(expression)
return expressions.Statement(
self.parser.parse(expression, lexer=self.lexer), self)
def copy(self, options):
opt = dict(self._options)
opt.update(options)
return YaqlEngine(self._lexer, self._parser, opt, self._factory)
class YaqlFactory(object):
def __init__(self, keyword_operator='=>'):
self._keyword_operator = keyword_operator
self.operators = self._standard_operators()
if keyword_operator:
self.operators.insert(0, (keyword_operator,
OperatorType.NAME_VALUE_PAIR))
@property
def keyword_operator(self):
return self._keyword_operator
# noinspection PyMethodMayBeStatic
def _standard_operators(self):
return [
('.', OperatorType.BINARY_LEFT_ASSOCIATIVE),
('?.', OperatorType.BINARY_LEFT_ASSOCIATIVE),
(),
('+', OperatorType.PREFIX_UNARY),
('-', OperatorType.PREFIX_UNARY),
(),
('=~', OperatorType.BINARY_LEFT_ASSOCIATIVE),
('!~', OperatorType.BINARY_LEFT_ASSOCIATIVE),
(),
('*', OperatorType.BINARY_LEFT_ASSOCIATIVE),
('/', OperatorType.BINARY_LEFT_ASSOCIATIVE),
('mod', OperatorType.BINARY_LEFT_ASSOCIATIVE),
(),
('+', OperatorType.BINARY_LEFT_ASSOCIATIVE),
('-', OperatorType.BINARY_LEFT_ASSOCIATIVE),
(),
('>', OperatorType.BINARY_LEFT_ASSOCIATIVE),
('<', OperatorType.BINARY_LEFT_ASSOCIATIVE),
('>=', OperatorType.BINARY_LEFT_ASSOCIATIVE),
('<=', OperatorType.BINARY_LEFT_ASSOCIATIVE),
('!=', OperatorType.BINARY_LEFT_ASSOCIATIVE, 'not_equal'),
('=', OperatorType.BINARY_LEFT_ASSOCIATIVE, 'equal'),
('in', OperatorType.BINARY_LEFT_ASSOCIATIVE),
(),
('not', OperatorType.PREFIX_UNARY),
(),
('and', OperatorType.BINARY_LEFT_ASSOCIATIVE),
(),
('or', OperatorType.BINARY_LEFT_ASSOCIATIVE),
(),
('->', OperatorType.BINARY_RIGHT_ASSOCIATIVE),
]
def insert_operator(self, existing_operator, existing_operator_binary,
new_operator, new_operator_type, create_group,
new_operator_alias=None):
binary_types = (OperatorType.BINARY_RIGHT_ASSOCIATIVE,
OperatorType.BINARY_LEFT_ASSOCIATIVE)
unary_types = (OperatorType.PREFIX_UNARY, OperatorType.SUFFIX_UNARY)
position = 0
if existing_operator is not None:
position = -1
for i, t in enumerate(self.operators):
if len(t) < 2 or t[0] != existing_operator:
continue
if existing_operator_binary and t[1] not in binary_types:
continue
if not existing_operator_binary and t[1] not in unary_types:
continue
position = i
break
if position < 0:
raise ValueError('Operator {0} is not found'.format(
existing_operator))
while position < len(self.operators) and len(
self.operators[position]) > 1:
position += 1
if create_group:
if position == len(self.operators):
self.operators.append(())
position += 1
else:
while position < len(self.operators) and len(
self.operators[position]) < 2:
position += 1
self.operators.insert(position, ())
self.operators.insert(
position, (new_operator, new_operator_type, new_operator_alias))
@staticmethod
def _name_generator():
value = 1
while True:
t = value
chars = []
while t:
chars.append(chr(ord('A') + t % 26))
t //= 26
yield ''.join(chars)
value += 1
def _build_operator_table(self, name_generator):
operators = {}
name_value_op = None
precedence = 1
for record in self.operators:
if not record:
precedence += 1
continue
up, bp, name, alias = operators.get(record[0], (0, 0, '', None))
if record[1] == OperatorType.NAME_VALUE_PAIR:
if name_value_op is not None:
raise exceptions.InvalidOperatorTableException(record[0])
name_value_op = record[0]
continue
if record[1] == OperatorType.PREFIX_UNARY:
if up:
raise exceptions.InvalidOperatorTableException(record[0])
up = precedence
elif record[1] == OperatorType.SUFFIX_UNARY:
if up:
raise exceptions.InvalidOperatorTableException(record[0])
up = -precedence
elif record[1] == OperatorType.BINARY_LEFT_ASSOCIATIVE:
if bp:
raise exceptions.InvalidOperatorTableException(record[0])
bp = precedence
elif record[1] == OperatorType.BINARY_RIGHT_ASSOCIATIVE:
if bp:
raise exceptions.InvalidOperatorTableException(record[0])
bp = -precedence
name = name or 'OP_' + next(name_generator)
operators[record[0]] = (
up, bp, name, record[2] if len(record) > 2 else None)
return YaqlOperators(operators, name_value_op)
# noinspection PyMethodMayBeStatic
def _create_lexer(self, operators):
return lexer.Lexer(operators)
# noinspection PyMethodMayBeStatic
def _create_parser(self, lexer_rules, operators):
return parser.Parser(lexer_rules, operators)
def create(self, options=None):
names = self._name_generator()
operators = self._build_operator_table(names)
lexer_rules = self._create_lexer(operators)
ply_lexer = lex.lex(object=lexer_rules, reflags=re.UNICODE)
ply_parser = yacc.yacc(
module=self._create_parser(lexer_rules, operators),
debug=False, tabmodule=None, write_tables=False)
return YaqlEngine(ply_lexer, ply_parser, options, self)

257
yaql/language/lexer.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2013 Mirantis, Inc.
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -12,166 +12,133 @@
# License for the specific language governing permissions and limitations
# under the License.
import ply.lex as lex
from yaql.language.exceptions import YaqlLexicalException
import codecs
import re
keywords = {
'true': 'TRUE',
'false': 'FALSE',
'null': 'NULL'
}
import six
keywords_to_val = {
'TRUE': True,
'FALSE': False,
'NULL': None
}
right_associative = [':']
from yaql.language import exceptions
unary_prefix = {
'-': "UNARY_MINUS",
'+': "UNARY_PLUS",
'~': "UNARY_TILDE",
'!': "UNARY_EXPL"
}
op_to_level = {
'abc': 0,
'|': 1,
'^': 2,
'&': 3,
'<': 4,
'>': 4,
'=': 5,
'!': 5,
'+': 6,
'-': 6,
'*': 7,
'/': 7,
'%': 7,
'.': 8
}
ops = {
(0, 'l'): "LVL0_LEFT",
(0, 'r'): "LVL0_RIGHT",
(1, 'l'): "LVL1_LEFT",
(1, 'r'): "LVL1_RIGHT",
(2, 'l'): "LVL2_LEFT",
(2, 'r'): "LVL2_RIGHT",
(3, 'l'): "LVL3_LEFT",
(3, 'r'): "LVL3_RIGHT",
(4, 'l'): "LVL4_LEFT",
(4, 'r'): "LVL4_RIGHT",
(5, 'l'): "LVL5_LEFT",
(5, 'r'): "LVL5_RIGHT",
(6, 'l'): "LVL6_LEFT",
(6, 'r'): "LVL6_RIGHT",
(7, 'l'): "LVL7_LEFT",
(7, 'r'): "LVL7_RIGHT",
(8, 'l'): "LVL8_LEFT",
(8, 'r'): "LVL8_RIGHT",
(9, 'l'): "LVL9_LEFT",
(9, 'r'): "LVL9_RIGHT"
}
NEVER_MATCHING_RE = '(?!x)x'
ESCAPE_SEQUENCE_RE = re.compile(r'''
( \\U........ # 8-digit hex escapes
| \\u.... # 4-digit hex escapes
| \\x.. # 2-digit hex escapes
| \\[0-7]{1,3} # Octal escapes
| \\N\{[^}]+\} # Unicode characters by name
| \\[\\'"abfnrtv] # Single-character escapes
)''', re.UNICODE | re.VERBOSE)
tokens = [
'STRING',
'QUOTED_STRING',
'NUMBER',
'FUNC',
'FILTER',
'NOT',
'DOLLAR'
] + list(keywords.values()) + list(ops.values()) + list(unary_prefix.values())
literals = "()],"
t_ignore = ' \t\r\n'
def decode_escapes(s):
def decode_match(match):
return codecs.decode(match.group(0), 'unicode-escape')
return ESCAPE_SEQUENCE_RE.sub(decode_match, s)
def t_DOLLAR(t):
"""
\\$\\w*
"""
return t
# noinspection PyPep8Naming
class Lexer(object):
t_ignore = ' \t\r\n'
t_INDEXER = '\\['
t_MAP = '{'
literals = '()],}'
keywords = {
'true': 'TRUE',
'false': 'FALSE',
'null': 'NULL'
}
def t_NUMBER(t):
"""
\\b\\d+(\\.?\\d+)?\\b
"""
if '.' in t.value:
t.value = float(t.value)
else:
t.value = int(t.value)
return t
keyword_to_val = {
'TRUE': True,
'FALSE': False,
'NULL': None
}
def __init__(self, yaql_operators):
self._operators_table = yaql_operators.operators
self.tokens = [
'KEYWORD_STRING',
'QUOTED_STRING',
'NUMBER',
'FUNC',
'DOLLAR',
'INDEXER',
'MAPPING',
'MAP'
] + list(self.keywords.values())
for op_symbol, op_record in six.iteritems(self._operators_table):
lexem_name = op_record[2]
setattr(self, 't_' + lexem_name, re.escape(op_symbol))
self.tokens.append(lexem_name)
self.t_MAPPING = re.escape(yaql_operators.name_value_op) \
if yaql_operators.name_value_op else NEVER_MATCHING_RE
def t_FUNC(t):
"""
\\b\\w+\\(|'(?:[^'\\\\]|\\\\.)*'\\(
"""
val = t.value[:-1].replace('\\', '').strip('\'')
t.value = val
return t
@staticmethod
def t_DOLLAR(t):
"""
\\$\\w*
"""
return t
@staticmethod
def t_NUMBER(t):
"""
\\b\\d+(\\.?\\d+)?\\b
"""
if '.' in t.value:
t.value = float(t.value)
else:
t.value = int(t.value)
return t
def t_FILTER(t):
"""
(?<!\\s)\\[
"""
return t
@staticmethod
def t_FUNC(t):
"""
\\b[^\\W\\d]\\w*\\(
"""
val = t.value[:-1]
t.value = val
return t
def t_KEYWORD_STRING(self, t):
"""
\\b[^\\W\\d]\\w*\\b
"""
if t.value in self._operators_table:
t.type = self._operators_table[t.value][2]
else:
t.type = self.keywords.get(t.value, 'KEYWORD_STRING')
t.value = self.keyword_to_val.get(t.type, t.value)
return t
def t_NOT(t):
"""
\\bnot\\b
"""
return t
@staticmethod
def t_QUOTED_STRING(t):
"""
'([^'\\\\]|\\\\.)*'
"""
t.value = decode_escapes(t.value[1:-1])
return t
@staticmethod
def t_DOUBLE_QUOTED_STRING(t):
"""
"([^"\\\\]|\\\\.)*"
"""
t.value = decode_escapes(t.value[1:-1])
t.type = 'QUOTED_STRING'
return t
def t_STRING(t):
"""
\\b\\w+\\b
"""
t.type = keywords.get(t.value, 'STRING')
t.value = keywords_to_val.get(t.type, t.value)
return t
@staticmethod
def t_QUOTED_VERBATIM_STRING(t):
"""
`([^`\\\\]|\\\\.)*`
"""
t.value = t.value[1:-1].replace('\\`', '`')
t.type = 'QUOTED_STRING'
return t
def t_QUOTED_STRING(t):
"""
'(?:[^'\\\\]|\\\\.)*'
"""
t.value = t.value[1:-1].replace('\\', '')
return t
def t_CHAR_ORB(t):
"""
[!@#%^&*=.:;`~\\-><+/]+
"""
if t.value in unary_prefix:
t.type = unary_prefix[t.value]
else:
t.type = get_orb_op_type(t.value[0], t.value[-1])
return t
def get_orb_op_type(first_char, last_char):
if first_char.isalpha() or first_char == '_':
level = op_to_level['abc']
else:
level = op_to_level.get(first_char, max(op_to_level.values()) + 1)
asc = 'r' if last_char in right_associative else 'l'
return ops.get((level, asc))
def t_error(t):
raise YaqlLexicalException(t.value[0], t.lexpos)
lexer = lex.lex()
@staticmethod
def t_error(t):
raise exceptions.YaqlLexicalException(t.value[0], t.lexpos)

361
yaql/language/parser.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2013 Mirantis, Inc.
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -12,201 +12,214 @@
# License for the specific language governing permissions and limitations
# under the License.
import tempfile
import six
import ply.yacc as yacc
from yaql.language import lexer, expressions, exceptions
from yaql.language import exceptions
from yaql.language import expressions
from yaql.language import utils
tokens = lexer.tokens
class Parser(object):
def __init__(self, lexer, yaql_operators):
self.tokens = lexer.tokens
self._aliases = {}
self._generate_operator_funcs(yaql_operators)
def _generate_operator_funcs(self, yaql_operators):
binary_doc = ''
unary_doc = ''
precedence_dict = {}
def p_value_to_const(p):
"""
value : STRING
| QUOTED_STRING
| NUMBER
| TRUE
| FALSE
| NULL
"""
p[0] = expressions.Constant(p[1])
for up, bp, op_name, op_alias in yaql_operators.operators.values():
self._aliases[op_name] = op_alias
if up:
l = precedence_dict.setdefault(
(abs(up), 'l' if up > 0 else 'r'), [])
l.append('UNARY_' + op_name if bp else op_name)
unary_doc += ('value : ' if not unary_doc else '\n| ')
spec_prefix = '{0} value' if up > 0 else 'value {0}'
if bp:
unary_doc += (spec_prefix + ' %prec UNARY_{0}').format(
op_name)
else:
unary_doc += spec_prefix.format(op_name)
if bp:
l = precedence_dict.setdefault(
(abs(bp), 'l' if bp > 0 else 'r'), [])
l.append(op_name)
binary_doc += ('value : ' if not binary_doc else '\n| ') + \
'value {0} value'.format(op_name)
# noinspection PyProtectedMember
def p_binary(this, p):
alias = this._aliases.get(p.slice[2].type)
p[0] = expressions.BinaryOperator(p[2], p[1], p[3], alias)
def p_value_to_dollar(p):
"""
value : DOLLAR
"""
p[0] = expressions.GetContextValue(expressions.Constant(p[1]))
# noinspection PyProtectedMember
def p_unary(this, p):
if p[1] in yaql_operators.operators:
alias = this._aliases.get(p.slice[1].type)
p[0] = expressions.UnaryOperator(p[1], p[2], alias)
else:
alias = this._aliases.get(p.slice[2].type)
p[0] = expressions.UnaryOperator(p[2], p[1], alias)
p_binary.__doc__ = binary_doc
self.p_binary = six.create_bound_method(p_binary, self)
p_unary.__doc__ = unary_doc
self.p_unary = six.create_bound_method(p_unary, self)
def p_val_to_function(p):
"""
value : func
"""
p[0] = p[1]
precedence = []
for i in range(1, len(precedence_dict) + 1):
for oa in ('r', 'l'):
value = precedence_dict.get((i, oa))
if value:
precedence.append(
(('left',) if oa is 'l' else ('right',)) +
tuple(value)
)
precedence.insert(1, ('left', 'LIST', 'INDEXER', 'MAP'))
precedence.reverse()
self.precedence = tuple(precedence)
@staticmethod
def p_value_to_const(p):
"""
value : QUOTED_STRING
| NUMBER
| TRUE
| FALSE
| NULL
"""
p[0] = expressions.Constant(p[1])
def p_method_no_args(p):
"""
func : value '.' FUNC ')'
"""
p[0] = expressions.Function(p[3], p[1])
@staticmethod
def p_keyword_constant(p):
"""
value : KEYWORD_STRING
"""
p[0] = expressions.KeywordConstant(p[1])
@staticmethod
def p_value_to_dollar(p):
"""
value : DOLLAR
"""
p[0] = expressions.GetContextValue(expressions.Constant(p[1]))
def p_arg_definition(p):
"""
arg : value
"""
p[0] = p[1]
@staticmethod
def p_val_in_parenthesis(p):
"""
value : '(' value ')'
"""
p[0] = expressions.Wrap(p[2])
@staticmethod
def p_args(p):
"""
args : arglist ',' named_arglist
| arglist
| named_arglist
"""
arg = ()
if len(p) >= 2:
arg = p[1]
if len(p) >= 4:
arg += p[3]
p[0] = arg
def p_arg_list(p):
"""
arg : arg ',' arg
"""
val_list = []
if isinstance(p[1], list):
val_list += p[1]
else:
val_list.append(p[1])
if isinstance(p[3], list):
val_list += p[3]
else:
val_list.append(p[3])
@staticmethod
def p_indexer(p):
"""
value : value INDEXER args ']'
"""
p[0] = expressions.IndexExpression(p[1], *p[3])
p[0] = val_list
@staticmethod
def p_list(p):
"""
value : INDEXER args ']' %prec LIST
"""
p[0] = expressions.ListExpression(*p[2])
@staticmethod
def p_empty_list(p):
"""
value : INDEXER ']' %prec LIST
"""
p[0] = expressions.ListExpression()
def p_method_w_args(p):
"""
func : value '.' FUNC arg ')'
"""
if isinstance(p[4], list):
arg = p[4]
else:
arg = [p[4]]
p[0] = expressions.Function(p[3], p[1], *arg)
@staticmethod
def p_map(p):
"""
value : MAP args '}'
"""
p[0] = expressions.MapExpression(*p[2])
@staticmethod
def p_empty_map(p):
"""
value : MAP '}'
"""
p[0] = expressions.MapExpression()
def p_function_no_args(p):
"""
func : FUNC ')'
"""
p[0] = expressions.Function(p[1])
@staticmethod
def p_val_to_function(p):
"""
value : func
"""
p[0] = p[1]
@staticmethod
def p_named_arg_definition(p):
"""
named_arg : value MAPPING value
"""
p[0] = expressions.MappingRuleExpression(p[1], p[3])
def p_function_w_args(p):
"""
func : FUNC arg ')'
"""
if isinstance(p[2], list):
arg = p[2]
else:
arg = [p[2]]
p[0] = expressions.Function(p[1], *arg)
@staticmethod
def p_arg_list(p):
"""
arglist : value
| arglist ',' value
|
| arglist ','
"""
if len(p) == 1:
p[0] = [utils.NO_VALUE]
elif len(p) == 2:
p[0] = [p[1]]
elif len(p) == 3:
p[0] = p[1] + [utils.NO_VALUE]
elif len(p) == 4:
p[0] = p[1] + [p[3]]
@staticmethod
def p_named_arg_list(p):
"""
named_arglist : named_arg
| named_arglist ',' named_arg
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[0] = p[1] + [p[3]]
def p_binary(p):
"""
value : value STRING value
| value LVL0_LEFT value
| value LVL0_RIGHT value
| value LVL1_LEFT value
| value LVL1_RIGHT value
| value LVL2_LEFT value
| value LVL2_RIGHT value
| value LVL3_LEFT value
| value LVL3_RIGHT value
| value LVL4_LEFT value
| value LVL4_RIGHT value
| value LVL5_LEFT value
| value LVL5_RIGHT value
| value LVL6_LEFT value
| value LVL6_RIGHT value
| value LVL7_LEFT value
| value LVL7_RIGHT value
| value LVL8_LEFT value
| value LVL8_RIGHT value
| value LVL9_LEFT value
| value LVL9_RIGHT value
| value UNARY_PLUS value
| value UNARY_MINUS value
| value UNARY_EXPL value
| value UNARY_TILDE value
"""
p[0] = expressions.BinaryOperator(p[2], p[1], p[3])
@staticmethod
def p_function(p):
"""
func : FUNC ')'
| FUNC args ')'
"""
arg = ()
if len(p) > 3:
arg = p[2]
p[0] = expressions.Function(p[1], *arg)
def p_unary_prefix(p):
"""
value : UNARY_TILDE value
| UNARY_PLUS value
| UNARY_EXPL value
| UNARY_MINUS value
| NOT value
"""
p[0] = expressions.UnaryOperator(p[1], p[2])
def p_val_in_parenthesis(p):
"""
value : '(' value ')'
"""
p[0] = expressions.Wrap(p[2])
def p_val_w_filter(p):
"""
value : value FILTER value ']'
"""
p[0] = expressions.Filter(p[1], p[3])
# def p_val_tuple(p):
# """
# value : value TUPLE value
# """
# p[0] = expressions.Tuple.create_tuple(p[1], p[3])
def p_error(p):
if p:
raise exceptions.YaqlGrammarException(p.value, p.lexpos)
else:
raise exceptions.YaqlGrammarException(None, None)
precedence = (
('left', lexer.ops[(0, 'l')], 'STRING', ','),
('right', lexer.ops[(0, 'r')]),
('left', lexer.ops[(1, 'l')]),
('right', lexer.ops[(1, 'r')]),
('left', lexer.ops[(2, 'l')]),
('right', lexer.ops[(2, 'r')]),
('left', lexer.ops[(3, 'l')]),
('right', lexer.ops[(3, 'r')]),
('left', lexer.ops[(4, 'l')]),
('right', lexer.ops[(4, 'r')]),
('left', lexer.ops[(5, 'l', )], 'NOT', 'UNARY_EXPL'),
('right', lexer.ops[(5, 'r')]),
('left', lexer.ops[(6, 'l')], 'UNARY_PLUS', 'UNARY_MINUS'),
('right', lexer.ops[(6, 'r')]),
('left', lexer.ops[(7, 'l')]),
('right', lexer.ops[(7, 'r')]),
('left', lexer.ops[(8, 'l')]),
('right', lexer.ops[(8, 'r')]),
('left', lexer.ops[(9, 'l')], 'UNARY_TILDE'),
('right', lexer.ops[(9, 'r')]),
)
parser = yacc.yacc(debug=False,
outputdir=tempfile.gettempdir(),
tabmodule='parser_table')
# parser = yacc.yacc()
def parse(expression):
return parser.parse(expression)
@staticmethod
def p_error(p):
if p:
raise exceptions.YaqlGrammarException(
p.lexer.lexdata, p.value, p.lexpos)
else:
raise exceptions.YaqlGrammarException(None, None, None)

185
yaql/language/runner.py Normal file
View File

@ -0,0 +1,185 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
import six
from yaql.language import exceptions
from yaql.language import expressions
from yaql.language import utils
from yaql.language import yaqltypes
def call(name, context, args, kwargs, engine, sender=utils.NO_VALUE,
data_context=None, return_context=False, use_convention=False):
if data_context is None:
data_context = context
if sender is utils.NO_VALUE:
predicate = lambda fd: fd.is_function
else:
predicate = lambda fd: fd.is_method
all_overloads = context.collect_functions(
name, predicate, use_convention=use_convention)
if not all_overloads:
if sender is utils.NO_VALUE:
raise exceptions.NoFunctionRegisteredException(name)
else:
raise exceptions.NoMethodRegisteredException(name, sender)
else:
delegate = choose_overload(name, all_overloads, engine, sender,
data_context, args, kwargs)
try:
result = delegate()
utils.limit_memory_usage(engine, (1, result[0]))
return result if return_context else result[0]
except StopIteration as e:
six.reraise(
exceptions.WrappedException,
exceptions.WrappedException(e),
sys.exc_info()[2])
def choose_overload(name, candidates, engine, sender, context, args, kwargs):
def raise_ambiguous():
if sender is utils.NO_VALUE:
raise exceptions.AmbiguousFunctionException(name)
else:
raise exceptions.AmbiguousMethodException(name, sender)
def raise_not_found():
if sender is utils.NO_VALUE:
raise exceptions.NoMatchingFunctionException(name)
else:
raise exceptions.NoMatchingMethodException(name, sender)
candidates2 = []
lazy_params = None
no_kwargs = None
if sender is not utils.NO_VALUE:
args = (sender,) + args
for level in candidates:
new_level = []
for c in level:
if no_kwargs is None:
no_kwargs = c.no_kwargs
args, kwargs = _translate_args(no_kwargs, args, kwargs)
elif no_kwargs != c.no_kwargs:
raise_ambiguous()
mapping = c.map_args(args, kwargs)
if mapping is None:
continue
pos, kwd = mapping
lazy = set()
for i, pos_arg in enumerate(pos):
if isinstance(pos_arg.value_type, yaqltypes.LazyParameterType):
lazy.add(i)
for key, value in six.iteritems(kwd):
if isinstance(value.value_type, yaqltypes.LazyParameterType):
lazy.add(key)
if lazy_params is None:
lazy_params = lazy
elif lazy_params != lazy:
raise_ambiguous()
new_level.append((c, mapping))
if new_level:
candidates2.append(new_level)
if len(candidates2) == 0:
raise_not_found()
arg_evaluator = lambda i, arg: (
arg(utils.NO_VALUE, context, engine)[0]
if (i not in lazy_params and isinstance(arg, expressions.Expression)
and not isinstance(arg, expressions.Constant))
else arg
)
args = tuple(arg_evaluator(i, arg) for i, arg in enumerate(args))
for key, value in six.iteritems(kwargs):
kwargs[key] = arg_evaluator(key, value)
delegate = None
winner_mapping = None
for level in candidates2:
for c, mapping in level:
try:
d = c.get_delegate(sender, engine, args, kwargs)
except exceptions.ArgumentException:
pass
else:
if delegate is not None:
if _is_specialization_of(winner_mapping, mapping):
continue
elif not _is_specialization_of(mapping, winner_mapping):
raise_ambiguous()
delegate = d
winner_mapping = mapping
if delegate is not None:
break
if delegate is None:
raise_not_found()
return lambda: delegate(context)
def _translate_args(without_kwargs, args, kwargs):
if without_kwargs:
if len(kwargs) > 0:
raise exceptions.ArgumentException(six.next(iter(kwargs)))
return args, {}
pos_args = []
kw_args = dict(kwargs)
for t in args:
if isinstance(t, expressions.MappingRuleExpression):
param_name = t.source
if isinstance(param_name, expressions.Constant):
param_name = param_name.value
if not isinstance(param_name, six.string_types):
raise exceptions.MappingTranslationException()
kw_args[param_name] = t.destination
else:
pos_args.append(t)
for key, value in six.iteritems(kwargs):
if key in kw_args:
raise exceptions.MappingTranslationException()
else:
kw_args[key] = value
return tuple(pos_args), kw_args
def _is_specialization_of(mapping1, mapping2):
args_mapping1, kwargs_mapping1 = mapping1
args_mapping2, kwargs_mapping2 = mapping2
res = False
for a1, a2 in six.moves.zip(args_mapping1, args_mapping2):
if a2.value_type.is_specialization_of(a1.value_type):
return False
elif a1.value_type.is_specialization_of(a2.value_type):
res = True
for key, a1 in six.iteritems(kwargs_mapping1):
a2 = kwargs_mapping2[key]
if a2.value_type.is_specialization_of(a1.value_type):
return False
elif a1.value_type.is_specialization_of(a2.value_type):
res = True
return res

421
yaql/language/specs.py Normal file
View File

@ -0,0 +1,421 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import inspect
import types
import six
from yaql.language import exceptions
from yaql.language import yaqltypes
from yaql.language import utils
NO_DEFAULT = utils.create_marker('<NoValue>')
class ParameterDefinition(object):
def __init__(self, name, value_type=None, position=None, alias=None,
default=None):
self.value_type = value_type
self.name = name
self.position = position
self.default = default
self.alias = alias
def __repr__(self):
return '{0} => position={1} value_type={2} default={3}'.format(
self.name, self.position, self.value_type, self.default)
def clone(self):
return ParameterDefinition(self.name, self.value_type,
self.position, self.alias, self.default)
class FunctionDefinition(object):
def __init__(self, name, parameters, payload, doc='',
is_function=True, is_method=False,
returns_context=False, no_kwargs=False):
self.is_method = is_method
self.is_function = is_function
self.name = name
self.parameters = parameters
self.payload = payload
self.doc = doc
self.returns_context = returns_context
self.no_kwargs = no_kwargs
def __call__(self, sender, engine, context):
return lambda *args, **kwargs: \
self.get_delegate(sender, engine, args, kwargs)(context)[0]
def clone(self):
parameters = dict(
(key, p.clone())
for key, p in six.iteritems(self.parameters))
res = FunctionDefinition(
self.name, parameters, self.payload,
self.doc, self.is_function, self.is_method,
self.returns_context, self.no_kwargs)
return res
def map_args(self, args, kwargs):
kwargs = dict(kwargs)
positional_args = len(args) * [
self.parameters.get('*', utils.NO_VALUE)]
max_dst_positional_args = len(args) + len(self.parameters)
positional_fix_table = max_dst_positional_args * [0]
keyword_args = {}
for p in six.itervalues(self.parameters):
if p.position is not None and isinstance(
p.value_type, yaqltypes.HiddenParameterType):
for index in range(p.position + 1, len(positional_fix_table)):
positional_fix_table[index] += 1
for key, p in six.iteritems(self.parameters):
arg_name = p.alias or p.name
if p.position is not None and key != '*':
arg_position = p.position - positional_fix_table[p.position]
if isinstance(p.value_type, yaqltypes.HiddenParameterType):
continue
elif arg_position < len(args) and args[arg_position] \
is not utils.NO_VALUE:
if arg_name in kwargs:
return None
positional_args[arg_position] = p
elif arg_name in kwargs:
keyword_args[arg_name] = p
del kwargs[arg_name]
elif p.default is NO_DEFAULT:
return None
elif arg_position < len(args) and args[arg_position]:
positional_args[arg_position] = p
elif p.position is None and key != '**':
if isinstance(p.value_type, yaqltypes.HiddenParameterType):
continue
elif arg_name in kwargs:
keyword_args[arg_name] = p
del kwargs[arg_name]
elif p.default is NO_DEFAULT:
return None
if len(kwargs) > 0:
if '**' in self.parameters:
argdef = self.parameters['**']
for key in six.iterkeys(kwargs):
keyword_args[key] = argdef
else:
return None
for i in range(len(positional_args)):
if positional_args[i] is utils.NO_VALUE:
return None
value = args[i]
if value is utils.NO_VALUE:
value = positional_args[i].default
if not positional_args[i].value_type.check(value):
return None
for kwd in six.iterkeys(kwargs):
if not keyword_args[kwd].value_type.check(kwargs[kwd]):
return None
return tuple(positional_args), keyword_args
def get_delegate(self, sender, engine, args, kwargs):
def checked(val, param):
if not param.value_type.check(val):
raise exceptions.ArgumentException(param.name)
def convert_arg_func(context):
try:
return param.value_type.convert(
val, sender, context, self, engine)
except exceptions.ArgumentValueException:
raise exceptions.ArgumentException(param.name)
return convert_arg_func
positional = 0
for arg_name, p in six.iteritems(self.parameters):
if p.position is not None and arg_name != '*':
positional += 1
positional_args = positional * [None]
positional_fix_table = positional * [0]
keyword_args = {}
for p in six.itervalues(self.parameters):
if p.position is not None and isinstance(
p.value_type, yaqltypes.HiddenParameterType):
for index in range(p.position + 1, positional):
positional_fix_table[index] += 1
for key, p in six.iteritems(self.parameters):
arg_name = p.alias or p.name
if p.position is not None and key != '*':
if isinstance(p.value_type, yaqltypes.HiddenParameterType):
positional_args[p.position] = checked(None, p)
positional -= 1
elif p.position - positional_fix_table[p.position] < len(
args) and args[p.position - positional_fix_table[
p.position]] is not utils.NO_VALUE:
if arg_name in kwargs:
raise exceptions.ArgumentException(p.name)
positional_args[p.position] = checked(
args[p.position - positional_fix_table[
p.position]], p)
elif arg_name in kwargs:
positional_args[p.position] = checked(
kwargs.pop(arg_name), p)
elif p.default is not NO_DEFAULT:
positional_args[p.position] = checked(p.default, p)
else:
raise exceptions.ArgumentException(p.name)
elif p.position is None and key != '**':
if isinstance(p.value_type, yaqltypes.HiddenParameterType):
keyword_args[key] = checked(None, p)
elif arg_name in kwargs:
keyword_args[key] = checked(kwargs.pop(arg_name), p)
elif p.default is not NO_DEFAULT:
keyword_args[key] = checked(p.default, p)
else:
raise exceptions.ArgumentException(p.name)
if len(args) > positional:
if '*' in self.parameters:
argdef = self.parameters['*']
positional_args.extend(
map(lambda t: checked(t, argdef), args[positional:]))
else:
raise exceptions.ArgumentException('*')
if len(kwargs) > 0:
if '**' in self.parameters:
argdef = self.parameters['**']
for key, value in six.iteritems(kwargs):
keyword_args[key] = checked(value, argdef)
else:
raise exceptions.ArgumentException('**')
def func(context):
new_context = context.create_child_context()
result = self.payload(
*tuple(map(lambda t: t(new_context),
positional_args)),
**dict(map(lambda t: (t[0], t[1](new_context)),
six.iteritems(keyword_args)))
)
if self.returns_context:
if isinstance(result, types.GeneratorType):
result_context = next(result)
return result, result_context
result_value, result_context = result
return result_value, result_context
else:
return result, new_context
return func
def is_valid_method(self):
min_position = len(self.parameters)
min_arg = None
for p in six.itervalues(self.parameters):
if p.position is not None and p.position < min_position and \
not isinstance(p.value_type,
yaqltypes.HiddenParameterType):
min_position = p.position
min_arg = p
return min_arg and not isinstance(
min_arg.value_type, yaqltypes.LazyParameterType)
def _get_function_definition(func):
if not hasattr(func, '__yaql_function__'):
fd = FunctionDefinition(None, {}, func, func.__doc__)
func.__yaql_function__ = fd
return func.__yaql_function__
def get_function_definition(func, name=None, function=None, method=None,
convention=None):
fd = _get_function_definition(func).clone()
if six.PY2:
spec = inspect.getargspec(func)
for arg in spec.args:
if arg not in fd.parameters:
parameter(arg, function_definition=fd)(func)
if spec.varargs and '*' not in fd.parameters:
parameter(spec.varargs, function_definition=fd)(func)
if spec.keywords and '**' not in fd.parameters:
parameter(spec.keywords, function_definition=fd)(func)
else:
spec = inspect.getfullargspec(func)
for arg in spec.args + spec.kwonlyargs:
if arg not in fd.parameters:
parameter(arg, function_definition=fd)(func)
if spec.varargs and '*' not in fd.parameters:
parameter(spec.varargs, function_definition=fd)(func)
if spec.varkw and '**' not in fd.parameters:
parameter(spec.varkw, function_definition=fd)(func)
if name is not None:
fd.name = name
elif fd.name is None:
if convention is not None:
fd.name = convention.convert_function_name(fd.payload.__name__)
else:
fd.name = fd.payload.__name__
if function is not None:
fd.is_function = function
if method is not None:
fd.is_method = method
if convention:
for p in six.itervalues(fd.parameters):
if p.alias is None:
p.alias = convention.convert_parameter_name(p.name)
return fd
def _parameter(name, value_type=None, nullable=None, alias=None,
function_definition=None):
def wrapper(func):
fd = function_definition or _get_function_definition(func)
if six.PY2:
spec = inspect.getargspec(func)
arg_name = name
if name == spec.keywords:
position = None
arg_name = '**'
elif name == spec.varargs:
position = len(spec.args)
arg_name = '*'
elif name not in spec.args:
raise exceptions.NoParameterFoundException(
function_name=fd.name or func.__name__,
param_name=name)
else:
position = spec.args.index(name)
default = NO_DEFAULT
if spec.defaults is not None and name in spec.args:
index = spec.args.index(name) - len(spec.args)
if index >= -len(spec.defaults):
default = spec.defaults[index]
else:
spec = inspect.getfullargspec(func)
arg_name = name
if name == spec.varkw:
position = None
arg_name = '**'
elif name == spec.varargs:
position = len(spec.args)
arg_name = '*'
elif name in spec.kwonlyargs:
position = None
elif name not in spec.args:
raise exceptions.NoParameterFoundException(
function_name=fd.name or func.__name__,
param_name=name)
else:
position = spec.args.index(name)
default = NO_DEFAULT
if spec.defaults is not None and name in spec.args:
index = spec.args.index(name) - len(spec.args)
if index >= -len(spec.defaults):
default = spec.defaults[index]
elif spec.kwonlydefaults is not None:
default = spec.kwonlydefaults.get(name, NO_DEFAULT)
if arg_name in fd.parameters:
raise exceptions.DuplicateParameterDecoratorException(
function_name=fd.name or func.__name__,
param_name=name)
yaql_type = value_type
p_nullable = nullable
if value_type is None:
if p_nullable is None:
p_nullable = True
if name == 'context':
yaql_type = yaqltypes.Context()
elif name == 'engine':
yaql_type = yaqltypes.Engine()
else:
base_type = object \
if default in (None, NO_DEFAULT, utils.NO_VALUE) \
else type(default)
yaql_type = yaqltypes.PythonType(base_type, p_nullable)
elif not isinstance(value_type, yaqltypes.SmartType):
if p_nullable is None:
p_nullable = default is None
yaql_type = yaqltypes.PythonType(value_type, p_nullable)
fd.parameters[arg_name] = ParameterDefinition(
name, yaql_type, position, alias, default
)
return func
return wrapper
def parameter(name, value_type=None, nullable=None, alias=None,
function_definition=None):
if value_type is not None and isinstance(
value_type, yaqltypes.HiddenParameterType):
raise ValueError('Use inject() for hidden parameters')
return _parameter(name, value_type, nullable=nullable, alias=alias,
function_definition=function_definition)
def inject(name, value_type=None, nullable=None, alias=None,
function_definition=None):
if value_type is not None and not isinstance(
value_type, yaqltypes.HiddenParameterType):
raise ValueError('Use parameter() for normal function parameters')
return _parameter(name, value_type, nullable=nullable, alias=alias,
function_definition=function_definition)
def name(function_name):
def wrapper(func):
fd = _get_function_definition(func)
fd.name = function_name
return func
return wrapper
def method(func):
fd = _get_function_definition(func)
fd.is_method = True
fd.is_function = False
return func
def extension_method(func):
fd = _get_function_definition(func)
fd.is_method = True
fd.is_function = True
return func
def returns_context(func):
fd = _get_function_definition(func)
fd.returns_context = True
return func
def no_kwargs(func):
fd = _get_function_definition(func)
fd.no_kwargs = True
return func

200
yaql/language/utils.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2013 Mirantis, Inc.
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -11,18 +11,194 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from six.moves import xrange
from yaql.language.exceptions import YaqlSequenceException
import collections
import sys
MAX_GENERATOR_ITEMS = 100000
import six
from yaql.language import exceptions
def limit(generator, _limit=MAX_GENERATOR_ITEMS):
res = []
for _ in xrange(_limit):
try:
res.append(next(generator))
except StopIteration:
return res
raise YaqlSequenceException(_limit)
def create_marker(msg):
class MarkerClass(object):
def __repr__(self):
return msg
return MarkerClass()
NO_VALUE = create_marker('<NoValue>')
def is_iterator(obj):
return isinstance(obj, collections.Iterator)
def is_iterable(obj):
return isinstance(obj, collections.Iterable) and not isinstance(
obj, six.string_types + (MappingType,))
def is_sequence(obj):
return isinstance(obj, collections.Sequence) and not isinstance(
obj, six.string_types)
def is_mutable(obj):
return isinstance(obj, (collections.MutableSequence,
collections.MutableSet,
collections.MutableMapping))
SequenceType = collections.Sequence
MutableSequenceType = collections.MutableSequence
SetType = collections.Set
MutableSetType = collections.MutableSet
MappingType = collections.Mapping
MutableMappingType = collections.MutableMapping
IterableType = collections.Iterable
IteratorType = collections.Iterator
def convert_input_data(obj):
if isinstance(obj, six.string_types):
return obj if isinstance(obj, six.text_type) else six.text_type(obj)
elif isinstance(obj, SequenceType):
return tuple(convert_input_data(t) for t in obj)
elif isinstance(obj, MappingType):
return FrozenDict((convert_input_data(key), convert_input_data(value))
for key, value in six.iteritems(obj))
elif isinstance(obj, MutableSetType):
return frozenset(convert_input_data(t) for t in obj)
elif isinstance(obj, IterableType):
return six.moves.map(convert_input_data, obj)
else:
return obj
def convert_output_data(obj, limit_func, engine):
if isinstance(obj, collections.Mapping):
result = {}
for key, value in limit_func(six.iteritems(obj)):
result[convert_output_data(key, limit_func, engine)] = \
convert_output_data(value, limit_func, engine)
return result
elif isinstance(obj, SetType):
set_type = list if convert_sets_to_lists(engine) else set
return set_type(convert_output_data(t, limit_func, engine)
for t in limit_func(obj))
elif isinstance(obj, (tuple, list)):
seq_type = list if convert_tuples_to_lists(engine) else type(obj)
return seq_type(convert_output_data(t, limit_func, engine)
for t in limit_func(obj))
elif is_iterable(obj):
return list(convert_output_data(t, limit_func, engine)
for t in limit_func(obj))
else:
return obj
def convert_sets_to_lists(engine):
return engine.options.get('yaql.convertSetsToLists', False)
def convert_tuples_to_lists(engine):
return engine.options.get('yaql.convertTuplesToLists', True)
class MappingRule(object):
def __init__(self, source, destination):
self.source = source
self.destination = destination
class FrozenDict(collections.Mapping):
def __init__(self, *args, **kwargs):
self._d = dict(*args, **kwargs)
self._hash = None
def __iter__(self):
return iter(self._d)
def __len__(self):
return len(self._d)
def __getitem__(self, key):
return self._d[key]
def get(self, key, default=None):
return self._d.get(key, default)
def __hash__(self):
if self._hash is None:
self._hash = 0
for pair in six.iteritems(self):
self._hash ^= hash(pair)
return self._hash
def __repr__(self):
return repr(self._d)
def memorize(collection, engine):
if not is_iterator(collection):
return collection
yielded = []
class RememberingIterator(six.Iterator):
def __init__(self):
self.seq = iter(collection)
self.index = 0
def __iter__(self):
return RememberingIterator()
def __next__(self):
if self.index < len(yielded):
self.index += 1
return yielded[self.index - 1]
else:
val = next(self.seq)
yielded.append(val)
limit_memory_usage(engine, (1, yielded))
self.index += 1
return val
return RememberingIterator()
def get_max_collection_size(engine):
return engine.options.get('yaql.limitIterators', -1)
def get_memory_quota(engine):
return engine.options.get('yaql.memoryQuota', -1)
def limit_iterable(iterable, engine):
count = get_max_collection_size(engine)
if count >= 0 and isinstance(iterable,
(SequenceType, MappingType, SetType)):
if len(iterable) > count:
raise exceptions.CollectionTooLargeException(count)
return iterable
def limiting_iterator():
for i, t in enumerate(iterable):
if 0 <= count <= i:
raise exceptions.CollectionTooLargeException(count)
yield t
return limiting_iterator()
def limit_memory_usage(engine, *args):
quota = get_memory_quota(engine)
if quota <= 0:
return
total = 0
for t in args:
total += t[0] * sys.getsizeof(t[1], 0)
if total > quota:
raise exceptions.MemoryQuotaExceededException()

382
yaql/language/yaqltypes.py Normal file
View File

@ -0,0 +1,382 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import six
from yaql.language import exceptions
from yaql.language import expressions
from yaql.language import utils
class HiddenParameterType(object):
# noinspection PyMethodMayBeStatic,PyUnusedLocal
def check(self, value):
return True
class LazyParameterType(object):
pass
class SmartType(object):
def __init__(self, nullable):
self.nullable = nullable
def check(self, value):
if value is None and not self.nullable:
return False
return True
def convert(self, value, sender, context, function_spec, engine):
if not self.check(value):
raise exceptions.ArgumentValueException()
utils.limit_memory_usage(engine, (1, value))
def is_specialization_of(self, other):
return False
class PythonType(SmartType):
def __init__(self, python_type, nullable=True, validators=None):
self.python_type = python_type
if not validators:
validators = [lambda _: True]
if not isinstance(validators, (list, tuple)):
validators = [validators]
self.validators = validators
super(PythonType, self).__init__(nullable)
def check(self, value):
if isinstance(value, expressions.Constant):
value = value.value
return super(PythonType, self).check(value) and (
value is None or isinstance(value, expressions.Expression) or (
isinstance(value, self.python_type) and all(
map(lambda t: t(value), self.validators))))
def convert(self, value, sender, context, function_spec, engine):
super(PythonType, self).convert(value, sender, context,
function_spec, engine)
if isinstance(value, expressions.Constant):
value = value.value
super(PythonType, self).convert(value, sender, context,
function_spec, engine)
return value
def is_specialization_of(self, other):
return (
isinstance(other, PythonType)
and issubclass(self.python_type, other.python_type)
and not issubclass(other.python_type, self.python_type)
)
class MappingRule(LazyParameterType, SmartType):
def __init__(self):
super(MappingRule, self).__init__(False)
def check(self, value):
return isinstance(value, expressions.MappingRuleExpression)
def convert(self, value, sender, context, function_spec, engine):
super(MappingRule, self).convert(value, sender, context,
function_spec, engine)
wrap = lambda func: lambda: func(sender, context, engine)[0]
return utils.MappingRule(wrap(value.source), wrap(value.destination))
class String(PythonType):
def __init__(self, nullable=False):
super(String, self).__init__(six.string_types, nullable=nullable)
def convert(self, value, sender, context, function_spec, engine):
value = super(String, self).convert(value, sender, context,
function_spec, engine)
return None if value is None else six.text_type(value)
class Iterable(PythonType):
def __init__(self, validators=None):
super(Iterable, self).__init__(
collections.Iterable, False, [
lambda t: not isinstance(t, six.string_types + (
utils.MappingType,))] + (validators or []))
def convert(self, value, sender, context, function_spec, engine):
res = super(Iterable, self).convert(value, sender, context,
function_spec, engine)
return utils.limit_iterable(res, engine)
class Iterator(Iterable):
def __init__(self, validators=None):
super(Iterator, self).__init__(
validators=[utils.is_iterator] + (validators or []))
class Sequence(PythonType):
def __init__(self, validators=None):
super(Sequence, self).__init__(
collections.Sequence, False, [
lambda t: not isinstance(t, six.string_types + (dict,))] + (
validators or []))
class Number(PythonType):
def __init__(self, nullable=False):
super(Number, self).__init__(
six.integer_types + (float,), nullable, [
lambda t: type(t) is not bool])
class Lambda(LazyParameterType, SmartType):
def __init__(self, with_context=False, method=False, return_context=False):
super(Lambda, self).__init__(True)
self.return_context = return_context
self.with_context = with_context
self.method = method
def check(self, value):
if self.method and isinstance(
value, expressions.Expression) and not value.uses_sender:
return False
return super(Lambda, self).check(value)
@staticmethod
def _publish_params(context, args, kwargs):
for i, param in enumerate(args):
context['$' + str(i + 1)] = param
for arg_name, arg_value in kwargs.items():
context['$' + arg_name] = arg_value
def _call(self, value, sender, context, engine, args, kwargs):
self._publish_params(context, args, kwargs)
if isinstance(value, expressions.Expression):
result = value(sender, context, engine)
elif six.callable(value):
result = value, context
else:
result = value, context
return result[0] if not self.return_context else result
def convert(self, value, sender, context, function_spec, engine):
super(Lambda, self).convert(value, sender, context,
function_spec, engine)
if value is None:
return None
def func(*args, **kwargs):
if self.method and self.with_context:
new_sender, new_context = args[:2]
args = args[2:]
elif self.method and not self.with_context:
new_sender, new_context = \
args[0], context.create_child_context()
args = args[1:]
elif not self.method and self.with_context:
new_sender, new_context = utils.NO_VALUE, args[0]
args = args[1:]
else:
new_sender, new_context = \
utils.NO_VALUE, context.create_child_context()
return self._call(value, new_sender, new_context,
engine, args, kwargs)
return func
class Super(HiddenParameterType, SmartType):
def __init__(self, with_context=False, method=None, return_context=False,
with_name=False):
self.return_context = return_context
self.with_context = with_context
self.method = method
self.with_name = with_name
super(Super, self).__init__(False)
@staticmethod
def _find_function_context(spec, context):
while context is not None:
funcs = context.get_functions(spec.name)
if funcs and spec in funcs:
return context
context = context.parent
raise exceptions.NoFunctionRegisteredException(
spec.name)
def convert(self, value, sender, context, function_spec, engine):
def func(*args, **kwargs):
function_context = self._find_function_context(
function_spec, context)
parent_function_context = function_context.parent
if parent_function_context is None:
raise exceptions.NoFunctionRegisteredException(
function_spec.name)
new_name = function_spec.name
if self.with_name:
new_name = args[0]
args = args[1:]
new_sender = sender
if self.method is True:
new_sender = args[0]
args = args[1:]
elif self.method is False:
new_sender = utils.NO_VALUE
if self.with_context:
new_context = args[0]
args = args[1:]
else:
new_context = context.create_child_context()
return parent_function_context(
new_name, engine, new_sender, new_context,
self.return_context)(*args, **kwargs)
return func
class Context(HiddenParameterType, SmartType):
def __init__(self):
super(Context, self).__init__(False)
def convert(self, value, sender, context, function_spec, engine):
return context
class Delegate(HiddenParameterType, SmartType):
def __init__(self, name=None, with_context=False, method=False,
return_context=False):
super(Delegate, self).__init__(False)
self.name = name
self.return_context = return_context
self.with_context = with_context
self.method = method
def convert(self, value, sender, context, function_spec, engine):
def func(*args, **kwargs):
name = self.name
if not name:
name = args[0]
args = args[1:]
new_sender = utils.NO_VALUE
if self.method:
new_sender = args[0]
args = args[1:]
if self.with_context:
new_context = args[0]
args = args[1:]
else:
new_context = context.create_child_context()
return new_context(
name, engine, new_sender, return_context=self.return_context,
use_convention=True)(*args, **kwargs)
return func
class Sender(HiddenParameterType, SmartType):
def __init__(self):
super(Sender, self).__init__(False)
def convert(self, value, sender, context, function_spec, engine):
return sender
class Engine(HiddenParameterType, SmartType):
def __init__(self):
super(Engine, self).__init__(False)
def convert(self, value, sender, context, function_spec, engine):
return engine
class FunctionDefinition(HiddenParameterType, SmartType):
def __init__(self):
super(FunctionDefinition, self).__init__(False)
def convert(self, value, sender, context, function_spec, engine):
return function_spec
class Constant(SmartType):
def __init__(self, nullable, expand=True):
self.expand = expand
super(Constant, self).__init__(nullable)
def check(self, value):
return super(Constant, self).check(value.value) and (
value is None or isinstance(value, expressions.Constant))
def convert(self, value, sender, context, function_spec, engine):
super(Constant, self).convert(value, sender, context,
function_spec, engine)
return value.value if self.expand else value
class YaqlExpression(LazyParameterType, SmartType):
def __init__(self):
super(YaqlExpression, self).__init__(False)
def check(self, value):
return isinstance(value, expressions.Expression)
def convert(self, value, sender, context, function_spec, engine):
super(YaqlExpression, self).convert(value, sender, context,
function_spec, engine)
return value
class StringConstant(Constant):
def __init__(self, nullable=False):
super(StringConstant, self).__init__(nullable)
def check(self, value):
return super(StringConstant, self).check(value) and (
value is None or isinstance(value.value, six.string_types))
class Keyword(Constant):
def __init__(self, expand=True):
super(Keyword, self).__init__(False, expand)
def check(self, value):
return isinstance(value, expressions.KeywordConstant)
class BooleanConstant(Constant):
def __init__(self, nullable=False, expand=True):
super(BooleanConstant, self).__init__(nullable, expand)
def check(self, value):
return super(BooleanConstant, self).check(value) and (
value is None or type(value.value) is bool)
class NumericConstant(Constant):
def __init__(self, nullable=False, expand=True):
super(NumericConstant, self).__init__(nullable, expand)
def check(self, value):
return super(NumericConstant, self).check(value) and (
value is None or isinstance(
value.value, six.integer_types + (float,)) and
type(value.value) is not bool)

41
yaql/legacy.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import yaql
from yaql.language import factory
from yaql.standard_library import legacy as std_legacy
class YaqlFactory(factory.YaqlFactory):
def __init__(self):
# noinspection PyTypeChecker
super(YaqlFactory, self).__init__(keyword_operator=None)
self.insert_operator(
'or', True, '=>',
factory.OperatorType.BINARY_LEFT_ASSOCIATIVE, True)
def create(self, options=None):
options = dict(options or {})
options['yaql.convertTuplesToLists'] = False
return super(YaqlFactory, self).create(options)
def create_context(*args, **kwargs):
tuples = kwargs.pop('tuples', True)
context = yaql.create_context(*args, **kwargs)
context = context.create_child_context()
std_legacy.register(context, tuples)
return context

0
yaql/standard_library/__init__.py Normal file
View File

52
yaql/standard_library/boolean.py Normal file
View File

@ -0,0 +1,52 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from yaql.language import specs
from yaql.language import yaqltypes
@specs.parameter('left', yaqltypes.Lambda())
@specs.parameter('right', yaqltypes.Lambda())
@specs.name('#operator_and')
def and_(left, right):
return left() and right()
@specs.parameter('left', yaqltypes.Lambda())
@specs.parameter('right', yaqltypes.Lambda())
@specs.name('#operator_or')
def or_(left, right):
return left() or right()
@specs.parameter('arg', bool)
@specs.name('#unary_operator_not')
def not_(arg):
return not arg
def bool_(value):
return bool(value)
def is_boolean(value):
return isinstance(value, bool)
def register(context):
context.register_function(and_)
context.register_function(or_)
context.register_function(not_)
context.register_function(bool_)
context.register_function(is_boolean)

76
yaql/standard_library/branching.py Normal file
View File

@ -0,0 +1,76 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from yaql.language import specs
from yaql.language import yaqltypes
@specs.parameter('args', yaqltypes.MappingRule())
@specs.no_kwargs
def switch(*args):
for mapping in args:
if mapping.source():
return mapping.destination()
@specs.parameter('args', yaqltypes.Lambda())
def select_case(*args):
index = 0
for f in args:
if f():
return index
index += 1
return index
@specs.parameter('args', yaqltypes.Lambda())
def select_all_cases(*args):
for i, f in enumerate(args):
if f():
yield i
@specs.parameter('args', yaqltypes.Lambda())
def examine(*args):
for f in args:
yield bool(f())
@specs.parameter('case', int)
@specs.parameter('args', yaqltypes.Lambda())
@specs.method
def switch_case(case, *args):
if 0 <= case < len(args):
return args[case]()
if len(args) == 0:
return None
return args[-1]()
@specs.parameter('args', yaqltypes.Lambda())
def coalesce(*args):
for f in args:
res = f()
if res is not None:
return res
return None
def register(context):
context.register_function(switch)
context.register_function(select_case)
context.register_function(switch_case)
context.register_function(select_all_cases)
context.register_function(examine)
context.register_function(coalesce)

558
yaql/standard_library/collections.py Normal file
View File

@ -0,0 +1,558 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
from yaql.language import specs
from yaql.language import utils
from yaql.language import yaqltypes
import yaql.standard_library.queries
@specs.parameter('args', nullable=True)
@specs.inject('delegate', yaqltypes.Delegate('to_list', method=True))
def list_(delegate, *args):
def rec(seq):
for t in seq:
if utils.is_iterator(t):
for t2 in rec(t):
yield t2
else:
yield t
return delegate(rec(args))
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
def to_list(collection):
if isinstance(collection, tuple):
return collection
return tuple(collection)
@specs.parameter('args', nullable=True)
@specs.name('#list')
def build_list(engine, *args):
utils.limit_memory_usage(engine, *((1, t) for t in args))
return tuple(args)
@specs.no_kwargs
@specs.parameter('args', utils.MappingRule)
def dict_(engine, *args):
result = {}
for t in args:
result[t.source] = t.destination
utils.limit_memory_usage(engine, (1, result))
return utils.FrozenDict(result)
@specs.parameter('items', yaqltypes.Iterable())
@specs.no_kwargs
def dict__(items, engine):
result = {}
for t in items:
it = iter(t)
key = next(it)
value = next(it)
result[key] = value
utils.limit_memory_usage(engine, (1, result))
return utils.FrozenDict(result)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('key_selector', yaqltypes.Lambda())
@specs.parameter('value_selector', yaqltypes.Lambda())
@specs.method
def to_dict(collection, engine, key_selector, value_selector=None):
result = {}
for t in collection:
key = key_selector(t)
value = t if value_selector is None else value_selector(t)
result[key] = value
utils.limit_memory_usage(engine, (1, result))
return result
@specs.parameter('d', utils.MappingType, alias='dict', nullable=True)
@specs.parameter('key', yaqltypes.Keyword())
@specs.name('#operator_.')
def dict_keyword_access(d, key):
return d[key]
@specs.parameter('collection', yaqltypes.Sequence())
@specs.parameter('attribute', yaqltypes.Keyword(expand=False))
@specs.inject('operator', yaqltypes.Delegate('#operator_.'))
@specs.name('#operator_.')
def collection_attribute(collection, attribute, operator):
return six.moves.map(
lambda t: operator(t, attribute), collection)
@specs.parameter('d', utils.MappingType, alias='dict')
@specs.name('#indexer')
def dict_indexer(d, key):
return d[key]
@specs.parameter('d', utils.MappingType, alias='dict')
@specs.name('#indexer')
def dict_indexer_with_default(d, key, default):
return d.get(key, default)
@specs.parameter('d', utils.MappingType, alias='dict')
@specs.name('get')
@specs.method
def dict_get(d, key, default=None):
return d.get(key, default)
@specs.parameter('d', utils.MappingType, alias='dict')
@specs.name('keys')
@specs.method
def dict_keys(d):
return six.iterkeys(d)
@specs.parameter('d', utils.MappingType, alias='dict')
@specs.name('values')
@specs.method
def dict_values(d):
return six.itervalues(d)
@specs.parameter('d', utils.MappingType, alias='dict')
@specs.name('items')
@specs.method
def dict_items(d):
return six.iteritems(d)
@specs.parameter('lst', yaqltypes.Sequence(), alias='list')
@specs.parameter('index', int, nullable=False)
@specs.name('#indexer')
def list_indexer(lst, index):
return lst[index]
@specs.parameter('value', nullable=True)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.name('#operator_in')
def in_(value, collection):
return value in collection
@specs.parameter('value', nullable=True)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.method
def contains(collection, value):
return value in collection
@specs.parameter('value', nullable=True)
@specs.parameter('d', utils.MappingType, alias='dict')
@specs.method
def contains_key(d, value):
return value in d
@specs.parameter('value', nullable=True)
@specs.parameter('d', utils.MappingType, alias='dict')
@specs.method
def contains_value(d, value):
return value in d.values()
@specs.parameter('left', yaqltypes.Iterable())
@specs.parameter('right', yaqltypes.Iterable())
@specs.name('#operator_+')
def combine_lists(left, right, engine):
if isinstance(left, tuple) and isinstance(right, tuple):
utils.limit_memory_usage(engine, (1, left), (1, right))
return left + right
elif isinstance(left, frozenset) and isinstance(right, frozenset):
utils.limit_memory_usage(engine, (1, left), (1, right))
return left.union(right)
return yaql.standard_library.queries.concat(left, right)
@specs.parameter('left', yaqltypes.Sequence())
@specs.parameter('right', int)
@specs.name('#operator_*')
def list_by_int(left, right, engine):
utils.limit_memory_usage(engine, (-right + 1, []), (right, left))
return left * right
@specs.parameter('left', int)
@specs.parameter('right', yaqltypes.Sequence())
@specs.name('#operator_*')
def int_by_list(left, right, engine):
return list_by_int(right, left, engine)
@specs.parameter('left', utils.MappingType)
@specs.parameter('right', utils.MappingType)
@specs.name('#operator_+')
def combine_dicts(left, right, engine):
utils.limit_memory_usage(engine, (1, left), (1, right))
d = dict(left)
d.update(right)
return utils.FrozenDict(d)
@specs.parameter('left', yaqltypes.Sequence())
@specs.parameter('right', yaqltypes.Sequence())
@specs.name('*equal')
def eq(left, right):
return left == right
@specs.parameter('left', yaqltypes.Sequence())
@specs.parameter('right', yaqltypes.Sequence())
@specs.name('*not_equal')
def neq(left, right):
return left != right
@specs.parameter('left', utils.MappingType)
@specs.parameter('right', utils.MappingType)
@specs.name('*equal')
def eq_dict(left, right):
return left == right
@specs.parameter('left', utils.MappingType)
@specs.parameter('right', utils.MappingType)
@specs.name('*not_equal')
def neq_dict(left, right):
return left != right
def is_list(arg):
return utils.is_sequence(arg)
def is_dict(arg):
return isinstance(arg, utils.MappingType)
def is_set(arg):
return isinstance(arg, utils.SetType)
@specs.parameter('d', utils.MappingType, alias='dict')
@specs.extension_method
@specs.name('len')
def dict_len(d):
return len(d)
@specs.parameter('sequence', yaqltypes.Sequence())
@specs.extension_method
@specs.name('len')
def sequence_len(sequence):
return len(sequence)
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('position', int)
@specs.parameter('count', int)
def delete(collection, position, count=1):
for i, t in enumerate(collection):
if count >= 0 and not position <= i < position + count:
yield t
elif count < 0 and not i >= position:
yield t
@specs.method
@specs.parameter('collection', yaqltypes.Iterable([
lambda t: not is_set(t)
]))
@specs.parameter('position', int)
@specs.parameter('count', int)
def replace(collection, position, value, count=1):
yielded = False
for i, t in enumerate(collection):
if (count >= 0 and position <= i < position + count
or count < 0 and i >= position):
if not yielded:
yielded = True
yield value
else:
yield t
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('position', int)
@specs.parameter('count', int)
@specs.parameter('values', yaqltypes.Iterable())
def replace_many(collection, position, values, count=1):
yielded = False
for i, t in enumerate(collection):
if (count >= 0 and position <= i < position + count
or count < 0 and i >= position):
if not yielded:
for v in values:
yield v
yielded = True
else:
yield t
@specs.method
@specs.name('delete')
@specs.parameter('d', utils.MappingType, alias='dict')
def delete_keys(d, *keys):
return delete_keys_seq(d, keys)
@specs.method
@specs.name('deleteAll')
@specs.parameter('d', utils.MappingType, alias='dict')
@specs.parameter('keys', yaqltypes.Iterable())
def delete_keys_seq(d, keys):
copy = dict(d)
for t in keys:
copy.pop(t, None)
return copy
@specs.method
@specs.parameter('collection', yaqltypes.Iterable(validators=[
lambda x: not isinstance(x, utils.SetType)]
))
@specs.parameter('value', nullable=True)
@specs.parameter('position', int)
@specs.name('insert')
def iter_insert(collection, position, value):
i = -1
for i, t in enumerate(collection):
if i == position:
yield value
yield t
if position > i:
yield value
@specs.method
@specs.parameter('collection', yaqltypes.Sequence())
@specs.parameter('value', nullable=True)
@specs.parameter('position', int)
@specs.name('insert')
def list_insert(collection, position, value):
copy = list(collection)
copy.insert(position, value)
return copy
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('values', yaqltypes.Iterable())
@specs.parameter('position', int)
def insert_many(collection, position, values):
i = -1
if position < 0:
for j in values:
yield j
for i, t in enumerate(collection):
if i == position:
for j in values:
yield j
yield t
if position > i:
for j in values:
yield j
@specs.parameter('s', utils.SetType, alias='set')
@specs.extension_method
@specs.name('len')
def set_len(s):
return len(s)
@specs.parameter('args', nullable=True)
@specs.inject('delegate', yaqltypes.Delegate('to_set', method=True))
def set_(delegate, *args):
def rec(seq):
for t in seq:
if utils.is_iterator(t):
for t2 in rec(t):
yield t2
else:
yield t
return delegate(rec(args))
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
def to_set(collection):
return frozenset(collection)
@specs.parameter('left', utils.SetType)
@specs.parameter('right', utils.SetType)
@specs.method
def union(left, right):
return left.union(right)
@specs.parameter('left', utils.SetType)
@specs.parameter('right', utils.SetType)
@specs.name('*equal')
def set_eq(left, right):
return left == right
@specs.parameter('left', utils.SetType)
@specs.parameter('right', utils.SetType)
@specs.name('*not_equal')
def set_neq(left, right):
return left != right
@specs.parameter('left', utils.SetType)
@specs.parameter('right', utils.SetType)
@specs.name('#operator_<')
def set_lt(left, right):
return left < right
@specs.parameter('left', utils.SetType)
@specs.parameter('right', utils.SetType)
@specs.name('#operator_<=')
def set_lte(left, right):
return left <= right
@specs.parameter('left', utils.SetType)
@specs.parameter('right', utils.SetType)
@specs.name('#operator_>=')
def set_gte(left, right):
return left >= right
@specs.parameter('left', utils.SetType)
@specs.parameter('right', utils.SetType)
@specs.name('#operator_>')
def set_gt(left, right):
return left > right
@specs.parameter('left', utils.SetType)
@specs.parameter('right', utils.SetType)
@specs.method
def intersect(left, right):
return left.intersection(right)
@specs.parameter('left', utils.SetType)
@specs.parameter('right', utils.SetType)
@specs.method
def difference(left, right):
return left.difference(right)
@specs.parameter('left', utils.SetType)
@specs.parameter('right', utils.SetType)
@specs.method
def symmetric_difference(left, right):
return left.symmetric_difference(right)
@specs.parameter('s', utils.SetType, alias='set')
@specs.method
@specs.name('add')
def set_add(s, *values):
return s.union(frozenset(values))
@specs.parameter('s', utils.SetType, alias='set')
@specs.method
@specs.name('remove')
def set_remove(s, *values):
return s.difference(frozenset(values))
def register(context, no_sets=False):
context.register_function(list_)
context.register_function(build_list)
context.register_function(to_list)
context.register_function(list_indexer)
context.register_function(dict_)
context.register_function(dict_, name='#map')
context.register_function(dict__)
context.register_function(to_dict)
context.register_function(dict_keyword_access)
context.register_function(dict_indexer)
context.register_function(dict_indexer_with_default)
context.register_function(dict_get)
context.register_function(dict_keys)
context.register_function(dict_values)
context.register_function(dict_items)
context.register_function(in_)
context.register_function(contains_key)
context.register_function(contains_value)
context.register_function(combine_lists)
context.register_function(collection_attribute)
context.register_function(list_by_int)
context.register_function(int_by_list)
context.register_function(combine_dicts)
context.register_function(eq)
context.register_function(neq)
context.register_function(eq_dict)
context.register_function(neq_dict)
context.register_function(is_dict)
context.register_function(is_list)
context.register_function(dict_len)
context.register_function(sequence_len)
context.register_function(delete)
context.register_function(delete_keys)
context.register_function(delete_keys_seq)
context.register_function(iter_insert)
context.register_function(list_insert)
context.register_function(replace)
context.register_function(replace_many)
context.register_function(insert_many)
context.register_function(contains)
if not no_sets:
context.register_function(is_set)
context.register_function(set_)
context.register_function(to_set)
context.register_function(set_len)
context.register_function(set_eq)
context.register_function(set_neq)
context.register_function(set_lt)
context.register_function(set_lte)
context.register_function(set_gt)
context.register_function(set_gte)
context.register_function(set_add)
context.register_function(set_remove)
context.register_function(union)
context.register_function(intersect)
context.register_function(difference)
context.register_function(
difference, name='#operator_-', function=True, method=False)
context.register_function(symmetric_difference)

164
yaql/standard_library/common.py Normal file
View File

@ -0,0 +1,164 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from yaql.language import specs
@specs.parameter('right', type(None), nullable=True)
@specs.parameter('left', nullable=False)
@specs.name('*equal')
def left_eq_null(left, right):
return False
@specs.parameter('right', type(None), nullable=True)
@specs.parameter('left', nullable=False)
@specs.name('#operator_<')
def left_lt_null(left, right):
return False
@specs.parameter('right', type(None), nullable=True)
@specs.parameter('left', nullable=False)
@specs.name('#operator_<=')
def left_lte_null(left, right):
return False
@specs.parameter('right', type(None), nullable=True)
@specs.parameter('left', nullable=False)
@specs.name('#operator_>')
def left_gt_null(left, right):
return True
@specs.parameter('right', type(None), nullable=True)
@specs.parameter('left', nullable=False)
@specs.name('#operator_>=')
def left_gte_null(left, right):
return True
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', nullable=False)
@specs.name('*equal')
def null_eq_right(left, right):
return False
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', nullable=False)
@specs.name('#operator_<')
def null_lt_right(left, right):
return True
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', nullable=False)
@specs.name('#operator_<=')
def null_lte_right(left, right):
return True
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', nullable=False)
@specs.name('#operator_>')
def null_gt_right(left, right):
return False
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', nullable=False)
@specs.name('#operator_>=')
def null_gte_right(left, right):
return False
@specs.parameter('right', type(None), nullable=True)
@specs.parameter('left', nullable=False)
@specs.name('*not_equal')
def left_neq_null(left, right):
return True
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', nullable=False)
@specs.name('*not_equal')
def null_neq_right(left, right):
return True
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', type(None), nullable=True)
@specs.name('*equal')
def null_eq_null(left, right):
return True
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', type(None), nullable=True)
@specs.name('*not_equal')
def null_neq_null(left, right):
return False
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', type(None), nullable=True)
@specs.name('#operator_<')
def null_lt_null(left, right):
return False
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', type(None), nullable=True)
@specs.name('#operator_<=')
def null_lte_null(left, right):
return True
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', type(None), nullable=True)
@specs.name('#operator_>')
def null_gt_null(left, right):
return False
@specs.parameter('left', type(None), nullable=True)
@specs.parameter('right', type(None), nullable=True)
@specs.name('#operator_>=')
def null_gte_null(left, right):
return True
def register(context):
context.register_function(left_eq_null)
context.register_function(left_neq_null)
context.register_function(left_lt_null)
context.register_function(left_lte_null)
context.register_function(left_gt_null)
context.register_function(left_gte_null)
context.register_function(null_eq_right)
context.register_function(null_neq_right)
context.register_function(null_lt_right)
context.register_function(null_lte_right)
context.register_function(null_gt_right)
context.register_function(null_gte_right)
context.register_function(null_eq_null)
context.register_function(null_neq_null)
context.register_function(null_lt_null)
context.register_function(null_lte_null)
context.register_function(null_gt_null)
context.register_function(null_gte_null)

123
yaql/standard_library/legacy.py Normal file
View File

@ -0,0 +1,123 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import six
from yaql.language import expressions
from yaql.language import specs
from yaql.language import utils
from yaql.language import yaqltypes
@specs.parameter('left', yaqltypes.YaqlExpression())
@specs.name('#operator_=>')
def build_tuple(left, right, context, engine):
if isinstance(left, expressions.BinaryOperator) and left.operator == '=>':
return left(utils.NO_VALUE, context, engine)[0] + (right,)
else:
return left(utils.NO_VALUE, context, engine)[0], right
@specs.parameter('tuples', tuple)
@specs.inject('delegate', yaqltypes.Super(with_name=True))
@specs.no_kwargs
@specs.extension_method
def dict_(delegate, *tuples):
return delegate('dict', tuples)
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
def to_list(collection):
return list(collection)
def tuple_(*args):
return args
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('index_expression', yaqltypes.Lambda())
def indexer(collection, index_expression):
if isinstance(collection, utils.SequenceType):
index = index_expression()
if isinstance(index, int) and not isinstance(index, bool):
return collection[index]
return six.moves.filter(index_expression, collection)
@specs.parameter('start', int)
@specs.parameter('stop', int, nullable=True)
@specs.extension_method
def range_(start, stop=None):
if stop is None:
return itertools.count(start)
else:
return six.moves.range(start, stop)
@specs.parameter('conditions', yaqltypes.Lambda())
def switch(*conditions):
for cond in conditions:
res = cond()
if not isinstance(res, tuple):
raise ValueError('switch() must have tuple parameters')
if len(res) != 2:
raise ValueError('switch() tuples must be of size 2')
if res[0]:
return res[1]
return None
@specs.parameter('mappings', yaqltypes.Lambda())
@specs.method
def as_(context, sender, *mappings):
for t in mappings:
tt = t(sender)
if not isinstance(tt, tuple):
raise ValueError('as() must have tuple parameters')
if len(tt) != 2:
raise ValueError('as() tuples must be of size 2')
context[tt[1]] = tt[0]
return sender
def _to_extension_method(name, context):
for spec in context.parent.get_functions(
name, lambda t: not t.is_function or not t.is_method,
use_convention=True):
spec = spec.clone()
spec.is_function = True
spec.is_method = True
context.register_function(spec)
def register(context, tuples):
if tuples:
context.register_function(build_tuple)
context.register_function(to_list)
context.register_function(tuple_)
context.register_function(dict_)
context.register_function(dict_, name='#map')
context.register_function(indexer, name='#indexer', exclusive=True)
context.register_function(range_)
context.register_function(switch, exclusive=True)
context.register_function(as_)
for t in ('get', 'list', 'bool', 'int', 'float', 'select', 'where',
'join', 'sum', 'take_while'):
_to_extension_method(t, context)

238
yaql/standard_library/math.py Normal file
View File

@ -0,0 +1,238 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import random
import six
from yaql.language import specs
from yaql.language import yaqltypes
@specs.parameter('left', yaqltypes.Number())
@specs.parameter('right', yaqltypes.Number())
@specs.name('#operator_+')
def binary_plus(left, right):
return left + right
@specs.parameter('left', yaqltypes.Number())
@specs.parameter('right', yaqltypes.Number())
@specs.name('#operator_-')
def binary_minus(left, right):
return left - right
@specs.parameter('left', yaqltypes.Number())
@specs.parameter('right', yaqltypes.Number())
@specs.name('#operator_*')
def multiplication(left, right):
return left * right
@specs.parameter('left', yaqltypes.Number())
@specs.parameter('right', yaqltypes.Number())
@specs.name('#operator_/')
def division(left, right):
if isinstance(left, six.integer_types) and isinstance(
right, six.integer_types):
return left // right
return left / right
@specs.parameter('left', yaqltypes.Number())
@specs.parameter('right', yaqltypes.Number())
@specs.name('#operator_mod')
def modulo(left, right):
return left % right
@specs.parameter('op', yaqltypes.Number())
@specs.name('#unary_operator_+')
def unary_plus(op):
return +op
@specs.parameter('op', yaqltypes.Number())
@specs.name('#unary_operator_-')
def unary_minus(op):
return -op
@specs.parameter('left', yaqltypes.Number())
@specs.parameter('right', yaqltypes.Number())
@specs.name('#operator_>')
def gt(left, right):
return left > right
@specs.parameter('left', yaqltypes.Number())
@specs.parameter('right', yaqltypes.Number())
@specs.name('#operator_>=')
def gte(left, right):
return left >= right
@specs.parameter('left', yaqltypes.Number())
@specs.parameter('right', yaqltypes.Number())
@specs.name('#operator_<')
def lt(left, right):
return left < right
@specs.parameter('left', yaqltypes.Number())
@specs.parameter('right', yaqltypes.Number())
@specs.name('#operator_<=')
def lte(left, right):
return left <= right
@specs.parameter('op', yaqltypes.Number())
def abs_(op):
return abs(op)
@specs.parameter('left', yaqltypes.Number())
@specs.parameter('right', yaqltypes.Number())
@specs.name('*equal')
def eq(left, right):
return left == right
@specs.parameter('left', yaqltypes.Number())
@specs.parameter('right', yaqltypes.Number())
@specs.name('*not_equal')
def neq(left, right):
return left != right
def int_(value):
return int(value)
def float_(value):
return float(value)
def random_():
return random.random()
def random__(from_, to_):
return random.randint(from_, to_)
@specs.parameter('left', int)
@specs.parameter('right', int)
def bitwise_and(left, right):
return left & right
@specs.parameter('left', int)
@specs.parameter('right', int)
def bitwise_or(left, right):
return left | right
@specs.parameter('left', int)
@specs.parameter('right', int)
def bitwise_xor(left, right):
return left ^ right
@specs.parameter('arg', int)
def bitwise_not(arg):
return ~arg
@specs.parameter('left', int)
@specs.parameter('right', int)
def shift_bits_right(left, right):
return left >> right
@specs.parameter('left', int)
@specs.parameter('right', int)
def shift_bits_left(left, right):
return left << right
@specs.parameter('a', nullable=True)
@specs.parameter('b', nullable=True)
@specs.inject('operator', yaqltypes.Delegate('#operator_>'))
def max_(a, b, operator):
if operator(b, a):
return b
return a
@specs.inject('operator', yaqltypes.Delegate('#operator_>'))
def min_(a, b, operator):
if operator(b, a):
return a
return b
@specs.parameter('a', yaqltypes.Number())
@specs.parameter('b', yaqltypes.Number())
@specs.parameter('c', yaqltypes.Number(nullable=True))
def pow_(a, b, c=None):
return pow(a, b, c)
@specs.parameter('num', yaqltypes.Number())
def sign(num):
if num > 0:
return 1
elif num < 0:
return -1
return 0
@specs.parameter('number', yaqltypes.Number())
@specs.parameter('ndigits', int)
def round_(number, ndigits=0):
return round(number, ndigits)
def register(context):
context.register_function(binary_plus)
context.register_function(binary_minus)
context.register_function(multiplication)
context.register_function(division)
context.register_function(modulo)
context.register_function(unary_plus)
context.register_function(unary_minus)
context.register_function(abs_)
context.register_function(gt)
context.register_function(gte)
context.register_function(lt)
context.register_function(lte)
context.register_function(eq)
context.register_function(neq)
context.register_function(int_)
context.register_function(float_)
context.register_function(random_)
context.register_function(random__)
context.register_function(bitwise_and)
context.register_function(bitwise_or)
context.register_function(bitwise_not)
context.register_function(bitwise_xor)
context.register_function(shift_bits_left)
context.register_function(shift_bits_right)
context.register_function(max_)
context.register_function(min_)
context.register_function(pow_)
context.register_function(sign)
context.register_function(round_)

671
yaql/standard_library/queries.py Normal file
View File

@ -0,0 +1,671 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import six
from yaql.language import specs
from yaql.language import utils
from yaql.language import yaqltypes
NO_VALUE = utils.create_marker('NoValue')
class OrderingIterable(utils.IterableType):
def __init__(self, collection, operator_lt, operator_gt):
self.collection = collection
self.operator_lt = operator_lt
self.operator_gt = operator_gt
self.order = []
self.sorted = None
def append_field(self, selector, is_ascending):
self.order.append((selector, is_ascending))
def __iter__(self):
if self.sorted is None:
self.do_sort()
return iter(self.sorted)
def do_sort(outer_self):
class Comparator(object):
@staticmethod
def compare(left, right):
result = 0
for t in outer_self.order:
a = t[0](left)
b = t[0](right)
if outer_self.operator_lt(a, b):
result = -1
elif outer_self.operator_gt(a, b):
result = 1
else:
continue
if not t[1]:
result *= -1
break
return result
def __init__(self, obj):
self.obj = obj
def __lt__(self, other):
return self.compare(self.obj, other.obj) < 0
def __gt__(self, other):
return self.compare(self.obj, other.obj) > 0
def __eq__(self, other):
return self.compare(self.obj, other.obj) == 0
def __le__(self, other):
return self.compare(self.obj, other.obj) <= 0
def __ge__(self, other):
return self.compare(self.obj, other.obj) >= 0
def __ne__(self, other):
return self.compare(self.obj, other.obj) != 0
outer_self.sorted = sorted(outer_self.collection, key=Comparator)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('predicate', yaqltypes.Lambda())
@specs.method
def where(collection, predicate):
return six.moves.filter(predicate, collection)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('selector', yaqltypes.Lambda())
@specs.method
def select(collection, selector):
return six.moves.map(selector, collection)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('count', int, nullable=False)
@specs.method
def skip(collection, count):
return itertools.islice(collection, count, None)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('count', int, nullable=False)
@specs.method
def limit(collection, count):
return itertools.islice(collection, count)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.extension_method
def append(collection, *args):
for t in collection:
yield t
for t in args:
yield t
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('key_selector', yaqltypes.Lambda())
@specs.extension_method
def distinct(engine, collection, key_selector=None):
distinct_values = set()
for t in collection:
key = t if key_selector is None else key_selector(t)
if key not in distinct_values:
distinct_values.add(key)
utils.limit_memory_usage(engine, (1, distinct_values))
yield t
@specs.parameter('collection', yaqltypes.Iterable())
@specs.extension_method
def enumerate_(collection, start=0):
for i, t in enumerate(collection, start):
yield [i, t]
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('predicate', yaqltypes.Lambda())
@specs.extension_method
def any_(collection, predicate=None):
for t in collection:
if predicate is None or predicate(t):
return True
return False
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('predicate', yaqltypes.Lambda())
@specs.extension_method
def all_(collection, predicate=None):
if predicate is None:
predicate = lambda x: bool(x)
for t in collection:
if not predicate(t):
return False
return True
@specs.parameter('collections', yaqltypes.Iterable())
@specs.extension_method
def concat(*collections):
for collection in collections:
for item in collection:
yield item
@specs.parameter('collection', utils.IteratorType)
@specs.name('len')
@specs.extension_method
def count_(collection):
count = 0
for t in collection:
count += 1
return count
@specs.parameter('collection', yaqltypes.Iterable())
@specs.method
def count(collection):
return count_(collection)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.method
def memorize(collection, engine):
return utils.memorize(collection, engine)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.inject('operator', yaqltypes.Delegate('#operator_+'))
@specs.method
def sum_(operator, collection, initial=utils.NO_VALUE):
return aggregate(collection, operator, initial)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.inject('func', yaqltypes.Delegate('max'))
@specs.method
def max_(func, collection, initial=utils.NO_VALUE):
return aggregate(collection, func, initial)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.inject('func', yaqltypes.Delegate('min'))
@specs.method
def min_(func, collection, initial=utils.NO_VALUE):
return aggregate(collection, func, initial)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('default', nullable=True)
@specs.method
def first(collection, default=NO_VALUE):
try:
return six.next(iter(collection))
except StopIteration:
if default is NO_VALUE:
raise
return default
@specs.parameter('collection', yaqltypes.Iterable())
@specs.method
def single(collection):
it = iter(collection)
result = six.next(it)
try:
six.next(it)
raise ValueError('Collection contains more than one item')
except StopIteration:
return result
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('default', nullable=True)
@specs.method
def last(collection, default=NO_VALUE):
if isinstance(collection, utils.SequenceType):
if len(collection) == 0:
if default is NO_VALUE:
raise StopIteration()
else:
return default
return collection[-1]
last_value = default
for t in collection:
last_value = t
if last_value is NO_VALUE:
raise StopIteration()
else:
return last_value
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('selector', yaqltypes.Lambda())
@specs.method
def select_many(collection, selector):
for item in collection:
inner = selector(item)
if utils.is_iterable(inner):
for t in inner:
yield t
else:
yield inner
@specs.parameter('stop', int)
def range_(stop):
return iter(six.moves.range(stop))
@specs.parameter('start', int)
@specs.parameter('stop', int)
@specs.parameter('step', int)
def range__(start, stop, step=1):
return iter(six.moves.range(start, stop, step))
@specs.parameter('start', int)
@specs.parameter('step', int)
def sequence(start=0, step=1):
return itertools.count(start, step)
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('selector', yaqltypes.Lambda())
@specs.inject('operator_gt', yaqltypes.Delegate('#operator_>'))
@specs.inject('operator_lt', yaqltypes.Delegate('#operator_<'))
@specs.method
def order_by(collection, selector, operator_lt, operator_gt):
oi = OrderingIterable(collection, operator_lt, operator_gt)
oi.append_field(selector, True)
return oi
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('selector', yaqltypes.Lambda())
@specs.inject('operator_gt', yaqltypes.Delegate('#operator_>'))
@specs.inject('operator_lt', yaqltypes.Delegate('#operator_<'))
@specs.method
def order_by_descending(collection, selector, operator_lt, operator_gt):
oi = OrderingIterable(collection, operator_lt, operator_gt)
oi.append_field(selector, False)
return oi
@specs.parameter('collection', OrderingIterable)
@specs.parameter('selector', yaqltypes.Lambda())
@specs.method
def then_by(collection, selector, context):
collection.append_field(selector, True)
collection.context = context
return collection
@specs.parameter('collection', OrderingIterable)
@specs.parameter('selector', yaqltypes.Lambda())
@specs.method
def then_by_descending(collection, selector, context):
collection.append_field(selector, False)
collection.context = context
return collection
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('key_selector', yaqltypes.Lambda())
@specs.parameter('value_selector', yaqltypes.Lambda())
@specs.parameter('aggregator', yaqltypes.Lambda())
@specs.method
def group_by(engine, collection, key_selector, value_selector=None,
aggregator=None):
groups = {}
if aggregator is None:
aggregator = lambda x: x
for t in collection:
value = t if value_selector is None else value_selector(t)
groups.setdefault(key_selector(t), []).append(value)
utils.limit_memory_usage(engine, (1, groups))
return select(six.iteritems(groups), aggregator)
@specs.method
@specs.parameter('collections', yaqltypes.Iterable())
def zip_(*collections):
return six.moves.zip(*collections)
@specs.method
@specs.parameter('collections', yaqltypes.Iterable())
def zip_longest(*collections, **kwargs):
return six.moves.zip_longest(
*collections, fillvalue=kwargs.pop('default', None))
@specs.method
@specs.parameter('collection1', yaqltypes.Iterable())
@specs.parameter('collection2', yaqltypes.Iterable())
@specs.parameter('predicate', yaqltypes.Lambda())
@specs.parameter('selector', yaqltypes.Lambda())
def join(collection1, collection2, predicate, selector):
for self_item in collection1:
for other_item in collection2:
if predicate(self_item, other_item):
yield selector(self_item, other_item)
@specs.method
@specs.parameter('obj', nullable=True)
@specs.parameter('times', int)
def repeat(obj, times=-1):
if times < 0:
return itertools.repeat(obj)
else:
return itertools.repeat(obj, times)
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
def cycle(collection):
return itertools.cycle(collection)
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('predicate', yaqltypes.Lambda())
def take_while(collection, predicate):
return itertools.takewhile(predicate, collection)
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('predicate', yaqltypes.Lambda())
def skip_while(collection, predicate):
return itertools.dropwhile(predicate, collection)
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.inject('operator', yaqltypes.Delegate('*equal'))
def index_of(collection, item, operator):
for i, t in enumerate(collection):
if operator(t, item):
return i
return -1
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.inject('operator', yaqltypes.Delegate('*equal'))
def last_index_of(collection, item, operator):
index = -1
for i, t in enumerate(collection):
if operator(t, item):
index = i
return index
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('predicate', yaqltypes.Lambda())
def index_where(collection, predicate):
for i, t in enumerate(collection):
if predicate(t):
return i
return -1
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('predicate', yaqltypes.Lambda())
def last_index_where(collection, predicate):
index = -1
for i, t in enumerate(collection):
if predicate(t):
index = i
return index
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('length', int)
@specs.inject('to_list', yaqltypes.Delegate('to_list', method=True))
def slice_(collection, length, to_list):
while True:
res = to_list(itertools.islice(collection, length))
if res:
yield res
else:
break
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('predicate', yaqltypes.Lambda())
@specs.inject('to_list', yaqltypes.Delegate('to_list', method=True))
def split_where(collection, predicate, to_list):
lst = to_list(collection)
start = 0
end = 0
while end < len(lst):
if predicate(lst[end]):
yield lst[start:end]
start = end + 1
end += 1
if start != end:
yield lst[start:end]
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('predicate', yaqltypes.Lambda())
@specs.inject('to_list', yaqltypes.Delegate('to_list', method=True))
def slice_where(collection, predicate, to_list):
lst = to_list(collection)
start = 0
end = 0
p1 = utils.NO_VALUE
while end < len(lst):
p2 = predicate(lst[end])
if p2 != p1 and p1 is not utils.NO_VALUE:
yield lst[start:end]
start = end
end += 1
p1 = p2
if start != end:
yield lst[start:end]
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('index', int)
@specs.inject('to_list', yaqltypes.Delegate('to_list', method=True))
def split_at(collection, index, to_list):
lst = to_list(collection)
return [lst[:index], lst[index:]]
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('selector', yaqltypes.Lambda())
def aggregate(collection, selector, seed=utils.NO_VALUE):
if seed is utils.NO_VALUE:
return six.moves.reduce(selector, collection)
else:
return six.moves.reduce(selector, collection, seed)
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.inject('to_list', yaqltypes.Delegate('to_list', method=True))
def reverse(collection, to_list):
return reversed(to_list(collection))
def _merge_dicts(dict1, dict2, list_merge_func, item_merger, max_levels=0):
result = {}
for key, value1 in six.iteritems(dict1):
result[key] = value1
if key in dict2:
value2 = dict2[key]
if max_levels != 1 and isinstance(value2, utils.MappingType):
if not isinstance(value1, utils.MappingType):
raise TypeError(
'Cannot merge {0} with {1}'.format(
type(value1), type(value2)))
result[key] = _merge_dicts(
value1, value2, list_merge_func,
0 if max_levels == 0 else max_levels - 1)
elif max_levels != 1 and utils.is_sequence(value2):
if not utils.is_sequence(value1):
raise TypeError(
'Cannot merge {0} with {1}'.format(
type(value1), type(value2)))
result[key] = list_merge_func(value1, value2)
else:
result[key] = item_merger(value1, value2)
for key2, value2 in six.iteritems(dict2):
if key2 not in result:
result[key2] = value2
return result
@specs.method
@specs.parameter('d', utils.MappingType, alias='dict')
@specs.parameter('another', utils.MappingType)
@specs.parameter('list_merger', yaqltypes.Lambda())
@specs.parameter('item_merger', yaqltypes.Lambda())
@specs.parameter('max_levels', int)
@specs.inject('to_list', yaqltypes.Delegate('to_list', method=True))
def merge_with(engine, to_list, d, another, list_merger=None,
item_merger=None, max_levels=0):
if list_merger is None:
list_merger = lambda lst1, lst2: to_list(
distinct(engine, lst1 + lst2))
if item_merger is None:
item_merger = lambda x, y: y
return _merge_dicts(d, another, list_merger, item_merger, max_levels)
def is_iterable(value):
return utils.is_iterable(value)
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('selector', yaqltypes.Lambda())
def accumulate(collection, selector, seed=utils.NO_VALUE):
it = iter(collection)
if seed is utils.NO_VALUE:
try:
seed = next(it)
except StopIteration:
raise TypeError(
'accumulate() of empty sequence with no initial value')
yield seed
total = seed
for x in it:
total = selector(total, x)
yield total
@specs.parameter('predicate', yaqltypes.Lambda())
@specs.parameter('next_', yaqltypes.Lambda())
@specs.parameter('selector', yaqltypes.Lambda())
def generate(initial, predicate, next_, selector=None):
while predicate(initial):
if selector is None:
yield initial
else:
yield selector(initial)
initial = next_(initial)
@specs.method
@specs.parameter('collection', yaqltypes.Iterable())
@specs.parameter('default', yaqltypes.Iterable())
def default_if_empty(engine, collection, default):
if isinstance(collection, (utils.SequenceType, utils.SetType)):
return default if len(collection) == 0 else collection
collection = memorize(collection, engine)
it = iter(collection)
try:
next(it)
return collection
except StopIteration:
return default
def register(context):
context.register_function(where)
context.register_function(where, name='filter')
context.register_function(select)
context.register_function(select, name='map')
context.register_function(limit)
context.register_function(limit, name='take')
context.register_function(skip)
context.register_function(append)
context.register_function(distinct)
context.register_function(enumerate_)
context.register_function(any_)
context.register_function(all_)
context.register_function(concat)
context.register_function(count_)
context.register_function(count)
context.register_function(memorize)
context.register_function(sum_)
context.register_function(min_)
context.register_function(max_)
context.register_function(first)
context.register_function(single)
context.register_function(last)
context.register_function(select_many)
context.register_function(range_)
context.register_function(range__)
context.register_function(order_by)
context.register_function(order_by_descending)
context.register_function(then_by)
context.register_function(then_by_descending)
context.register_function(group_by)
context.register_function(join)
context.register_function(zip_)
context.register_function(zip_longest)
context.register_function(repeat)
context.register_function(cycle)
context.register_function(take_while)
context.register_function(skip_while)
context.register_function(index_of)
context.register_function(last_index_of)
context.register_function(index_where)
context.register_function(last_index_where)
context.register_function(slice_)
context.register_function(split_where)
context.register_function(slice_where)
context.register_function(split_at)
context.register_function(aggregate)
context.register_function(aggregate, name='reduce')
context.register_function(accumulate)
context.register_function(reverse)
context.register_function(merge_with)
context.register_function(is_iterable)
context.register_function(sequence)
context.register_function(generate)
context.register_function(default_if_empty)

204
yaql/standard_library/regex.py Normal file
View File

@ -0,0 +1,204 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
import six
from yaql.language import specs
from yaql.language import yaqltypes
REGEX_TYPE = type(re.compile('.'))
@specs.parameter('pattern', yaqltypes.String())
def regex(pattern, ignore_case=False, multi_line=False, dot_all=False):
flags = re.UNICODE
if ignore_case:
flags |= re.IGNORECASE
if multi_line:
flags |= re.MULTILINE
if dot_all:
flags |= re.DOTALL
return re.compile(pattern, flags)
@specs.parameter('regexp', REGEX_TYPE)
@specs.parameter('string', yaqltypes.String())
@specs.method
def matches(regexp, string):
return regexp.search(string) is not None
@specs.parameter('regexp', REGEX_TYPE)
@specs.parameter('string', yaqltypes.String())
@specs.name('#operator_=~')
def matches_operator_regex(string, regexp):
return regexp.search(string) is not None
@specs.parameter('pattern', yaqltypes.String())
@specs.parameter('string', yaqltypes.String())
@specs.name('#operator_=~')
def matches_operator_string(string, pattern):
return re.search(pattern, string) is not None
@specs.parameter('regexp', REGEX_TYPE)
@specs.parameter('string', yaqltypes.String())
@specs.name('#operator_!~')
def not_matches_operator_regex(string, regexp):
return regexp.search(string) is None
@specs.parameter('pattern', yaqltypes.String())
@specs.parameter('string', yaqltypes.String())
@specs.name('#operator_!~')
def not_matches_operator_string(string, pattern):
return re.search(pattern, string) is None
def _publish_match(context, match):
rec = {
'value': match.group(),
'start': match.start(0),
'end': match.end(0)
}
context['$1'] = rec
for i, t in enumerate(match.groups(), 1):
rec = {
'value': t,
'start': match.start(i),
'end': match.end(i)
}
context['$' + str(i + 1)] = rec
for key, value, in six.itervalues(match.groupdict()):
rec = {
'value': value,
'start': match.start(value),
'end': match.end(value)
}
context['$' + key] = rec
@specs.parameter('regexp', REGEX_TYPE)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('selector', yaqltypes.Lambda(with_context=True))
@specs.method
def search(context, regexp, string, selector=None):
res = regexp.search(string)
if res is None:
return None
if selector is None:
return res.group()
_publish_match(context, res)
return selector(context)
@specs.parameter('regexp', REGEX_TYPE)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('selector', yaqltypes.Lambda(with_context=True))
@specs.method
def search_all(context, regexp, string, selector=None):
for res in regexp.finditer(string):
new_context = context.create_child_context()
if selector is None:
yield res.group()
else:
_publish_match(new_context, res)
yield selector(new_context)
@specs.parameter('regexp', REGEX_TYPE)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('max_split', int)
@specs.method
def split(regexp, string, max_split=0):
return regexp.split(string, max_split)
@specs.parameter('regexp', REGEX_TYPE)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('max_split', int)
@specs.method
@specs.name('split')
def split_string(string, regexp, max_split=0):
return regexp.split(string, max_split)
@specs.parameter('regexp', REGEX_TYPE)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('repl', yaqltypes.String())
@specs.parameter('count', int)
@specs.method
def replace(regexp, string, repl, count=0):
return regexp.sub(repl, string, count)
@specs.parameter('regexp', REGEX_TYPE)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('repl', yaqltypes.String())
@specs.parameter('count', int)
@specs.method
@specs.name('replace')
def replace_string(string, regexp, repl, count=0):
return replace(regexp, string, repl, count)
@specs.parameter('regexp', REGEX_TYPE)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('repl', yaqltypes.Lambda(with_context=True))
@specs.parameter('count', int)
@specs.method
def replace_by(context, regexp, string, repl, count=0):
def repl_func(match):
new_context = context.create_child_context()
_publish_match(context, match)
return repl(new_context)
return regexp.sub(repl_func, string, count)
@specs.parameter('regexp', REGEX_TYPE)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('repl', yaqltypes.Lambda(with_context=True))
@specs.parameter('count', int)
@specs.method
@specs.name('replaceBy')
def replace_by_string(context, string, regexp, repl, count=0):
return replace_by(context, regexp, string, repl, count)
@specs.parameter('string', yaqltypes.String())
def escape_regex(string):
return re.escape(string)
def register(context):
context.register_function(regex)
context.register_function(matches)
context.register_function(matches_operator_string)
context.register_function(matches_operator_regex)
context.register_function(not_matches_operator_string)
context.register_function(not_matches_operator_regex)
context.register_function(search)
context.register_function(search_all)
context.register_function(split)
context.register_function(split_string)
context.register_function(replace)
context.register_function(replace_by)
context.register_function(replace_string)
context.register_function(replace_by_string)
context.register_function(escape_regex)

369
yaql/standard_library/strings.py Normal file
View File

@ -0,0 +1,369 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import string as string_module
import six
from yaql.language import specs
from yaql.language import utils
from yaql.language import yaqltypes
@specs.parameter('left', yaqltypes.String())
@specs.parameter('right', yaqltypes.String())
@specs.name('#operator_>')
def gt(left, right):
return left > right
@specs.parameter('left', yaqltypes.String())
@specs.parameter('right', yaqltypes.String())
@specs.name('#operator_<')
def lt(left, right):
return left < right
@specs.parameter('left', yaqltypes.String())
@specs.parameter('right', yaqltypes.String())
@specs.name('#operator_>=')
def gte(left, right):
return left > right
@specs.parameter('left', yaqltypes.String())
@specs.parameter('right', yaqltypes.String())
@specs.name('#operator_<=')
def lte(left, right):
return left < right
@specs.parameter('left', yaqltypes.String())
@specs.parameter('right', yaqltypes.String())
@specs.name('*equal')
def eq(left, right):
return left == right
@specs.parameter('left', yaqltypes.String())
@specs.parameter('right', yaqltypes.String())
@specs.name('*not_equal')
def neq(left, right):
return left != right
@specs.parameter('args', yaqltypes.String())
def concat(*args):
return ''.join(args)
@specs.parameter('string', yaqltypes.String())
@specs.method
def to_upper(string):
return string.upper()
@specs.parameter('string', yaqltypes.String())
def len_(string):
return len(string)
@specs.parameter('string', yaqltypes.String())
@specs.method
def to_lower(string):
return string.lower()
@specs.parameter('string', yaqltypes.String())
@specs.parameter('separator', yaqltypes.String(nullable=True))
@specs.parameter('max_splits', int)
@specs.method
def split(string, separator=None, max_splits=-1):
return string.split(separator, max_splits)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('separator', yaqltypes.String(nullable=True))
@specs.parameter('max_splits', int)
@specs.method
def right_split(string, separator=None, max_splits=-1):
return string.rsplit(separator, max_splits)
@specs.parameter('sequence', yaqltypes.Iterable())
@specs.parameter('separator', yaqltypes.String())
@specs.method
def join(sequence, separator):
return separator.join(sequence)
@specs.parameter('value', nullable=True)
def str_(value):
if value is None:
return 'null'
elif value is True:
return 'true'
elif value is False:
return 'false'
else:
return six.text_type(value)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('chars', yaqltypes.String(nullable=True))
@specs.method
def trim(string, chars=None):
return string.strip(chars)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('chars', yaqltypes.String(nullable=True))
@specs.method
def trim_left(string, chars=None):
return string.lstrip(chars)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('chars', yaqltypes.String(nullable=True))
@specs.method
def trim_right(string, chars=None):
return string.rstrip(chars)
@specs.parameter('string', yaqltypes.String(nullable=True))
@specs.parameter('chars', yaqltypes.String(nullable=True))
@specs.extension_method
def norm(string, chars=None):
if string is None:
return None
value = string.strip(chars)
return None if not value else value
@specs.parameter('string', yaqltypes.String(nullable=True))
@specs.parameter('trim_spaces', bool, alias='trim')
@specs.parameter('chars', yaqltypes.String(nullable=True))
@specs.extension_method
def is_empty(string, trim_spaces=True, chars=None):
if string is None:
return True
if trim_spaces:
string = string.strip(chars)
return not string
@specs.parameter('string', yaqltypes.String())
@specs.parameter('old', yaqltypes.String())
@specs.parameter('new', yaqltypes.String())
@specs.parameter('count', int)
@specs.method
def replace(string, old, new, count=-1):
return string.replace(old, new, count)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('replacements', utils.MappingType)
@specs.parameter('count', int)
@specs.inject('str_func', yaqltypes.Delegate('str'))
@specs.method
@specs.name('replace')
def replace_with_dict(string, str_func, replacements, count=-1):
for key, value in six.iteritems(replacements):
string = string.replace(str_func(key), str_func(value), count)
return string
@specs.parameter('__format_string__', yaqltypes.String())
@specs.extension_method
def format_(__format_string__, *args, **kwargs):
return __format_string__.format(*args, **kwargs)
@specs.parameter('left', yaqltypes.String())
@specs.parameter('right', int)
@specs.name('#operator_*')
def string_by_int(left, right, engine):
utils.limit_memory_usage(engine, (-right + 1, u''), (right, left))
return left * right
@specs.parameter('left', yaqltypes.String())
@specs.parameter('right', yaqltypes.String())
@specs.name('#operator_in')
def in_(left, right):
return left in right
@specs.parameter('left', int)
@specs.parameter('right', yaqltypes.String())
@specs.name('#operator_*')
def int_by_string(left, right, engine):
return string_by_int(right, left, engine)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('start', int)
@specs.parameter('length', int)
@specs.method
def substring(string, start, length=-1):
if length < 0:
length = len(string)
if start < 0:
start += len(string)
return string[start:start + length]
@specs.parameter('string', yaqltypes.String())
@specs.parameter('sub', yaqltypes.String())
@specs.parameter('start', int)
@specs.method
def index_of(string, sub, start=0):
return string.find(sub, start)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('sub', yaqltypes.String())
@specs.parameter('start', int)
@specs.parameter('length', int)
@specs.method
def index_of_(string, sub, start, length):
if length < 0:
length = len(string)
if start < 0:
start += len(string)
return string.find(sub, start, length)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('sub', yaqltypes.String())
@specs.parameter('start', int)
@specs.method
def last_index_of(string, sub, start=0):
return string.rfind(sub, start)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('sub', yaqltypes.String())
@specs.parameter('start', int)
@specs.parameter('length', int)
@specs.method
def last_index_of_(string, sub, start, length):
if length < 0:
length = len(string)
if start < 0:
start += len(string)
return string.rfind(sub, start, length)
@specs.parameter('string', yaqltypes.String())
@specs.method
def to_char_array(string):
return tuple(string)
def characters(
digits=False, hexdigits=False,
ascii_lowercase=False, ascii_uppercase=False,
ascii_letters=False, letters=False,
octdigits=False, punctuation=False, printable=False,
lowercase=False, uppercase=False, whitespace=False):
string = ''
if digits:
string += string_module.digits
if hexdigits:
string += string_module.hexdigits
if ascii_lowercase:
string += string_module.ascii_lowercase
if ascii_uppercase:
string += string_module.ascii_uppercase
if ascii_letters:
string += string_module.ascii_letters
if letters:
string += string_module.letters
if octdigits:
string += string_module.octdigits
if punctuation:
string += string_module.punctuation
if printable:
string += string_module.printable
if lowercase:
string += string_module.lowercase
if uppercase:
string += string_module.uppercase
if whitespace:
string += string_module.whitespace
return tuple(set(string))
def is_string(arg):
return isinstance(arg, six.string_types)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('prefixes', yaqltypes.String())
@specs.method
def starts_with(string, *prefixes):
return string.startswith(prefixes)
@specs.parameter('string', yaqltypes.String())
@specs.parameter('suffixes', yaqltypes.String())
@specs.method
def ends_with(string, *suffixes):
return string.endswith(suffixes)
@specs.parameter('num', yaqltypes.Number(nullable=True))
def hex_(num):
return hex(num)
def register(context):
context.register_function(gt)
context.register_function(lt)
context.register_function(gte)
context.register_function(lte)
context.register_function(eq)
context.register_function(neq)
context.register_function(len_)
context.register_function(to_lower)
context.register_function(to_upper)
context.register_function(split)
context.register_function(right_split)
context.register_function(join)
context.register_function(str_)
context.register_function(concat)
context.register_function(concat, name='#operator_+')
context.register_function(trim)
context.register_function(trim_left)
context.register_function(trim_right)
context.register_function(replace)
context.register_function(replace_with_dict)
context.register_function(format_)
context.register_function(is_empty)
context.register_function(string_by_int)
context.register_function(int_by_string)
context.register_function(substring)
context.register_function(index_of)
context.register_function(index_of_)
context.register_function(last_index_of)
context.register_function(last_index_of_)
context.register_function(to_char_array)
context.register_function(characters)
context.register_function(is_string)
context.register_function(norm)
context.register_function(in_)
context.register_function(starts_with)
context.register_function(ends_with)
context.register_function(hex_)

121
yaql/standard_library/system.py Normal file
View File

@ -0,0 +1,121 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import six
from yaql.language import specs
from yaql.language import utils
from yaql.language import yaqltypes
@specs.parameter('name', yaqltypes.StringConstant())
@specs.name('#get_context_data')
def get_context_data(name, context):
return context[name]
@specs.parameter('sender', yaqltypes.Lambda(with_context=True,
return_context=True))
@specs.parameter('expr', yaqltypes.Lambda(with_context=True, method=True,
return_context=True))
@specs.name('#operator_.')
@specs.returns_context
def op_dot(context, sender, expr):
return expr(*sender(context))
@specs.parameter('sender',
yaqltypes.Lambda(with_context=True, return_context=True))
@specs.parameter('expr', yaqltypes.YaqlExpression())
@specs.inject('operator', yaqltypes.Delegate('#operator_.', with_context=True))
@specs.name('#operator_?.')
def elvis_operator(context, operator, sender, expr):
sender, context = sender(context)
if sender is None:
return None
return operator(context, sender, expr)
@specs.parameter('sequence', yaqltypes.Iterable())
@specs.parameter('args', yaqltypes.String())
@specs.method
def unpack(sequence, context, *args):
lst = tuple(itertools.islice(sequence, len(args) + 1))
if 0 < len(args) != len(lst):
raise ValueError('Cannot unpack {0} elements into {1}'.format(
len(lst), len(args)))
if len(args) > 0:
for i in range(len(lst)):
context[args[i]] = lst[i]
else:
for i, t in enumerate(sequence, 1):
context[str(i)] = t
return lst
def with_(context, *args):
for i, t in enumerate(args, 1):
context[str(i)] = t
@specs.inject('__context__', yaqltypes.Context())
def let(__context__, *args, **kwargs):
for i, value in enumerate(args, 1):
__context__[str(i)] = value
for key, value in six.iteritems(kwargs):
__context__[key] = value
@specs.parameter('name', yaqltypes.String())
@specs.parameter('func', yaqltypes.Lambda())
def def_(name, func, context):
@specs.name(name)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
context.register_function(wrapper)
@specs.parameter('left', yaqltypes.Lambda(return_context=True))
@specs.parameter('right', yaqltypes.Lambda(with_context=True))
@specs.name('#operator_->')
def send_context(left, right):
context = left()[1]
return right(context)
@specs.method
@specs.parameter('condition', yaqltypes.Lambda())
@specs.parameter('message', yaqltypes.String())
def assert_(engine, obj, condition, message=u'Assertion failed'):
if utils.is_iterator(obj):
obj = utils.memorize(obj, engine)
if not condition(obj):
raise AssertionError(message)
return obj
def register(context):
context.register_function(get_context_data)
context.register_function(op_dot)
context.register_function(unpack)
context.register_function(with_)
context.register_function(send_context)
context.register_function(let)
context.register_function(def_)
context.register_function(elvis_operator)
context.register_function(assert_)

89
yaql/tests/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2013 Mirantis, Inc.
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -11,24 +11,85 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import types
import unittest
import testtools
import yaql
from yaql.language.utils import limit
from yaql.language import factory
from yaql import legacy
class YaqlTest(unittest.TestCase):
class TestCase(testtools.TestCase):
_default_engine = None
_default_legacy_engine = None
engine_options = {
'yaql.limitIterators': 100,
'yaql.memoryQuota': 20000,
'yaql.convertTuplesToLists': True,
'yaql.convertSetsToLists': True
}
legacy_engine_options = {
'yaql.limitIterators': 100,
'yaql.memoryQuota': 20000,
}
def create_engine(self):
func = TestCase._default_engine
if func is None:
engine_factory = factory.YaqlFactory()
TestCase._default_engine = func = engine_factory.create(
options=self.engine_options)
return func
def create_legacy_engine(self):
func = TestCase._default_legacy_engine
if func is None:
engine_factory = legacy.YaqlFactory()
TestCase._default_legacy_engine = func = engine_factory.create(
options=self.legacy_engine_options)
return func
@property
def context(self):
if self._context is None:
self._context = yaql.create_context()
return self._context
@property
def legacy_context(self):
if self._legacy_context is None:
self._legacy_context = legacy.create_context()
return self._legacy_context
@context.setter
def context(self, value):
self._context = value
@property
def engine(self):
if self._engine is None:
self._engine = self.create_engine()
return self._engine
@property
def legacy_engine(self):
if self._legacy_engine is None:
self._legacy_engine = self.create_legacy_engine()
return self._legacy_engine
def setUp(self):
self.context = yaql.create_context()
self._context = None
self._engine = None
self._legacy_context = None
self._legacy_engine = None
super(TestCase, self).setUp()
def eval(self, expression, data=None, context=None):
res = yaql.parse(expression).evaluate(data=data,
context=context or self.context)
if isinstance(res, types.GeneratorType):
return limit(res)
else:
return res
expr = self.engine(expression)
return expr.evaluate(data=data, context=context or self.context)
def assertEval(self, value, expression, data=None, context=None):
self.assertEquals(value, self.eval(expression, data, context))
def legacy_eval(self, expression, data=None, context=None):
expr = self.legacy_engine(expression)
return expr.evaluate(data=data, context=context or self.legacy_context)

49
yaql/tests/test_arithmetic.py
View File

@ -1,49 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from yaql.tests import YaqlTest
class TestArithmetic(YaqlTest):
def test_int_arithmetic(self):
self.assertEquals(20, self.eval('15+5'))
self.assertEquals(20, self.eval('15+10-5'))
self.assertEquals(20, self.eval('15+10-5*2+10/2'))
self.assertEquals(2, self.eval('5/2'))
def test_float_arithmetic(self):
self.assertEquals(10.0, self.eval('5.0 * 2'))
self.assertEquals(10.0, self.eval('5 * 2.0'))
self.assertEquals(2.5, self.eval('5/2.0'))
self.assertEquals(2.5, self.eval('5.0/2'))
def test_mix_binary_unary(self):
self.assertEquals(15, self.eval('20 + -5'))
self.assertEquals(-25, self.eval('-20 + -5'))
self.assertEquals(-25, self.eval('-20 - +5'))
def test_int_conversion(self):
self.assertNotEquals(int, type(self.eval('123.45')))
self.assertEquals(int, type(self.eval('int(123.45)')))
def test_float_conversion(self):
self.assertNotEquals(float, type(self.eval('123')))
self.assertEquals(float, type(self.eval('float(123)')))
def test_random(self):
self.assertTrue(0 < self.eval('random()') < 1)
if __name__ == '__main__':
unittest.main()

55
yaql/tests/test_boolean.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2014 Mirantis, Inc.
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -12,47 +12,28 @@
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from yaql.tests import YaqlTest
import yaql.tests
class TestBooleans(YaqlTest):
class TestBoolean(yaql.tests.TestCase):
def test_and(self):
self.assertEval(True, 'true and true')
self.assertEval(False, 'true and false')
self.assertEval(False, 'false and true')
self.assertEval(False, 'false and false')
self.assertTrue(self.eval('true and true'))
self.assertFalse(self.eval('true and false'))
self.assertFalse(self.eval('false and false'))
self.assertFalse(self.eval('false and true'))
self.assertEqual(12, self.eval('true and 12'))
def test_or(self):
self.assertEval(True, 'true or true')
self.assertEval(True, 'true or false')
self.assertEval(True, 'false or true')
self.assertEval(False, 'false or false')
self.assertTrue(self.eval('true or true'))
self.assertTrue(self.eval('true or false'))
self.assertFalse(self.eval('false or false'))
self.assertTrue(self.eval('false or true'))
self.assertEqual(12, self.eval('12 or true'))
def test_not(self):
self.assertEval(True, 'not false')
self.assertEval(False, 'not true')
self.assertEval(True, 'not (not true)')
self.assertFalse(self.eval('not true'))
self.assertTrue(self.eval('not false'))
def test_excl(self):
self.assertEval(True, '!false')
self.assertEval(False, '!true')
self.assertEval(True, '!(!true)')
def test_bool_precedence(self):
self.assertEval(True, 'true and not false')
self.assertEval(True, 'not true or not false')
@unittest.skip(
"Custom precedence for 'or' and 'and' operators is not defined")
def test_incorrect_boolean_precedence(self):
self.assertEval(True, "true or (true and false)") # works
self.assertEval(True, "true or true and false") # breaks
def test_boolean_conversion(self):
self.assertNotEquals(bool, type(self.eval('abcd')))
self.assertEquals(bool, type(self.eval('bool(abcd)')))
if __name__ == '__main__':
unittest.main()
def test_lazy(self):
self.assertEqual(1, self.eval('$ or 10/($-1)', data=1))
self.assertEqual(0, self.eval('$ and 10/$', data=0))

55
yaql/tests/test_branching.py Normal file
View File

@ -0,0 +1,55 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import yaql.tests
class TestBranching(yaql.tests.TestCase):
def test_switch(self):
expr = 'switch($ < 10 => 1, $ >= 10 and $ < 100 => 2, $ >= 100 => 3)'
self.assertEqual(3, self.eval(expr, data=123))
self.assertEqual(2, self.eval(expr, data=50))
self.assertEqual(1, self.eval(expr, data=-123))
def test_select_case(self):
expr = 'selectCase($ < 10, $ >= 10 and $ < 100)'
self.assertEqual(2, self.eval(expr, data=123))
self.assertEqual(1, self.eval(expr, data=50))
self.assertEqual(0, self.eval(expr, data=-123))
def test_select_all_cases(self):
expr = 'selectAllCases($ < 10, $ > 5)'
self.assertEqual([0], self.eval(expr, data=1))
self.assertEqual([0, 1], self.eval(expr, data=7))
self.assertEqual([1], self.eval(expr, data=12))
def test_examine(self):
expr = 'examine($ < 10, $ > 5)'
self.assertEqual([True, False], self.eval(expr, data=1))
self.assertEqual([True, True], self.eval(expr, data=7))
self.assertEqual([False, True], self.eval(expr, data=12))
def test_switch_case(self):
expr = "$.switchCase('a', 'b', 'c')"
self.assertEqual('a', self.eval(expr, data=0))
self.assertEqual('b', self.eval(expr, data=1))
self.assertEqual('c', self.eval(expr, data=3))
self.assertEqual('c', self.eval(expr, data=30))
self.assertEqual('c', self.eval(expr, data=-30))
def test_coalesce(self):
self.assertEqual(2, self.eval('coalesce($, 2)', data=None))
self.assertEqual(1, self.eval('coalesce($, 2)', data=1))
self.assertEqual(2, self.eval('coalesce($, $, 2)', data=None))
self.assertIsNone(self.eval('coalesce($)', data=None))

488
yaql/tests/test_collections.py Normal file
View File

@ -0,0 +1,488 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from yaql.language import exceptions
import yaql.tests
class TestCollections(yaql.tests.TestCase):
def test_list(self):
self.assertEqual([], self.eval('list()'))
self.assertEqual([1, 2, 3], self.eval('list(1, 2, 3)'))
self.assertEqual([1, 2, [3, 4]], self.eval('list(1, 2, list(3, 4))'))
def test_list_expr(self):
self.assertEqual([1, 2, 3], self.eval('[1,2,3]'))
self.assertEqual([2, 4], self.eval('[1,[2]][1] + [3, [4]][1]'))
self.assertEqual([1, 2, 3, 4], self.eval('[1,2] + [3, 4]'))
self.assertEqual(2, self.eval('([1,2] + [3, 4])[1]'))
self.assertEqual([], self.eval('[]'))
def test_list_from_iterator(self):
iterator = (i for i in range(3))
self.assertEqual([0, 1, 2], self.eval('list($)', data=iterator))
def test_to_list(self):
data = (i for i in range(3))
self.assertEqual([0, 1, 2], self.eval('$.toList()', data=data))
data = [0, 1, 2]
self.assertEqual([0, 1, 2], self.eval('$.toList()', data=data))
data = (0, 1, 2)
self.assertEqual([0, 1, 2], self.eval('$.toList()', data=data))
data = {'a': 1, 'b': 2}
self.assertTrue(self.eval('isList($.keys().toList())', data=data))
def test_list_concatenates_and_flatten_generators(self):
generator_func = lambda: (i for _ in range(2) for i in range(3))
self.context['$seq1'] = generator_func()
self.context['$seq2'] = generator_func()
self.assertEqual([0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2],
self.eval('list($seq1, $seq2)'))
def test_indexer_list_access(self):
data = [1, 2, 3]
self.assertEqual(1, self.eval('$[0]', data=data))
self.assertEqual(3, self.eval('$[-1]', data=data))
self.assertEqual(2, self.eval('$[-1-1]', data=data))
self.assertRaises(IndexError,
self.eval, "$[3]", data=data)
self.assertRaises(IndexError,
self.eval, "$[-4]", data=data)
self.assertRaises(exceptions.NoMatchingFunctionException,
self.eval, "$[a]", data=data)
def test_dict(self):
self.assertEqual(
{'b c': 13, 'a': 2, 4: 5, None: None, True: False, 8: 8},
self.eval("dict(a => 2, 'b c' => 13, 4 => 5, "
"null => null, true => false, 2+6=>8)"))
def test_dict_expr(self):
self.assertEqual(
{'b c': 13, 'a': 2, 4: 5, None: None, True: False, 8: 8},
self.eval("{a => 2, 'b c' => 13, 4 => 5, "
"null => null, true => false, 2+6=>8}"))
self.assertEqual({'b': 2, 'a': 1}, self.eval('{a => 1} + {b=>2}'))
self.assertEqual({}, self.eval('{}'))
def test_dict_from_sequence(self):
self.assertEqual({'a': 1, 'b': 2},
self.eval("dict(list(list(a, 1), list('b', 2)))"))
generator = (i for i in (('a', 1), ['b', 2]))
self.assertEqual({'a': 1, 'b': 2},
self.eval('dict($)', data=generator))
def test_to_dict(self):
self.assertEqual({1: 1, 2: 4, 3: 9},
self.eval('$.toDict($, $*$)', data=[1, 2, 3]))
def test_keyword_dict_access(self):
data = {'a': 12, 'b c': 44}
self.assertEqual(12, self.eval('$.a', data=data))
self.assertRaises(exceptions.NoMatchingFunctionException,
self.eval, "$.'b c'", data=data)
self.assertRaises(exceptions.NoMatchingFunctionException,
self.eval, "$.123", data=data)
self.assertRaises(KeyError,
self.eval, "$.b", data=data)
def test_keyword_collection_access(self):
data = [{'a': 2}, {'a': 4}]
self.assertEqual([2, 4], self.eval('$.a', data=data))
def test_indexer_dict_access(self):
data = {'a': 12, 'b c': 44}
self.assertEqual(12, self.eval('$[a]', data=data))
self.assertEqual(44, self.eval("$['b c']", data=data))
self.assertRaises(KeyError,
self.eval, "$[b]", data=data)
def test_indexer_dict_access_with(self):
data = {'a': 12, 'b c': 44}
self.assertEqual(55, self.eval('$[c, 55]', data=data))
self.assertEqual(66, self.eval('$[c, default => 66]', data=data))
def test_list_eq(self):
self.assertTrue(self.eval('[c, 55]=[c, 55]'))
self.assertFalse(self.eval('[c, 55]=[55, c]'))
self.assertFalse(self.eval('[c, 55]=null'))
self.assertFalse(self.eval('null = [c, 55]'))
def test_list_neq(self):
self.assertFalse(self.eval('[c, 55] != [c, 55]'))
self.assertTrue(self.eval('[c, 55] != [55, c]'))
self.assertTrue(self.eval('[c, 55] != null'))
self.assertTrue(self.eval('null != [c, 55]'))
def test_dict_eq(self):
self.assertTrue(self.eval('{a => [c, 55]} = {a => [c, 55]}'))
self.assertTrue(self.eval('{[c, 55] => a} = {[c, 55] => a}'))
self.assertFalse(self.eval('{[c, 55] => a} = {[c, 56] => a}'))
self.assertFalse(self.eval('{[c, 55] => a, b => 1} = {[c, 55] => a}'))
self.assertFalse(self.eval('{[c, 55] => a} = null'))
def test_dict_neq(self):
self.assertFalse(self.eval('{a => [c, 55]} != {a => [c, 55]}'))
self.assertFalse(self.eval('{[c, 55] => a} != {[c, 55] => a}'))
self.assertTrue(self.eval('{[c, 55] => a} != {[c, 56] => a}'))
self.assertTrue(self.eval('{[c, 55] => a, b => 1} != {[c, 55] => a}'))
self.assertTrue(self.eval('{[c, 55] => a} != null'))
def test_dict_get(self):
data = {'a': 12, 'b c': 44}
self.assertEqual(12, self.eval('$.get(a)', data=data))
self.assertIsNone(self.eval('$.get(b)', data=data))
self.assertEqual(50, self.eval('$.get(c, 50)', data=data))
def test_dict_keys(self):
data = {'a': 12, 'b': 44}
self.assertItemsEqual(['a', 'b'], self.eval('$.keys()', data=data))
def test_dict_values(self):
data = {'a': 12, 'b': 44}
self.assertItemsEqual([12, 44], self.eval('$.values()', data=data))
def test_dict_items(self):
data = {'a': 12, 'b': 44}
self.assertItemsEqual([['a', 12], ['b', 44]],
self.eval('$.items()', data=data))
self.assertEqual(data, self.eval('dict($.items())', data=data))
def test_in(self):
data = {'a': 12, 'b': 44}
self.assertTrue(self.eval('44 in $.values()', data=data))
self.assertTrue(self.eval('24 in $.values().select(2*$)', data=data))
self.assertTrue(self.eval('{a => 2} in {{a => 2} => {b => 3}}.keys()'))
self.assertTrue(self.eval('5 in set(1, 2, 5)'))
self.assertTrue(self.eval('[1, 2] in set([1, 2], 5)'))
self.assertTrue(self.eval('5 in [1, 2, 5]'))
self.assertTrue(self.eval('[1, 2] in {[1, 2] => [3, 4]}.keys()'))
self.assertRaises(
exceptions.NoMatchingFunctionException,
self.eval, 'a in $', data=data)
def test_contains(self):
data = {'a': 12, 'b': 44}
self.assertTrue(self.eval('$.containsKey(a)', data=data))
self.assertTrue(self.eval('$.values().contains(44)', data=data))
self.assertFalse(self.eval('$.containsKey(null)', data=data))
self.assertTrue(self.eval(
'$.values().select(2*$).contains(24)', data=data))
self.assertTrue(self.eval('set(1, 2, 5).contains(5)'))
self.assertTrue(self.eval('[1, 2, 5].contains(5)'))
self.assertTrue(
self.eval('{{a => 2} => {b => 3}}.containsKey({a => 2})'))
self.assertTrue(self.eval('{[1, 2] => [3, 4]}.containsKey([1, 2])'))
self.assertTrue(self.eval('{[1, 2] => [3, 4]}.containsValue([3, 4])'))
self.assertTrue(self.eval('set([1, 2], 5).contains([1, 2])'))
def test_list_addition(self):
self.assertEqual(
[1, 2, 3, 4],
self.eval('list(1, 2) + list(3, 4)'))
self.assertEqual(
[1, 2, 6, 8],
self.eval('list(1, 2) + list(3, 4).select($ * 2)'))
def test_dict_addition(self):
self.assertEqual(
{'a': 1, 'b': 2},
self.eval('dict(a => 1) + dict(b => 2)'))
def test_list_multiplication(self):
self.assertItemsEqual(
[1, 2, 1, 2, 1, 2],
self.eval('3 * [1, 2]'))
self.assertItemsEqual(
[1, 2, 1, 2, 1, 2],
self.eval('[1, 2] * 3'))
def test_dict_list_key(self):
self.assertEqual(
3,
self.eval('dict($ => 3).get($)', data=[1, 2]))
self.assertEqual(
3,
self.eval('dict($ => 3).get($)', data=[1, [2]]))
def test_dict_dict_key(self):
self.assertEqual(
3,
self.eval('dict($ => 3).get($)', data={'a': 1}))
def test_delete(self):
self.assertEqual(
[2, 3, 4],
self.eval('[1, 2, 3, 4].delete(0)'))
self.assertEqual(
[3, 4],
self.eval('[1, 2, 3, 4].delete(0, 2)'))
self.assertEqual(
[4],
self.eval('[1, 2, 3, 4].delete(0, 2).delete(0)'))
self.assertEqual(
[1],
self.eval('[1, 2, 3, 4].delete(1, -1)'))
self.assertEqual(
[1, 2, 3, 4],
self.eval('[1, 2, 3, 4].delete(0, 0)'))
self.assertEqual(
[],
self.eval('[1, 2, 3, 4].delete(0, -1)'))
def test_insert(self):
self.assertEqual(
[1, 'a', 2],
self.eval('[1, 2].insert(1, a)'))
self.assertEqual(
[1, ['a', 'b'], 2],
self.eval('[1, 2].insert(1, [a, b])'))
self.assertEqual(
[1, 'a', 2],
self.eval('[1, 2].insert(-1, a)'))
self.assertEqual(
[1, 2, 'a'],
self.eval('[1, 2].insert(100, a)'))
self.assertEqual(
['b', 'a'],
self.eval('[].insert(0, a).insert(0, b)'))
self.assertRaises(
exceptions.NoMatchingMethodException,
self.eval, 'set(a, b).insert(1, a)')
def test_insert_iter(self):
self.assertEqual(
[1, 'a', 2],
self.eval('[1, 2].select($).insert(1, a)'))
self.assertEqual(
[1, ['a', 'b'], 2],
self.eval('[1, 2].select($).insert(1, [a, b])'))
self.assertEqual(
[1, 2],
self.eval('[1, 2].select($).insert(-1, a)'))
self.assertEqual(
[1, 2, 'a'],
self.eval('[1, 2].select($).insert(100, a)'))
self.assertEqual(
['b', 'a'],
self.eval('[].select($).insert(0, a).insert(0, b)'))
self.assertEqual(
['a', 'a', 'b'],
self.eval('set(a, b).orderBy($).insert(1, a)'))
def test_insert_many(self):
self.assertEqual(
[1, 'a', 'b', 2],
self.eval('[1, 2].insertMany(1, [a, b])'))
self.assertEqual(
['a', 'b', 1, 2],
self.eval('[1, 2].insertMany(-1, [a, b])'))
self.assertEqual(
[1, 2, 'a', 'b'],
self.eval('[1, 2].insertMany(100, [a, b])'))
self.assertEqual(
['a', 'a', 'b', 'b'],
self.eval('[].insertMany(0, [a, b]).insertMany(1, [a, b])'))
def test_replace(self):
self.assertEqual(
[None, 2, 3, 4],
self.eval('[1, 2, 3, 4].replace(0, null)'))
self.assertEqual(
[None, 3, 4],
self.eval('[1, 2, 3, 4].replace(0, null, 2)'))
self.assertEqual(
[1, 7],
self.eval('[1, 2, 3, 4].replace(1, 7, -1)'))
self.assertRaises(
exceptions.NoMatchingMethodException,
self.eval, 'set(1, 2, 3, 4).replace(1, 7)')
self.assertEqual(
[1, 7, 3, 4],
self.eval('set(4, 2, 3, 1).orderBy($).replace(1, 7)'))
def test_replace_many(self):
self.assertEqual(
[7, 8, 2, 3, 4],
self.eval('[1, 2, 3, 4].replaceMany(0, [7, 8])'))
self.assertEqual(
[7, 8, 3, 4],
self.eval('[1, 2, 3, 4].replaceMany(0, [7, 8], 2)'))
self.assertEqual(
[1, 7, 8],
self.eval('[1, 2, 3, 4].replaceMany(1, [7, 8], -1)'))
def test_delete_dict(self):
data = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
self.assertEqual(
{'a': 1, 'd': 4},
self.eval('$.delete(b, c)', data=data))
def test_delete_all(self):
data = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
self.assertEqual(
{'a': 1, 'd': 4},
self.eval('$.deleteAll([b, c])', data=data))
def test_set(self):
self.assertItemsEqual([2, 1, 3], self.eval('set(1, 2, 3, 2, 1)'))
self.assertEqual([[1, 2, 3, 2, 1]], self.eval('set([1, 2, 3, 2, 1])'))
self.assertEqual([], self.eval('set()'))
self.assertEqual(
[{'a': {'b': 'c'}}],
self.eval('set({a => {b => c}})'))
def test_set_from_iterator(self):
self.assertItemsEqual([2, 1, 3], self.eval('set([1, 2, 3].select($))'))
def test_to_set(self):
self.assertItemsEqual(
[2, 1, 3], self.eval('[1, 2, 3].select($).toSet()'))
self.assertItemsEqual(
[2, 1, 3], self.eval('[1, 2, 3].toSet()'))
def test_set_len(self):
self.assertEqual(3, self.eval('set(1, 2, 3).len()'))
self.assertEqual(3, self.eval('len(set(1, 2, 3))'))
def test_set_addition(self):
self.assertItemsEqual(
[4, 3, 2, 1],
self.eval('set(1, 2, 3) + set(4, 2, 3)'))
self.assertTrue(
self.eval('isSet(set(1, 2, 3) + set(4, 2, 3))'))
def test_set_union(self):
self.assertItemsEqual(
[4, 3, 2, 1],
self.eval('set(1, 2, 3).union(set(4, 2, 3))'))
def test_set_eq(self):
self.assertTrue(self.eval('set(1, 2, 3) = set(3, 2, 1)'))
self.assertFalse(self.eval('set(1, 2, 3) = set(1, 2, 3, 4)'))
def test_set_neq(self):
self.assertFalse(self.eval('set(1, 2, 3) != set(3, 2, 1)'))
self.assertTrue(self.eval('set(1, 2, 3) != set(1, 2, 3, 4)'))
def test_set_lt(self):
self.assertTrue(self.eval('set(1, 2, 3) < set(1, 2, 3, 4)'))
self.assertFalse(self.eval('set(1, 2, 3) < set(1, 2, 5)'))
def test_set_gt(self):
self.assertTrue(self.eval('set(1, 2, 3, 4) > set(1, 2, 3)'))
self.assertFalse(self.eval('set(1, 2, 3) > set(1, 2, 3)'))
def test_set_gte(self):
self.assertFalse(self.eval('set(1, 2, 4) >= set(1, 2, 3)'))
self.assertTrue(self.eval('set(1, 2, 3) >= set(1, 2, 3)'))
def test_set_lte(self):
self.assertFalse(self.eval('set(1, 2, 3) <= set(1, 2, 4)'))
self.assertTrue(self.eval('set(1, 2, 3) <= set(1, 2, 3)'))
def test_set_difference(self):
self.assertItemsEqual(
[4, 1],
self.eval('set(1, 2, 3, 4).difference(set(2, 3))'))
def test_set_subtraction(self):
self.assertItemsEqual(
[4, 1],
self.eval('set(1, 2, 3, 4) - set(2, 3)'))
self.assertTrue(
self.eval('isSet(set(1, 2, 3, 4) - set(2, 3))'))
def test_set_symmetric_difference(self):
self.assertItemsEqual(
[4, 1, 5],
self.eval('set(1, 2, 3, 4).symmetricDifference(set(2, 3, 5))'))
def test_set_add(self):
self.assertItemsEqual(
[4, 1, 2, 3],
self.eval('set(1, 2, 3).add(4)'))
self.assertItemsEqual(
[4, 1, 2, 3, 5],
self.eval('set(1, 2, 3).add(4, 5)'))
self.assertItemsEqual(
[1, 3, 2, [1, 2]],
self.eval('set(1, 2, 3).add([1, 2])'))
self.assertItemsEqual(
[4, 1, None, 2, 3, 5],
self.eval('set(1, 2, 3).add(4, 5, null)'))
self.assertTrue(
self.eval('isSet(set(1, 2, 3).add(4, 5, null))'))
def test_set_remove(self):
self.assertItemsEqual(
[1, 3],
self.eval('set(1, 2, 3).remove(2)'))
self.assertItemsEqual(
[3, None],
self.eval('set(1, 2, null, 3).remove(1, 2, 5)'))
self.assertItemsEqual(
[3],
self.eval('set(1, 2, null, 3).remove(1, 2, 5, null)'))
self.assertItemsEqual(
[1, 3, 2],
self.eval('set(1, 2, 3, [1, 2]).remove([1, 2])'))
self.assertTrue(
self.eval('isSet(set(1, 2, 3, [1, 2]).remove([1, 2]))'))

69
yaql/tests/test_common.py Normal file
View File

@ -0,0 +1,69 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import yaql.tests
class TestCommon(yaql.tests.TestCase):
def test_null(self):
self.assertIsNone(self.eval('null'))
def test_true(self):
res = self.eval('true')
self.assertTrue(res)
self.assertIsInstance(res, bool)
def test_false(self):
res = self.eval('false')
self.assertFalse(res)
self.assertIsInstance(res, bool)
def test_string(self):
self.assertEqual('True', self.eval('True'))
self.assertEqual('some string', self.eval("'some string'"))
def test_null_to_null(self):
self.assertTrue(self.eval('null = null'))
self.assertFalse(self.eval('null != null'))
self.assertTrue(self.eval('null <= null'))
self.assertTrue(self.eval('null >= null'))
self.assertFalse(self.eval('null < null'))
self.assertFalse(self.eval('null > null'))
def test_ordering(self):
self.assertTrue(self.eval('null < 0'))
self.assertTrue(self.eval('null < true'))
self.assertTrue(self.eval('null < false'))
self.assertTrue(self.eval('null < a'))
self.assertTrue(self.eval('null <= 0'))
self.assertFalse(self.eval('null > 0'))
self.assertFalse(self.eval('null >= 0'))
self.assertTrue(self.eval('null != 0'))
self.assertTrue(self.eval('null != false'))
self.assertFalse(self.eval('null = false'))
self.assertFalse(self.eval('null = 0'))
self.assertFalse(self.eval('0 < null'))
self.assertFalse(self.eval('0 <= null'))
self.assertTrue(self.eval('0 >= null'))
self.assertTrue(self.eval('0 > null'))
def test_max(self):
self.assertEqual(5, self.eval('max(1, 5)'))
self.assertEqual(-1, self.eval('max(null, -1)'))
self.assertIsNone(self.eval('max(null, null)'))
def test_min(self):
self.assertEqual(1, self.eval('min(1, 5)'))
self.assertIsNone(self.eval('min(null, -1)'))
self.assertIsNone(self.eval('min(null, null)'))

52
yaql/tests/test_const.py
View File

@ -1,52 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from yaql.tests import YaqlTest
class TestConst(YaqlTest):
def test_string_constant(self):
self.assertEquals("abc", self.eval("abc"))
self.assertEquals("100", self.eval("'100'"))
self.assertEquals('some mw const', self.eval("'some mw const'"))
self.assertEquals('i\'m fine', self.eval("'i\\'m fine'"))
def test_numeric_constant(self):
self.assertEquals(0, self.eval('0'))
self.assertEquals(100, self.eval('100'))
self.assertEquals(0, self.eval("0"))
self.assertEquals(100, self.eval("100"))
def test_negative_constant(self):
self.assertEquals(-10, self.eval('-10'))
def test_boolean_constant(self):
self.assertEquals(True, self.eval('true'))
self.assertEquals(False, self.eval('false'))
self.assertNotEquals(True, self.eval('True'))
self.assertNotEquals(False, self.eval('False'))
def test_r_multiline(self):
self.assertEval(10, '3 +\r 7')
def test_n_multiline(self):
self.assertEval(10, '3 +\n 7')
def test_rn_multiline(self):
self.assertEval(10, '3\r\n +\r\n 7')
if __name__ == '__main__':
unittest.main()

152
yaql/tests/test_containers.py
View File

@ -1,152 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import types
from yaql.language.exceptions import YaqlException
from yaql.language.exceptions import YaqlExecutionException
from yaql.language.exceptions import YaqlSequenceException
from yaql.tests import YaqlTest
import yaql.tests.testdata
class TestCollections(YaqlTest):
def test_get_by_index(self):
int_list = [1, 2, 3, 4, 5, 6]
self.assertEquals(4, self.eval('$[3]', int_list))
def test_where_by_index(self):
int_list = [1, 2, 3, 4, 5, 6]
self.assertRaises(YaqlException, self.eval, '$.where(3)', int_list)
def test_filter_by_predicate(self):
int_list = [1, 2, 3, 4, 5, 6]
self.assertEquals([4, 5, 6], list(self.eval('$[$>3]', int_list)))
def test_filter_by_non_boolean_predicate(self):
int_list = [1, 2, 3, 4, 5, 6]
self.assertRaises(YaqlException, self.eval, '$.where($+1)', int_list)
def test_list_definition(self):
self.assertEquals([1, 2, 3], self.eval('list(1,2,3)'))
def test_dict_definition(self):
self.assertEval({'key1': 'value', 'key2': 100},
'dict(key1=>value, key2=>100)')
def test_wrong_dict_definition(self):
self.assertRaises(YaqlExecutionException, self.eval, 'dict(a,b,c)')
self.assertRaises(YaqlExecutionException, self.eval,
'dict(a=>b=>c, a=>d=>e)')
def test_in(self):
int_list = [1, 2, 3, 4, 5, 6]
self.assertTrue(self.eval('4 in $', int_list))
def test_not_in(self):
int_list = [1, 2, 3, 4, 5, 6]
self.assertFalse(self.eval('7 in $', int_list))
def test_iterable_property_attribution(self):
data = yaql.tests.testdata.users
expression = "$.email"
self.assertEquals(
['user1@example.com',
'user2@example.com',
'user3@example.com'],
self.eval(expression, data))
def test_iterable_property_attribution_2(self):
data = yaql.tests.testdata.data
expression = "$.users.email"
self.assertEquals(
['user1@example.com',
'user2@example.com',
'user3@example.com'],
self.eval(expression, data))
def test_iterable_dictionary_attribution(self):
data = yaql.tests.testdata.data
expression = "$.services.'com.mirantis.murano.yaql.name'"
self.assertEquals(['Service1', 'Service2',
'Service3', 'Service4'],
self.eval(expression, data))
def test_join(self):
data = yaql.tests.testdata.data
expression = "$.services.join($.users, " \
"$1.'com.mirantis.murano.yaql.owner'=$2.id, " \
"dict(service_name=>" \
"$1.'com.mirantis.murano.yaql.name', " \
"user_name=>$2.email))"
value = self.eval(expression, data=data)
self.assertEqual('Service1', value[0]['service_name'])
self.assertEqual('Service2', value[1]['service_name'])
self.assertEqual('Service3', value[2]['service_name'])
self.assertEqual('Service4', value[3]['service_name'])
self.assertEqual('user1@example.com', value[0]['user_name'])
self.assertEqual('user1@example.com', value[1]['user_name'])
self.assertEqual('user2@example.com', value[2]['user_name'])
self.assertEqual('user3@example.com', value[3]['user_name'])
def test_select(self):
data = [1, 2, 3, 4]
expression = "$.select($*10)"
self.assertEval([10, 20, 30, 40], expression, data)
def test_data_sum(self):
data = [1, 2, 3, 4]
expression = "$.sum()"
self.assertEval(10, expression, data)
def test_method_sum(self):
expression = "list(1,2,3,4).sum()"
self.assertEval(10, expression)
def test_function_sum(self):
expression = "sum(list(1,2,3,4))"
self.assertEval(10, expression)
def test_range_const(self):
expression = "range(0,4)"
self.assertEval([0, 1, 2, 3], expression)
def test_range_computed(self):
expression = "range(1+2, 10-4)"
self.assertEval([3, 4, 5], expression)
def test_take_while(self):
data = [1, 2, 3, 4]
self.assertEval([1, 2], "$.take_while($<3)", data)
def test_infinite_random_loop(self):
val = self.eval("range(0).select(random()).take_while($<0.99)")
for v in val:
self.assertTrue(0 < v < 0.99)
def test_generator_limiting(self):
# do not use self.eval here as it uses limiting on its own
v = yaql.parse('range(0, 10)').evaluate()
self.assertTrue(isinstance(v, types.GeneratorType))
v2 = yaql.parse('range(0, 10).list()').evaluate()
self.assertTrue(isinstance(v2, list))
v3 = yaql.parse('range(0).list()')
self.assertRaises(YaqlSequenceException, v3.evaluate)
def test_select_for_each(self):
data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
expression = "$.for_each(switch(($>5)=>$, " \
"($>2)=>('_'+string($)), true=>0))"
self.assertEval([0, 0, "_3", "_4", "_5", 6, 7, 8, 9, 10], expression,
data)

168
yaql/tests/test_engine.py Normal file
View File

@ -0,0 +1,168 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import yaql
from yaql.language import exceptions
from yaql.language import specs
from yaql.language import yaqltypes
from yaql.language import utils
from yaql import tests
class TestEngine(tests.TestCase):
def test_no_function_registered(self):
self.assertRaises(
exceptions.NoFunctionRegisteredException,
self.eval, 'kjhfksjdhfk()')
def test_no_method_registered(self):
self.assertRaises(
exceptions.NoMethodRegisteredException,
self.eval, '[1,2].kjhfksjdhfk($)')
def test_no_matching_function(self):
self.assertRaises(
exceptions.NoMatchingFunctionException,
self.eval, 'len(1, 2, 3)')
def test_mapping_translation_exception(self):
self.context.register_function(
lambda *args, **kwargs: 1, name='f')
self.assertRaises(
exceptions.MappingTranslationException,
self.eval, 'f(2+2 => 4)')
def test_no_matching_method(self):
self.assertRaises(
exceptions.NoMatchingMethodException,
self.eval, '[1, 2].select(1, 2, 3)')
def test_duplicate_parameters(self):
def raises():
@specs.parameter('p')
@specs.parameter('p')
def f(p):
return p
self.assertRaises(
exceptions.DuplicateParameterDecoratorException,
raises)
def test_invalid_parameter(self):
def raises():
@specs.parameter('x')
def f(p):
return p
self.assertRaises(
exceptions.NoParameterFoundException,
raises)
def test_lexical_error(self):
self.assertRaises(
exceptions.YaqlLexicalException,
self.eval, '1 ? 2')
def test_grammar_error(self):
self.assertRaises(
exceptions.YaqlGrammarException,
self.eval, '1 2')
self.assertRaises(
exceptions.YaqlGrammarException,
self.eval, '(2')
def test_invalid_method(self):
self.assertRaises(
exceptions.InvalidMethodException,
self.context.register_function, lambda: 1, name='f', method=True)
@specs.parameter('x', yaqltypes.Lambda())
def func(x):
return x
self.assertRaises(
exceptions.InvalidMethodException,
self.context.register_function, func, name='f2', method=True)
def test_function_definition(self):
def func(a, b, *args, **kwargs):
return a, b, args, kwargs
fd = specs.get_function_definition(func)
self.assertEqual(
(1, 2, (5, 7), {'kw1': 'x', 'kw2': None}),
fd(utils.NO_VALUE, self.engine, self.context)(
1, 2, 5, 7, kw1='x', kw2=None))
self.assertEqual(
(1, 5, (), {}),
fd(utils.NO_VALUE, self.engine, self.context)(1, b=5))
def test_eval(self):
self.assertEqual(
120,
yaql.eval(
'let(inp => $) -> [1, 2, 3].select($ + $inp).reduce($1 * $2)',
data=3)
)
def test_skip_args(self):
def func(a=11, b=22, c=33):
return a, b, c
self.context.register_function(func)
self.assertEqual([1, 22, 33], self.eval('func(1,,)'))
self.assertEqual([11, 1, 33], self.eval('func(,1,)'))
self.assertEqual([11, 22, 1], self.eval('func(,,1)'))
self.assertEqual([0, 22, 1], self.eval('func(0,,1)'))
self.assertEqual([11, 22, 33], self.eval('func(,,)'))
self.assertEqual([11, 22, 33], self.eval('func(,)'))
self.assertEqual([11, 22, 33], self.eval('func()'))
self.assertEqual([11, 1, 4], self.eval('func(,1, c=>4)'))
self.assertRaises(
exceptions.NoMatchingFunctionException,
self.eval, 'func(,1, b=>4)')
self.assertRaises(
exceptions.NoMatchingFunctionException,
self.eval, 'func(,1,, c=>4)')
def test_super(self):
@specs.parameter('string', yaqltypes.String())
@specs.inject('base', yaqltypes.Super())
def len2(string, base):
return 2 * base(string)
context = self.context.create_child_context()
context.register_function(len2, name='len')
self.assertEqual(6, self.eval('len(abc)', context=context))
def test_delegate_factory(self):
@specs.parameter('name', yaqltypes.String())
@specs.inject('__df__', yaqltypes.Delegate())
def call_func(__df__, name, *args, **kwargs):
return __df__(name, *args, **kwargs)
context = self.context.create_child_context()
context.register_function(call_func)
self.assertEqual(
[1, 2],
self.eval('callFunc(list, 1, 2)', context=context))
self.assertEqual(
6,
self.eval("callFunc('#operator_*', 2, 3)", context=context))

107
yaql/tests/test_execution_chains.py
View File

@ -1,107 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
import unittest
from yaql.language.exceptions import YaqlException, YaqlExecutionException
from yaql.tests import YaqlTest
from yaql.language.engine import context_aware, parameter
def f4(self):
return 'f4({0})'.format(self)
@context_aware
def f3(self, context):
context.register_function(f4, 'f4')
return 'f3({0})'.format(self)
@context_aware
def f2(self, context):
context.register_function(f3, 'f3')
return 'f2({0})'.format(self)
@context_aware
def f1(self, context):
context.register_function(f2, 'f2')
return 'f1({0})'.format(self)
@context_aware
def override_with_caps(self, context):
context.register_function(lambda self: "data is: " + self.upper(), 'print')
return self
def _print(self):
return "data is: %s" % self
@parameter('self', arg_type=six.string_types)
def print_string(self):
return "print %s" % self
class TestExecutionChain(YaqlTest):
def setUp(self):
super(TestExecutionChain, self).setUp()
self.context.register_function(f1, 'f1')
self.context.register_function(_print, 'print')
self.context.register_function(override_with_caps, 'caps_on')
def test_chain1(self):
expression = 'f1(abc).f2().f3()'
self.assertEquals('f3(f2(f1(abc)))', self.eval(expression))
def test_chain2(self):
expression = 'abc.f1().f2().f3().f4()'
self.assertEquals('f4(f3(f2(f1(abc))))', self.eval(expression))
def test_chain3(self):
expression = 'abc.f2().f3()'
self.assertRaises(YaqlException, self.eval, expression)
def test_chain4(self):
expression = 'abc.f4().f3().f2().f1()'
self.assertRaises(YaqlException, self.eval, expression)
def test_chain5(self):
expression = 'abc.f1() + abc.f2()'
self.assertRaises(YaqlException, self.eval, expression)
def test_override(self):
expression1 = 'abc.print()'
expression2 = 'abc.caps_on().print()'
self.assertEquals("data is: abc", self.eval(expression1))
self.assertEquals("data is: ABC", self.eval(expression2))
def test_override_and_forget(self):
expression = "abc.caps_on().print() + ', but ' + abc.print()"
self.assertEquals("data is: ABC, but data is: abc",
self.eval(expression))
def test_self_validation(self):
good_expression = "abc.print_string()"
wrong_expression = "123.print_string()"
self.context.register_function(print_string)
self.assertEval("print abc", good_expression) # self is valid string
self.assertRaises(YaqlExecutionException, self.eval, wrong_expression)
if __name__ == '__main__':
unittest.main()

106
yaql/tests/test_legacy.py Normal file
View File

@ -0,0 +1,106 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import yaql.tests
class TestLegacy(yaql.tests.TestCase):
def setUp(self):
super(TestLegacy, self).setUp()
self.eval = self.legacy_eval
def test_tuples(self):
self.assertEqual((1, 2), self.eval('1 => 2'))
self.assertEqual((None, 'a b'), self.eval('null => "a b"'))
self.assertEqual((1, 2, 3), self.eval('1 => 2 => 3'))
self.assertEqual(((1, 2), 3), self.eval('(1 => 2) => 3'))
self.assertEqual((1, (2, 3)), self.eval('1 => (2 => 3)'))
def test_tuples_func(self):
self.assertEqual((1, 2), self.eval('tuple(1, 2)'))
self.assertEqual((None,), self.eval('tuple(null)'))
self.assertEqual(tuple(), self.eval('tuple()'))
def test_dict(self):
self.assertEqual(
{'a': 'b', 1: 2, None: None},
self.eval('dict(1 => 2, a => b, null => null)'))
self.assertEqual({}, self.eval('dict()'))
def test_list(self):
self.assertEqual([1, 2, 'a', None], self.eval('list(1, 2, a, null)'))
self.assertEqual([], self.eval('list()'))
self.assertEqual([1, 2], self.eval('[1, 2].select($).list()'))
def test_dict_get(self):
self.assertEqual(5, self.eval("get($, 'a b')", data={'a b': 5}))
self.assertEqual(5, self.eval("$.get('a b')", data={'a b': 5}))
def test_int(self):
self.assertEqual(5, self.eval("'5'.int()"))
self.assertEqual(5, self.eval("5.2.int()"))
self.assertEqual(5, self.eval("int('5')"))
self.assertEqual(5, self.eval("int(5.2)"))
def test_float(self):
self.assertAlmostEqual(5.1, self.eval("'5.1'.float()"))
self.assertAlmostEqual(5.0, self.eval("5.float()"))
self.assertAlmostEqual(5.1, self.eval("float('5.1')"))
self.assertAlmostEqual(5.0, self.eval("float(5)"))
def test_bool(self):
self.assertFalse(self.eval("null.bool()"))
self.assertFalse(self.eval("''.bool()"))
self.assertFalse(self.eval("0.bool()"))
self.assertFalse(self.eval("false.bool()"))
self.assertFalse(self.eval("[].bool()"))
self.assertFalse(self.eval("{}.bool()"))
self.assertTrue(self.eval("' '.bool()"))
self.assertTrue(self.eval("x.bool()"))
self.assertTrue(self.eval("1.bool()"))
self.assertTrue(self.eval("true.bool()"))
self.assertTrue(self.eval("[1].bool()"))
self.assertTrue(self.eval("{a=>b}.bool()"))
def test_filter(self):
self.assertEqual(2, self.eval("list(1,2,3)[1]"))
self.assertEqual(3, self.eval("list(1,2,3)[$]", data=2))
self.assertEqual([1, 3], self.eval("list(1,2,3)[$ != 2]"))
self.assertEqual([], self.eval("list()[$ > 0]"))
def test_sum(self):
self.assertEqual(6, self.eval('list(1,2,3).sum()'))
self.assertEqual(6, self.eval('sum(list(1,2,3))'))
def test_range(self):
self.assertEqual([2, 3, 4, 5], self.eval('range(2).take(4)'))
self.assertEqual([1, 2, 3], self.eval('range(1, 4)'))
self.assertEqual([2, 3, 4, 5], self.eval('2.range().take(4)'))
self.assertEqual([1, 2, 3], self.eval('1.range(4)'))
def test_take_while(self):
self.assertEqual([1, 2], self.eval('[1, 2, 3, 4].takeWhile($ < 3)'))
self.assertEqual([1, 2], self.eval('takeWhile([1, 2, 3, 4], $ < 3)'))
def test_switch(self):
expr = 'switch($ < 10 => 1, $ >= 10 and $ < 100 => 2, $ >= 100 => 3)'
self.assertEqual(3, self.eval(expr, data=123))
self.assertEqual(2, self.eval(expr, data=50))
self.assertEqual(1, self.eval(expr, data=-123))
def test_as(self):
self.assertEqual(
[3, 6],
self.eval('[1, 2].as(sum($) => a).select($ * $a)'))

192
yaql/tests/test_math.py Normal file
View File

@ -0,0 +1,192 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import yaql.tests
class TestMath(yaql.tests.TestCase):
def test_binary_plus_int(self):
res = self.eval('2 + 3')
self.assertEqual(5, res)
self.assertIsInstance(res, int)
def test_binary_plus_float(self):
res = self.eval('2 + 3.0')
self.assertEqual(5, res)
self.assertIsInstance(res, float)
res = self.eval('2.3+3.5')
self.assertEqual(5.8, res)
self.assertIsInstance(res, float)
def test_binary_minus_int(self):
res = self.eval('12 -3')
self.assertEqual(9, res)
self.assertIsInstance(res, int)
def test_binary_minus_float(self):
res = self.eval('1 - 2.1')
self.assertEqual(-1.1, res)
self.assertIsInstance(res, float)
res = self.eval('123.321- 0.321')
self.assertEqual(123.0, res)
self.assertIsInstance(res, float)
def test_multiplication_int(self):
res = self.eval('3 * 2')
self.assertEqual(6, res)
self.assertIsInstance(res, int)
self.assertEqual(-6, self.eval('3 * -2'))
self.assertEqual(6, self.eval('-3 * -2'))
def test_multiplication_float(self):
res = self.eval('3.0 * 2.0')
self.assertEqual(6.0, res)
self.assertIsInstance(res, float)
self.assertAlmostEqual(-6.51, self.eval('3.1 * -2.1'))
self.assertAlmostEqual(6.51, self.eval('-3.1 * -2.1'))
def test_division(self):
self.assertEqual(3, self.eval('7 / 2'))
self.assertEqual(-4, self.eval('7 / -2'))
self.assertAlmostEqual(2.5, self.eval('5 / 2.0'))
self.assertAlmostEqual(2.5, self.eval('5.0 / 2'))
self.assertAlmostEqual(-2.5, self.eval('-5.0 / 2.0'))
self.assertRaises(ZeroDivisionError, self.eval, '7 / 0')
self.assertRaises(ZeroDivisionError, self.eval, '7 / -0.0')
def test_brackets(self):
self.assertEqual(-4, self.eval('1 - (2) - 3'))
self.assertEqual(2, self.eval('1 - (2 - 3)'))
def test_unary_minus(self):
self.assertEqual(-4, self.eval('-4'))
self.assertEqual(-12.0, self.eval('-12.0'))
self.assertEqual(4, self.eval('3--1'))
self.assertEqual(2, self.eval('3+-1'))
self.assertAlmostEqual(4.3, self.eval('3.2 - -1.1'))
self.assertEqual(2, self.eval('-(1-3)'))
def test_unary_plus(self):
self.assertEqual(4, self.eval('+4'))
self.assertEqual(12.0, self.eval('+12.0'))
self.assertEqual(2, self.eval('3-+1'))
self.assertEqual(4, self.eval('3++1'))
self.assertAlmostEqual(2.1, self.eval('3.2 - +1.1'))
def test_modulo_int(self):
res = self.eval('9 mod 5')
self.assertEqual(4, res)
self.assertIsInstance(res, int)
self.assertEqual(-1, self.eval('9 mod -5'))
def test_modulo_float(self):
res = self.eval('9.0 mod 5')
self.assertEqual(4.0, res)
self.assertIsInstance(res, float)
res = self.eval('9 mod 5.0')
self.assertEqual(4.0, res)
self.assertIsInstance(res, float)
res = self.eval('9.0 mod 5.0')
self.assertEqual(4.0, res)
self.assertIsInstance(res, float)
self.assertAlmostEqual(-1.1, self.eval('9.1 mod -5.1'))
def test_abs(self):
self.assertEqual(4, self.eval('abs(-4)'))
self.assertEqual(4, self.eval('abs(4)'))
self.assertEqual(4.4, self.eval('abs(-4.4)'))
def test_gt(self):
res = self.eval('5 > 3')
self.assertIsInstance(res, bool)
self.assertTrue(res)
self.assertFalse(self.eval('3 > 3'))
def test_lt(self):
res = self.eval('3 < 5')
self.assertIsInstance(res, bool)
self.assertTrue(res)
self.assertFalse(self.eval('3 < 3'))
def test_gte(self):
res = self.eval('5 >= 3')
self.assertIsInstance(res, bool)
self.assertTrue(res)
self.assertTrue(self.eval('3 >= 3'))
self.assertFalse(self.eval('2 >= 3'))
def test_lte(self):
res = self.eval('3 <= 5')
self.assertIsInstance(res, bool)
self.assertTrue(res)
self.assertTrue(self.eval('3 <= 3'))
self.assertFalse(self.eval('3 <= 2'))
def test_eq(self):
self.assertTrue(self.eval('5 = 5'))
self.assertFalse(self.eval('5 = 6'))
def test_neq(self):
self.assertFalse(self.eval('5 != 5'))
self.assertTrue(self.eval('5 != 6'))
def test_zero_division(self):
self.assertRaises(ZeroDivisionError, self.eval, '0/0')
def test_random(self):
self.assertTrue(self.eval('with(random()) -> $ >= 0 and $ < 1'))
self.assertTrue(self.eval('with(random(2, 5)) -> $ >= 2 and $ <= 5'))
def test_float(self):
self.assertAlmostEqual(-1.23, self.eval("float('-1.23')"))
def test_bitwise_or(self):
self.assertEqual(3, self.eval('bitwiseOr(1, 3)'))
self.assertEqual(3, self.eval('bitwiseOr(1, 2)'))
def test_bitwise_and(self):
self.assertEqual(1, self.eval('bitwiseAnd(1, 3)'))
self.assertEqual(0, self.eval('bitwiseAnd(1, 2)'))
def test_bitwise_xor(self):
self.assertEqual(2, self.eval('bitwiseXor(1, 3)'))
self.assertEqual(3, self.eval('bitwiseXor(1, 2)'))
def test_bitwise_not(self):
self.assertEqual(-2, self.eval('bitwiseNot(1)'))
def test_shift_bits_left(self):
self.assertEqual(32, self.eval('shiftBitsLeft(1, 5)'))
def test_shift_bits_right(self):
self.assertEqual(2, self.eval('shiftBitsRight(32, 4)'))
self.assertEqual(0, self.eval('shiftBitsRight(32, 6)'))
def test_pow(self):
self.assertEqual(32, self.eval('pow(2, 5)'))
self.assertEqual(4, self.eval('pow(2, 5, 7)'))
def test_sign(self):
self.assertEqual(1, self.eval('sign(123)'))
self.assertEqual(-1, self.eval('sign(-123)'))
self.assertEqual(0, self.eval('sign(0)'))
def test_round(self):
self.assertAlmostEqual(2.0, self.eval('round(2.3)'))
self.assertAlmostEqual(2.3, self.eval('round(2.345, 1)'))

51
yaql/tests/test_ns.py
View File

@ -1,51 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from yaql.functions import ns
from yaql.tests import YaqlTest
class TestNS(YaqlTest):
def setUp(self):
def foo(self):
return "bar: %s" % self
super(TestNS, self).setUp()
ns.add_to_context(self.context)
namespace = ns.Namespace("com.example.yaql.namespace", 'name',
'composite name', 'function_name')
ns.get_resolver(self.context).register('nms', namespace)
self.context.\
register_function(foo, 'com.example.yaql.namespace.function_name')
def test_resolve(self):
self.assertEval("com.example.yaql.namespace.name", 'nms:name')
self.assertEval("com.example.yaql.namespace.composite name",
"nms:'composite name'")
self.assertEval("com.example.yaql.namespace.function_name",
'nms:function_name')
def test_unable_to_resolve(self):
self.assertRaises(ns.NamespaceResolutionException, self.eval,
'nms2:name')
def test_unable_to_validate(self):
self.assertRaises(ns.NamespaceValidationException, self.eval,
'nms:line')
def test_namespace_function_call(self):
self.assertEval("bar: abc", 'abc.nms:function_name()')
if __name__ == '__main__':
unittest.main()

68
yaql/tests/test_objects.py
View File

@ -1,68 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from yaql.language.engine import parameter
from yaql.language.exceptions import YaqlExecutionException
from yaql.tests import YaqlTest
class Foobar():
def __init__(self, prompt):
self.prompt = prompt
@parameter('value')
def foo(self, value):
return "%s: %s" % (self.prompt, str(value).upper())
def bar(self, another_value):
return "%s: %s" % (self.prompt, str(another_value).lower())
class TestObjects(YaqlTest):
def test_registering_decorated_class_method(self):
self.context.register_function(Foobar.foo, 'foo')
self.assertEquals(1, len(self.context.get_functions('foo', 2)))
def test_registering_undecorated_class_method(self):
self.context.register_function(Foobar.bar, 'bar')
self.assertEquals(1, len(self.context.get_functions('bar', 2)))
def test_calling_decorated_class_method(self):
self.context.register_function(Foobar.foo, 'foo')
self.context.register_function(Foobar.bar, 'bar')
expression = '$.foo(aBc)'
expression2 = '$.bar(aBc)'
data = Foobar('foobar')
self.assertEquals('foobar: ABC', self.eval(expression, data))
self.assertEquals('foobar: abc', self.eval(expression2, data))
def test_calling_undecorated_class_method(self):
self.context.register_function(Foobar.foo, 'foo')
self.context.register_function(Foobar.bar, 'bar')
expression = '$.bar(aBc)'
data = Foobar("foobar")
self.assertEquals('foobar: abc', self.eval(expression, data))
#TODO(ruhe): figure out why it fails on py34 only
@unittest.skip("passes py27, fails on py34")
def test_calling_decorated_class_methods_for_invalid_objects(self):
self.context.register_function(Foobar.foo, 'foo')
expression = '$.foo(aBc)'
self.assertRaises(YaqlExecutionException, self.eval, expression, 'str')
self.assertRaises(YaqlExecutionException, self.eval, expression,
object())
if __name__ == '__main__':
unittest.main()

445
yaql/tests/test_queries.py Normal file
View File

@ -0,0 +1,445 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from yaql.language import exceptions
import yaql.tests
class TestQueries(yaql.tests.TestCase):
def test_where(self):
data = [1, 2, 3, 4, 5, 6]
self.assertEqual([4, 5, 6], self.eval('$.where($ > 3)', data=data))
def test_select(self):
data = [1, 2, 3]
self.assertEqual([1, 4, 9], self.eval('$.select($ * $)', data=data))
def test_skip(self):
data = [1, 2, 3, 4]
self.assertEqual([2, 3, 4], self.eval('$.skip(1)', data=data))
def test_limit(self):
data = [1, 2, 3, 4]
self.assertEqual([1, 2], self.eval('$.limit(2)', data=data))
self.assertEqual([1, 2], self.eval('$.take(2)', data=data))
def test_append(self):
data = [1, 2]
self.assertEqual([1, 2, 3, 4], self.eval('$.append(3, 4)', data=data))
def test_complex_query(self):
data = [1, 2, 3, 4, 5, 6]
self.assertEqual(
[4],
self.eval('$.where($ < 4).select($ * $).skip(1).limit(1)',
data=data))
def test_distinct(self):
data = [1, 2, 3, 2, 4, 8]
self.assertEqual([1, 2, 3, 4, 8], self.eval('$.distinct()', data=data))
self.assertEqual([1, 2, 3, 4, 8], self.eval('distinct($)', data=data))
def test_distinct_with_selector(self):
data = [['a', 1], ['b', 2], ['c', 1], ['d', 3], ['e', 2]]
self.assertItemsEqual([['a', 1], ['b', 2], ['d', 3]],
self.eval('$.distinct($[1])', data=data))
self.assertItemsEqual([['a', 1], ['b', 2], ['d', 3]],
self.eval('distinct($, $[1])', data=data))
def test_any(self):
self.assertFalse(self.eval('$.any()', data=[]))
self.assertTrue(self.eval('$.any()', data=[0]))
def test_all(self):
self.assertTrue(self.eval('$.all()', data=[]))
self.assertFalse(self.eval('$.all()', data=[1, 0]))
self.assertTrue(self.eval('$.all()', data=[1, 2]))
self.assertFalse(self.eval('$.all($ > 1)', data=[2, 1]))
self.assertTrue(self.eval('$.all($ > 1)', data=[2, 3]))
def test_enumerate(self):
data = [1, 2, 3]
self.assertEqual([[0, 1], [1, 2], [2, 3]],
self.eval('$.enumerate()', data=data))
self.assertEqual([[3, 1], [4, 2], [5, 3]],
self.eval('$.enumerate(3)', data=data))
self.assertEqual([[0, 1], [1, 2], [2, 3]],
self.eval('enumerate($)', data=data))
self.assertEqual([[3, 1], [4, 2], [5, 3]],
self.eval('enumerate($, 3)', data=data))
def test_concat(self):
data = [1, 2, 3]
self.assertEqual(
[1, 2, 3, 2, 4, 6],
self.eval('$.select($).concat($.select(2 * $))', data=data))
self.assertEqual(
[1, 2, 3, 2, 4, 6, 1, 2, 3],
self.eval('concat($, $.select(2 * $), $)', data=data))
def test_len(self):
data = [1, 2, 3]
self.assertEqual(3, self.eval('len($)', data=data))
self.assertEqual(3, self.eval('$.len()', data=data))
self.assertEqual(3, self.eval('$.count()', data=data))
self.assertRaises(
exceptions.FunctionResolutionError,
self.eval, 'count($)', data=data)
def test_sum(self):
data = range(4)
self.assertEqual(6, self.eval('$.sum()', data=data))
def test_memorize(self):
generator_func = lambda: (i for i in range(3))
self.assertRaises(
TypeError,
self.eval, '$.len() + $.sum()', data=generator_func())
self.assertEqual(
6,
self.eval('let($.memorize()) -> $.len() + $.sum()',
data=generator_func()))
def test_first(self):
self.assertEqual(2, self.eval('list(2, 3).first()'))
self.assertEqual(4, self.eval('list(2, 3).select($ * 2).first()'))
self.assertIsNone(self.eval('list().first(null)'))
self.assertRaises(StopIteration, self.eval, 'list().first()')
def test_single(self):
self.assertEqual(2, self.eval('list(2).single()'))
self.assertRaises(StopIteration, self.eval, 'list().single()')
self.assertRaises(ValueError, self.eval, 'list(1, 2).single()')
def test_last(self):
self.assertEqual(3, self.eval('list(2, 3).last()'))
self.assertEqual(6, self.eval('list(2, 3).select($ * 2).last()'))
self.assertIsNone(self.eval('list().last(null)'))
self.assertRaises(StopIteration, self.eval, 'list().last()')
def test_range(self):
self.assertEqual([0, 1], self.eval('range(2)'))
self.assertEqual([1, 2, 3], self.eval('range(1, 4)'))
self.assertEqual([4, 3, 2], self.eval('range(4, 1, -1)'))
def test_select_many(self):
self.assertEqual([0, 0, 1, 0, 1, 2],
self.eval('range(4).selectMany(range($))'))
def test_select_many_scalar(self):
# check that string is not interpreted as a sequence and that
# selectMany works when selector returns scalar
self.assertEqual(
['xx', 'xx'],
self.eval('range(2).selectMany(xx)'))
def test_order_by(self):
self.assertEqual(
[1, 2, 3, 4],
self.eval('$.orderBy($)', data=[4, 2, 1, 3]))
self.assertEqual(
[4, 3, 2, 1],
self.eval('$.orderByDescending($)', data=[4, 2, 1, 3]))
def test_order_by_multilevel(self):
self.assertEqual(
[[1, 0], [1, 5], [2, 2]],
self.eval(
'$.orderBy($[0]).thenBy($[1])',
data=[[2, 2], [1, 5], [1, 0]]))
self.assertEqual(
[[1, 5], [1, 0], [2, 2]],
self.eval(
'$.orderBy($[0]).thenByDescending($[1])',
data=[[2, 2], [1, 5], [1, 0]]))
self.assertEqual(
[[2, 2], [1, 0], [1, 5]],
self.eval(
'$.orderByDescending($[0]).thenBy($[1])',
data=[[2, 2], [1, 5], [1, 0]]))
self.assertEqual(
[[2, 2], [1, 5], [1, 0]],
self.eval(
'$.orderByDescending($[0]).thenByDescending($[1])',
data=[[2, 2], [1, 5], [1, 0]]))
def test_group_by(self):
data = {'a': 1, 'b': 2, 'c': 1, 'd': 3, 'e': 2}
self.assertItemsEqual(
[
[1, [['a', 1], ['c', 1]]],
[2, [['b', 2], ['e', 2]]],
[3, [['d', 3]]]
],
self.eval('$.items().orderBy($[0]).groupBy($[1])', data=data))
self.assertItemsEqual(
[[1, ['a', 'c']], [2, ['b', 'e']], [3, ['d']]],
self.eval('$.items().orderBy($[0]).groupBy($[1], $[0])',
data=data))
self.assertItemsEqual(
[[1, 'ac'], [2, 'be'], [3, 'd']],
self.eval('$.items().orderBy($[0]).'
'groupBy($[1], $[0], [$[0], $[1].sum()])', data=data))
self.assertItemsEqual(
[[1, ['a', 1, 'c', 1]], [2, ['b', 2, 'e', 2]], [3, ['d', 3]]],
self.eval('$.items().orderBy($[0]).'
'groupBy($[1],, [$[0], $[1].sum()])',
data=data))
self.assertItemsEqual(
[[1, ['a', 1, 'c', 1]], [2, ['b', 2, 'e', 2]], [3, ['d', 3]]],
self.eval('$.items().orderBy($[0]).'
'groupBy($[1], aggregator => [$[0], $[1].sum()])',
data=data))
def test_join(self):
self.assertEqual(
[[2, 1], [3, 1], [3, 2], [4, 1], [4, 2], [4, 3]],
self.eval('$.join($, $1 > $2, [$1, $2])', data=[1, 2, 3, 4]))
self.assertEqual(
[[1, 3], [1, 4], [2, 3], [2, 4]],
self.eval('[1,2].join([3, 4], true, [$1, $2])'))
def test_zip(self):
self.assertEqual(
[[1, 4], [2, 5]],
self.eval('[1, 2, 3].zip([4, 5])'))
self.assertEqual(
[[1, 4, 6], [2, 5, 7]],
self.eval('[1, 2, 3].zip([4, 5], [6, 7, 8])'))
def test_zip_longest(self):
self.assertEqual(
[[1, 4], [2, 5], [3, None]],
self.eval('[1, 2, 3].zipLongest([4, 5])'))
self.assertEqual(
[[1, 4, 6], [2, 5, None], [3, None, None]],
self.eval('[1, 2, 3].zipLongest([4, 5], [6])'))
self.assertEqual(
[[1, 4], [2, 5], [3, 0]],
self.eval('[1, 2, 3].zipLongest([4, 5], default => 0)'))
def test_repeat(self):
self.assertEqual(
[None, None],
self.eval('null.repeat(2)'))
self.assertEqual(
[1, 1, 1, 1, 1],
self.eval('1.repeat().limit(5)'))
def test_cycle(self):
self.assertEqual(
[1, 2, 1, 2, 1],
self.eval('[1, 2].cycle().take(5)'))
def test_take_while(self):
self.assertEqual(
[1, 2, 3],
self.eval('[1, 2, 3, 4, 5].takeWhile($ < 4)'))
def test_skip_while(self):
self.assertEqual(
[4, 5],
self.eval('[1, 2, 3, 4, 5].skipWhile($ < 4)'))
def test_index_of(self):
self.assertEqual(1, self.eval('[1, 2, 3, 2, 1].indexOf(2)'))
self.assertEqual(-1, self.eval('[1, 2, 3, 2, 1].indexOf(22)'))
def test_last_index_of(self):
self.assertEqual(3, self.eval('[1, 2, 3, 2, 1].lastIndexOf(2)'))
self.assertEqual(-1, self.eval('[1, 2, 3, 2, 1].lastIndexOf(22)'))
def test_index_where(self):
self.assertEqual(1, self.eval('[1, 2, 3, 2, 1].indexWhere($ = 2)'))
self.assertEqual(-1, self.eval('[1, 2, 3, 2, 1].indexWhere($ = 22)'))
def test_last_index_where(self):
self.assertEqual(3, self.eval('[1, 2, 3, 2, 1].lastIndexWhere($ = 2)'))
self.assertEqual(
-1, self.eval('[1, 2, 3, 2, 1].lastIndexWhere($ = 22)'))
def test_slice(self):
self.assertEqual(
[[1, 2], [3, 4], [5]],
self.eval('range(1, 6).slice(2)'))
def test_split(self):
self.assertEqual(
[[], [2, 3], [5]],
self.eval('range(1, 6).splitWhere($ mod 3 = 1)'))
def test_split_at(self):
self.assertEqual(
[[1, 2], [3, 4, 5]],
self.eval('range(1, 6).splitAt(2)'))
def test_slice_where(self):
self.assertEqual(
[['a', 'a'], ['b'], ['a', 'a']],
self.eval('[a,a,b,a,a].sliceWhere($ != a)'))
def test_aggregate(self):
self.assertEqual(
'aabaa',
self.eval('[a,a,b,a,a].aggregate($1 + $2)'))
self.assertRaises(
TypeError,
self.eval, '[].aggregate($1 + $2)')
self.assertEqual(
1,
self.eval('[].aggregate($1 + $2, 1)'))
self.assertEqual(
'aabaa',
self.eval('[a,a,b,a,a].reduce($1 + $2)'))
self.assertEqual(
0,
self.eval('[].reduce(max($1, $2), 0)'))
def test_accumulate(self):
self.assertEqual(
['a', 'aa', u'aab', 'aaba', 'aabaa'],
self.eval('[a,a,b,a,a].accumulate($1 + $2)'))
self.assertEqual(
[1],
self.eval('[].accumulate($1 + $2, 1)'))
def test_default_if_empty(self):
self.assertEqual(
[1, 2],
self.eval('[].defaultIfEmpty([1, 2])'))
self.assertEqual(
[3, 4],
self.eval('[3, 4].defaultIfEmpty([1, 2])'))
self.assertEqual(
[1, 2],
self.eval('[].select($).defaultIfEmpty([1, 2])'))
self.assertEqual(
[3, 4],
self.eval('[3, 4].select($).defaultIfEmpty([1, 2])'))
def test_generate(self):
self.assertEqual(
[0, 2, 4, 6, 8],
self.eval('generate(0, $ < 10, $ + 2)'))
self.assertEqual(
[0, 4, 16, 36, 64],
self.eval('generate(0, $ < 10, $ + 2, $ * $)'))
def test_max(self):
self.assertEqual(
0,
self.eval('[].max(0)'))
self.assertRaises(
TypeError,
self.eval, '[].max()')
self.assertEqual(
234,
self.eval('[44, 234, 23].max()'))
def test_min(self):
self.assertEqual(
0,
self.eval('[].min(0)'))
self.assertRaises(
TypeError,
self.eval, '[].min()')
self.assertEqual(
23,
self.eval('[44, 234, 23].min()'))
def test_reverse(self):
self.assertEqual(
[9, 4, 1],
self.eval('range(1, 4).select($*$).reverse()'))
def test_merge_with(self):
dict1 = {'a': 1, 'b': 'x', 'c': [1, 2], 'x': {'a': 1}}
dict2 = {'d': 5, 'b': 'y', 'c': [2, 3], 'x': {'b': 2}}
self.assertEqual(
{'a': 1, 'c': [1, 2, 3], 'b': 'y', 'd': 5, 'x': {'a': 1, 'b': 2}},
self.eval(
'$.d1.mergeWith($.d2)',
data={'d1': dict1, 'd2': dict2}))
dict1 = {'a': 1, 'b': 2, 'c': [1, 2]}
dict2 = {'d': 5, 'b': 3, 'c': [2, 3]}
self.assertEqual(
{'a': 1, 'c': [1, 2, 2, 3], 'b': 3, 'd': 5},
self.eval(
'$.d1.mergeWith($.d2, $1 + $2)',
data={'d1': dict1, 'd2': dict2}))
self.assertEqual(
{'a': 1, 'b': 3, 'c': [2, 3], 'd': 5},
self.eval(
'$.d1.mergeWith($.d2, $1 + $2, maxLevels => 1)',
data={'d1': dict1, 'd2': dict2}))
self.assertEqual(
{'a': 1, 'b': 2, 'c': [1, 2, 3], 'd': 5},
self.eval(
'$.d1.mergeWith($.d2,, min($1, $2))',
data={'d1': dict1, 'd2': dict2}))
def test_infinite_collections(self):
self.assertRaises(
exceptions.CollectionTooLargeException,
self.eval, 'len(list(sequence()))')
self.assertRaises(
exceptions.CollectionTooLargeException,
self.eval, 'list(sequence())')
self.assertRaises(
exceptions.CollectionTooLargeException,
self.eval, 'len(dict(sequence().select([$, $])))')
self.assertRaises(
exceptions.CollectionTooLargeException,
self.eval, 'dict(sequence().select([$, $]))')
self.assertRaises(
exceptions.CollectionTooLargeException,
self.eval, 'sequence()')
self.assertRaises(
exceptions.CollectionTooLargeException,
self.eval, 'set(sequence())')

136
yaql/tests/test_regex.py Normal file
View File

@ -0,0 +1,136 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import yaql.tests
class TestRegex(yaql.tests.TestCase):
def test_matches(self):
self.assertTrue(self.eval("regex('a.b').matches(axb)"))
self.assertFalse(self.eval("regex('a.b').matches(abx)"))
def test_matches_operator_regex(self):
self.assertTrue(self.eval("axb =~ regex('a.b')"))
self.assertFalse(self.eval("abx =~ regex('a.b')"))
def test_not_matches_operator_regex(self):
self.assertFalse(self.eval("axb !~ regex('a.b')"))
self.assertTrue(self.eval("abx !~ regex('a.b')"))
def test_matches_operator_string(self):
self.assertTrue(self.eval("axb =~ 'a.b'"))
self.assertFalse(self.eval("abx =~ 'a.b'"))
def test_not_matches_operator_string(self):
self.assertFalse(self.eval("axb !~ 'a.b'"))
self.assertTrue(self.eval("abx !~ 'a.b'"))
def test_search(self):
self.assertEqual(
'24.16',
self.eval(r"regex(`(\d+)\.?(\d+)?`).search('a24.16b')"))
def test_search_with_selector(self):
self.assertEqual(
'24.16 = 24(2-4) + 16(5-7)',
self.eval(
r"regex(`(\d+)\.?(\d+)?`).search("r"'aa24.16bb', "
r"format('{0} = {1}({2}-{3}) + {4}({5}-{6})', "
r"$.value, $2.value, $2.start, $2.end, "
r"$3.value, $3.start, $3.end))"))
def test_search_all(self):
self.assertEqual(
['24', '16'],
self.eval(r"regex(`\d+`).searchAll('a24.16b')"))
def test_search_all_with_selector(self):
self.assertEqual(
['24!', '16!'],
self.eval(r"regex(`\d+`).searchAll('a24.16b', $.value+'!')"))
def test_split(self):
self.assertEqual(
['Words', 'words', 'words', ''],
self.eval(r"regex(`\W+`).split('Words, words, words.')"))
self.assertEqual(
['Words', ', ', 'words', ', ', 'words', '.', ''],
self.eval(r"regex(`(\W+)`).split('Words, words, words.')"))
self.assertEqual(
['Words', 'words, words.'],
self.eval(r"regex(`\W+`).split('Words, words, words.', 1)"))
self.assertEqual(
['0', '3', '9'],
self.eval(r"regex('[a-f]+', ignoreCase => true).split('0a3B9')"))
def test_split_on_string(self):
self.assertEqual(
['Words', 'words', 'words', ''],
self.eval(r"'Words, words, words.'.split(regex(`\W+`))"))
self.assertEqual(
['Words', ', ', 'words', ', ', 'words', '.', ''],
self.eval(r"'Words, words, words.'.split(regex(`(\W+)`))"))
self.assertEqual(
['Words', 'words, words.'],
self.eval(r"'Words, words, words.'.split(regex(`\W+`), 1)"))
self.assertEqual(
['0', '3', '9'],
self.eval(r"'0a3B9'.split(regex('[a-f]+', ignoreCase => true))"))
def test_replace(self):
self.assertEqual(
'axxbxx',
self.eval(r"regex(`\d+`).replace(a12b23, xx)"))
self.assertEqual(
'axxb23',
self.eval(r"regex(`\d+`).replace(a12b23, xx, 1)"))
def test_replace_on_string(self):
self.assertEqual(
'axxbxx',
self.eval(r"a12b23.replace(regex(`\d+`), xx)"))
self.assertEqual(
'axxb23',
self.eval(r"a12b23.replace(regex(`\d+`), xx, 1)"))
def test_replace_by(self):
self.assertEqual(
'axxbyy',
self.eval(r"regex(`\d+`).replaceBy(a12b23, "
r"let(a => int($.value)) -> switch("
r"$a < 20 => xx, true => yy))"))
self.assertEqual(
'axxb23',
self.eval(r"regex(`\d+`).replaceBy(a12b23, "
r"let(a => int($.value)) -> switch("
r"$a < 20 => xx, true => yy), 1)"))
def test_replace_by_on_string(self):
self.assertEqual(
'axxbyy',
self.eval(r"a12b23.replaceBy(regex(`\d+`), "
r"with(int($.value)) -> switch("
r"$ < 20 => xx, true => yy))"))
self.assertEqual(
'axxb23',
self.eval(r"a12b23.replaceBy(regex(`\d+`), "
r"let(a => int($.value)) -> switch("
r"$a < 20 => xx, true => yy), 1)"))
def test_escape_regex(self):
self.assertEqual(
'\\[',
self.eval(r"escapeRegex('[')"))

138
yaql/tests/test_resolution.py Normal file
View File

@ -0,0 +1,138 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from yaql.language import exceptions
from yaql.language import specs
from yaql.language import yaqltypes
import yaql.tests
class TestResolution(yaql.tests.TestCase):
def test_resolve_parameter_count_single_layer(self):
def f1(a):
return a
def f2(a, b):
return a + b
self.context.register_function(f1, name='f')
self.context.register_function(f2, name='f')
self.assertEqual(12, self.eval('f(12)'))
self.assertEqual(25, self.eval('f(12, 13)'))
def test_resolve_parameter_count_multi_layer(self):
def f1(a):
return a
def f2(a, b):
return a + b
context1 = self.context.create_child_context()
context1.register_function(f1, name='f')
context2 = context1.create_child_context()
context2.register_function(f2, name='f')
self.assertEqual(12, self.eval('f(12)', context=context2))
self.assertEqual(25, self.eval('f(12, 13)', context=context2))
def test_layer_override(self):
def f1(a):
return a
def f2(a):
return -a
context1 = self.context.create_child_context()
context1.register_function(f1, name='f')
context2 = context1.create_child_context()
context2.register_function(f2, name='f')
self.assertEqual(-12, self.eval('f(12)', context=context2))
def test_single_layer_ambiguity(self):
def f1(a):
return a
def f2(a):
return -a
context1 = self.context.create_child_context()
context1.register_function(f1, name='f')
context1.register_function(f2, name='f')
self.assertRaises(
exceptions.AmbiguousFunctionException,
self.eval, 'f(12)', context=context1)
def test_single_layer_laziness_ambiguity(self):
@specs.parameter('a', yaqltypes.Lambda())
def f1(a):
return a()
def f2(a):
return -a
def f3(a, b):
return a + b
context1 = self.context.create_child_context()
context1.register_function(f1, name='f')
context1.register_function(f2, name='f')
context1.register_function(f3, name='f')
self.assertRaises(
exceptions.AmbiguousFunctionException,
self.eval, 'f(2 * $)', data=3, context=context1)
self.assertEqual(25, self.eval('f(12, 13)', context=context1))
def test_multi_layer_laziness_ambiguity(self):
@specs.parameter('a', yaqltypes.Lambda())
def f1(a, b):
return a() + b
@specs.parameter('a', yaqltypes.Lambda())
def f2(a, b):
return a() + b
@specs.parameter('b', yaqltypes.Lambda())
def f3(a, b):
return -a - b()
@specs.parameter('a', yaqltypes.Lambda())
def f4(a, b):
return -a() + b
context1 = self.context.create_child_context()
context1.register_function(f1, name='foo')
context1.register_function(f2, name='bar')
context2 = context1.create_child_context()
context2.register_function(f3, name='foo')
context2.register_function(f4, name='bar')
self.assertRaises(
exceptions.AmbiguousFunctionException,
self.eval, 'foo(12, 13)', context=context2)
self.assertEqual(
1,
self.eval('bar(12, 13)', context=context2))
def test_ambiguous_method(self):
self.context.register_function(
lambda c, s: 1, name='select', method=True)
self.assertRaises(
exceptions.AmbiguousMethodException,
self.eval, '[1,2].select($)')

194
yaql/tests/test_strings.py
View File

@ -1,4 +1,6 @@
# Copyright (c) 2014 Mirantis, Inc.
# -*- coding: utf-8 -*-
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -11,37 +13,183 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from yaql.language.exceptions import YaqlExecutionException
from yaql.tests import YaqlTest
from yaql.language import exceptions
import yaql.tests
import unittest
class TestStrings(yaql.tests.TestCase):
def test_scalar(self):
self.assertEqual("some \ttext", self.eval("'some \\ttext'"))
self.assertEqual(r"\\", self.eval(r"'\\\\'"))
self.assertEqual("some \"text\"", self.eval(r'"some \"text\""'))
def test_verbatim_strings(self):
self.assertEqual('c:\\f\\x', self.eval(r"`c:\f\x`"))
self.assertEqual('`', self.eval(r"`\``"))
self.assertEqual('\\n', self.eval(r"`\n`"))
self.assertEqual(r"\\", self.eval(r"`\\`"))
class TestStrings(YaqlTest):
def test_string_concat(self):
expression = "abc + cdef + ' qw er'"
self.assertEquals('abccdef qw er', self.eval(expression))
def test_len(self):
self.assertEqual(3, self.eval('len(abc)'))
def test_string_to_list(self):
expression = "abc.asList()"
expression2 = "abc.asList()[1]"
self.assertEquals(['a', 'b', 'c'], self.eval(expression))
self.assertEquals('b', self.eval(expression2))
def test_to_upper(self):
self.assertEqual('QQ', self.eval('qq.toUpper()'))
self.assertEqual(u'ПРИВЕТ', self.eval(u'Привет.toUpper()'))
def test_string_conversion_function(self):
self.assertEval("42", "string(42)")
def test_to_lower(self):
self.assertEqual('qq', self.eval('QQ.toLower()'))
self.assertEqual(u'привет', self.eval(u'Привет.toLower()'))
def test_string_conversion_method(self):
self.assertEval("42", "42.to_string()")
def test_eq(self):
self.assertTrue(self.eval('a = a'))
self.assertFalse(self.eval('a = b'))
def test_string_conversion_method_as_function(self):
self.assertEval("42", "to_string(42)")
def test_neq(self):
self.assertFalse(self.eval('a != a'))
self.assertTrue(self.eval('a != b'))
def test_unable_to_call_string_as_method(self):
self.assertRaises(YaqlExecutionException, self.eval, "42.string")
def test_is_string(self):
self.assertTrue(self.eval('isString(abc)'))
self.assertFalse(self.eval('isString(null)'))
self.assertFalse(self.eval('isString(123)'))
self.assertFalse(self.eval('isString(true)'))
def test_split(self):
self.assertEqual(
['some', 'text'],
self.eval("$.split('\\n')", data='some\ntext'))
if __name__ == '__main__':
unittest.main()
def test_rsplit(self):
self.assertEqual(
['one\ntwo', 'three'],
self.eval("$.rightSplit('\\n', 1)", data='one\ntwo\nthree'))
def test_join(self):
self.assertEqual('some-text', self.eval("[some, text].join('-')"))
def test_is_empty(self):
self.assertTrue(self.eval("isEmpty('')"))
self.assertTrue(self.eval("isEmpty(null)"))
self.assertTrue(self.eval("null.isEmpty()"))
self.assertTrue(self.eval("isEmpty(' ')"))
self.assertFalse(self.eval("isEmpty(' x')"))
def test_norm(self):
self.assertIsNone(self.eval("norm('')"))
self.assertIsNone(self.eval("norm(null)"))
self.assertIsNone(self.eval("norm(' ')"))
self.assertEqual('x', self.eval("norm(' x')"))
def test_replace(self):
self.assertEqual('AxxD', self.eval("ABBD.replace(B, x)"))
self.assertEqual('AxxD', self.eval("ABxD.replace(B, x, 1)"))
def test_replace_with_dict(self):
self.assertEqual(
'Az1D',
self.eval('AxyD.replace({x => z, y => 1})'))
self.assertEqual(
'Ayfalse2D!', self.eval(
"A122Dnull.replace({1 => y, 2 => false, null => '!'}, 1)"))
def test_in(self):
self.assertTrue(self.eval("B in ABC"))
self.assertFalse(self.eval("D in ABC"))
def test_str(self):
self.assertEqual('null', self.eval('str(null)'))
self.assertEqual('true', self.eval('str(true)'))
self.assertEqual('false', self.eval('str(false)'))
self.assertEqual('12', self.eval("str('12')"))
def test_join_seq(self):
self.assertEqual(
'text-1-null-true',
self.eval("[text, 1, null, true].select(str($)).join('-')"))
def test_concat_plus(self):
self.assertEqual('abc', self.eval("a +b + c"))
def test_concat_func(self):
self.assertEqual('abc', self.eval("concat(a, b, c)"))
def test_format(self):
self.assertEqual('a->b', self.eval("'{0}->{x}'.format(a, x => b)"))
self.assertEqual('a->b', self.eval("format('{0}->{x}', a, x => b)"))
def test_trim(self):
self.assertEqual('x', self.eval("' x '.trim()"))
self.assertEqual('x', self.eval("'abxba'.trim(ab)"))
def test_trim_left(self):
self.assertEqual('x ', self.eval("' x '.trimLeft()"))
self.assertEqual('xba', self.eval("'abxba'.trimLeft(ab)"))
def test_trim_right(self):
self.assertEqual(' x', self.eval("' x '.trimRight()"))
self.assertEqual('abx', self.eval("'abxba'.trimRight(ab)"))
def test_multiplication(self):
self.assertEqual('xxx', self.eval("x * 3"))
self.assertEqual('xxx', self.eval("3 * x"))
def test_substring(self):
data = 'abcdef'
self.assertEqual('cdef', self.eval('$.substring(2)', data=data))
self.assertEqual('ef', self.eval('$.substring(-2)', data=data))
self.assertEqual('cde', self.eval('$.substring(2, 3)', data=data))
self.assertEqual('de', self.eval('$.substring(-3, 2)', data=data))
self.assertEqual('bcdef', self.eval('$.substring(1, -1)', data=data))
self.assertEqual('bcdef', self.eval('$.substring(-5, -1)', data=data))
def test_index_of(self):
data = 'abcdefedcba'
self.assertEqual(2, self.eval('$.indexOf(c)', data=data))
self.assertEqual(2, self.eval('$.indexOf(c, 2)', data=data))
self.assertEqual(-1, self.eval('$.indexOf(x)', data=data))
self.assertEqual(5, self.eval('$.indexOf(f, 3)', data=data))
self.assertEqual(2, self.eval('$.indexOf(c, 0, 3)', data=data))
self.assertEqual(-1, self.eval('$.indexOf(c, 0, 2)', data=data))
self.assertEqual(9, self.eval('$.indexOf(b, 2, -1)', data=data))
def test_last_index_of(self):
data = 'abcdefedcbabc'
self.assertEqual(12, self.eval('$.lastIndexOf(c)', data=data))
self.assertEqual(2, self.eval('$.lastIndexOf(c, 0, 4)', data=data))
self.assertEqual(-1, self.eval('$.lastIndexOf(c, 3, 4)', data=data))
def test_max(self):
self.assertEqual('z', self.eval('max(a, z)'))
def test_min(self):
self.assertEqual('a', self.eval('min(a, z)'))
def test_to_char_array(self):
self.assertEqual(['a', 'b', 'c'], self.eval('abc.toCharArray()'))
def test_characters(self):
self.assertItemsEqual(
['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'],
self.eval('characters(octdigits => true, digits => true)'))
def test_starts_with(self):
self.assertTrue(self.eval("ABC.startsWith(A)"))
self.assertTrue(self.eval("ABC.startsWith(B, A)"))
self.assertFalse(self.eval("ABC.startsWith(C)"))
self.assertRaises(
exceptions.NoMatchingMethodException,
self.eval, "ABC.startsWith(null)")
def test_ends_with(self):
self.assertTrue(self.eval("ABC.endsWith(C)"))
self.assertTrue(self.eval("ABC.endsWith(B, C)"))
self.assertFalse(self.eval("ABC.endsWith(B)"))
self.assertRaises(
exceptions.NoMatchingMethodException,
self.eval, "ABC.endsWith(null)")
def test_hex(self):
self.assertEqual('0xff', self.eval('hex(255)'))
self.assertEqual('-0x2a', self.eval('hex(-42)'))

226
yaql/tests/test_system.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2014 Mirantis, Inc.
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -12,197 +12,57 @@
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from yaql.tests import YaqlTest
import yaql
from yaql.language.engine import parameter
from yaql.language.exceptions import YaqlExecutionException
import yaql.tests
class TestSystem(YaqlTest):
class TestSystem(yaql.tests.TestCase):
def test_def(self):
self.assertEqual(
[1, 4, 9],
self.eval('def(sq, $*$) -> $.select(sq($))', data=[1, 2, 3]))
self.assertEqual(
[1, 4, 9],
self.eval('def(sq, $arg * $arg) -> $.select(sq(arg => $))',
data=[1, 2, 3]))
def test_string_concat(self):
self.assertEquals("abcqwe", self.eval('abc + qwe'))
self.assertEquals("abc qwe", self.eval("abc + ' ' + qwe"))
def test_def_recursion(self):
self.assertEqual(24, self.eval(
'def(rec, switch($ = 1 => 1, true => $*rec($-1))) -> rec($)',
data=4))
def test_get_context_data(self):
obj = object()
self.assertEquals(obj, self.eval('$', obj))
def test_elvis_dict(self):
self.assertEqual(1, self.eval('$?.a', data={'a': 1}))
self.assertIsNone(self.eval('$?.a', data=None))
def test_get_object_attribution(self):
class Foo(object):
def __init__(self, value):
self.bar = value
def test_elvis_method(self):
self.assertEqual([2, 3], self.eval('$?.select($+1)', data=[1, 2]))
self.assertIsNone(self.eval('$?.select($+1)', data=None))
foo = Foo(42)
self.assertEquals(42, self.eval('$.bar', foo))
bar = Foo(foo)
self.assertEquals(42, self.eval('$.bar.bar', bar))
def test_unpack(self):
self.assertEqual(
5, self.eval('[2, 3].unpack() -> $1 + $2'))
def test_missing_object_property_attribution(self):
class Foo(object):
def __init__(self, value):
self.bar = value
def test_unpack_with_names(self):
self.assertEqual(
5, self.eval('[2, 3].unpack(a, b) -> $a + $b'))
foo = Foo(42)
self.assertRaises(YaqlExecutionException,
self.eval, '$.foo.missing', foo)
self.assertRaises(YaqlExecutionException,
self.eval, '$.foo.missing', {'foo': 'bar'})
self.assertRaises(
ValueError,
self.eval, '[2, 3].unpack(a, b, c) -> $a + $b')
def test_int_bool_resolving(self):
@parameter('param', arg_type=int)
def int_func(param):
return "int: " + str(param)
self.assertRaises(
ValueError,
self.eval, '[2, 3].unpack(a) -> $a')
@parameter('param', arg_type=bool)
def bool_func(param):
return "bool: " + str(param)
def test_assert(self):
self.assertEqual(
[3, 4],
self.eval('[2, 3].assert(len($) > 1).select($ + 1)'))
context1 = yaql.create_context(False)
context2 = yaql.create_context(False)
context3 = yaql.create_context(False)
context4 = yaql.create_context(False)
self.assertRaises(
AssertionError,
self.eval, '[2].assert(len($) > 1).select($ + 1)')
context1.register_function(int_func, 'foo')
context2.register_function(bool_func, 'foo')
context3.register_function(int_func, 'foo')
context3.register_function(bool_func, 'foo')
context4.register_function(bool_func, 'foo')
context4.register_function(int_func, 'foo')
self.assertEquals("int: 1", self.eval('foo(1)', context=context1))
self.assertEquals("int: 0", self.eval('foo(0)', context=context1))
self.assertRaises(YaqlExecutionException,
self.eval, "foo('1')", context=context1)
self.assertRaises(YaqlExecutionException,
self.eval, 'foo(1)', context=context2)
self.assertEquals("bool: True",
self.eval('foo(true)', context=context2))
self.assertEquals("bool: False",
self.eval('foo(false)', context=context2))
self.assertRaises(YaqlExecutionException,
self.eval, "foo(1)", context=context2)
self.assertRaises(YaqlExecutionException,
self.eval, 'foo(0)', context=context2)
self.assertRaises(YaqlExecutionException,
self.eval, 'foo(True)', context=context2)
self.assertRaises(YaqlExecutionException,
self.eval, "foo('true')", context=context2)
self.assertEquals("int: 1", self.eval('foo(1)', context=context3))
self.assertEquals("int: 0", self.eval('foo(0)', context=context3))
self.assertEquals("bool: True",
self.eval('foo(true)', context=context3))
self.assertEquals("bool: False",
self.eval('foo(false)', context=context3))
self.assertEquals("int: 1", self.eval('foo(1)', context=context4))
self.assertEquals("int: 0", self.eval('foo(0)', context=context4))
self.assertEquals("bool: True",
self.eval('foo(true)', context=context4))
self.assertEquals("bool: False",
self.eval('foo(false)', context=context4))
def test_get_dict_attribution(self):
d = {
'key1': 'string1',
'key2': {
'inner': {
'last': 42,
'lastString': 'string'
}
},
'composite key': 3
}
self.assertEquals('string1', self.eval('$.key1', d))
self.assertEquals('string', self.eval('$.key2.inner.lastString', d))
self.assertEquals(42, self.eval('$.key2.inner.last', d))
self.assertEquals(3, self.eval("$.'composite key'", d))
def test_missing_key_dict_attributions(self):
d = {
'key1': 'string1',
'key2': {
'inner': {
'last': 42,
'lastString': 'string'
}
},
'composite key': 3
}
self.assertEquals(None, self.eval("$.'missing key'", d))
self.assertEquals(None, self.eval("$.key2.missing", d))
def test_function_call(self):
def foo():
return 42
self.context.register_function(foo, 'test')
self.assertEquals(42, self.eval("test()"))
def test_composite_function_call_1(self):
def foo():
return 42
self.context.register_function(foo, 'long.namespace.based.name')
self.assertEval(42, "'long.namespace.based.name'()")
def test_composite_function_call_2(self):
def foo():
return 42
self.context.register_function(foo, 'some spaced name\'s')
self.assertEval(42, "'some spaced name\\'s'()")
def test_return_same_function(self):
def foo(bar):
return bar
self.context.register_function(foo, 'foo')
self.assertEquals('bar', self.eval('foo(bar)'))
def test_return_same_method(self):
def foo(self):
return self
self.context.register_function(foo, 'foo')
self.assertEquals('bar', self.eval('bar.foo()'))
def test_self_reordering(self):
def concat_right(self, arg):
return self + ',' + arg
@parameter('self', is_self=True)
def concat_left(arg, self):
return arg + ',' + self
self.context.register_function(concat_right, 'concat1')
self.context.register_function(concat_left, 'concat2')
self.assertEquals('abc,qwe', self.eval('abc.concat1(qwe)'))
self.assertEquals('qwe,abc', self.eval('abc.concat2(qwe)'))
def test_parenthesis(self):
expression = '(2+3)*2'
self.assertEquals(10, self.eval(expression))
def test_as(self):
@parameter('f', lazy=True)
def foo(self, f):
return (self, f())
self.context.register_function(foo)
expression = "(random()).as($*10=>random_by_ten).foo($random_by_ten)"
v = self.eval(expression)
self.assertTrue(v[1] == v[0] * 10)
def test_switch(self):
expression = "$.switch(($>5)=>$, ($>2)=>('_'+string($)), true=>0)"
self.assertEval(10, expression, 10)
self.assertEval("_4", expression, 4)
self.assertEval(0, expression, 1)
if __name__ == '__main__':
unittest.main()
self.assertEqual(
3,
self.eval('[2].select($ + 1).assert(len($) = 1).first()'))

58
yaql/tests/test_tuples.py
View File

@ -1,58 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from yaql.tests import YaqlTest
class TestTuples(YaqlTest):
def test_build_tuple(self):
expression = 'a=>b'
self.assertEquals(('a', 'b'), self.eval(expression))
def test_build_triple_tuple(self):
expression = 'a=>b=>c'
self.assertEquals(('a', 'b', 'c'), self.eval(expression))
def test_build_5x_tuple(self):
expression = 'a=>b=>c=>d=>e'
self.assertEquals(('a', 'b', 'c', 'd', 'e'), self.eval(expression))
def test_build_nested_tuple1(self):
expression = 'a=>(b=>c)'
self.assertEquals(('a', ('b', 'c')), self.eval(expression))
def test_build_nested_tuple2(self):
expression = '(a=>b)=>(c=>d)'
self.assertEquals((('a', 'b'), ('c', 'd')), self.eval(expression))
def test_build_nested_tuple3(self):
expression = '(a=>b)=>(c=>d)=>(e=>f)'
self.assertEquals((('a', 'b'), ('c', 'd'), ('e', 'f')),
self.eval(expression))
def test_build_nested_tuple4(self):
expression = 'a=>(b=>c)=>(d=>(e=>f=>g=>h))=>i'
self.assertEquals(('a', ('b', 'c'), ('d', ('e', 'f', 'g', 'h')), 'i'),
self.eval(expression))
def test_tuple_precedence(self):
expression1 = 'a=>2+3'
expression2 = '2+3=>a'
self.assertEquals(('a', 5), self.eval(expression1))
self.assertEquals((5, 'a'), self.eval(expression2))
if __name__ == '__main__':
unittest.main()

77
yaql/tests/test_varargs.py
View File

@ -1,77 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from yaql.language.engine import parameter
from yaql.tests import YaqlTest
def foo(*args):
return [arg.upper() for arg in args]
def bar(self, *args):
return [arg + self for arg in args]
@parameter('predicates', lazy=True, function_only=True)
def buz(*predicates):
i = 1
for predicate in predicates:
if predicate(i):
yield i
else:
yield 0
i += 1
@parameter('predicates', lazy=True, function_only=True)
def qux(self, *predicates):
return [predicate(self) for predicate in predicates]
class TestVarArgs(YaqlTest):
def setUp(self):
super(TestVarArgs, self).setUp()
self.context.register_function(foo)
self.context.register_function(bar)
self.context.register_function(buz)
self.context.register_function(qux)
def test_varargs_only(self):
expression = "foo(abc, cde, qwerty)"
self.assertEval(['ABC', 'CDE', 'QWERTY'], expression)
def test_combined_args_and_varargs_as_method(self):
expression = "data.bar(abc, cde, qwerty)"
self.assertEval(['abcdata', 'cdedata', 'qwertydata'], expression)
def test_combined_args_and_varargs_as_function(self):
expression = "bar(data, abc, cde, qwerty)"
self.assertEval(['abcdata', 'cdedata', 'qwertydata'], expression)
def test_predicate_varargs_only(self):
expression = "buz($>0, $!=2, $=3, $<4)"
self.assertEval([1, 0, 3, 0], expression)
def test_predicate_args_and_varargs_as_method(self):
expression = "10.qux($*2, $/2, $/5.0, $.to_string(), string($))"
self.assertEval([20, 5, 2.0, "10", "10"], expression)
def test_predicate_args_and_varargs_as_function(self):
expression = "qux(10, $*2, $/2, $/5.0, $.to_string(), string($))"
self.assertEval([20, 5, 2.0, "10", "10"], expression)
if __name__ == '__main__':
unittest.main()

71
yaql/tests/testdata.py
View File

@ -1,71 +0,0 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def process_customer(customer):
return customer.email
class Customer():
def __init__(self, _id, email):
self.id = _id
self.email = email
self.list_prop = [1, 2, 3, 4]
ns = {'com.examples.test.Symbol': 'Some Test NS-based data'}
users = [Customer(1, 'user1@example.com'),
Customer(2, 'user2@example.com'),
Customer(3, 'user3@example.com')]
services = [
{
'com.mirantis.murano.yaql.name': 'Service1',
'com.mirantis.murano.yaql.version': '1.5.3.1237',
'com.mirantis.murano.yaql.position': 1,
'com.mirantis.murano.yaql.description': 'Some Windows service',
'com.mirantis.murano.yaql.owner': 1,
'com.mirantis.murano.yaql.parent_service': 'com.mirantis.murano.'
'examples.Service0'
},
{
'com.mirantis.murano.yaql.name': 'Service2',
'com.mirantis.murano.yaql.version': '2.1',
'com.mirantis.murano.yaql.position': 2,
'com.mirantis.murano.yaql.description': 'Another Windows service',
'com.mirantis.murano.yaql.owner': 1,
'com.mirantis.murano.yaql.parent_service': None
},
{
'com.mirantis.murano.yaql.name': 'Service3',
'com.mirantis.murano.yaql.version': None,
'com.mirantis.murano.yaql.position': 3,
'com.mirantis.murano.yaql.description': 'Some Linux service',
'com.mirantis.murano.yaql.owner': 2,
'com.mirantis.murano.yaql.parent_service': None
},
{
'com.mirantis.murano.yaql.name': 'Service4',
'com.mirantis.murano.yaql.version': '1.0',
'com.mirantis.murano.yaql.position': 4,
'com.mirantis.murano.yaql.description': 'Some MacOS service',
'com.mirantis.murano.yaql.owner': 3,
'com.mirantis.murano.yaql.parent_service': 'com.mirantis.murano.'
'examples.Service0'
},
]
data = {'users': users, 'services': services, 'ns': ns}