From 53e71f05ecd6e25e9460a5ded7535c5340e38fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Thu, 17 Dec 2015 16:36:28 +0100 Subject: [PATCH] Advanced tags support It allows to search resources using advanced queries like: solar resource show --tag 'location=node1 & resource=hosts_file' solar resource show --tag 'resource=hosts_file | riak=*' DocImpact Change-Id: I25cf1522bf83b7909b9d60cfe0baf4665b81ef27 --- examples/riak/riak_service.yaml | 12 ++--- solar/core/resource/resource.py | 22 +++++--- solar/core/tags_set_parser.py | 69 ++++++++++++++++++++----- solar/test/test_operations_with_tags.py | 36 +++++++++++-- tox.ini | 2 +- 5 files changed, 112 insertions(+), 29 deletions(-) diff --git a/examples/riak/riak_service.yaml b/examples/riak/riak_service.yaml index acf931d2..ad2c983d 100644 --- a/examples/riak/riak_service.yaml +++ b/examples/riak/riak_service.yaml @@ -14,21 +14,21 @@ resources: ip: {{node}}::ip updates: - - with_tags: ['resource=hosts_file'] + - with_tags: 'resource=hosts_file' values: - hosts:name: + hosts:name: - riak_service{{index}}::riak_hostname::NO_EVENTS hosts:ip: - riak_service{{index}}::ip::NO_EVENTS - - with_tags: ['resource=haproxy_service_config', 'service=riak', 'protocol=http'] + - with_tags: 'resource=haproxy_service_config & service=riak & protocol=http' values: backends:server: - riak_service{{index}}::riak_hostname backends:port: - riak_service{{index}}::riak_port_http - - with_tags: ['resource=haproxy_service_config', 'service=riak', 'protocol=tcp'] + - with_tags: 'resource=haproxy_service_config & service=riak & protocol=tcp' values: backends:server: - riak_service{{index}}::riak_hostname @@ -37,8 +37,8 @@ updates: events: - type: depends_on - parent: - with_tags: ['resource=hosts_file', 'location={{node}}'] + parent: + with_tags: 'resource=hosts_file & location={{node}}' action: run state: success depend_action: riak_service{{index}}.run diff --git a/solar/core/resource/resource.py b/solar/core/resource/resource.py index 9e4e8351..3d3b3e48 100644 --- a/solar/core/resource/resource.py +++ b/solar/core/resource/resource.py @@ -15,6 +15,7 @@ from copy import deepcopy from hashlib import md5 +import itertools import json import os from uuid import uuid4 @@ -25,6 +26,8 @@ import networkx from solar.core.signals import get_mapping +from solar.core.tags_set_parser import Expression +from solar.core.tags_set_parser import get_string_tokens from solar.core import validation from solar.dblayer.model import StrInt from solar.dblayer.solar_models import CommitedResource @@ -327,13 +330,18 @@ def load_all(): return [Resource(r) for r in DBResource.multi_get(candids)] -def load_by_tags(tags): - tags = set(tags) - candids_all = set() - for tag in tags: - candids = DBResource.tags.filter(tag) - candids_all.update(set(candids)) - return [Resource(r) for r in DBResource.multi_get(candids_all)] +def load_by_tags(query): + if isinstance(query, (list, set, tuple)): + query = '|'.join(query) + + parsed_tags = get_string_tokens(query) + r_with_tags = [DBResource.tags.filter(tag) for tag in parsed_tags] + r_with_tags = set(itertools.chain(*r_with_tags)) + candids = [Resource(r) for r in DBResource.multi_get(r_with_tags)] + + nodes = filter( + lambda n: Expression(query, n.tags).evaluate(), candids) + return nodes def validate_resources(): diff --git a/solar/core/tags_set_parser.py b/solar/core/tags_set_parser.py index 12799787..fb901269 100644 --- a/solar/core/tags_set_parser.py +++ b/solar/core/tags_set_parser.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +import re + from ply import lex from ply import yacc @@ -23,9 +25,13 @@ tokens = ( "AND", "OR", "LPAREN", - "RPAREN") + "RPAREN", + "ANY", + "EQ") t_STRING = r'[A-Za-z0-9-_/\\]+' +t_EQ = r'=' +t_ANY = r'\*' t_AND = '&|,' t_OR = r'\|' t_LPAREN = r'\(' @@ -60,10 +66,23 @@ class ScalarWrapper(object): return self.value -def p_expression_logical_op(p): - """Parser +class AnyWrapper(object): - expression : expression AND expression + def __init__(self, value): + global expression + # convert all tags from key=value to key=* + tags = map(lambda s: re.sub('=\w+', '=*', s), expression.tags) + self.value = (set([value]) <= set(tags)) + + def evaluate(self): + return self.value + + def __call__(self): + return self.value + + +def p_expression_logical_op(p): + """expression : expression AND expression | expression OR expression """ result, arg1, op, arg2 = p @@ -76,18 +95,28 @@ def p_expression_logical_op(p): def p_expression_string(p): - """Parser + """expression : STRING""" + p[0] = ScalarWrapper(p[1] + '=') - expression : STRING + +def p_expression_assign(p): + """expression : STRING EQ STRING + | STRING EQ """ - p[0] = ScalarWrapper(p[1]) + if len(p) == 3: + last = '' + else: + last = p[3] + p[0] = ScalarWrapper(p[1] + p[2] + last) + + +def p_expression_assign_any(p): + """expression : STRING EQ ANY""" + p[0] = AnyWrapper(p[1] + p[2] + p[3]) def p_expression_group(p): - """Parser - - expression : LPAREN expression RPAREN - """ + """expression : LPAREN expression RPAREN""" p[0] = p[2] @@ -113,10 +142,26 @@ class Expression(object): lexer = lex.lex() -parser = yacc.yacc(debug=False, write_tables=False) +parser = yacc.yacc(debug=False, write_tables=False, errorlog=yacc.NullLogger()) expression = None +def get_string_tokens(txt): + lexer.input(txt) + parsed = [] + token_part = '' + for token in lexer: + if token.type in ['STRING', 'ANY', 'EQ']: + token_part += token.value + else: + if token_part: + parsed.append(token_part) + token_part = '' + if token_part: + parsed.append(token_part) + return parsed + + def parse(expr): global parser global expression diff --git a/solar/test/test_operations_with_tags.py b/solar/test/test_operations_with_tags.py index af4ede47..026f300c 100644 --- a/solar/test/test_operations_with_tags.py +++ b/solar/test/test_operations_with_tags.py @@ -21,16 +21,23 @@ from solar.dblayer.solar_models import Resource @fixture def tagged_resources(): - tags = ['n1', 'n2', 'n3'] + base_tags = ['n1=x', 'n2'] + tags = base_tags + ['node=t1'] t1 = Resource.from_dict('t1', {'name': 't1', 'tags': tags, 'base_path': 'x'}) t1.save_lazy() + tags = base_tags + ['node=t2'] t2 = Resource.from_dict('t2', {'name': 't2', 'tags': tags, 'base_path': 'x'}) t2.save_lazy() + tags = base_tags + ['node=t3'] t3 = Resource.from_dict('t3', {'name': 't3', 'tags': tags, 'base_path': 'x'}) t3.save_lazy() + tags = ['node=t3'] + t4 = Resource.from_dict('t4', + {'name': 't4', 'tags': tags, 'base_path': 'x'}) + t4.save_lazy() ModelMeta.save_all_lazy() return [t1, t2, t3] @@ -42,5 +49,28 @@ def test_add_remove_tags(tagged_resources): for res in loaded: res.remove_tags('n1') - assert len(resource.load_by_tags(set(['n1']))) == 0 - assert len(resource.load_by_tags(set(['n2']))) == 3 + assert len(resource.load_by_tags(set(['n1=']))) == 0 + assert len(resource.load_by_tags(set(['n2=']))) == 3 + + +def test_filter_with_and(tagged_resources): + loaded = resource.load_by_tags('node=t1 & n1=x') + assert len(loaded) == 1 + loaded = resource.load_by_tags('node=t1,n1=*') + assert len(loaded) == 1 + loaded = resource.load_by_tags('n2,n1=*') + assert len(loaded) == 3 + loaded = resource.load_by_tags('node=* & n1=x') + assert len(loaded) == 3 + + +def test_filter_with_or(tagged_resources): + loaded = resource.load_by_tags('node=t1 | node=t2') + assert len(loaded) == 2 + loaded = resource.load_by_tags('node=t1 | node=t2 | node=t3') + assert len(loaded) == 4 + + +def test_with_brackets(tagged_resources): + loaded = resource.load_by_tags('(node=t1 | node=t2 | node=t3) & n1=x') + assert len(loaded) == 3 diff --git a/tox.ini b/tox.ini index 2a38889f..f4fc34be 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ envdir = devenv usedevelop = True [flake8] -ignore = H101,H236,E731 +ignore = H101,H236,E731,H405 exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,__init__.py,docs show-pep8 = True show-source = True