From fb7201b5d5f917fd11cce3fe007c2f48051af5c2 Mon Sep 17 00:00:00 2001 From: Evgeniy L Date: Tue, 29 Dec 2015 21:08:02 +0300 Subject: [PATCH] Add yaql template language to spaces description --- bareon_dynamic_allocator/allocators.py | 30 +++++-- bareon_dynamic_allocator/parser.py | 82 +++++++++++++++++++ bareon_dynamic_allocator/yaql_parser.py | 35 -------- .../example_2_disks.yaml | 5 +- .../simple_schema.yaml | 68 ++++++--------- 5 files changed, 133 insertions(+), 87 deletions(-) create mode 100644 bareon_dynamic_allocator/parser.py delete mode 100644 bareon_dynamic_allocator/yaql_parser.py diff --git a/bareon_dynamic_allocator/allocators.py b/bareon_dynamic_allocator/allocators.py index afc56b6..7df0400 100644 --- a/bareon_dynamic_allocator/allocators.py +++ b/bareon_dynamic_allocator/allocators.py @@ -13,16 +13,15 @@ # under the License. import itertools -import six +import six +import numpy as np +from termcolor import colored +from oslo_log import log from scipy.optimize import linprog from scipy.ndimage.interpolation import shift -from termcolor import colored - -import numpy as np - -from oslo_log import log +from bareon_dynamic_allocator.parser import Parser LOG = log.getLogger(__name__) @@ -89,20 +88,33 @@ class Space(object): for k, v in six.iteritems(kwargs): setattr(self, k, v) + # If no min_size specified set it to 0 + if not kwargs.get('min_size'): + self.min_size = 0 + + # Exact size can be repreneted as min_size and max_size + if kwargs.get('size'): + self.min_size = kwargs.get('size') + self.max_size = kwargs.get('size') + class DynamicAllocator(object): def __init__(self, hw_info, schema): + LOG.debug('Hardware information: \n%s', hw_info) + LOG.debug('Spaces schema: \n%s', schema) self.disks = [Disk(**disk) for disk in hw_info['disks']] - self.spaces = [Space(**space) for space in schema] + rendered_spaces = Parser(schema, hw_info).parse() + LOG.debug('Rendered spaces schema: \n%s', rendered_spaces) + self.spaces = [Space(**space) for space in rendered_spaces] # Add fake volume Unallocated, in order to be able # to have only volumes with minimal size, without # additional space allocation - self.lp = DynamicAllocationLinearProgram(self.disks, self.spaces) + self.solver = DynamicAllocationLinearProgram(self.disks, self.spaces) def generate_static(self): - sizes = self.lp.solve() + sizes = self.solver.solve() return sizes diff --git a/bareon_dynamic_allocator/parser.py b/bareon_dynamic_allocator/parser.py new file mode 100644 index 0000000..ccdb9e2 --- /dev/null +++ b/bareon_dynamic_allocator/parser.py @@ -0,0 +1,82 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then +# License for the specific language governing permissions and limitations +# under the License. + + +import yaml +import yaql +import six +import re + + +def seq_iter(obj): + if isinstance(obj, dict): + for k, v in six.iteritems(obj): + yield k, v + elif isinstance(obj, list): + for i in xrange(len(obj)): + yield i, obj[i] + + +class YAQLParser(object): + engine_options = { + 'yaql.limitIterators': 100, + 'yaql.treatSetsAsLists': True, + 'yaql.memoryQuota': 10000 + } + + def __init__(self, data, context): + self.factory = yaql.YaqlFactory() + self.parser = self.factory.create(options=self.engine_options) + self.context = context + self.data = data + + def parse(self): + return self.parser(self.data).evaluate(self.context) + + +class NoopParser(object): + + def __init__(self, data, _): + self.data = data + + def parse(self): + return self.data + + +class Parser(object): + + yaql_re = re.compile(r'^\s*yaql\s*=\s*') + + def __init__(self, template, context): + self.template = template + self.context = context + + def parse(self): + return self._walk(self.template) + + def _walk(self, node): + if isinstance(node, six.string_types): + return self.get_parser(node).parse() + + for key, item in seq_iter(node): + node[key] = self._walk(item) + + return node + + def get_parser(self, node): + if self.yaql_re.match(node): + wo_prefix = self.yaql_re.sub('', node) + return YAQLParser(wo_prefix, self.context) + + return NoopParser(node, self.context) diff --git a/bareon_dynamic_allocator/yaql_parser.py b/bareon_dynamic_allocator/yaql_parser.py deleted file mode 100644 index 768aec1..0000000 --- a/bareon_dynamic_allocator/yaql_parser.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See then -# License for the specific language governing permissions and limitations -# under the License. - - -import yaml -import yaql - -data = yaml.load(open('/Users/eli/job/bareon-dynamic-allocator/etc/bareon-dynamic-allocator/example_2_disks.yaml').read()) - - -engine_options = { - 'yaql.limitIterators': 100, - 'yaql.treatSetsAsLists': True, - 'yaql.memoryQuota': 10000 -} - - -factory = yaql.YaqlFactory() -parser = factory.create(options=engine_options) - -from pprint import pprint -pprint(parser('$.disks.where($.type = "hdd" or ($.type = "ssd" and $.dev = "/dev/sda")) ').evaluate(data)) -pprint(parser('$.disks.where(($.type = "ssd" and $.dev = "/dev/sda") or $.type = "hdd") ').evaluate(data)) -print parser('$.get(ram1, 1000) + 2').evaluate(data) diff --git a/etc/bareon-dynamic-allocator/example_2_disks.yaml b/etc/bareon-dynamic-allocator/example_2_disks.yaml index ceddd1f..bbc78ba 100644 --- a/etc/bareon-dynamic-allocator/example_2_disks.yaml +++ b/etc/bareon-dynamic-allocator/example_2_disks.yaml @@ -1,10 +1,11 @@ +ram: 2 disks: - id: /dev/disk/by-id/id-for-sda path: /dev/disk/by-path/path-for-sda dev: /dev/sda type: hdd vendor: Hitachi - size: 30 + size: 40 - id: /dev/disk/by-id/id-for-sdb path: /dev/disk/by-path/path-for-sdb @@ -16,6 +17,6 @@ disks: - id: /dev/disk/by-id/id-for-sdc path: /dev/disk/by-path/path-for-sdc dev: /dev/sdc - type: sdd + type: ssd vendor: Samsung size: 40 diff --git a/etc/bareon-dynamic-allocator/simple_schema.yaml b/etc/bareon-dynamic-allocator/simple_schema.yaml index e91c36e..f7d6cba 100644 --- a/etc/bareon-dynamic-allocator/simple_schema.yaml +++ b/etc/bareon-dynamic-allocator/simple_schema.yaml @@ -4,56 +4,42 @@ max_size: 20 mount: / fs_type: ext4 - weight: 1 + +- id: swap + type: lv + # Calc swap size according to RAM. + # | RAM | Recommended swap space | + # |--------------+-----------------------------| + # | <= 2GB | 2 times the amount of RAM | + # | > 2GB – 8GB | Equal to the amount of RAM | + # | > 8GB – 64GB | 0.5 times the amount of RAM | + # | > 64GB | 4GB of swap space | + # Source https://access.redhat.com/site/documentation/en-US/ + # Red_Hat_Enterprise_Linux/6/html/Installation_Guide/ + # s2-diskpartrecommend-ppc.html#id4394007 + size: | + yaql=let(ram => $.get(ram, 1024)) -> + selectCase( + $ram <= 2048, + $ram > 2048 and $ram < 8192, + $ram > 8192 and $ram < 65536). + switchCase( + $ram * 2, + $ram, + $ram / 2, + 4096) + fs_type: swap - id: test type: lv min_size: 0 - # max_size: 30 mount: /test fs_type: ext4 - weight: 0.2 - -- id: swap - type: lv - min_size: 0 - # max_size: 5 - fs_type: swap - weight: 0.5 + best_with_disks: | + yaql=$.disks.where($.type = "ssd") - id: partition type: partition min_size: 0 fs_type: ext4 mount: /var/www - weight: 0.5 -############# - -# - id: root2 -# type: lv -# min_size: 10 -# mount: / -# fs_type: ext4 -# weight: 1 - -# - id: test2 -# type: lv -# min_size: 30 -# mount: /test -# fs_type: ext4 -# weight: 1 - -# - id: swap2 -# # A single partition -# type: partition -# min_size: 5 -# max_size: 5 -# fs_type: swap - -# - id: swap2 -# # A set of partitions allocated -# # over several disks -# type: partitions -# min_size: 5 -# max_size: 5 -# fs_type: swap