diff --git a/shade/_utils.py b/shade/_utils.py index ae2065496..b5811d8fb 100644 --- a/shade/_utils.py +++ b/shade/_utils.py @@ -15,6 +15,7 @@ import contextlib import inspect import netifaces +import re import six import time @@ -535,3 +536,167 @@ def shade_exceptions(error_message=None): if error_message is None: error_message = str(e) raise exc.OpenStackCloudException(error_message) + + +def safe_dict_min(key, data): + """Safely find the minimum for a given key in a list of dict objects. + + This will find the minimum integer value for specific dictionary key + across a list of dictionaries. The values for the given key MUST be + integers, or string representations of an integer. + + The dictionary key does not have to be present in all (or any) + of the elements/dicts within the data set. + + :param string key: The dictionary key to search for the minimum value. + :param list data: List of dicts to use for the data set. + + :returns: None if the field was not not found in any elements, or + the minimum value for the field otherwise. + """ + min_value = None + for d in data: + if (key in d) and (d[key] is not None): + try: + val = int(d[key]) + except ValueError: + raise exc.OpenStackCloudException( + "Search for minimum value failed. " + "Value for {key} is not an integer: {value}".format( + key=key, value=d[key]) + ) + if (min_value is None) or (val < min_value): + min_value = val + return min_value + + +def safe_dict_max(key, data): + """Safely find the maximum for a given key in a list of dict objects. + + This will find the maximum integer value for specific dictionary key + across a list of dictionaries. The values for the given key MUST be + integers, or string representations of an integer. + + The dictionary key does not have to be present in all (or any) + of the elements/dicts within the data set. + + :param string key: The dictionary key to search for the maximum value. + :param list data: List of dicts to use for the data set. + + :returns: None if the field was not not found in any elements, or + the maximum value for the field otherwise. + """ + max_value = None + for d in data: + if (key in d) and (d[key] is not None): + try: + val = int(d[key]) + except ValueError: + raise exc.OpenStackCloudException( + "Search for maximum value failed. " + "Value for {key} is not an integer: {value}".format( + key=key, value=d[key]) + ) + if (max_value is None) or (val > max_value): + max_value = val + return max_value + + +def parse_range(value): + """Parse a numerical range string. + + Breakdown a range expression into its operater and numerical parts. + This expression must be a string. Valid values must be an integer string, + optionally preceeded by one of the following operators:: + + - "<" : Less than + - ">" : Greater than + - "<=" : Less than or equal to + - ">=" : Greater than or equal to + + Some examples of valid values and function return values:: + + - "1024" : returns (None, 1024) + - "<5" : returns ("<", 5) + - ">=100" : returns (">=", 100) + + :param string value: The range expression to be parsed. + + :returns: A tuple with the operator string (or None if no operator + was given) and the integer value. None is returned if parsing failed. + """ + if value is None: + return None + + range_exp = re.match('(<|>|<=|>=){0,1}(\d+)$', value) + if range_exp is None: + return None + + op = range_exp.group(1) + num = int(range_exp.group(2)) + return (op, num) + + +def range_filter(data, key, range_exp): + """Filter a list by a single range expression. + + :param list data: List of dictionaries to be searched. + :param string key: Key name to search within the data set. + :param string range_exp: The expression describing the range of values. + + :returns: A list subset of the original data set. + :raises: OpenStackCloudException on invalid range expressions. + """ + filtered = [] + range_exp = str(range_exp).upper() + + if range_exp == "MIN": + key_min = safe_dict_min(key, data) + if key_min is None: + return [] + for d in data: + if int(d[key]) == key_min: + filtered.append(d) + return filtered + elif range_exp == "MAX": + key_max = safe_dict_max(key, data) + if key_max is None: + return [] + for d in data: + if int(d[key]) == key_max: + filtered.append(d) + return filtered + + # Not looking for a min or max, so a range or exact value must + # have been supplied. + val_range = parse_range(range_exp) + + # If parsing the range fails, it must be a bad value. + if val_range is None: + raise exc.OpenStackCloudException( + "Invalid range value: {value}".format(value=range_exp)) + + op = val_range[0] + if op: + # Range matching + for d in data: + d_val = int(d[key]) + if op == '<': + if d_val < val_range[1]: + filtered.append(d) + elif op == '>': + if d_val > val_range[1]: + filtered.append(d) + elif op == '<=': + if d_val <= val_range[1]: + filtered.append(d) + elif op == '>=': + if d_val >= val_range[1]: + filtered.append(d) + return filtered + else: + # Exact number match + for d in data: + if int(d[key]) == val_range[1]: + filtered.append(d) + return filtered diff --git a/shade/openstackcloud.py b/shade/openstackcloud.py index 4e00ac947..4cd51526a 100644 --- a/shade/openstackcloud.py +++ b/shade/openstackcloud.py @@ -351,6 +351,53 @@ class OpenStackCloud(object): ret.update(self._get_project_param_dict(project)) return ret + def range_search(self, data, filters): + """Perform integer range searches across a list of dictionaries. + + Given a list of dictionaries, search across the list using the given + dictionary keys and a range of integer values for each key. Only + dictionaries that match ALL search filters across the entire original + data set will be returned. + + It is not a requirement that each dictionary contain the key used + for searching. Those without the key will be considered non-matching. + + The range values must be string values and is either a set of digits + representing an integer for matching, or a range operator followed by + a set of digits representing an integer for matching. If a range + operator is not given, exact value matching will be used. Valid + operators are one of: <,>,<=,>= + + :param list data: List of dictionaries to be searched. + :param dict filters: Dict describing the one or more range searches to + perform. If more than one search is given, the result will be the + members of the original data set that match ALL searches. An + example of filtering by multiple ranges:: + + {"vcpus": "<=5", "ram": "<=2048", "disk": "1"} + + :returns: A list subset of the original data set. + :raises: OpenStackCloudException on invalid range expressions. + """ + filtered = [] + + for key, range_value in filters.items(): + # We always want to operate on the full data set so that + # calculations for minimum and maximum are correct. + results = _utils.range_filter(data, key, range_value) + + if not filtered: + # First set of results + filtered = results + else: + # The combination of all searches should be the intersection of + # all result sets from each search. So adjust the current set + # of filtered data by computing its intersection with the + # latest result set. + filtered = [r for r in results for f in filtered if r == f] + + return filtered + @_utils.cache_on_arguments() def list_projects(self): """List Keystone Projects. diff --git a/shade/tests/functional/test_range_search.py b/shade/tests/functional/test_range_search.py new file mode 100644 index 000000000..eb31b7984 --- /dev/null +++ b/shade/tests/functional/test_range_search.py @@ -0,0 +1,129 @@ +# Copyright (c) 2016 IBM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + + +import shade +from shade import exc +from shade.tests import base + + +class TestRangeSearch(base.TestCase): + + def setUp(self): + super(TestRangeSearch, self).setUp() + self.cloud = shade.openstack_cloud(cloud='devstack') + + def test_range_search_bad_range(self): + flavors = self.cloud.list_flavors() + self.assertRaises(exc.OpenStackCloudException, + self.cloud.range_search, flavors, {"ram": "<1a0"}) + + def test_range_search_exact(self): + flavors = self.cloud.list_flavors() + result = self.cloud.range_search(flavors, {"ram": "4096"}) + self.assertIsInstance(result, list) + self.assertEqual(1, len(result)) + self.assertEqual("m1.medium", result[0]['name']) + + def test_range_search_min(self): + flavors = self.cloud.list_flavors() + result = self.cloud.range_search(flavors, {"ram": "MIN"}) + self.assertIsInstance(result, list) + self.assertEqual(1, len(result)) + self.assertEqual("m1.tiny", result[0]['name']) + + def test_range_search_max(self): + flavors = self.cloud.list_flavors() + result = self.cloud.range_search(flavors, {"ram": "MAX"}) + self.assertIsInstance(result, list) + self.assertEqual(1, len(result)) + self.assertEqual("m1.xlarge", result[0]['name']) + + def test_range_search_lt(self): + flavors = self.cloud.list_flavors() + result = self.cloud.range_search(flavors, {"ram": "<4096"}) + self.assertIsInstance(result, list) + self.assertEqual(2, len(result)) + flavor_names = [r['name'] for r in result] + self.assertIn("m1.tiny", flavor_names) + self.assertIn("m1.small", flavor_names) + + def test_range_search_gt(self): + flavors = self.cloud.list_flavors() + result = self.cloud.range_search(flavors, {"ram": ">4096"}) + self.assertIsInstance(result, list) + self.assertEqual(2, len(result)) + flavor_names = [r['name'] for r in result] + self.assertIn("m1.large", flavor_names) + self.assertIn("m1.xlarge", flavor_names) + + def test_range_search_le(self): + flavors = self.cloud.list_flavors() + result = self.cloud.range_search(flavors, {"ram": "<=4096"}) + self.assertIsInstance(result, list) + self.assertEqual(3, len(result)) + flavor_names = [r['name'] for r in result] + self.assertIn("m1.tiny", flavor_names) + self.assertIn("m1.small", flavor_names) + self.assertIn("m1.medium", flavor_names) + + def test_range_search_ge(self): + flavors = self.cloud.list_flavors() + result = self.cloud.range_search(flavors, {"ram": ">=4096"}) + self.assertIsInstance(result, list) + self.assertEqual(3, len(result)) + flavor_names = [r['name'] for r in result] + self.assertIn("m1.medium", flavor_names) + self.assertIn("m1.large", flavor_names) + self.assertIn("m1.xlarge", flavor_names) + + def test_range_search_multi_1(self): + flavors = self.cloud.list_flavors() + result = self.cloud.range_search(flavors, + {"ram": "MIN", "vcpus": "MIN"}) + self.assertIsInstance(result, list) + self.assertEqual(1, len(result)) + self.assertEqual("m1.tiny", result[0]['name']) + + def test_range_search_multi_2(self): + flavors = self.cloud.list_flavors() + result = self.cloud.range_search(flavors, + {"ram": "<8192", "vcpus": "MIN"}) + self.assertIsInstance(result, list) + self.assertEqual(2, len(result)) + flavor_names = [r['name'] for r in result] + # All of these should have 1 vcpu + self.assertIn("m1.tiny", flavor_names) + self.assertIn("m1.small", flavor_names) + + def test_range_search_multi_3(self): + flavors = self.cloud.list_flavors() + result = self.cloud.range_search(flavors, + {"ram": ">=4096", "vcpus": "<6"}) + self.assertIsInstance(result, list) + self.assertEqual(2, len(result)) + flavor_names = [r['name'] for r in result] + self.assertIn("m1.medium", flavor_names) + self.assertIn("m1.large", flavor_names) + + def test_range_search_multi_4(self): + flavors = self.cloud.list_flavors() + result = self.cloud.range_search(flavors, + {"ram": ">=4096", "vcpus": "MAX"}) + self.assertIsInstance(result, list) + self.assertEqual(1, len(result)) + # This is the only result that should have max vcpu + self.assertEqual("m1.xlarge", result[0]['name']) diff --git a/shade/tests/unit/test__utils.py b/shade/tests/unit/test__utils.py index afd93010e..dff7956e9 100644 --- a/shade/tests/unit/test__utils.py +++ b/shade/tests/unit/test__utils.py @@ -12,11 +12,23 @@ # License for the specific language governing permissions and limitations # under the License. +import testtools from shade import _utils +from shade import exc from shade.tests.unit import base +RANGE_DATA = [ + dict(id=1, key1=1, key2=5), + dict(id=2, key1=1, key2=20), + dict(id=3, key1=2, key2=10), + dict(id=4, key1=2, key2=30), + dict(id=5, key1=3, key2=40), + dict(id=6, key1=3, key2=40), +] + + class TestUtils(base.TestCase): def test__filter_list_name_or_id(self): @@ -148,3 +160,157 @@ class TestUtils(base.TestCase): ) retval = _utils.normalize_volumes([vol]) self.assertEqual([expected], retval) + + def test_safe_dict_min_ints(self): + """Test integer comparison""" + data = [{'f1': 3}, {'f1': 2}, {'f1': 1}] + retval = _utils.safe_dict_min('f1', data) + self.assertEqual(1, retval) + + def test_safe_dict_min_strs(self): + """Test integer as strings comparison""" + data = [{'f1': '3'}, {'f1': '2'}, {'f1': '1'}] + retval = _utils.safe_dict_min('f1', data) + self.assertEqual(1, retval) + + def test_safe_dict_min_None(self): + """Test None values""" + data = [{'f1': 3}, {'f1': None}, {'f1': 1}] + retval = _utils.safe_dict_min('f1', data) + self.assertEqual(1, retval) + + def test_safe_dict_min_key_missing(self): + """Test missing key for an entry still works""" + data = [{'f1': 3}, {'x': 2}, {'f1': 1}] + retval = _utils.safe_dict_min('f1', data) + self.assertEqual(1, retval) + + def test_safe_dict_min_key_not_found(self): + """Test key not found in any elements returns None""" + data = [{'f1': 3}, {'f1': 2}, {'f1': 1}] + retval = _utils.safe_dict_min('doesnotexist', data) + self.assertIsNone(retval) + + def test_safe_dict_min_not_int(self): + """Test non-integer key value raises OSCE""" + data = [{'f1': 3}, {'f1': "aaa"}, {'f1': 1}] + with testtools.ExpectedException( + exc.OpenStackCloudException, + "Search for minimum value failed. " + "Value for f1 is not an integer: aaa" + ): + _utils.safe_dict_min('f1', data) + + def test_safe_dict_max_ints(self): + """Test integer comparison""" + data = [{'f1': 3}, {'f1': 2}, {'f1': 1}] + retval = _utils.safe_dict_max('f1', data) + self.assertEqual(3, retval) + + def test_safe_dict_max_strs(self): + """Test integer as strings comparison""" + data = [{'f1': '3'}, {'f1': '2'}, {'f1': '1'}] + retval = _utils.safe_dict_max('f1', data) + self.assertEqual(3, retval) + + def test_safe_dict_max_None(self): + """Test None values""" + data = [{'f1': 3}, {'f1': None}, {'f1': 1}] + retval = _utils.safe_dict_max('f1', data) + self.assertEqual(3, retval) + + def test_safe_dict_max_key_missing(self): + """Test missing key for an entry still works""" + data = [{'f1': 3}, {'x': 2}, {'f1': 1}] + retval = _utils.safe_dict_max('f1', data) + self.assertEqual(3, retval) + + def test_safe_dict_max_key_not_found(self): + """Test key not found in any elements returns None""" + data = [{'f1': 3}, {'f1': 2}, {'f1': 1}] + retval = _utils.safe_dict_max('doesnotexist', data) + self.assertIsNone(retval) + + def test_safe_dict_max_not_int(self): + """Test non-integer key value raises OSCE""" + data = [{'f1': 3}, {'f1': "aaa"}, {'f1': 1}] + with testtools.ExpectedException( + exc.OpenStackCloudException, + "Search for maximum value failed. " + "Value for f1 is not an integer: aaa" + ): + _utils.safe_dict_max('f1', data) + + def test_parse_range_None(self): + self.assertIsNone(_utils.parse_range(None)) + + def test_parse_range_invalid(self): + self.assertIsNone(_utils.parse_range("1024") + self.assertIsInstance(retval, tuple) + self.assertEqual(">", retval[0]) + self.assertEqual(1024, retval[1]) + + def test_parse_range_le(self): + retval = _utils.parse_range("<=1024") + self.assertIsInstance(retval, tuple) + self.assertEqual("<=", retval[0]) + self.assertEqual(1024, retval[1]) + + def test_parse_range_ge(self): + retval = _utils.parse_range(">=1024") + self.assertIsInstance(retval, tuple) + self.assertEqual(">=", retval[0]) + self.assertEqual(1024, retval[1]) + + def test_range_filter_min(self): + retval = _utils.range_filter(RANGE_DATA, "key1", "min") + self.assertIsInstance(retval, list) + self.assertEqual(2, len(retval)) + self.assertEqual(RANGE_DATA[:2], retval) + + def test_range_filter_max(self): + retval = _utils.range_filter(RANGE_DATA, "key1", "max") + self.assertIsInstance(retval, list) + self.assertEqual(2, len(retval)) + self.assertEqual(RANGE_DATA[-2:], retval) + + def test_range_filter_range(self): + retval = _utils.range_filter(RANGE_DATA, "key1", "<3") + self.assertIsInstance(retval, list) + self.assertEqual(4, len(retval)) + self.assertEqual(RANGE_DATA[:4], retval) + + def test_range_filter_exact(self): + retval = _utils.range_filter(RANGE_DATA, "key1", "2") + self.assertIsInstance(retval, list) + self.assertEqual(2, len(retval)) + self.assertEqual(RANGE_DATA[2:4], retval) + + def test_range_filter_invalid_int(self): + with testtools.ExpectedException( + exc.OpenStackCloudException, + "Invalid range value: <1A0" + ): + _utils.range_filter(RANGE_DATA, "key1", "<1A0") + + def test_range_filter_invalid_op(self): + with testtools.ExpectedException( + exc.OpenStackCloudException, + "Invalid range value: <>100" + ): + _utils.range_filter(RANGE_DATA, "key1", "<>100") diff --git a/shade/tests/unit/test_shade.py b/shade/tests/unit/test_shade.py index 30b10ed96..8602da98b 100644 --- a/shade/tests/unit/test_shade.py +++ b/shade/tests/unit/test_shade.py @@ -29,6 +29,16 @@ from shade.tests import fakes from shade.tests.unit import base +RANGE_DATA = [ + dict(id=1, key1=1, key2=5), + dict(id=2, key1=1, key2=20), + dict(id=3, key1=2, key2=10), + dict(id=4, key1=2, key2=30), + dict(id=5, key1=3, key2=40), + dict(id=6, key1=3, key2=40), +] + + class TestShade(base.TestCase): def setUp(self): @@ -623,3 +633,36 @@ class TestShade(base.TestCase): mock_nova.client.get.return_value = ('200', body) self.assertTrue(self.cloud._has_nova_extension('NMN')) self.assertFalse(self.cloud._has_nova_extension('invalid')) + + def test_range_search(self): + filters = {"key1": "min", "key2": "20"} + retval = self.cloud.range_search(RANGE_DATA, filters) + self.assertIsInstance(retval, list) + self.assertEqual(1, len(retval)) + self.assertEqual([RANGE_DATA[1]], retval) + + def test_range_search_2(self): + filters = {"key1": "<=2", "key2": ">10"} + retval = self.cloud.range_search(RANGE_DATA, filters) + self.assertIsInstance(retval, list) + self.assertEqual(2, len(retval)) + self.assertEqual([RANGE_DATA[1], RANGE_DATA[3]], retval) + + def test_range_search_3(self): + filters = {"key1": "2", "key2": "min"} + retval = self.cloud.range_search(RANGE_DATA, filters) + self.assertIsInstance(retval, list) + self.assertEqual(0, len(retval)) + + def test_range_search_4(self): + filters = {"key1": "max", "key2": "min"} + retval = self.cloud.range_search(RANGE_DATA, filters) + self.assertIsInstance(retval, list) + self.assertEqual(0, len(retval)) + + def test_range_search_5(self): + filters = {"key1": "min", "key2": "min"} + retval = self.cloud.range_search(RANGE_DATA, filters) + self.assertIsInstance(retval, list) + self.assertEqual(1, len(retval)) + self.assertEqual([RANGE_DATA[0]], retval)