Merge "Add range search functionality"

shade/_utils.py

@@ -15,6 +15,7 @@
 import contextlib
 import inspect
 import netifaces
+import re
 import six
 import time
 
@@ -535,3 +536,167 @@ def shade_exceptions(error_message=None):
         if error_message is None:
             error_message = str(e)
         raise exc.OpenStackCloudException(error_message)
+
+
+def safe_dict_min(key, data):
+    """Safely find the minimum for a given key in a list of dict objects.
+
+    This will find the minimum integer value for specific dictionary key
+    across a list of dictionaries. The values for the given key MUST be
+    integers, or string representations of an integer.
+
+    The dictionary key does not have to be present in all (or any)
+    of the elements/dicts within the data set.
+
+    :param string key: The dictionary key to search for the minimum value.
+    :param list data: List of dicts to use for the data set.
+
+    :returns: None if the field was not not found in any elements, or
+        the minimum value for the field otherwise.
+    """
+    min_value = None
+    for d in data:
+        if (key in d) and (d[key] is not None):
+            try:
+                val = int(d[key])
+            except ValueError:
+                raise exc.OpenStackCloudException(
+                    "Search for minimum value failed. "
+                    "Value for {key} is not an integer: {value}".format(
+                        key=key, value=d[key])
+                )
+            if (min_value is None) or (val < min_value):
+                min_value = val
+    return min_value
+
+
+def safe_dict_max(key, data):
+    """Safely find the maximum for a given key in a list of dict objects.
+
+    This will find the maximum integer value for specific dictionary key
+    across a list of dictionaries. The values for the given key MUST be
+    integers, or string representations of an integer.
+
+    The dictionary key does not have to be present in all (or any)
+    of the elements/dicts within the data set.
+
+    :param string key: The dictionary key to search for the maximum value.
+    :param list data: List of dicts to use for the data set.
+
+    :returns: None if the field was not not found in any elements, or
+        the maximum value for the field otherwise.
+    """
+    max_value = None
+    for d in data:
+        if (key in d) and (d[key] is not None):
+            try:
+                val = int(d[key])
+            except ValueError:
+                raise exc.OpenStackCloudException(
+                    "Search for maximum value failed. "
+                    "Value for {key} is not an integer: {value}".format(
+                        key=key, value=d[key])
+                )
+            if (max_value is None) or (val > max_value):
+                max_value = val
+    return max_value
+
+
+def parse_range(value):
+    """Parse a numerical range string.
+
+    Breakdown a range expression into its operater and numerical parts.
+    This expression must be a string. Valid values must be an integer string,
+    optionally preceeded by one of the following operators::
+
+        - "<"  : Less than
+        - ">"  : Greater than
+        - "<=" : Less than or equal to
+        - ">=" : Greater than or equal to
+
+    Some examples of valid values and function return values::
+
+        - "1024"  : returns (None, 1024)
+        - "<5"    : returns ("<", 5)
+        - ">=100" : returns (">=", 100)
+
+    :param string value: The range expression to be parsed.
+
+    :returns: A tuple with the operator string (or None if no operator
+        was given) and the integer value. None is returned if parsing failed.
+    """
+    if value is None:
+        return None
+
+    range_exp = re.match('(<|>|<=|>=){0,1}(\d+)$', value)
+    if range_exp is None:
+        return None
+
+    op = range_exp.group(1)
+    num = int(range_exp.group(2))
+    return (op, num)
+
+
+def range_filter(data, key, range_exp):
+    """Filter a list by a single range expression.
+
+    :param list data: List of dictionaries to be searched.
+    :param string key: Key name to search within the data set.
+    :param string range_exp: The expression describing the range of values.
+
+    :returns: A list subset of the original data set.
+    :raises: OpenStackCloudException on invalid range expressions.
+    """
+    filtered = []
+    range_exp = str(range_exp).upper()
+
+    if range_exp == "MIN":
+        key_min = safe_dict_min(key, data)
+        if key_min is None:
+            return []
+        for d in data:
+            if int(d[key]) == key_min:
+                filtered.append(d)
+        return filtered
+    elif range_exp == "MAX":
+        key_max = safe_dict_max(key, data)
+        if key_max is None:
+            return []
+        for d in data:
+            if int(d[key]) == key_max:
+                filtered.append(d)
+        return filtered
+
+    # Not looking for a min or max, so a range or exact value must
+    # have been supplied.
+    val_range = parse_range(range_exp)
+
+    # If parsing the range fails, it must be a bad value.
+    if val_range is None:
+        raise exc.OpenStackCloudException(
+            "Invalid range value: {value}".format(value=range_exp))
+
+    op = val_range[0]
+    if op:
+        # Range matching
+        for d in data:
+            d_val = int(d[key])
+            if op == '<':
+                if d_val < val_range[1]:
+                    filtered.append(d)
+            elif op == '>':
+                if d_val > val_range[1]:
+                    filtered.append(d)
+            elif op == '<=':
+                if d_val <= val_range[1]:
+                    filtered.append(d)
+            elif op == '>=':
+                if d_val >= val_range[1]:
+                    filtered.append(d)
+        return filtered
+    else:
+        # Exact number match
+        for d in data:
+            if int(d[key]) == val_range[1]:
+                filtered.append(d)
+        return filtered

shade/openstackcloud.py

@@ -351,6 +351,53 @@ class OpenStackCloud(object):
         ret.update(self._get_project_param_dict(project))
         return ret
 
+    def range_search(self, data, filters):
+        """Perform integer range searches across a list of dictionaries.
+
+        Given a list of dictionaries, search across the list using the given
+        dictionary keys and a range of integer values for each key. Only
+        dictionaries that match ALL search filters across the entire original
+        data set will be returned.
+
+        It is not a requirement that each dictionary contain the key used
+        for searching. Those without the key will be considered non-matching.
+
+        The range values must be string values and is either a set of digits
+        representing an integer for matching, or a range operator followed by
+        a set of digits representing an integer for matching. If a range
+        operator is not given, exact value matching will be used. Valid
+        operators are one of: <,>,<=,>=
+
+        :param list data: List of dictionaries to be searched.
+        :param dict filters: Dict describing the one or more range searches to
+            perform. If more than one search is given, the result will be the
+            members of the original data set that match ALL searches. An
+            example of filtering by multiple ranges::
+
+                {"vcpus": "<=5", "ram": "<=2048", "disk": "1"}
+
+        :returns: A list subset of the original data set.
+        :raises: OpenStackCloudException on invalid range expressions.
+        """
+        filtered = []
+
+        for key, range_value in filters.items():
+            # We always want to operate on the full data set so that
+            # calculations for minimum and maximum are correct.
+            results = _utils.range_filter(data, key, range_value)
+
+            if not filtered:
+                # First set of results
+                filtered = results
+            else:
+                # The combination of all searches should be the intersection of
+                # all result sets from each search. So adjust the current set
+                # of filtered data by computing its intersection with the
+                # latest result set.
+                filtered = [r for r in results for f in filtered if r == f]
+
+        return filtered
+
     @_utils.cache_on_arguments()
     def list_projects(self):
         """List Keystone Projects.

shade/tests/functional/test_range_search.py

@@ -0,0 +1,129 @@
+# Copyright (c) 2016 IBM
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import shade
+from shade import exc
+from shade.tests import base
+
+
+class TestRangeSearch(base.TestCase):
+
+    def setUp(self):
+        super(TestRangeSearch, self).setUp()
+        self.cloud = shade.openstack_cloud(cloud='devstack')
+
+    def test_range_search_bad_range(self):
+        flavors = self.cloud.list_flavors()
+        self.assertRaises(exc.OpenStackCloudException,
+                          self.cloud.range_search, flavors, {"ram": "<1a0"})
+
+    def test_range_search_exact(self):
+        flavors = self.cloud.list_flavors()
+        result = self.cloud.range_search(flavors, {"ram": "4096"})
+        self.assertIsInstance(result, list)
+        self.assertEqual(1, len(result))
+        self.assertEqual("m1.medium", result[0]['name'])
+
+    def test_range_search_min(self):
+        flavors = self.cloud.list_flavors()
+        result = self.cloud.range_search(flavors, {"ram": "MIN"})
+        self.assertIsInstance(result, list)
+        self.assertEqual(1, len(result))
+        self.assertEqual("m1.tiny", result[0]['name'])
+
+    def test_range_search_max(self):
+        flavors = self.cloud.list_flavors()
+        result = self.cloud.range_search(flavors, {"ram": "MAX"})
+        self.assertIsInstance(result, list)
+        self.assertEqual(1, len(result))
+        self.assertEqual("m1.xlarge", result[0]['name'])
+
+    def test_range_search_lt(self):
+        flavors = self.cloud.list_flavors()
+        result = self.cloud.range_search(flavors, {"ram": "<4096"})
+        self.assertIsInstance(result, list)
+        self.assertEqual(2, len(result))
+        flavor_names = [r['name'] for r in result]
+        self.assertIn("m1.tiny", flavor_names)
+        self.assertIn("m1.small", flavor_names)
+
+    def test_range_search_gt(self):
+        flavors = self.cloud.list_flavors()
+        result = self.cloud.range_search(flavors, {"ram": ">4096"})
+        self.assertIsInstance(result, list)
+        self.assertEqual(2, len(result))
+        flavor_names = [r['name'] for r in result]
+        self.assertIn("m1.large", flavor_names)
+        self.assertIn("m1.xlarge", flavor_names)
+
+    def test_range_search_le(self):
+        flavors = self.cloud.list_flavors()
+        result = self.cloud.range_search(flavors, {"ram": "<=4096"})
+        self.assertIsInstance(result, list)
+        self.assertEqual(3, len(result))
+        flavor_names = [r['name'] for r in result]
+        self.assertIn("m1.tiny", flavor_names)
+        self.assertIn("m1.small", flavor_names)
+        self.assertIn("m1.medium", flavor_names)
+
+    def test_range_search_ge(self):
+        flavors = self.cloud.list_flavors()
+        result = self.cloud.range_search(flavors, {"ram": ">=4096"})
+        self.assertIsInstance(result, list)
+        self.assertEqual(3, len(result))
+        flavor_names = [r['name'] for r in result]
+        self.assertIn("m1.medium", flavor_names)
+        self.assertIn("m1.large", flavor_names)
+        self.assertIn("m1.xlarge", flavor_names)
+
+    def test_range_search_multi_1(self):
+        flavors = self.cloud.list_flavors()
+        result = self.cloud.range_search(flavors,
+                                         {"ram": "MIN", "vcpus": "MIN"})
+        self.assertIsInstance(result, list)
+        self.assertEqual(1, len(result))
+        self.assertEqual("m1.tiny", result[0]['name'])
+
+    def test_range_search_multi_2(self):
+        flavors = self.cloud.list_flavors()
+        result = self.cloud.range_search(flavors,
+                                         {"ram": "<8192", "vcpus": "MIN"})
+        self.assertIsInstance(result, list)
+        self.assertEqual(2, len(result))
+        flavor_names = [r['name'] for r in result]
+        # All of these should have 1 vcpu
+        self.assertIn("m1.tiny", flavor_names)
+        self.assertIn("m1.small", flavor_names)
+
+    def test_range_search_multi_3(self):
+        flavors = self.cloud.list_flavors()
+        result = self.cloud.range_search(flavors,
+                                         {"ram": ">=4096", "vcpus": "<6"})
+        self.assertIsInstance(result, list)
+        self.assertEqual(2, len(result))
+        flavor_names = [r['name'] for r in result]
+        self.assertIn("m1.medium", flavor_names)
+        self.assertIn("m1.large", flavor_names)
+
+    def test_range_search_multi_4(self):
+        flavors = self.cloud.list_flavors()
+        result = self.cloud.range_search(flavors,
+                                         {"ram": ">=4096", "vcpus": "MAX"})
+        self.assertIsInstance(result, list)
+        self.assertEqual(1, len(result))
+        # This is the only result that should have max vcpu
+        self.assertEqual("m1.xlarge", result[0]['name'])

shade/tests/unit/test__utils.py

@@ -12,11 +12,23 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import testtools
 
 from shade import _utils
+from shade import exc
 from shade.tests.unit import base
 
 
+RANGE_DATA = [
+    dict(id=1, key1=1, key2=5),
+    dict(id=2, key1=1, key2=20),
+    dict(id=3, key1=2, key2=10),
+    dict(id=4, key1=2, key2=30),
+    dict(id=5, key1=3, key2=40),
+    dict(id=6, key1=3, key2=40),
+]
+
+
 class TestUtils(base.TestCase):
 
     def test__filter_list_name_or_id(self):
@@ -148,3 +160,157 @@ class TestUtils(base.TestCase):
         )
         retval = _utils.normalize_volumes([vol])
         self.assertEqual([expected], retval)
+
+    def test_safe_dict_min_ints(self):
+        """Test integer comparison"""
+        data = [{'f1': 3}, {'f1': 2}, {'f1': 1}]
+        retval = _utils.safe_dict_min('f1', data)
+        self.assertEqual(1, retval)
+
+    def test_safe_dict_min_strs(self):
+        """Test integer as strings comparison"""
+        data = [{'f1': '3'}, {'f1': '2'}, {'f1': '1'}]
+        retval = _utils.safe_dict_min('f1', data)
+        self.assertEqual(1, retval)
+
+    def test_safe_dict_min_None(self):
+        """Test None values"""
+        data = [{'f1': 3}, {'f1': None}, {'f1': 1}]
+        retval = _utils.safe_dict_min('f1', data)
+        self.assertEqual(1, retval)
+
+    def test_safe_dict_min_key_missing(self):
+        """Test missing key for an entry still works"""
+        data = [{'f1': 3}, {'x': 2}, {'f1': 1}]
+        retval = _utils.safe_dict_min('f1', data)
+        self.assertEqual(1, retval)
+
+    def test_safe_dict_min_key_not_found(self):
+        """Test key not found in any elements returns None"""
+        data = [{'f1': 3}, {'f1': 2}, {'f1': 1}]
+        retval = _utils.safe_dict_min('doesnotexist', data)
+        self.assertIsNone(retval)
+
+    def test_safe_dict_min_not_int(self):
+        """Test non-integer key value raises OSCE"""
+        data = [{'f1': 3}, {'f1': "aaa"}, {'f1': 1}]
+        with testtools.ExpectedException(
+            exc.OpenStackCloudException,
+            "Search for minimum value failed. "
+            "Value for f1 is not an integer: aaa"
+        ):
+            _utils.safe_dict_min('f1', data)
+
+    def test_safe_dict_max_ints(self):
+        """Test integer comparison"""
+        data = [{'f1': 3}, {'f1': 2}, {'f1': 1}]
+        retval = _utils.safe_dict_max('f1', data)
+        self.assertEqual(3, retval)
+
+    def test_safe_dict_max_strs(self):
+        """Test integer as strings comparison"""
+        data = [{'f1': '3'}, {'f1': '2'}, {'f1': '1'}]
+        retval = _utils.safe_dict_max('f1', data)
+        self.assertEqual(3, retval)
+
+    def test_safe_dict_max_None(self):
+        """Test None values"""
+        data = [{'f1': 3}, {'f1': None}, {'f1': 1}]
+        retval = _utils.safe_dict_max('f1', data)
+        self.assertEqual(3, retval)
+
+    def test_safe_dict_max_key_missing(self):
+        """Test missing key for an entry still works"""
+        data = [{'f1': 3}, {'x': 2}, {'f1': 1}]
+        retval = _utils.safe_dict_max('f1', data)
+        self.assertEqual(3, retval)
+
+    def test_safe_dict_max_key_not_found(self):
+        """Test key not found in any elements returns None"""
+        data = [{'f1': 3}, {'f1': 2}, {'f1': 1}]
+        retval = _utils.safe_dict_max('doesnotexist', data)
+        self.assertIsNone(retval)
+
+    def test_safe_dict_max_not_int(self):
+        """Test non-integer key value raises OSCE"""
+        data = [{'f1': 3}, {'f1': "aaa"}, {'f1': 1}]
+        with testtools.ExpectedException(
+            exc.OpenStackCloudException,
+            "Search for maximum value failed. "
+            "Value for f1 is not an integer: aaa"
+        ):
+            _utils.safe_dict_max('f1', data)
+
+    def test_parse_range_None(self):
+        self.assertIsNone(_utils.parse_range(None))
+
+    def test_parse_range_invalid(self):
+        self.assertIsNone(_utils.parse_range("<invalid"))
+
+    def test_parse_range_int_only(self):
+        retval = _utils.parse_range("1024")
+        self.assertIsInstance(retval, tuple)
+        self.assertIsNone(retval[0])
+        self.assertEqual(1024, retval[1])
+
+    def test_parse_range_lt(self):
+        retval = _utils.parse_range("<1024")
+        self.assertIsInstance(retval, tuple)
+        self.assertEqual("<", retval[0])
+        self.assertEqual(1024, retval[1])
+
+    def test_parse_range_gt(self):
+        retval = _utils.parse_range(">1024")
+        self.assertIsInstance(retval, tuple)
+        self.assertEqual(">", retval[0])
+        self.assertEqual(1024, retval[1])
+
+    def test_parse_range_le(self):
+        retval = _utils.parse_range("<=1024")
+        self.assertIsInstance(retval, tuple)
+        self.assertEqual("<=", retval[0])
+        self.assertEqual(1024, retval[1])
+
+    def test_parse_range_ge(self):
+        retval = _utils.parse_range(">=1024")
+        self.assertIsInstance(retval, tuple)
+        self.assertEqual(">=", retval[0])
+        self.assertEqual(1024, retval[1])
+
+    def test_range_filter_min(self):
+        retval = _utils.range_filter(RANGE_DATA, "key1", "min")
+        self.assertIsInstance(retval, list)
+        self.assertEqual(2, len(retval))
+        self.assertEqual(RANGE_DATA[:2], retval)
+
+    def test_range_filter_max(self):
+        retval = _utils.range_filter(RANGE_DATA, "key1", "max")
+        self.assertIsInstance(retval, list)
+        self.assertEqual(2, len(retval))
+        self.assertEqual(RANGE_DATA[-2:], retval)
+
+    def test_range_filter_range(self):
+        retval = _utils.range_filter(RANGE_DATA, "key1", "<3")
+        self.assertIsInstance(retval, list)
+        self.assertEqual(4, len(retval))
+        self.assertEqual(RANGE_DATA[:4], retval)
+
+    def test_range_filter_exact(self):
+        retval = _utils.range_filter(RANGE_DATA, "key1", "2")
+        self.assertIsInstance(retval, list)
+        self.assertEqual(2, len(retval))
+        self.assertEqual(RANGE_DATA[2:4], retval)
+
+    def test_range_filter_invalid_int(self):
+        with testtools.ExpectedException(
+            exc.OpenStackCloudException,
+            "Invalid range value: <1A0"
+        ):
+            _utils.range_filter(RANGE_DATA, "key1", "<1A0")
+
+    def test_range_filter_invalid_op(self):
+        with testtools.ExpectedException(
+            exc.OpenStackCloudException,
+            "Invalid range value: <>100"
+        ):
+            _utils.range_filter(RANGE_DATA, "key1", "<>100")

shade/tests/unit/test_shade.py

@@ -29,6 +29,16 @@ from shade.tests import fakes
 from shade.tests.unit import base
 
 
+RANGE_DATA = [
+    dict(id=1, key1=1, key2=5),
+    dict(id=2, key1=1, key2=20),
+    dict(id=3, key1=2, key2=10),
+    dict(id=4, key1=2, key2=30),
+    dict(id=5, key1=3, key2=40),
+    dict(id=6, key1=3, key2=40),
+]
+
+
 class TestShade(base.TestCase):
 
     def setUp(self):
@@ -623,3 +633,36 @@ class TestShade(base.TestCase):
         mock_nova.client.get.return_value = ('200', body)
         self.assertTrue(self.cloud._has_nova_extension('NMN'))
         self.assertFalse(self.cloud._has_nova_extension('invalid'))
+
+    def test_range_search(self):
+        filters = {"key1": "min", "key2": "20"}
+        retval = self.cloud.range_search(RANGE_DATA, filters)
+        self.assertIsInstance(retval, list)
+        self.assertEqual(1, len(retval))
+        self.assertEqual([RANGE_DATA[1]], retval)
+
+    def test_range_search_2(self):
+        filters = {"key1": "<=2", "key2": ">10"}
+        retval = self.cloud.range_search(RANGE_DATA, filters)
+        self.assertIsInstance(retval, list)
+        self.assertEqual(2, len(retval))
+        self.assertEqual([RANGE_DATA[1], RANGE_DATA[3]], retval)
+
+    def test_range_search_3(self):
+        filters = {"key1": "2", "key2": "min"}
+        retval = self.cloud.range_search(RANGE_DATA, filters)
+        self.assertIsInstance(retval, list)
+        self.assertEqual(0, len(retval))
+
+    def test_range_search_4(self):
+        filters = {"key1": "max", "key2": "min"}
+        retval = self.cloud.range_search(RANGE_DATA, filters)
+        self.assertIsInstance(retval, list)
+        self.assertEqual(0, len(retval))
+
+    def test_range_search_5(self):
+        filters = {"key1": "min", "key2": "min"}
+        retval = self.cloud.range_search(RANGE_DATA, filters)
+        self.assertIsInstance(retval, list)
+        self.assertEqual(1, len(retval))
+        self.assertEqual([RANGE_DATA[0]], retval)