From e888d8e5cd2a7bf1f1ffd4de875e8c649751157a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 7 Sep 2016 13:51:15 -0500 Subject: [PATCH] Add support for jmespath filter expressions ansible upstream recently added a filter using jmespath to allow for arbitrary querying of complex data structures. While reviewing that patch, it seemed quite flexible and lovely. It's also apparently also used in boto for similar reasons, so we're following a pretty well tested solution - and since it's now a depend on ansible, the extra depend shouldn't be too much of a burden for most folks. Change-Id: Ia4bf455f0e32f29a6fce79c71fecce7b0ed57ea5 --- ...add-jmespath-support-f47b7a503dbbfda1.yaml | 4 + requirements.txt | 1 + shade/_utils.py | 13 ++- shade/openstackcloud.py | 84 +++++++++++++++---- shade/tests/functional/test_users.py | 4 + shade/tests/unit/test__utils.py | 7 ++ 6 files changed, 93 insertions(+), 20 deletions(-) create mode 100644 releasenotes/notes/add-jmespath-support-f47b7a503dbbfda1.yaml diff --git a/releasenotes/notes/add-jmespath-support-f47b7a503dbbfda1.yaml b/releasenotes/notes/add-jmespath-support-f47b7a503dbbfda1.yaml new file mode 100644 index 000000000..2d157a3c8 --- /dev/null +++ b/releasenotes/notes/add-jmespath-support-f47b7a503dbbfda1.yaml @@ -0,0 +1,4 @@ +--- +features: + - All get and search functions can now take a jmespath expression in their + filters parameter. diff --git a/requirements.txt b/requirements.txt index 54e1d450c..e229015b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ pbr>=0.11,<2.0 munch decorator +jmespath jsonpatch ipaddress os-client-config>=1.20.0 diff --git a/shade/_utils.py b/shade/_utils.py index 5e87615cf..7d63fcd39 100644 --- a/shade/_utils.py +++ b/shade/_utils.py @@ -14,6 +14,7 @@ import contextlib import inspect +import jmespath import munch import netifaces import re @@ -73,7 +74,7 @@ def _filter_list(data, name_or_id, filters): key if a value for name_or_id is given. :param string name_or_id: The name or ID of the entity being filtered. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -83,6 +84,8 @@ def _filter_list(data, name_or_id, filters): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. """ if name_or_id: identifier_matches = [] @@ -96,6 +99,9 @@ def _filter_list(data, name_or_id, filters): if not filters: return data + if isinstance(filters, six.string_types): + return jmespath.search(filters, data) + def _dict_filter(f, d): if not d: return False @@ -129,8 +135,11 @@ def _get_entity(func, name_or_id, filters, **kwargs): and returns a list of entities to filter. :param string name_or_id: The name or ID of the entity being filtered or a dict - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" """ # Sometimes in the control flow of shade, we already have an object # fetched. Rather than then needing to pull the name or id out of that diff --git a/shade/openstackcloud.py b/shade/openstackcloud.py index 08af74bc3..1e9780838 100644 --- a/shade/openstackcloud.py +++ b/shade/openstackcloud.py @@ -464,7 +464,7 @@ class OpenStackCloud(object): operators are one of: <,>,<=,>= :param list data: List of dictionaries to be searched. - :param dict filters: Dict describing the one or more range searches to + :param filters: Dict describing the one or more range searches to perform. If more than one search is given, the result will be the members of the original data set that match ALL searches. An example of filtering by multiple ranges:: @@ -634,7 +634,10 @@ class OpenStackCloud(object): """Seach Keystone users. :param string name: user name or id. - :param dict filters: a dict containing additional filters to use. + :param filters: a dict containing additional filters to use. + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: a list of ``munch.Munch`` containing the users @@ -648,7 +651,10 @@ class OpenStackCloud(object): """Get exactly one Keystone user. :param string name_or_id: user name or id. - :param dict filters: a dict containing additional filters to use. + :param filters: a dict containing additional filters to use. + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: a single ``munch.Munch`` containing the user description. @@ -1939,7 +1945,7 @@ class OpenStackCloud(object): """Get a keypair by name or ID. :param name_or_id: Name or ID of the keypair. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -1949,6 +1955,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A keypair ``munch.Munch`` or None if no matching keypair is found. @@ -1960,7 +1969,7 @@ class OpenStackCloud(object): """Get a network by name or ID. :param name_or_id: Name or ID of the network. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -1970,6 +1979,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A network ``munch.Munch`` or None if no matching network is found. @@ -1981,7 +1993,7 @@ class OpenStackCloud(object): """Get a router by name or ID. :param name_or_id: Name or ID of the router. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -1991,6 +2003,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A router ``munch.Munch`` or None if no matching router is found. @@ -2002,7 +2017,7 @@ class OpenStackCloud(object): """Get a subnet by name or ID. :param name_or_id: Name or ID of the subnet. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -2023,7 +2038,7 @@ class OpenStackCloud(object): """Get a port by name or ID. :param name_or_id: Name or ID of the port. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -2033,6 +2048,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A port ``munch.Munch`` or None if no matching port is found. @@ -2043,7 +2061,7 @@ class OpenStackCloud(object): """Get a volume by name or ID. :param name_or_id: Name or ID of the volume. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -2053,6 +2071,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A volume ``munch.Munch`` or None if no matching volume is found. @@ -2064,7 +2085,7 @@ class OpenStackCloud(object): """Get a flavor by name or ID. :param name_or_id: Name or ID of the flavor. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -2074,6 +2095,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :param get_extra: Whether or not the list_flavors call should get the extra flavor specs. @@ -2091,7 +2115,7 @@ class OpenStackCloud(object): """Get a security group by name or ID. :param name_or_id: Name or ID of the security group. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -2101,6 +2125,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A security group ``munch.Munch`` or None if no matching security group is found. @@ -2147,7 +2174,7 @@ class OpenStackCloud(object): """Get a server by name or ID. :param name_or_id: Name or ID of the server. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -2157,6 +2184,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A server ``munch.Munch`` or None if no matching server is found. @@ -2175,13 +2205,16 @@ class OpenStackCloud(object): """Get a server group by name or ID. :param name_or_id: Name or ID of the server group. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: { 'policy': 'affinity', } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A server groups dict or None if no matching server group is found. @@ -2194,7 +2227,7 @@ class OpenStackCloud(object): """Get an image by name or ID. :param name_or_id: Name or ID of the image. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -2204,6 +2237,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: An image ``munch.Munch`` or None if no matching image is found @@ -2255,7 +2291,7 @@ class OpenStackCloud(object): """Get a floating IP by ID :param id: ID of the floating IP. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -2265,6 +2301,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A floating IP ``munch.Munch`` or None if no matching floating IP is found. @@ -3395,7 +3434,7 @@ class OpenStackCloud(object): """Get a volume by name or ID. :param name_or_id: Name or ID of the volume snapshot. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -3405,6 +3444,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A volume ``munch.Munch`` or None if no matching volume is found. @@ -5922,8 +5964,11 @@ class OpenStackCloud(object): """Get a zone by name or ID. :param name_or_id: Name or ID of the zone - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A zone dict or None if no matching zone is found. @@ -6187,7 +6232,7 @@ class OpenStackCloud(object): ClusterTemplate is the new name for BayModel. :param name_or_id: Name or ID of the ClusterTemplate. - :param dict filters: + :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: @@ -6197,6 +6242,9 @@ class OpenStackCloud(object): 'gender': 'Female' } } + OR + A string containing a jmespath expression for further filtering. + Example:: "[?last_name==`Smith`] | [?other.gender]==`Female`]" :returns: A ClusterTemplate dict or None if no matching ClusterTemplate is found. diff --git a/shade/tests/functional/test_users.py b/shade/tests/functional/test_users.py index 47ea68930..54b32d1bd 100644 --- a/shade/tests/functional/test_users.py +++ b/shade/tests/functional/test_users.py @@ -67,6 +67,10 @@ class TestUsers(base.BaseFunctionalTestCase): users = self.operator_cloud.search_users(filters={'enabled': True}) self.assertIsNotNone(users) + def test_search_users_jmespath(self): + users = self.operator_cloud.search_users(filters="[?enabled]") + self.assertIsNotNone(users) + def test_create_user(self): user_name = self.user_prefix + '_create' user_email = 'nobody@nowhere.com' diff --git a/shade/tests/unit/test__utils.py b/shade/tests/unit/test__utils.py index 8fae4ff71..3ef87fcb4 100644 --- a/shade/tests/unit/test__utils.py +++ b/shade/tests/unit/test__utils.py @@ -45,6 +45,13 @@ class TestUtils(base.TestCase): ret = _utils._filter_list(data, 'donald', {'other': 'duck'}) self.assertEqual([el1], ret) + def test__filter_list_filter_jmespath(self): + el1 = dict(id=100, name='donald', other='duck') + el2 = dict(id=200, name='donald', other='trump') + data = [el1, el2] + ret = _utils._filter_list(data, 'donald', "[?other == `duck`]") + self.assertEqual([el1], ret) + def test__filter_list_dict1(self): el1 = dict(id=100, name='donald', last='duck', other=dict(category='duck'))