openstack/aodh
Code Issues Proposed changes

Fixes non-string metadata query issue

Browse Source 
Based on current wsme implement, all the metadata query will get
the metadata value as string type. As a result, all the non-string
metadata query will not get correct result. Since the data is store
in database with correct data type. The fix will try to eval the
value before setting so as to get the correct data type.

Fixes bug 1200577

Change-Id: I8114e816da123b9dc08f32f9022db0e1b9fc2e5a
This commit is contained in:
Fei Long Wang 2013-07-27 23:44:29 +08:00
parent 748c2da238
commit 0236768b7e
6 changed files with 488 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
ceilometer/api/controllers/v2.py
View File

@ -1,6 +1,7 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 New Dream Network, LLC (DreamHost)
# Copyright 2013 IBM Corp.
#
# Author: Doug Hellmann <doug.hellmann@dreamhost.com>
# Angus Salkeld <asalkeld@redhat.com>
@ -30,6 +31,7 @@
# [PUT ] /meters/<meter> -- update the meter (not the samples)
# [DELETE] /meters/<meter> -- delete the meter and samples
#
import ast
import datetime
import inspect
import pecan
@ -42,6 +44,7 @@ from wsme import types as wtypes
from ceilometer.openstack.common import context
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log
from ceilometer.openstack.common import strutils
from ceilometer.openstack.common import timeutils
from ceilometer import sample
from ceilometer import storage
@ -118,17 +121,80 @@ class Query(_Base):
value = wtypes.text
"The value to compare against the stored data"
type = wtypes.text
"The data type of value to compare against the stored data"
def __repr__(self):
# for logging calls
return '<Query %r %s %r>' % (self.field, self.op, self.value)
return '<Query %r %s %r %s>' % (self.field,
self.op,
self.value,
self.type)
@classmethod
def sample(cls):
return cls(field='resource_id',
op='eq',
value='bd9431c1-8d69-4ad3-803a-8d4a6b89fd36',
type='string'
)
def _get_value_as_type(self):
"""Convert metadata value to the specified data type.
This method is called during metadata query to help convert the
querying metadata to the data type specified by user. If there is no
data type given, the metadata will be parsed by ast.literal_eval to
try to do a smart converting.
NOTE (flwang) Using "_" as prefix to avoid an InvocationError raised
from wsmeext/sphinxext.py. It's OK to call it outside the Query class.
Because the "public" side of that class is actually the outside of the
API, and the "private" side is the API implementation. The method is
only used in the API implementation, so it's OK.
:returns: metadata value converted with the specified data type.
"""
try:
converted_value = self.value
if not self.type:
try:
converted_value = ast.literal_eval(self.value)
except ValueError:
msg = _('Failed to convert the metadata value %s'
' automatically') % (self.value)
LOG.debug(msg)
else:
if self.type == 'integer':
converted_value = int(self.value)
elif self.type == 'float':
converted_value = float(self.value)
elif self.type == 'boolean':
converted_value = strutils.bool_from_string(self.value)
elif self.type == 'string':
converted_value = self.value
else:
# For now, this method only support integer, float,
# boolean and and string as the metadata type. A TypeError
# will be raised for any other type.
raise TypeError()
except ValueError:
msg = _('Failed to convert the metadata value %(value)s'
' to the expected data type %(type)s.') % \
{'value': self.value, 'type': self.type}
raise wsme.exc.ClientSideError(msg)
except TypeError:
msg = _('The data type %s is not supported. The supported'
' data type list is: integer, float, boolean and'
' string.') % (self.type)
raise wsme.exc.ClientSideError(msg)
except Exception:
msg = _('Unexpected exception converting %(value)s to'
' the expected data type %(type)s.') % \
{'value': self.value, 'type': self.type}
raise wsme.exc.ClientSideError(msg)
return converted_value
def _sanitize_query(q):
'''Check the query to see if:
@ -189,7 +255,7 @@ def _query_to_kwargs(query, db_func):
elif i.field == 'search_offset':
stamp['search_offset'] = i.value
elif i.field.startswith('metadata.'):
metaquery[i.field] = i.value
metaquery[i.field] = i._get_value_as_type()
else:
trans[translation.get(i.field, i.field)] = i.value

218
ceilometer/openstack/common/strutils.py Normal file
View File

@ -0,0 +1,218 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
System-level utilities and helper functions.
"""
import re
import sys
import unicodedata
import six
from ceilometer.openstack.common.gettextutils import _ # noqa
# Used for looking up extensions of text
# to their 'multiplied' byte amount
BYTE_MULTIPLIERS = {
'': 1,
't': 1024 ** 4,
'g': 1024 ** 3,
'm': 1024 ** 2,
'k': 1024,
}
BYTE_REGEX = re.compile(r'(^-?\d+)(\D*)')
TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
def int_from_bool_as_string(subject):
"""Interpret a string as a boolean and return either 1 or 0.
Any string value in:
('True', 'true', 'On', 'on', '1')
is interpreted as a boolean True.
Useful for JSON-decoded stuff and config file parsing
"""
return bool_from_string(subject) and 1 or 0
def bool_from_string(subject, strict=False):
"""Interpret a string as a boolean.
A case-insensitive match is performed such that strings matching 't',
'true', 'on', 'y', 'yes', or '1' are considered True and, when
`strict=False`, anything else is considered False.
Useful for JSON-decoded stuff and config file parsing.
If `strict=True`, unrecognized values, including None, will raise a
ValueError which is useful when parsing values passed in from an API call.
Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
"""
if not isinstance(subject, six.string_types):
subject = str(subject)
lowered = subject.strip().lower()
if lowered in TRUE_STRINGS:
return True
elif lowered in FALSE_STRINGS:
return False
elif strict:
acceptable = ', '.join(
"'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
msg = _("Unrecognized value '%(val)s', acceptable values are:"
" %(acceptable)s") % {'val': subject,
'acceptable': acceptable}
raise ValueError(msg)
else:
return False
def safe_decode(text, incoming=None, errors='strict'):
"""Decodes incoming str using `incoming` if they're not already unicode.
:param incoming: Text's current encoding
:param errors: Errors handling policy. See here for valid
values http://docs.python.org/2/library/codecs.html
:returns: text or a unicode `incoming` encoded
representation of it.
:raises TypeError: If text is not an isntance of str
"""
if not isinstance(text, six.string_types):
raise TypeError("%s can't be decoded" % type(text))
if isinstance(text, six.text_type):
return text
if not incoming:
incoming = (sys.stdin.encoding or
sys.getdefaultencoding())
try:
return text.decode(incoming, errors)
except UnicodeDecodeError:
# Note(flaper87) If we get here, it means that
# sys.stdin.encoding / sys.getdefaultencoding
# didn't return a suitable encoding to decode
# text. This happens mostly when global LANG
# var is not set correctly and there's no
# default encoding. In this case, most likely
# python will use ASCII or ANSI encoders as
# default encodings but they won't be capable
# of decoding non-ASCII characters.
#
# Also, UTF-8 is being used since it's an ASCII
# extension.
return text.decode('utf-8', errors)
def safe_encode(text, incoming=None,
encoding='utf-8', errors='strict'):
"""Encodes incoming str/unicode using `encoding`.
If incoming is not specified, text is expected to be encoded with
current python's default encoding. (`sys.getdefaultencoding`)
:param incoming: Text's current encoding
:param encoding: Expected encoding for text (Default UTF-8)
:param errors: Errors handling policy. See here for valid
values http://docs.python.org/2/library/codecs.html
:returns: text or a bytestring `encoding` encoded
representation of it.
:raises TypeError: If text is not an isntance of str
"""
if not isinstance(text, six.string_types):
raise TypeError("%s can't be encoded" % type(text))
if not incoming:
incoming = (sys.stdin.encoding or
sys.getdefaultencoding())
if isinstance(text, six.text_type):
return text.encode(encoding, errors)
elif text and encoding != incoming:
# Decode text before encoding it with `encoding`
text = safe_decode(text, incoming, errors)
return text.encode(encoding, errors)
return text
def to_bytes(text, default=0):
"""Converts a string into an integer of bytes.
Looks at the last characters of the text to determine
what conversion is needed to turn the input text into a byte number.
Supports "B, K(B), M(B), G(B), and T(B)". (case insensitive)
:param text: String input for bytes size conversion.
:param default: Default return value when text is blank.
"""
match = BYTE_REGEX.search(text)
if match:
magnitude = int(match.group(1))
mult_key_org = match.group(2)
if not mult_key_org:
return magnitude
elif text:
msg = _('Invalid string format: %s') % text
raise TypeError(msg)
else:
return default
mult_key = mult_key_org.lower().replace('b', '', 1)
multiplier = BYTE_MULTIPLIERS.get(mult_key)
if multiplier is None:
msg = _('Unknown byte multiplier: %s') % mult_key_org
raise TypeError(msg)
return magnitude * multiplier
def to_slug(value, incoming=None, errors="strict"):
"""Normalize string.
Convert to lowercase, remove non-word characters, and convert spaces
to hyphens.
Inspired by Django's `slugify` filter.
:param value: Text to slugify
:param incoming: Text's current encoding
:param errors: Errors handling policy. See here for valid
values http://docs.python.org/2/library/codecs.html
:returns: slugified unicode representation of `value`
:raises TypeError: If text is not an instance of str
"""
value = safe_decode(value, incoming, errors)
# NOTE(aababilov): no need to use safe_(encode|decode) here:
# encodings are always "ascii", error handling is always "ignore"
# and types are always known (first: unicode; second: str)
value = unicodedata.normalize("NFKD", value).encode(
"ascii", "ignore").decode("ascii")
value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
return SLUGIFY_HYPHENATE_RE.sub("-", value)

3
ceilometer/tests/api.py
View File

@ -152,9 +152,10 @@ class FunctionalTest(db_test_base.TestBase):
query_params = {'q.field': [],
'q.value': [],
'q.op': [],
'q.type': [],
}
for query in q:
for name in ['field', 'op', 'value']:
for name in ['field', 'op', 'value', 'type']:
query_params['q.%s' % name].append(query.get(name, ''))
all_params = {}
all_params.update(params)

1
openstack-common.conf
View File

@ -17,6 +17,7 @@ module=notifier
module=policy
module=rpc
module=service
module=strutils
module=threadgroup
module=timeutils
base=ceilometer

106
tests/api/v2/test_list_meters_scenarios.py
View File

@ -1,6 +1,7 @@
# -*- encoding: utf-8 -*-
#
# Copyright 2012 Red Hat, Inc.
# Copyright 2013 IBM Corp.
#
# Author: Angus Salkeld <asalkeld@redhat.com>
#
@ -60,7 +61,10 @@ class TestListMeters(FunctionalTest,
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample'},
'tag': 'self.sample',
'size': 123,
'util': 0.75,
'is_public': True},
source='test_source'),
sample.Sample(
'meter.test',
@ -72,7 +76,10 @@ class TestListMeters(FunctionalTest,
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 11, 40),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample1'},
'tag': 'self.sample1',
'size': 0,
'util': 0.47,
'is_public': False},
source='test_source'),
sample.Sample(
'meter.mine',
@ -84,7 +91,10 @@ class TestListMeters(FunctionalTest,
'resource-id2',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample2'},
'tag': 'self.sample2',
'size': 456,
'util': 0.64,
'is_public': False},
source='test_source'),
sample.Sample(
'meter.test',
@ -96,7 +106,10 @@ class TestListMeters(FunctionalTest,
'resource-id3',
timestamp=datetime.datetime(2012, 7, 2, 10, 42),
resource_metadata={'display_name': 'test-server',
'tag': 'self.sample3'},
'tag': 'self.sample3',
'size': 0,
'util': 0.75,
'is_public': False},
source='test_source'),
sample.Sample(
'meter.mine',
@ -112,8 +125,11 @@ class TestListMeters(FunctionalTest,
'properties': {
'prop_1': 'prop_value',
'prop_2': {'sub_prop_1':
'sub_prop_value'}}
'sub_prop_value'}
},
'size': 0,
'util': 0.58,
'is_public': True},
source='test_source')]:
msg = rpc.meter_message_from_counter(
cnt,
@ -173,6 +189,86 @@ class TestListMeters(FunctionalTest,
self.assertEqual(set(r['counter_name'] for r in data),
set(['meter.test']))
def test_list_meters_query_integer_metadata(self):
data = self.get_json('/meters/meter.test',
q=[{'field': 'metadata.size',
'op': 'eq',
'value': '0',
'type': 'integer'}]
)
self.assertEquals(2, len(data))
self.assertEquals(set(r['resource_id'] for r in data),
set(['resource-id',
'resource-id3']))
self.assertEquals(set(r['counter_name'] for r in data),
set(['meter.test']))
self.assertEquals(set(r['resource_metadata']['size'] for r in data),
set(['0']))
def test_list_meters_query_float_metadata(self):
data = self.get_json('/meters/meter.test',
q=[{'field': 'metadata.util',
'op': 'eq',
'value': '0.75',
'type': 'float'}]
)
self.assertEquals(2, len(data))
self.assertEquals(set(r['resource_id'] for r in data),
set(['resource-id',
'resource-id3']))
self.assertEquals(set(r['counter_name'] for r in data),
set(['meter.test']))
self.assertEquals(set(r['resource_metadata']['util'] for r in data),
set(['0.75']))
def test_list_meters_query_boolean_metadata(self):
data = self.get_json('/meters/meter.mine',
q=[{'field': 'metadata.is_public',
'op': 'eq',
'value': 'False',
'type': 'boolean'}]
)
self.assertEquals(1, len(data))
self.assertEquals(set(r['resource_id'] for r in data),
set(['resource-id2']))
self.assertEquals(set(r['counter_name'] for r in data),
set(['meter.mine']))
self.assertEquals(set(r['resource_metadata']['is_public'] for r
in data), set(['False']))
def test_list_meters_query_string_metadata(self):
data = self.get_json('/meters/meter.test',
q=[{'field': 'metadata.tag',
'op': 'eq',
'value': 'self.sample'}]
)
self.assertEquals(1, len(data))
self.assertEquals(set(r['resource_id'] for r in data),
set(['resource-id']))
self.assertEquals(set(r['counter_name'] for r in data),
set(['meter.test']))
self.assertEquals(set(r['resource_metadata']['tag'] for r in data),
set(['self.sample']))
def test_list_meters_query_integer_float_metadata_without_type(self):
data = self.get_json('/meters/meter.test',
q=[{'field': 'metadata.size',
'op': 'eq',
'value': '0'},
{'field': 'metadata.util',
'op': 'eq',
'value': '0.75'}]
)
self.assertEquals(1, len(data))
self.assertEquals(set(r['resource_id'] for r in data),
set(['resource-id3']))
self.assertEquals(set(r['counter_name'] for r in data),
set(['meter.test']))
self.assertEquals(set(r['resource_metadata']['size'] for r in data),
set(['0']))
self.assertEquals(set(r['resource_metadata']['util'] for r in data),
set(['0.75']))
def test_with_resource(self):
data = self.get_json('/meters', q=[{'field': 'resource_id',
'value': 'resource-id',

98
tests/api/v2/test_query.py Normal file
View File

@ -0,0 +1,98 @@
# Copyright 2013 OpenStack Foundation.
# All Rights Reserved.
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Test the methods related to query."""
import wsme
from ceilometer.api.controllers.v2 import Query
from ceilometer.tests import base as tests_base
class TestQuery(tests_base.TestCase):
def test_get_value_as_type_with_integer(self):
query = Query(field='metadata.size',
op='eq',
value='123',
type='integer')
expected = 123
self.assertEqual(query._get_value_as_type(), expected)
def test_get_value_as_type_with_float(self):
query = Query(field='metadata.size',
op='eq',
value='123.456',
type='float')
expected = 123.456
self.assertEqual(query._get_value_as_type(), expected)
def test_get_value_as_type_with_boolean(self):
query = Query(field='metadata.is_public',
op='eq',
value='True',
type='boolean')
expected = True
self.assertEqual(query._get_value_as_type(), expected)
def test_get_value_as_type_with_string(self):
query = Query(field='metadata.name',
op='eq',
value='linux',
type='string')
expected = 'linux'
self.assertEqual(query._get_value_as_type(), expected)
def test_get_value_as_type_with_integer_without_type(self):
query = Query(field='metadata.size',
op='eq',
value='123')
expected = 123
self.assertEqual(query._get_value_as_type(), expected)
def test_get_value_as_type_with_float_without_type(self):
query = Query(field='metadata.size',
op='eq',
value='123.456')
expected = 123.456
self.assertEqual(query._get_value_as_type(), expected)
def test_get_value_as_type_with_boolean_without_type(self):
query = Query(field='metadata.is_public',
op='eq',
value='True')
expected = True
self.assertEqual(query._get_value_as_type(), expected)
def test_get_value_as_type_with_string_without_type(self):
query = Query(field='metadata.name',
op='eq',
value='linux')
expected = 'linux'
self.assertEqual(query._get_value_as_type(), expected)
def test_get_value_as_type_with_bad_type(self):
query = Query(field='metadata.size',
op='eq',
value='123.456',
type='blob')
self.assertRaises(wsme.exc.ClientSideError, query._get_value_as_type)
def test_get_value_as_type_with_bad_value(self):
query = Query(field='metadata.size',
op='eq',
value='fake',
type='integer')
self.assertRaises(wsme.exc.ClientSideError, query._get_value_as_type)