Avoid false negatives on message signature comparison

Fixes bug 1262255

Previously, samples for the 'instance.scheduled' meter were always
rejected with a false negative on the message signature verification.

The problem occured because certain resource metadata (such as the
block_device_mapping) in the 'scheduler.run_instance.scheduled'
notification are realized as a list of dict, and are thus similarly
encoded in the corresponding sample payload.

However the message signature computation then became indeterminate,
as the unicode representation of equivalent dicts is not guaranteed
equal (e.g. depends on insertion order in the case of hash collisions).

Now, we avoid false negatives by explicitly re-ordering insertion
into such dicts.

Change-Id: I77f7d89229518cf040608d7eb3307e2257bce07a

ceilometer/tests/publisher/test_rpc_publisher.py

@@ -95,6 +95,20 @@ class TestSignature(test.BaseTestCase):
             'not-so-secret')
         self.assertTrue(rpc.verify_signature(data, 'not-so-secret'))
 
+    def test_verify_signature_nested_list_of_dict(self):
+        small = 1
+        big = 1 << 64
+        nested = {small: 99, big: 42}
+        data = {'a': 'A',
+                'b': 'B',
+                'nested': {'list': [nested]}}
+        data['message_signature'] = rpc.compute_signature(
+            data,
+            'not-so-secret')
+        # the keys 1 and 1<<64 cause a hash collision on 64bit platforms
+        data['nested']['list'] = [{big: 42, small: 99}]
+        self.assertTrue(rpc.verify_signature(data, 'not-so-secret'))
+
     def test_verify_signature_nested_json(self):
         data = {'a': 'A',
                 'b': 'B',

ceilometer/tests/test_utils.py

@@ -68,6 +68,20 @@ class TestUtils(test.BaseTestCase):
                                  ('nested.a', 'A'),
                                  ('nested.b', 'B')])
 
+    def test_recursive_keypairs_with_list_of_dict(self):
+        small = 1
+        big = 1 << 64
+        expected = [('a', 'A'),
+                    ('b', 'B'),
+                    ('nested:list', ['{%d: 99, %dL: 42}' % (small, big)])]
+        # the keys 1 and 1<<64 cause a hash collision on 64bit platforms
+        for nested in [{small: 99, big: 42}, {big: 42, small: 99}]:
+            data = {'a': 'A',
+                    'b': 'B',
+                    'nested': {'list': [nested]}}
+            pairs = list(utils.recursive_keypairs(data))
+            self.assertEqual(pairs, expected)
+
     def test_decimal_to_dt_with_none_parameter(self):
         self.assertEqual(utils.decimal_to_dt(None), None)
 

ceilometer/utils.py

@@ -36,8 +36,17 @@ def recursive_keypairs(d, separator=':'):
             # When doing a pair of JSON encode/decode operations to the tuple,
             # the tuple would become list. So we have to generate the value as
             # list here.
-            yield name, list(map(lambda x: unicode(x).encode('utf-8'),
-                                 value))
+
+            # in the special case of the list item itself being a dict,
+            # create an equivalent dict with a predictable insertion order
+            # to avoid inconsistencies in the message signature computation
+            # for equivalent payloads modulo ordering
+            first = lambda i: i[0]
+            m = map(lambda x: unicode(dict(sorted(x.items(), key=first))
+                                      if isinstance(x, dict)
+                                      else x).encode('utf-8'),
+                    value)
+            yield name, list(m)
         else:
             yield name, value