diff --git a/doc/source/objects.rst b/doc/source/objects.rst new file mode 100644 index 000000000..8f2e528bc --- /dev/null +++ b/doc/source/objects.rst @@ -0,0 +1,120 @@ +.. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Versioned Objects +================= + +Zun uses the `oslo.versionedobjects library +`_ to +construct an object model that can be communicated via RPC. These objects have +a version history and functionality to convert from one version to a previous +version. This allows for 2 different levels of the code to still pass objects +to each other, as in the case of rolling upgrades. + +Object Version Testing +---------------------- + +In order to ensure object versioning consistency is maintained, +oslo.versionedobjects has a fixture to aid in testing object versioning. +`oslo.versionedobjects.fixture.ObjectVersionChecker +`_ +generates fingerprints of each object, which is a combination of the current +version number of the object, along with a hash of the RPC-critical parts of +the object (fields and remotable methods). + +The tests hold a static mapping of the fingerprints of all objects. When an +object is changed, the hash generated in the test will differ from that held in +the static mapping. This will signal to the developer that the version of the +object needs to be increased. Following this version increase, the fingerprint +that is then generated by the test can be copied to the static mapping in the +tests. This symbolizes that if the code change is approved, this is the new +state of the object to compare against. + +Object Change Example +''''''''''''''''''''' + +The following example shows the unit test workflow when changing an object +(Cluster was updated to hold a new 'foo' field):: + + tox -e py27 zun.tests.unit.objects.test_objects + +This results in a unit test failure with the following output: + + .. code-block:: python + + testtools.matchers._impl.MismatchError: !=: + reference = {'Container': '1.0-35edde13ad178e9419e7ea8b6d580bcd'} + actual = {'Container': '1.0-22b40e8eed0414561ca921906b189820'} + + .. code-block:: console + + : Fields or remotable methods in some objects have changed. Make sure the versions of the objects has been bumped, and update the hashes in the static fingerprints tree (object_data). For more information, read http://docs.openstack.org/developer/zun/objects.html. + +This is an indication that me adding the 'foo' field to Cluster means I need +to bump the version of Cluster, so I increase the version and add a comment +saying what I changed in the new version: + + .. code-block:: python + + @base.ZunObjectRegistry.register + class Container(base.ZunPersistentObject, base.ZunObject, + base.ZunObjectDictCompat): + # Version 1.0: Initial version + # Version 1.1: Add container_id column + # Version 1.2: Add memory column + # Version 1.3: Add task_state column + # Version 1.4: Add cpu, workdir, ports, hostname and labels columns + # Version 1.5: Add meta column + # Version 1.6: Add addresses column + # Version 1.7: Add host column + # Version 1.8: Add restart_policy + # Version 1.9: Add status_detail column + # Version 1.10: Add tty, stdin_open + # Version 1.11: Add image_driver + VERSION = '1.11' + +Now that I have updated the version, I will run the tests again and let the +test tell me the fingerprint that I now need to put in the static tree: + + .. code-block:: python + + testtools.matchers._impl.MismatchError: !=: + reference = {'Container': '1.10-35edde13ad178e9419e7ea8b6d580bcd'} + actual = {'Container': '1.11-ddffeb42cb5472decab6d73534fe103f'} + +I can now copy the new fingerprint needed +(1.11-ddffeb42cb5472decab6d73534fe103f), to the object_data map within +zun/tests/unit/objects/test_objects.py: + + .. code-block:: python + + object_data = { + 'Container': '1.11-ddffeb42cb5472decab6d73534fe103f', + 'Image': '1.0-0b976be24f4f6ee0d526e5c981ce0633', + 'NUMANode': '1.0-cba878b70b2f8b52f1e031b41ac13b4e', + 'NUMATopology': '1.0-b54086eda7e4b2e6145ecb6ee2c925ab', + 'ResourceClass': '1.0-2c41abea55d0f7cb47a97bdb345b37fd', + 'ResourceProvider': '1.0-92b427359d5a4cf9ec6c72cbe630ee24', + 'ZunService': '1.0-2a19ab9987a746621b2ada02d8aadf22', + } + +Running the unit tests now shows no failure. + +If I did not update the version, and rather just copied the new hash to the +object_data map, the review would show the hash (but not the version) was +updated in object_data. At that point, a reviewer should point this out, and +mention that the object version needs to be updated. + +If a remotable method were added/changed, the same process is followed, because +this will also cause a hash change. diff --git a/zun/tests/unit/objects/test_objects.py b/zun/tests/unit/objects/test_objects.py new file mode 100644 index 000000000..56e7110f0 --- /dev/null +++ b/zun/tests/unit/objects/test_objects.py @@ -0,0 +1,457 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +import gettext + +import mock +from oslo_versionedobjects import exception as object_exception +from oslo_versionedobjects import fields +from oslo_versionedobjects import fixture + +from zun.common import context as zun_context +from zun.objects import base +from zun.tests import base as test_base + +gettext.install('zun') + + +@base.ZunObjectRegistry.register +class MyObj(base.ZunPersistentObject, base.ZunObject): + VERSION = '1.0' + + fields = {'foo': fields.IntegerField(), + 'bar': fields.StringField(), + 'missing': fields.StringField(), + } + + def obj_load_attr(self, attrname): + setattr(self, attrname, 'loaded!') + + @base.remotable_classmethod + def query(cls, context): + obj = cls(context) + obj.foo = 1 + obj.bar = 'bar' + obj.obj_reset_changes() + return obj + + @base.remotable + def marco(self, context): + return 'polo' + + @base.remotable + def update_test(self, context): + if context.project_id == 'alternate': + self.bar = 'alternate-context' + else: + self.bar = 'updated' + + @base.remotable + def save(self, context): + self.obj_reset_changes() + + @base.remotable + def refresh(self, context): + self.foo = 321 + self.bar = 'refreshed' + self.obj_reset_changes() + + @base.remotable + def modify_save_modify(self, context): + self.bar = 'meow' + self.save(context) + self.foo = 42 + + +class MyObj2(object): + @classmethod + def obj_name(cls): + return 'MyObj' + + @base.remotable_classmethod + def get(cls, *args, **kwargs): + pass + + +@base.ZunObjectRegistry.register_if(False) +class TestSubclassedObject(MyObj): + fields = {'new_field': fields.StringField()} + + +class _TestObject(object): + def test_hydration_type_error(self): + primitive = {'versioned_object.name': 'MyObj', + 'versioned_object.namespace': 'zun', + 'versioned_object.version': '1.0', + 'versioned_object.data': {'foo': 'a'}} + self.assertRaises(ValueError, MyObj.obj_from_primitive, primitive) + + def test_hydration(self): + primitive = {'versioned_object.name': 'MyObj', + 'versioned_object.namespace': 'zun', + 'versioned_object.version': '1.0', + 'versioned_object.data': {'foo': 1}} + obj = MyObj.obj_from_primitive(primitive) + self.assertEqual(1, obj.foo) + + def test_hydration_bad_ns(self): + primitive = {'versioned_object.name': 'MyObj', + 'versioned_object.namespace': 'foo', + 'versioned_object.version': '1.0', + 'versioned_object.data': {'foo': 1}} + self.assertRaises(object_exception.UnsupportedObjectError, + MyObj.obj_from_primitive, primitive) + + def test_dehydration(self): + expected = {'versioned_object.name': 'MyObj', + 'versioned_object.namespace': 'zun', + 'versioned_object.version': '1.0', + 'versioned_object.data': {'foo': 1}} + obj = MyObj(self.context) + obj.foo = 1 + obj.obj_reset_changes() + self.assertEqual(expected, obj.obj_to_primitive()) + + def test_get_updates(self): + obj = MyObj(self.context) + self.assertEqual({}, obj.obj_get_changes()) + obj.foo = 123 + self.assertEqual({'foo': 123}, obj.obj_get_changes()) + obj.bar = 'test' + self.assertEqual({'foo': 123, 'bar': 'test'}, obj.obj_get_changes()) + obj.obj_reset_changes() + self.assertEqual({}, obj.obj_get_changes()) + + def test_object_property(self): + obj = MyObj(self.context, foo=1) + self.assertEqual(1, obj.foo) + + def test_object_property_type_error(self): + obj = MyObj(self.context) + + def fail(): + obj.foo = 'a' + self.assertRaises(ValueError, fail) + + def test_load(self): + obj = MyObj(self.context) + self.assertEqual('loaded!', obj.bar) + + def test_load_in_base(self): + @base.ZunObjectRegistry.register_if(False) + class Foo(base.ZunPersistentObject, base.ZunObject): + fields = {'foobar': fields.IntegerField()} + obj = Foo(self.context) + # NOTE(danms): Can't use assertRaisesRegexp() because of py26 + raised = False + ex = None + try: + obj.foobar + except NotImplementedError as e: + raised = True + ex = e + self.assertTrue(raised) + self.assertIn('foobar', str(ex)) + + def test_loaded_in_primitive(self): + obj = MyObj(self.context) + obj.foo = 1 + obj.obj_reset_changes() + self.assertEqual('loaded!', obj.bar) + expected = {'versioned_object.name': 'MyObj', + 'versioned_object.namespace': 'zun', + 'versioned_object.version': '1.0', + 'versioned_object.changes': ['bar'], + 'versioned_object.data': {'foo': 1, + 'bar': 'loaded!'}} + self.assertEqual(expected, obj.obj_to_primitive()) + + def test_changes_in_primitive(self): + obj = MyObj(self.context) + obj.foo = 123 + self.assertEqual(set(['foo']), obj.obj_what_changed()) + primitive = obj.obj_to_primitive() + self.assertIn('versioned_object.changes', primitive) + obj2 = MyObj.obj_from_primitive(primitive) + self.assertEqual(set(['foo']), obj2.obj_what_changed()) + obj2.obj_reset_changes() + self.assertEqual(set(), obj2.obj_what_changed()) + + def test_unknown_objtype(self): + self.assertRaises(object_exception.UnsupportedObjectError, + base.ZunObject.obj_class_from_name, 'foo', '1.0') + + def test_with_alternate_context(self): + context1 = zun_context.RequestContext('foo', 'foo') + context2 = zun_context.RequestContext('bar', project_id='alternate') + obj = MyObj.query(context1) + obj.update_test(context2) + self.assertEqual('alternate-context', obj.bar) + + def test_orphaned_object(self): + obj = MyObj.query(self.context) + obj._context = None + self.assertRaises(object_exception.OrphanedObjectError, + obj.update_test) + + def test_changed_1(self): + obj = MyObj.query(self.context) + obj.foo = 123 + self.assertEqual(set(['foo']), obj.obj_what_changed()) + obj.update_test(self.context) + self.assertEqual(set(['foo', 'bar']), obj.obj_what_changed()) + self.assertEqual(123, obj.foo) + + def test_changed_2(self): + obj = MyObj.query(self.context) + obj.foo = 123 + self.assertEqual(set(['foo']), obj.obj_what_changed()) + obj.save(self.context) + self.assertEqual(set([]), obj.obj_what_changed()) + self.assertEqual(123, obj.foo) + + def test_changed_3(self): + obj = MyObj.query(self.context) + obj.foo = 123 + self.assertEqual(set(['foo']), obj.obj_what_changed()) + obj.refresh(self.context) + self.assertEqual(set([]), obj.obj_what_changed()) + self.assertEqual(321, obj.foo) + self.assertEqual('refreshed', obj.bar) + + def test_changed_4(self): + obj = MyObj.query(self.context) + obj.bar = 'something' + self.assertEqual(set(['bar']), obj.obj_what_changed()) + obj.modify_save_modify(self.context) + self.assertEqual(set(['foo']), obj.obj_what_changed()) + self.assertEqual(42, obj.foo) + self.assertEqual('meow', obj.bar) + + def test_static_result(self): + obj = MyObj.query(self.context) + self.assertEqual('bar', obj.bar) + result = obj.marco(self.context) + self.assertEqual('polo', result) + + def test_updates(self): + obj = MyObj.query(self.context) + self.assertEqual(1, obj.foo) + obj.update_test(self.context) + self.assertEqual('updated', obj.bar) + + def test_base_attributes(self): + dt = datetime.datetime(1955, 11, 5) + datatime = fields.DateTimeField() + obj = MyObj(self.context) + obj.created_at = dt + obj.updated_at = dt + expected = {'versioned_object.name': 'MyObj', + 'versioned_object.namespace': 'zun', + 'versioned_object.version': '1.0', + 'versioned_object.changes': + ['created_at', 'updated_at'], + 'versioned_object.data': + {'created_at': datatime.stringify(dt), + 'updated_at': datatime.stringify(dt)} + } + actual = obj.obj_to_primitive() + # versioned_object.changes is built from a set and order is undefined + self.assertEqual(sorted(expected['versioned_object.changes']), + sorted(actual['versioned_object.changes'])) + del expected['versioned_object.changes'],\ + actual['versioned_object.changes'] + self.assertEqual(expected, actual) + + def test_contains(self): + obj = MyObj(self.context) + self.assertNotIn('foo', obj) + obj.foo = 1 + self.assertIn('foo', obj) + self.assertNotIn('does_not_exist', obj) + + def test_obj_attr_is_set(self): + obj = MyObj(self.context, foo=1) + self.assertTrue(obj.obj_attr_is_set('foo')) + self.assertFalse(obj.obj_attr_is_set('bar')) + self.assertRaises(AttributeError, obj.obj_attr_is_set, 'bang') + + def test_get(self): + obj = MyObj(self.context, foo=1) + # Foo has value, should not get the default + self.assertEqual(1, getattr(obj, 'foo', 2)) + # Foo has value, should return the value without error + self.assertEqual(1, getattr(obj, 'foo')) + # Bar without a default should lazy-load + self.assertEqual('loaded!', getattr(obj, 'bar')) + # Bar now has a default, but loaded value should be returned + self.assertEqual('loaded!', getattr(obj, 'bar', 'not-loaded')) + # Invalid attribute should raise AttributeError + self.assertFalse(hasattr(obj, 'nothing')) + + def test_object_inheritance(self): + base_fields = list(base.ZunPersistentObject.fields.keys()) + myobj_fields = ['foo', 'bar', 'missing'] + base_fields + myobj3_fields = ['new_field'] + self.assertTrue(issubclass(TestSubclassedObject, MyObj)) + self.assertEqual(len(MyObj.fields), len(myobj_fields)) + self.assertEqual(set(MyObj.fields.keys()), set(myobj_fields)) + self.assertEqual(len(TestSubclassedObject.fields), + len(myobj_fields) + len(myobj3_fields)) + self.assertEqual(set(TestSubclassedObject.fields.keys()), + set(myobj_fields) | set(myobj3_fields)) + + def test_get_changes(self): + obj = MyObj(self.context) + self.assertEqual({}, obj.obj_get_changes()) + obj.foo = 123 + self.assertEqual({'foo': 123}, obj.obj_get_changes()) + obj.bar = 'test' + self.assertEqual({'foo': 123, 'bar': 'test'}, obj.obj_get_changes()) + obj.obj_reset_changes() + self.assertEqual({}, obj.obj_get_changes()) + + def test_obj_fields(self): + @base.ZunObjectRegistry.register_if(False) + class TestObj(base.ZunPersistentObject, base.ZunObject): + fields = {'foo': fields.IntegerField()} + obj_extra_fields = ['bar'] + + @property + def bar(self): + return 'this is bar' + + obj = TestObj(self.context) + self.assertEqual(set(['created_at', 'updated_at', 'foo', 'bar']), + set(obj.obj_fields)) + + def test_obj_constructor(self): + obj = MyObj(self.context, foo=123, bar='abc') + self.assertEqual(123, obj.foo) + self.assertEqual('abc', obj.bar) + self.assertEqual(set(['foo', 'bar']), obj.obj_what_changed()) + + +class TestObject(test_base.TestCase, _TestObject): + pass + + +# This is a static dictionary that holds all fingerprints of the versioned +# objects registered with the ZunRegistry. Each fingerprint contains +# the version of the object and an md5 hash of RPC-critical parts of the +# object (fields and remotable methods). If either the version or hash +# change, the static tree needs to be updated. +# For more information on object version testing, read +# http://docs.openstack.org/developer/zun/objects.html +object_data = { + 'Container': '1.11-ddffeb42cb5472decab6d73534fe103f', + 'Image': '1.0-0b976be24f4f6ee0d526e5c981ce0633', + 'MyObj': '1.0-34c4b1aadefd177b13f9a2f894cc23cd', + 'NUMANode': '1.0-cba878b70b2f8b52f1e031b41ac13b4e', + 'NUMATopology': '1.0-b54086eda7e4b2e6145ecb6ee2c925ab', + 'ResourceClass': '1.0-2c41abea55d0f7cb47a97bdb345b37fd', + 'ResourceProvider': '1.0-92b427359d5a4cf9ec6c72cbe630ee24', + 'ZunService': '1.0-2a19ab9987a746621b2ada02d8aadf22', +} + + +class TestObjectVersions(test_base.TestCase): + def test_versions(self): + # Test the versions of current objects with the static tree above. + # This ensures that any incompatible object changes require a version + # bump. + classes = base.ZunObjectRegistry.obj_classes() + checker = fixture.ObjectVersionChecker(obj_classes=classes) + + expected, actual = checker.test_hashes(object_data) + self.assertEqual(expected, actual, + "Fields or remotable methods in some objects have " + "changed. Make sure the versions of the objects has " + "been bumped, and update the hashes in the static " + "fingerprints tree (object_data). For more " + "information, read http://docs.openstack.org/" + "developer/zun/objects.html.") + + +class TestObjectSerializer(test_base.TestCase): + + def test_object_serialization(self): + ser = base.ZunObjectSerializer() + obj = MyObj(self.context) + primitive = ser.serialize_entity(self.context, obj) + self.assertIn('versioned_object.name', primitive) + obj2 = ser.deserialize_entity(self.context, primitive) + self.assertIsInstance(obj2, MyObj) + self.assertEqual(self.context, obj2._context) + + def test_object_serialization_iterables(self): + ser = base.ZunObjectSerializer() + obj = MyObj(self.context) + for iterable in (list, tuple, set): + thing = iterable([obj]) + primitive = ser.serialize_entity(self.context, thing) + self.assertEqual(1, len(primitive)) + for item in primitive: + self.assertFalse(isinstance(item, base.ZunObject)) + thing2 = ser.deserialize_entity(self.context, primitive) + self.assertEqual(1, len(thing2)) + for item in thing2: + self.assertIsInstance(item, MyObj) + + @mock.patch('zun.objects.base.ZunObject.indirection_api') + def _test_deserialize_entity_newer(self, obj_version, backported_to, + mock_indirection_api, + my_version='1.6'): + ser = base.ZunObjectSerializer() + mock_indirection_api.object_backport_versions.side_effect \ + = NotImplementedError() + mock_indirection_api.object_backport.return_value = 'backported' + + @base.ZunObjectRegistry.register + class MyTestObj(MyObj): + VERSION = my_version + + obj = MyTestObj() + obj.VERSION = obj_version + primitive = obj.obj_to_primitive() + result = ser.deserialize_entity(self.context, primitive) + if backported_to is None: + self.assertFalse(mock_indirection_api.object_backport.called) + else: + self.assertEqual('backported', result) + mock_indirection_api.object_backport.assert_called_with( + self.context, primitive, backported_to) + + def test_deserialize_entity_newer_version_backports_level1(self): + "Test object with unsupported (newer) version" + self._test_deserialize_entity_newer('11.5', '1.6') + + def test_deserialize_entity_newer_version_backports_level2(self): + "Test object with unsupported (newer) version" + self._test_deserialize_entity_newer('1.25', '1.6') + + def test_deserialize_entity_same_revision_does_not_backport(self): + "Test object with supported revision" + self._test_deserialize_entity_newer('1.6', None) + + def test_deserialize_entity_newer_revision_does_not_backport_zero(self): + "Test object with supported revision" + self._test_deserialize_entity_newer('1.6.0', None) + + def test_deserialize_entity_newer_revision_does_not_backport(self): + "Test object with supported (newer) revision" + self._test_deserialize_entity_newer('1.6.1', None) + + def test_deserialize_entity_newer_version_passes_revision(self): + "Test object with unsupported (newer) version and revision" + self._test_deserialize_entity_newer('1.7', '1.6.1', my_version='1.6.1')