port pci device pool from nova to zun

port pci device pool from nova to zun

Change-Id: If4be745285ddebe78cc69b7e682d83fe74fa3629
Partially-Implements: blueprint support-pcipassthroughfilter

zun/db/sqlalchemy/alembic/versions/ff7b9665d504_add_pci_stats_to_compute_node.py

@@ -0,0 +1,33 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+"""add pci stats to compute node
+
+Revision ID: ff7b9665d504
+Revises: fc27c7415d9c
+Create Date: 2017-09-26 13:49:11.470002
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'ff7b9665d504'
+down_revision = 'fc27c7415d9c'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('compute_node',
+                  sa.Column('pci_stats', sa.Text()))

zun/db/sqlalchemy/models.py

@@ -319,6 +319,9 @@ class ComputeNode(Base):
     os = Column(String(64), nullable=True)
     kernel_version = Column(String(128), nullable=True)
     labels = Column(JSONEncodedDict)
+    # Json string PCI Stats
+    # '[{"vendor_id":"8086", "product_id":"1234", "count":3 }, ...]'
+    pci_stats = Column(Text)
 
 
 class Capsule(Base):

zun/objects/__init__.py

@@ -16,6 +16,7 @@ from zun.objects import container
 from zun.objects import image
 from zun.objects import numa
 from zun.objects import pci_device
+from zun.objects import pci_device_pool
 from zun.objects import resource_class
 from zun.objects import resource_provider
 from zun.objects import volume_mapping
@@ -32,6 +33,7 @@ ResourceClass = resource_class.ResourceClass
 ComputeNode = compute_node.ComputeNode
 Capsule = capsule.Capsule
 PciDevice = pci_device.PciDevice
+PciDevicePool = pci_device_pool.PciDevicePool
 
 __all__ = (
     Container,
@@ -45,4 +47,5 @@ __all__ = (
     ComputeNode,
     Capsule,
     PciDevice,
+    PciDevicePool,
 )

zun/objects/compute_node.py

@@ -10,11 +10,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from oslo_serialization import jsonutils
 from oslo_versionedobjects import fields
 
 from zun.db import api as dbapi
 from zun.objects import base
 from zun.objects.numa import NUMATopology
+from zun.objects import pci_device_pool
 
 
 @base.ZunObjectRegistry.register
@@ -27,7 +29,8 @@ class ComputeNode(base.ZunPersistentObject, base.ZunObject):
     # Version 1.5: Add host labels info
     # Version 1.6: Add mem_used to compute node
     # Version 1.7: Change get_by_hostname to get_by_name
-    VERSION = '1.7'
+    # Version 1.8: Add pci_device_pools to compute node
+    VERSION = '1.8'
 
     fields = {
         'uuid': fields.UUIDField(read_only=True, nullable=False),
@@ -48,12 +51,18 @@ class ComputeNode(base.ZunPersistentObject, base.ZunObject):
         'os': fields.StringField(nullable=True),
         'kernel_version': fields.StringField(nullable=True),
         'labels': fields.DictOfStringsField(nullable=True),
+        # NOTE(pmurray): the pci_device_pools field maps to the
+        # pci_stats field in the database
+        'pci_device_pools': fields.ListOfObjectsField('PciDevicePool',
+                                                      nullable=True),
     }
 
     @staticmethod
     def _from_db_object(context, compute_node, db_compute_node):
         """Converts a database entity to a formal object."""
-        for field in compute_node.fields:
+        special_cases = set(['pci_device_pools'])
+        fields = set(compute_node.fields) - special_cases
+        for field in fields:
             if field == 'numa_topology':
                 numa_obj = NUMATopology._from_dict(
                     db_compute_node['numa_topology'])
@@ -61,6 +70,10 @@ class ComputeNode(base.ZunPersistentObject, base.ZunObject):
             else:
                 setattr(compute_node, field, db_compute_node[field])
 
+        pci_stats = db_compute_node.get('pci_stats')
+        if pci_stats is not None:
+            pci_stats = pci_device_pool.from_pci_stats(pci_stats)
+        compute_node.pci_device_pools = pci_stats
         compute_node.obj_reset_changes(recursive=True)
         return compute_node
 
@@ -70,6 +83,14 @@ class ComputeNode(base.ZunPersistentObject, base.ZunObject):
         return [ComputeNode._from_db_object(context, cls(context), obj)
                 for obj in db_objects]
 
+    @staticmethod
+    def _convert_pci_stats_to_db_format(updates):
+        if 'pci_device_pools' in updates:
+            pools = updates.pop('pci_device_pools')
+            if pools is not None:
+                pools = jsonutils.dumps(pools.obj_to_primitive())
+            updates['pci_stats'] = pools
+
     @base.remotable
     def create(self, context):
         """Create a compute node record in the DB.
@@ -82,6 +103,7 @@ class ComputeNode(base.ZunPersistentObject, base.ZunObject):
         if numa_obj is not None:
             values['numa_topology'] = numa_obj._to_dict()
 
+        self._convert_pci_stats_to_db_format(values)
         db_compute_node = dbapi.create_compute_node(context, values)
         self._from_db_object(context, self, db_compute_node)
 

zun/objects/pci_device_pool.py

@@ -0,0 +1,93 @@
+# Copyright (c) 2017 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+
+from oslo_serialization import jsonutils
+from oslo_versionedobjects import fields
+import six
+
+from zun.objects import base
+
+
+@base.ZunObjectRegistry.register
+class PciDevicePool(base.ZunObject):
+    # Version 1.0: Initial version
+    VERSION = '1.0'
+
+    fields = {
+        'product_id': fields.StringField(),
+        'vendor_id': fields.StringField(),
+        'numa_node': fields.IntegerField(nullable=True),
+        'tags': fields.DictOfNullableStringsField(),
+        'count': fields.IntegerField(),
+        }
+
+    # NOTE(pmurray): before this object existed the pci device pool data was
+    # stored as a dict. For backward compatibility we need to be able to read
+    # it in from a dict
+    @classmethod
+    def from_dict(cls, value):
+        pool_dict = copy.copy(value)
+        pool = cls()
+        pool.vendor_id = pool_dict.pop("vendor_id")
+        pool.product_id = pool_dict.pop("product_id")
+        pool.numa_node = pool_dict.pop("numa_node", None)
+        pool.count = pool_dict.pop("count")
+        pool.tags = pool_dict
+        return pool
+
+    # NOTE(sbauza): Before using objects, pci stats was a list of
+    # dictionaries not having tags. For compatibility with other modules, let's
+    # create a reversible method
+    def to_dict(self):
+        pci_pool = base.obj_to_primitive(self)
+        tags = pci_pool.pop('tags', {})
+        for k, v in tags.items():
+            pci_pool[k] = v
+        return pci_pool
+
+
+@base.ZunObjectRegistry.register
+class PciDevicePoolList(base.ObjectListBase, base.ZunObject):
+    # Version 1.0: Initial version
+    VERSION = '1.0'
+    fields = {'objects': fields.ListOfObjectsField('PciDevicePool'), }
+
+
+def from_pci_stats(pci_stats):
+    """Create and return a PciDevicePoolList from the data stored in the db,
+
+    which can be either the serialized object, or, prior to the creation of the
+    device pool objects, a simple dict or a list of such dicts.
+    """
+    pools = []
+    if isinstance(pci_stats, six.string_types):
+        try:
+            pci_stats = jsonutils.loads(pci_stats)
+        except (ValueError, TypeError):
+            pci_stats = None
+    if pci_stats:
+        # Check for object-ness, or old-style storage format.
+        if 'zun_object.namespace' in pci_stats:
+            return PciDevicePoolList.obj_from_primitive(pci_stats)
+        else:
+            # This can be either a dict or a list of dicts
+            if isinstance(pci_stats, list):
+                pools = [PciDevicePool.from_dict(stat)
+                         for stat in pci_stats]
+            else:
+                pools = [PciDevicePool.from_dict(pci_stats)]
+    return PciDevicePoolList(objects=pools)

zun/tests/unit/fake_pci_device_pools.py

@@ -0,0 +1,38 @@
+# Copyright 2017 IBM Corp.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from zun.objects import pci_device_pool
+
+
+# This represents the format that PCI device pool info was stored in the DB
+# before this info was made into objects.
+fake_pool_dict = {
+    'product_id': 'fake-product',
+    'vendor_id': 'fake-vendor',
+    'numa_node': 1,
+    't1': 'v1',
+    't2': 'v2',
+    'count': 2,
+    }
+
+fake_pool = pci_device_pool.PciDevicePool(count=5,
+                                          product_id='foo',
+                                          vendor_id='bar',
+                                          numa_node=0,
+                                          tags={'t1': 'v1', 't2': 'v2'})
+fake_pool_primitive = fake_pool.obj_to_primitive()
+
+fake_pool_list = pci_device_pool.PciDevicePoolList(objects=[fake_pool])
+fake_pool_list_primitive = fake_pool_list.obj_to_primitive()

zun/tests/unit/objects/test_objects.py

@@ -353,9 +353,11 @@ object_data = {
     'ResourceClass': '1.1-d661c7675b3cd5b8c3618b68ba64324e',
     'ResourceProvider': '1.0-92b427359d5a4cf9ec6c72cbe630ee24',
     'ZunService': '1.1-b1549134bfd5271daec417ca8cabc77e',
-    'ComputeNode': '1.7-9b700eb146e9978d84e9ccc5849d90e2',
     'Capsule': '1.2-1f8b3716ef272c9d9cb55390f6a7cdc3',
-    'PciDevice': '1.0-19fdd11935cda5e92947913f081d9edd'
+    'PciDevice': '1.0-19fdd11935cda5e92947913f081d9edd',
+    'ComputeNode': '1.8-5f5f893df0b514c88a3abaec1dfbb89e',
+    'PciDevicePool': '1.0-3f5ddc3ff7bfa14da7f6c7e9904cc000',
+    'PciDevicePoolList': '1.0-15ecf022a68ddbb8c2a6739cfc9f8f5e'
 }
 
 

zun/tests/unit/objects/test_pci_device_pool.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2017 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+
+from zun import objects
+from zun.objects import pci_device_pool
+from zun.tests import base
+from zun.tests.unit import fake_pci_device_pools as fake_pci
+
+
+class _TestPciDevicePoolObject(object):
+
+    def test_pci_pool_from_dict_not_distructive(self):
+        test_dict = copy.copy(fake_pci.fake_pool_dict)
+        objects.PciDevicePool.from_dict(test_dict)
+        self.assertEqual(fake_pci.fake_pool_dict, test_dict)
+
+    def test_pci_pool_from_dict(self):
+        pool_obj = objects.PciDevicePool.from_dict(fake_pci.fake_pool_dict)
+        self.assertEqual(pool_obj.product_id, 'fake-product')
+        self.assertEqual(pool_obj.vendor_id, 'fake-vendor')
+        self.assertEqual(pool_obj.numa_node, 1)
+        self.assertEqual(pool_obj.tags, {'t1': 'v1', 't2': 'v2'})
+        self.assertEqual(pool_obj.count, 2)
+
+    def test_pci_pool_from_dict_bad_tags(self):
+        bad_dict = copy.deepcopy(fake_pci.fake_pool_dict)
+        bad_dict['bad'] = {'foo': 'bar'}
+        self.assertRaises(ValueError,
+                          objects.PciDevicePool.from_dict,
+                          value=bad_dict)
+
+    def test_pci_pool_from_dict_no_tags(self):
+        dict_notag = copy.copy(fake_pci.fake_pool_dict)
+        dict_notag.pop('t1')
+        dict_notag.pop('t2')
+        pool_obj = objects.PciDevicePool.from_dict(dict_notag)
+        self.assertEqual(pool_obj.tags, {})
+
+    def test_pci_pool_to_dict(self):
+        tags = {'t1': 'foo', 't2': 'bar'}
+        pool_obj = objects.PciDevicePool(product_id='pid', tags=tags)
+        pool_dict = pool_obj.to_dict()
+        self.assertEqual({'product_id': 'pid',
+                          't1': 'foo',
+                          't2': 'bar'}, pool_dict)
+
+    def test_pci_pool_to_dict_no_tags(self):
+        pool_obj = objects.PciDevicePool(product_id='pid', tags={})
+        pool_dict = pool_obj.to_dict()
+        self.assertEqual({'product_id': 'pid'}, pool_dict)
+
+    def test_pci_pool_to_dict_with_tags_unset(self):
+        pool_obj = objects.PciDevicePool(product_id='pid')
+        pool_dict = pool_obj.to_dict()
+        self.assertEqual({'product_id': 'pid'}, pool_dict)
+
+
+class TestConvertPciStats(base.TestCase):
+    def test_from_pci_stats_obj(self):
+        prim = fake_pci.fake_pool_list_primitive
+        pools = pci_device_pool.from_pci_stats(prim)
+        self.assertEqual(len(pools), 1)
+
+    def test_from_pci_stats_dict(self):
+        prim = fake_pci.fake_pool_dict
+        pools = pci_device_pool.from_pci_stats(prim)
+        self.assertEqual(len(pools), 1)
+
+    def test_from_pci_stats_list_of_dicts(self):
+        prim = fake_pci.fake_pool_dict
+        pools = pci_device_pool.from_pci_stats([prim, prim])
+        self.assertEqual(len(pools), 2)
+
+    def test_from_pci_stats_bad(self):
+        prim = "not a valid json string for an object"
+        pools = pci_device_pool.from_pci_stats(prim)
+        self.assertEqual(len(pools), 0)