diff --git a/doc/source/model.rst b/doc/source/model.rst index 2d7d19c0f..508d4a54f 100644 --- a/doc/source/model.rst +++ b/doc/source/model.rst @@ -238,3 +238,35 @@ POV. is_enabled=bool(), is_domain=bool(), properties=dict()) + +Volume +------ + +A volume from cinder. + +.. code-block:: python + + Volume = dict( + location=Location(), + id=str(), + name=str(), + description=str(), + size=int(), + attachments=list(), + status=str(), + migration_status=str() or None, + host=str() or None, + replication_driver=str() or None, + replication_status=str() or None, + replication_extended_status=str() or None, + snapshot_id=str() or None, + created_at=str(), + updated_at=str() or None, + source_volume_id=str() or None, + consistencygroup_id=str() or None, + volume_type=str() or None, + metadata=dict(), + is_bootable=bool(), + is_encrypted=bool(), + can_multiattach=bool(), + properties=dict()) diff --git a/shade/_normalize.py b/shade/_normalize.py index b4c66a0df..c919aff97 100644 --- a/shade/_normalize.py +++ b/shade/_normalize.py @@ -513,7 +513,6 @@ class Normalizer(object): def _normalize_project(self, project): - ret = munch.Munch() # Copy incoming project because of shared dicts in unittests project = project.copy() @@ -563,3 +562,99 @@ class Normalizer(object): ret.setdefault(key, val) return ret + + def _normalize_volumes(self, volumes): + """Normalize the structure of volumes + + This makes tenants from cinder v1 look like volumes from v2. + + :param list projects: A list of volumes to normalize + + :returns: A list of normalized dicts. + """ + ret = [] + for volume in volumes: + ret.append(self._normalize_volume(volume)) + return ret + + def _normalize_volume(self, volume): + + volume = volume.copy() + + # Discard noise + volume.pop('links', None) + volume.pop('NAME_ATTR', None) + volume.pop('HUMAN_ID', None) + volume.pop('human_id', None) + + volume_id = volume.pop('id') + + name = volume.pop('display_name', None) + name = volume.pop('name', name) + + description = volume.pop('display_description', None) + description = volume.pop('description', description) + + is_bootable = _to_bool(volume.pop('bootable', True)) + is_encrypted = _to_bool(volume.pop('encrypted', False)) + can_multiattach = _to_bool(volume.pop('multiattach', False)) + + project_id = _pop_or_get( + volume, 'os-vol-tenant-attr:tenant_id', None, self.strict_mode) + az = volume.pop('availability_zone', None) + + location = self._get_current_location(project_id=project_id, zone=az) + + host = _pop_or_get( + volume, 'os-vol-host-attr:host', None, self.strict_mode) + replication_extended_status = _pop_or_get( + volume, 'os-volume-replication:extended_status', + None, self.strict_mode) + + migration_status = _pop_or_get( + volume, 'os-vol-mig-status-attr:migstat', None, self.strict_mode) + migration_status = volume.pop('migration_status', migration_status) + _pop_or_get(volume, 'user_id', None, self.strict_mode) + source_volume_id = _pop_or_get( + volume, 'source_volid', None, self.strict_mode) + replication_driver = _pop_or_get( + volume, 'os-volume-replication:driver_data', + None, self.strict_mode) + + ret = munch.Munch( + location=location, + id=volume_id, + name=name, + description=description, + size=_pop_int(volume, 'size'), + attachments=volume.pop('attachments', []), + status=volume.pop('status'), + migration_status=migration_status, + host=host, + replication_driver=replication_driver, + replication_status=volume.pop('replication_status', None), + replication_extended_status=replication_extended_status, + snapshot_id=volume.pop('snapshot_id', None), + created_at=volume.pop('created_at'), + updated_at=volume.pop('updated_at', None), + source_volume_id=source_volume_id, + consistencygroup_id=volume.pop('consistencygroup_id', None), + volume_type=volume.pop('volume_type', None), + metadata=volume.pop('metadata', {}), + is_bootable=is_bootable, + is_encrypted=is_encrypted, + can_multiattach=can_multiattach, + properties=volume.copy(), + ) + + # Backwards compat + if not self.strict_mode: + ret['display_name'] = name + ret['display_description'] = description + ret['bootable'] = is_bootable + ret['encrypted'] = is_encrypted + ret['multiattach'] = can_multiattach + ret['availability_zone'] = az + for key, val in ret['properties'].items(): + ret.setdefault(key, val) + return ret diff --git a/shade/_utils.py b/shade/_utils.py index 489fec176..f30b3ae93 100644 --- a/shade/_utils.py +++ b/shade/_utils.py @@ -210,30 +210,6 @@ def normalize_users(users): return meta.obj_list_to_dict(ret) -def normalize_volumes(volumes): - ret = [] - for vol in volumes: - new_vol = vol.copy() - name = vol.get('name', vol.get('display_name')) - description = vol.get('description', vol.get('display_description')) - new_vol['name'] = name - new_vol['display_name'] = name - new_vol['description'] = description - new_vol['display_description'] = description - # For some reason, cinder v1 uses strings for bools for these fields. - # Cinder v2 uses booleans. - for field in ('bootable', 'multiattach'): - if field in new_vol and isinstance(new_vol[field], - six.string_types): - if new_vol[field] is not None: - if new_vol[field].lower() == 'true': - new_vol[field] = True - elif new_vol[field].lower() == 'false': - new_vol[field] = False - ret.append(new_vol) - return meta.obj_list_to_dict(ret) - - def normalize_domains(domains): ret = [ dict( diff --git a/shade/openstackcloud.py b/shade/openstackcloud.py index 11b0de2b3..0212e32cb 100644 --- a/shade/openstackcloud.py +++ b/shade/openstackcloud.py @@ -1518,7 +1518,7 @@ class OpenStackCloud(_normalize.Normalizer): warnings.warn('cache argument to list_volumes is deprecated. Use ' 'invalidate instead.') with _utils.shade_exceptions("Error fetching volume list"): - return _utils.normalize_volumes( + return self._normalize_volumes( self.manager.submit_task(_tasks.VolumeList())) @_utils.cache_on_arguments() @@ -3338,7 +3338,7 @@ class OpenStackCloud(_normalize.Normalizer): raise OpenStackCloudException( "Error in creating volume, please check logs") - return _utils.normalize_volumes([volume])[0] + return self._normalize_volume(volume) def delete_volume(self, name_or_id=None, wait=True, timeout=None): """Delete a volume. @@ -3592,7 +3592,10 @@ class OpenStackCloud(_normalize.Normalizer): raise OpenStackCloudException( "Error in creating volume snapshot, please check logs") - return _utils.normalize_volumes([snapshot])[0] + # TODO(mordred) need to normalize snapshots. We were normalizing them + # as volumes, which is an error. They need to be normalized as + # volume snapshots, which are completely different objects + return snapshot def get_volume_snapshot_by_id(self, snapshot_id): """Takes a snapshot_id and gets a dict of the snapshot @@ -3612,7 +3615,7 @@ class OpenStackCloud(_normalize.Normalizer): ) ) - return _utils.normalize_volumes([snapshot])[0] + return self._normalize_volume(snapshot) def get_volume_snapshot(self, name_or_id, filters=None): """Get a volume by name or ID. @@ -3703,7 +3706,7 @@ class OpenStackCloud(_normalize.Normalizer): """ with _utils.shade_exceptions("Error getting a list of snapshots"): - return _utils.normalize_volumes( + return self._normalize_volumes( self.manager.submit_task( _tasks.VolumeSnapshotList( detailed=detailed, search_opts=search_opts))) diff --git a/shade/tests/unit/test_caching.py b/shade/tests/unit/test_caching.py index d749d5785..2ab556b6a 100644 --- a/shade/tests/unit/test_caching.py +++ b/shade/tests/unit/test_caching.py @@ -22,7 +22,6 @@ import testtools import warlock import shade.openstackcloud -from shade import _utils from shade import exc from shade import meta from shade.tests import fakes @@ -177,14 +176,14 @@ class TestMemoryCache(base.TestCase): def test_list_volumes(self, cinder_mock): fake_volume = fakes.FakeVolume('volume1', 'available', 'Volume 1 Display Name') - fake_volume_dict = _utils.normalize_volumes( - [meta.obj_to_dict(fake_volume)])[0] + fake_volume_dict = self.cloud._normalize_volume( + meta.obj_to_dict(fake_volume)) cinder_mock.volumes.list.return_value = [fake_volume] self.assertEqual([fake_volume_dict], self.cloud.list_volumes()) fake_volume2 = fakes.FakeVolume('volume2', 'available', 'Volume 2 Display Name') - fake_volume2_dict = _utils.normalize_volumes( - [meta.obj_to_dict(fake_volume2)])[0] + fake_volume2_dict = self.cloud._normalize_volume( + meta.obj_to_dict(fake_volume2)) cinder_mock.volumes.list.return_value = [fake_volume, fake_volume2] self.assertEqual([fake_volume_dict], self.cloud.list_volumes()) self.cloud.list_volumes.invalidate(self.cloud) @@ -195,14 +194,14 @@ class TestMemoryCache(base.TestCase): def test_list_volumes_creating_invalidates(self, cinder_mock): fake_volume = fakes.FakeVolume('volume1', 'creating', 'Volume 1 Display Name') - fake_volume_dict = _utils.normalize_volumes( - [meta.obj_to_dict(fake_volume)])[0] + fake_volume_dict = self.cloud._normalize_volume( + meta.obj_to_dict(fake_volume)) cinder_mock.volumes.list.return_value = [fake_volume] self.assertEqual([fake_volume_dict], self.cloud.list_volumes()) fake_volume2 = fakes.FakeVolume('volume2', 'available', 'Volume 2 Display Name') - fake_volume2_dict = _utils.normalize_volumes( - [meta.obj_to_dict(fake_volume2)])[0] + fake_volume2_dict = self.cloud._normalize_volume( + meta.obj_to_dict(fake_volume2)) cinder_mock.volumes.list.return_value = [fake_volume, fake_volume2] self.assertEqual([fake_volume_dict, fake_volume2_dict], self.cloud.list_volumes()) @@ -211,8 +210,8 @@ class TestMemoryCache(base.TestCase): def test_create_volume_invalidates(self, cinder_mock): fake_volb4 = fakes.FakeVolume('volume1', 'available', 'Volume 1 Display Name') - fake_volb4_dict = _utils.normalize_volumes( - [meta.obj_to_dict(fake_volb4)])[0] + fake_volb4_dict = self.cloud._normalize_volume( + meta.obj_to_dict(fake_volb4)) cinder_mock.volumes.list.return_value = [fake_volb4] self.assertEqual([fake_volb4_dict], self.cloud.list_volumes()) volume = dict(display_name='junk_vol', @@ -220,8 +219,8 @@ class TestMemoryCache(base.TestCase): display_description='test junk volume') fake_vol = fakes.FakeVolume('12345', 'creating', '') fake_vol_dict = meta.obj_to_dict(fake_vol) - fake_vol_dict = _utils.normalize_volumes( - [meta.obj_to_dict(fake_vol)])[0] + fake_vol_dict = self.cloud._normalize_volume( + meta.obj_to_dict(fake_vol)) cinder_mock.volumes.create.return_value = fake_vol cinder_mock.volumes.list.return_value = [fake_volb4, fake_vol] diff --git a/shade/tests/unit/test_create_volume_snapshot.py b/shade/tests/unit/test_create_volume_snapshot.py index 445945eb7..91af862ae 100644 --- a/shade/tests/unit/test_create_volume_snapshot.py +++ b/shade/tests/unit/test_create_volume_snapshot.py @@ -20,7 +20,6 @@ Tests for the `create_volume_snapshot` command. """ from mock import patch -from shade import _utils from shade import meta from shade import OpenStackCloud from shade.tests import fakes @@ -47,8 +46,8 @@ class TestCreateVolumeSnapshot(base.TestCase): build_snapshot, fake_snapshot] self.assertEqual( - _utils.normalize_volumes( - [meta.obj_to_dict(fake_snapshot)])[0], + self.cloud._normalize_volume( + meta.obj_to_dict(fake_snapshot)), self.cloud.create_volume_snapshot(volume_id='1234', wait=True) ) diff --git a/shade/tests/unit/test_normalize.py b/shade/tests/unit/test_normalize.py index 178b61b96..a0705f0d6 100644 --- a/shade/tests/unit/test_normalize.py +++ b/shade/tests/unit/test_normalize.py @@ -14,7 +14,6 @@ import mock -from shade import _utils from shade.tests.unit import base RAW_SERVER_DICT = { @@ -803,36 +802,204 @@ class TestUtils(base.TestCase): def test_normalize_volumes_v1(self): vol = dict( + id='55db9e89-9cb4-4202-af88-d8c4a174998e', display_name='test', display_description='description', bootable=u'false', # unicode type multiattach='true', # str type + status='in-use', + created_at='2015-08-27T09:49:58-05:00', ) - expected = dict( - name=vol['display_name'], - display_name=vol['display_name'], - description=vol['display_description'], - display_description=vol['display_description'], - bootable=False, - multiattach=True, - ) - retval = _utils.normalize_volumes([vol]) - self.assertEqual([expected], retval) + expected = { + 'attachments': [], + 'availability_zone': None, + 'bootable': False, + 'can_multiattach': True, + 'consistencygroup_id': None, + 'created_at': vol['created_at'], + 'description': vol['display_description'], + 'display_description': vol['display_description'], + 'display_name': vol['display_name'], + 'encrypted': False, + 'host': None, + 'id': '55db9e89-9cb4-4202-af88-d8c4a174998e', + 'is_bootable': False, + 'is_encrypted': False, + 'location': { + 'cloud': '_test_cloud_', + 'project': { + 'domain_id': None, + 'domain_name': None, + 'id': mock.ANY, + 'name': 'admin'}, + 'region_name': u'RegionOne', + 'zone': None}, + 'metadata': {}, + 'migration_status': None, + 'multiattach': True, + 'name': vol['display_name'], + 'properties': {}, + 'replication_driver': None, + 'replication_extended_status': None, + 'replication_status': None, + 'size': 0, + 'snapshot_id': None, + 'source_volume_id': None, + 'status': vol['status'], + 'updated_at': None, + 'volume_type': None, + } + retval = self.cloud._normalize_volume(vol) + self.assertEqual(expected, retval.toDict()) def test_normalize_volumes_v2(self): vol = dict( + id='55db9e89-9cb4-4202-af88-d8c4a174998e', + name='test', + description='description', + bootable=False, + multiattach=True, + status='in-use', + created_at='2015-08-27T09:49:58-05:00', + availability_zone='my-zone', + ) + vol['os-vol-tenant-attr:tenant_id'] = 'my-project' + expected = { + 'attachments': [], + 'availability_zone': vol['availability_zone'], + 'bootable': False, + 'can_multiattach': True, + 'consistencygroup_id': None, + 'created_at': vol['created_at'], + 'description': vol['description'], + 'display_description': vol['description'], + 'display_name': vol['name'], + 'encrypted': False, + 'host': None, + 'id': '55db9e89-9cb4-4202-af88-d8c4a174998e', + 'is_bootable': False, + 'is_encrypted': False, + 'location': { + 'cloud': '_test_cloud_', + 'project': { + 'domain_id': None, + 'domain_name': None, + 'id': vol['os-vol-tenant-attr:tenant_id'], + 'name': None}, + 'region_name': u'RegionOne', + 'zone': vol['availability_zone']}, + 'metadata': {}, + 'migration_status': None, + 'multiattach': True, + 'name': vol['name'], + 'os-vol-tenant-attr:tenant_id': vol[ + 'os-vol-tenant-attr:tenant_id'], + 'properties': { + 'os-vol-tenant-attr:tenant_id': vol[ + 'os-vol-tenant-attr:tenant_id']}, + 'replication_driver': None, + 'replication_extended_status': None, + 'replication_status': None, + 'size': 0, + 'snapshot_id': None, + 'source_volume_id': None, + 'status': vol['status'], + 'updated_at': None, + 'volume_type': None, + } + retval = self.cloud._normalize_volume(vol) + self.assertEqual(expected, retval.toDict()) + + def test_normalize_volumes_v1_strict(self): + vol = dict( + id='55db9e89-9cb4-4202-af88-d8c4a174998e', display_name='test', display_description='description', + bootable=u'false', # unicode type + multiattach='true', # str type + status='in-use', + created_at='2015-08-27T09:49:58-05:00', + ) + expected = { + 'attachments': [], + 'can_multiattach': True, + 'consistencygroup_id': None, + 'created_at': vol['created_at'], + 'description': vol['display_description'], + 'host': None, + 'id': '55db9e89-9cb4-4202-af88-d8c4a174998e', + 'is_bootable': False, + 'is_encrypted': False, + 'location': { + 'cloud': '_test_cloud_', + 'project': { + 'domain_id': None, + 'domain_name': None, + 'id': mock.ANY, + 'name': 'admin'}, + 'region_name': u'RegionOne', + 'zone': None}, + 'metadata': {}, + 'migration_status': None, + 'name': vol['display_name'], + 'properties': {}, + 'replication_driver': None, + 'replication_extended_status': None, + 'replication_status': None, + 'size': 0, + 'snapshot_id': None, + 'source_volume_id': None, + 'status': vol['status'], + 'updated_at': None, + 'volume_type': None, + } + retval = self.strict_cloud._normalize_volume(vol) + self.assertEqual(expected, retval.toDict()) + + def test_normalize_volumes_v2_strict(self): + vol = dict( + id='55db9e89-9cb4-4202-af88-d8c4a174998e', + name='test', + description='description', bootable=False, multiattach=True, + status='in-use', + created_at='2015-08-27T09:49:58-05:00', + availability_zone='my-zone', ) - expected = dict( - name=vol['display_name'], - display_name=vol['display_name'], - description=vol['display_description'], - display_description=vol['display_description'], - bootable=False, - multiattach=True, - ) - retval = _utils.normalize_volumes([vol]) - self.assertEqual([expected], retval) + vol['os-vol-tenant-attr:tenant_id'] = 'my-project' + expected = { + 'attachments': [], + 'can_multiattach': True, + 'consistencygroup_id': None, + 'created_at': vol['created_at'], + 'description': vol['description'], + 'host': None, + 'id': '55db9e89-9cb4-4202-af88-d8c4a174998e', + 'is_bootable': False, + 'is_encrypted': False, + 'location': { + 'cloud': '_test_cloud_', + 'project': { + 'domain_id': None, + 'domain_name': None, + 'id': vol['os-vol-tenant-attr:tenant_id'], + 'name': None}, + 'region_name': u'RegionOne', + 'zone': vol['availability_zone']}, + 'metadata': {}, + 'migration_status': None, + 'name': vol['name'], + 'properties': {}, + 'replication_driver': None, + 'replication_extended_status': None, + 'replication_status': None, + 'size': 0, + 'snapshot_id': None, + 'source_volume_id': None, + 'status': vol['status'], + 'updated_at': None, + 'volume_type': None, + } + retval = self.strict_cloud._normalize_volume(vol) + self.assertEqual(expected, retval.toDict())