openstack/shade
Code Issues Proposed changes

Normalize volumes

Browse Source 
We had a normalization function in _utils but it was nor documented nor
did it support strict mode. Document the normalization and  be more explicit
about which things we support and don't.

Change-Id: I360af3abcfd69afebd941c5d6e359a84dc956283
This commit is contained in:
Monty Taylor 2016-11-13 11:25:50 -06:00
parent 89cea034fc
commit 697da6fd4e
No known key found for this signature in database
GPG Key ID: 7BAE94BC7141A594
7 changed files with 338 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
doc/source/model.rst
View File

@ -238,3 +238,35 @@ POV.
is_enabled=bool(),
is_domain=bool(),
properties=dict())
Volume
------
A volume from cinder.
.. code-block:: python
Volume = dict(
location=Location(),
id=str(),
name=str(),
description=str(),
size=int(),
attachments=list(),
status=str(),
migration_status=str() or None,
host=str() or None,
replication_driver=str() or None,
replication_status=str() or None,
replication_extended_status=str() or None,
snapshot_id=str() or None,
created_at=str(),
updated_at=str() or None,
source_volume_id=str() or None,
consistencygroup_id=str() or None,
volume_type=str() or None,
metadata=dict(),
is_bootable=bool(),
is_encrypted=bool(),
can_multiattach=bool(),
properties=dict())

97
shade/_normalize.py
View File

@ -513,7 +513,6 @@ class Normalizer(object):
def _normalize_project(self, project):
ret = munch.Munch()
# Copy incoming project because of shared dicts in unittests
project = project.copy()
@ -563,3 +562,99 @@ class Normalizer(object):
ret.setdefault(key, val)
return ret
def _normalize_volumes(self, volumes):
"""Normalize the structure of volumes
This makes tenants from cinder v1 look like volumes from v2.
:param list projects: A list of volumes to normalize
:returns: A list of normalized dicts.
"""
ret = []
for volume in volumes:
ret.append(self._normalize_volume(volume))
return ret
def _normalize_volume(self, volume):
volume = volume.copy()
# Discard noise
volume.pop('links', None)
volume.pop('NAME_ATTR', None)
volume.pop('HUMAN_ID', None)
volume.pop('human_id', None)
volume_id = volume.pop('id')
name = volume.pop('display_name', None)
name = volume.pop('name', name)
description = volume.pop('display_description', None)
description = volume.pop('description', description)
is_bootable = _to_bool(volume.pop('bootable', True))
is_encrypted = _to_bool(volume.pop('encrypted', False))
can_multiattach = _to_bool(volume.pop('multiattach', False))
project_id = _pop_or_get(
volume, 'os-vol-tenant-attr:tenant_id', None, self.strict_mode)
az = volume.pop('availability_zone', None)
location = self._get_current_location(project_id=project_id, zone=az)
host = _pop_or_get(
volume, 'os-vol-host-attr:host', None, self.strict_mode)
replication_extended_status = _pop_or_get(
volume, 'os-volume-replication:extended_status',
None, self.strict_mode)
migration_status = _pop_or_get(
volume, 'os-vol-mig-status-attr:migstat', None, self.strict_mode)
migration_status = volume.pop('migration_status', migration_status)
_pop_or_get(volume, 'user_id', None, self.strict_mode)
source_volume_id = _pop_or_get(
volume, 'source_volid', None, self.strict_mode)
replication_driver = _pop_or_get(
volume, 'os-volume-replication:driver_data',
None, self.strict_mode)
ret = munch.Munch(
location=location,
id=volume_id,
name=name,
description=description,
size=_pop_int(volume, 'size'),
attachments=volume.pop('attachments', []),
status=volume.pop('status'),
migration_status=migration_status,
host=host,
replication_driver=replication_driver,
replication_status=volume.pop('replication_status', None),
replication_extended_status=replication_extended_status,
snapshot_id=volume.pop('snapshot_id', None),
created_at=volume.pop('created_at'),
updated_at=volume.pop('updated_at', None),
source_volume_id=source_volume_id,
consistencygroup_id=volume.pop('consistencygroup_id', None),
volume_type=volume.pop('volume_type', None),
metadata=volume.pop('metadata', {}),
is_bootable=is_bootable,
is_encrypted=is_encrypted,
can_multiattach=can_multiattach,
properties=volume.copy(),
)
# Backwards compat
if not self.strict_mode:
ret['display_name'] = name
ret['display_description'] = description
ret['bootable'] = is_bootable
ret['encrypted'] = is_encrypted
ret['multiattach'] = can_multiattach
ret['availability_zone'] = az
for key, val in ret['properties'].items():
ret.setdefault(key, val)
return ret

24
shade/_utils.py
View File

@ -210,30 +210,6 @@ def normalize_users(users):
return meta.obj_list_to_dict(ret)
def normalize_volumes(volumes):
ret = []
for vol in volumes:
new_vol = vol.copy()
name = vol.get('name', vol.get('display_name'))
description = vol.get('description', vol.get('display_description'))
new_vol['name'] = name
new_vol['display_name'] = name
new_vol['description'] = description
new_vol['display_description'] = description
# For some reason, cinder v1 uses strings for bools for these fields.
# Cinder v2 uses booleans.
for field in ('bootable', 'multiattach'):
if field in new_vol and isinstance(new_vol[field],
six.string_types):
if new_vol[field] is not None:
if new_vol[field].lower() == 'true':
new_vol[field] = True
elif new_vol[field].lower() == 'false':
new_vol[field] = False
ret.append(new_vol)
return meta.obj_list_to_dict(ret)
def normalize_domains(domains):
ret = [
dict(

13
shade/openstackcloud.py
View File

@ -1518,7 +1518,7 @@ class OpenStackCloud(_normalize.Normalizer):
warnings.warn('cache argument to list_volumes is deprecated. Use '
'invalidate instead.')
with _utils.shade_exceptions("Error fetching volume list"):
return _utils.normalize_volumes(
return self._normalize_volumes(
self.manager.submit_task(_tasks.VolumeList()))
@_utils.cache_on_arguments()
@ -3338,7 +3338,7 @@ class OpenStackCloud(_normalize.Normalizer):
raise OpenStackCloudException(
"Error in creating volume, please check logs")
return _utils.normalize_volumes([volume])[0]
return self._normalize_volume(volume)
def delete_volume(self, name_or_id=None, wait=True, timeout=None):
"""Delete a volume.
@ -3592,7 +3592,10 @@ class OpenStackCloud(_normalize.Normalizer):
raise OpenStackCloudException(
"Error in creating volume snapshot, please check logs")
return _utils.normalize_volumes([snapshot])[0]
# TODO(mordred) need to normalize snapshots. We were normalizing them
# as volumes, which is an error. They need to be normalized as
# volume snapshots, which are completely different objects
return snapshot
def get_volume_snapshot_by_id(self, snapshot_id):
"""Takes a snapshot_id and gets a dict of the snapshot
@ -3612,7 +3615,7 @@ class OpenStackCloud(_normalize.Normalizer):
)
)
return _utils.normalize_volumes([snapshot])[0]
return self._normalize_volume(snapshot)
def get_volume_snapshot(self, name_or_id, filters=None):
"""Get a volume by name or ID.
@ -3703,7 +3706,7 @@ class OpenStackCloud(_normalize.Normalizer):
"""
with _utils.shade_exceptions("Error getting a list of snapshots"):
return _utils.normalize_volumes(
return self._normalize_volumes(
self.manager.submit_task(
_tasks.VolumeSnapshotList(
detailed=detailed, search_opts=search_opts)))

25
shade/tests/unit/test_caching.py
View File

@ -22,7 +22,6 @@ import testtools
import warlock
import shade.openstackcloud
from shade import _utils
from shade import exc
from shade import meta
from shade.tests import fakes
@ -177,14 +176,14 @@ class TestMemoryCache(base.TestCase):
def test_list_volumes(self, cinder_mock):
fake_volume = fakes.FakeVolume('volume1', 'available',
'Volume 1 Display Name')
fake_volume_dict = _utils.normalize_volumes(
[meta.obj_to_dict(fake_volume)])[0]
fake_volume_dict = self.cloud._normalize_volume(
meta.obj_to_dict(fake_volume))
cinder_mock.volumes.list.return_value = [fake_volume]
self.assertEqual([fake_volume_dict], self.cloud.list_volumes())
fake_volume2 = fakes.FakeVolume('volume2', 'available',
'Volume 2 Display Name')
fake_volume2_dict = _utils.normalize_volumes(
[meta.obj_to_dict(fake_volume2)])[0]
fake_volume2_dict = self.cloud._normalize_volume(
meta.obj_to_dict(fake_volume2))
cinder_mock.volumes.list.return_value = [fake_volume, fake_volume2]
self.assertEqual([fake_volume_dict], self.cloud.list_volumes())
self.cloud.list_volumes.invalidate(self.cloud)
@ -195,14 +194,14 @@ class TestMemoryCache(base.TestCase):
def test_list_volumes_creating_invalidates(self, cinder_mock):
fake_volume = fakes.FakeVolume('volume1', 'creating',
'Volume 1 Display Name')
fake_volume_dict = _utils.normalize_volumes(
[meta.obj_to_dict(fake_volume)])[0]
fake_volume_dict = self.cloud._normalize_volume(
meta.obj_to_dict(fake_volume))
cinder_mock.volumes.list.return_value = [fake_volume]
self.assertEqual([fake_volume_dict], self.cloud.list_volumes())
fake_volume2 = fakes.FakeVolume('volume2', 'available',
'Volume 2 Display Name')
fake_volume2_dict = _utils.normalize_volumes(
[meta.obj_to_dict(fake_volume2)])[0]
fake_volume2_dict = self.cloud._normalize_volume(
meta.obj_to_dict(fake_volume2))
cinder_mock.volumes.list.return_value = [fake_volume, fake_volume2]
self.assertEqual([fake_volume_dict, fake_volume2_dict],
self.cloud.list_volumes())
@ -211,8 +210,8 @@ class TestMemoryCache(base.TestCase):
def test_create_volume_invalidates(self, cinder_mock):
fake_volb4 = fakes.FakeVolume('volume1', 'available',
'Volume 1 Display Name')
fake_volb4_dict = _utils.normalize_volumes(
[meta.obj_to_dict(fake_volb4)])[0]
fake_volb4_dict = self.cloud._normalize_volume(
meta.obj_to_dict(fake_volb4))
cinder_mock.volumes.list.return_value = [fake_volb4]
self.assertEqual([fake_volb4_dict], self.cloud.list_volumes())
volume = dict(display_name='junk_vol',
@ -220,8 +219,8 @@ class TestMemoryCache(base.TestCase):
display_description='test junk volume')
fake_vol = fakes.FakeVolume('12345', 'creating', '')
fake_vol_dict = meta.obj_to_dict(fake_vol)
fake_vol_dict = _utils.normalize_volumes(
[meta.obj_to_dict(fake_vol)])[0]
fake_vol_dict = self.cloud._normalize_volume(
meta.obj_to_dict(fake_vol))
cinder_mock.volumes.create.return_value = fake_vol
cinder_mock.volumes.list.return_value = [fake_volb4, fake_vol]

5
shade/tests/unit/test_create_volume_snapshot.py
View File

@ -20,7 +20,6 @@ Tests for the `create_volume_snapshot` command.
"""
from mock import patch
from shade import _utils
from shade import meta
from shade import OpenStackCloud
from shade.tests import fakes
@ -47,8 +46,8 @@ class TestCreateVolumeSnapshot(base.TestCase):
build_snapshot, fake_snapshot]
self.assertEqual(
_utils.normalize_volumes(
[meta.obj_to_dict(fake_snapshot)])[0],
self.cloud._normalize_volume(
meta.obj_to_dict(fake_snapshot)),
self.cloud.create_volume_snapshot(volume_id='1234', wait=True)
)

209
shade/tests/unit/test_normalize.py
View File

@ -14,7 +14,6 @@
import mock
from shade import _utils
from shade.tests.unit import base
RAW_SERVER_DICT = {
@ -803,36 +802,204 @@ class TestUtils(base.TestCase):
def test_normalize_volumes_v1(self):
vol = dict(
id='55db9e89-9cb4-4202-af88-d8c4a174998e',
display_name='test',
display_description='description',
bootable=u'false', # unicode type
multiattach='true', # str type
status='in-use',
created_at='2015-08-27T09:49:58-05:00',
)
expected = dict(
name=vol['display_name'],
display_name=vol['display_name'],
description=vol['display_description'],
display_description=vol['display_description'],
bootable=False,
multiattach=True,
)
retval = _utils.normalize_volumes([vol])
self.assertEqual([expected], retval)
expected = {
'attachments': [],
'availability_zone': None,
'bootable': False,
'can_multiattach': True,
'consistencygroup_id': None,
'created_at': vol['created_at'],
'description': vol['display_description'],
'display_description': vol['display_description'],
'display_name': vol['display_name'],
'encrypted': False,
'host': None,
'id': '55db9e89-9cb4-4202-af88-d8c4a174998e',
'is_bootable': False,
'is_encrypted': False,
'location': {
'cloud': '_test_cloud_',
'project': {
'domain_id': None,
'domain_name': None,
'id': mock.ANY,
'name': 'admin'},
'region_name': u'RegionOne',
'zone': None},
'metadata': {},
'migration_status': None,
'multiattach': True,
'name': vol['display_name'],
'properties': {},
'replication_driver': None,
'replication_extended_status': None,
'replication_status': None,
'size': 0,
'snapshot_id': None,
'source_volume_id': None,
'status': vol['status'],
'updated_at': None,
'volume_type': None,
}
retval = self.cloud._normalize_volume(vol)
self.assertEqual(expected, retval.toDict())
def test_normalize_volumes_v2(self):
vol = dict(
id='55db9e89-9cb4-4202-af88-d8c4a174998e',
name='test',
description='description',
bootable=False,
multiattach=True,
status='in-use',
created_at='2015-08-27T09:49:58-05:00',
availability_zone='my-zone',
)
vol['os-vol-tenant-attr:tenant_id'] = 'my-project'
expected = {
'attachments': [],
'availability_zone': vol['availability_zone'],
'bootable': False,
'can_multiattach': True,
'consistencygroup_id': None,
'created_at': vol['created_at'],
'description': vol['description'],
'display_description': vol['description'],
'display_name': vol['name'],
'encrypted': False,
'host': None,
'id': '55db9e89-9cb4-4202-af88-d8c4a174998e',
'is_bootable': False,
'is_encrypted': False,
'location': {
'cloud': '_test_cloud_',
'project': {
'domain_id': None,
'domain_name': None,
'id': vol['os-vol-tenant-attr:tenant_id'],
'name': None},
'region_name': u'RegionOne',
'zone': vol['availability_zone']},
'metadata': {},
'migration_status': None,
'multiattach': True,
'name': vol['name'],
'os-vol-tenant-attr:tenant_id': vol[
'os-vol-tenant-attr:tenant_id'],
'properties': {
'os-vol-tenant-attr:tenant_id': vol[
'os-vol-tenant-attr:tenant_id']},
'replication_driver': None,
'replication_extended_status': None,
'replication_status': None,
'size': 0,
'snapshot_id': None,
'source_volume_id': None,
'status': vol['status'],
'updated_at': None,
'volume_type': None,
}
retval = self.cloud._normalize_volume(vol)
self.assertEqual(expected, retval.toDict())
def test_normalize_volumes_v1_strict(self):
vol = dict(
id='55db9e89-9cb4-4202-af88-d8c4a174998e',
display_name='test',
display_description='description',
bootable=u'false', # unicode type
multiattach='true', # str type
status='in-use',
created_at='2015-08-27T09:49:58-05:00',
)
expected = {
'attachments': [],
'can_multiattach': True,
'consistencygroup_id': None,
'created_at': vol['created_at'],
'description': vol['display_description'],
'host': None,
'id': '55db9e89-9cb4-4202-af88-d8c4a174998e',
'is_bootable': False,
'is_encrypted': False,
'location': {
'cloud': '_test_cloud_',
'project': {
'domain_id': None,
'domain_name': None,
'id': mock.ANY,
'name': 'admin'},
'region_name': u'RegionOne',
'zone': None},
'metadata': {},
'migration_status': None,
'name': vol['display_name'],
'properties': {},
'replication_driver': None,
'replication_extended_status': None,
'replication_status': None,
'size': 0,
'snapshot_id': None,
'source_volume_id': None,
'status': vol['status'],
'updated_at': None,
'volume_type': None,
}
retval = self.strict_cloud._normalize_volume(vol)
self.assertEqual(expected, retval.toDict())
def test_normalize_volumes_v2_strict(self):
vol = dict(
id='55db9e89-9cb4-4202-af88-d8c4a174998e',
name='test',
description='description',
bootable=False,
multiattach=True,
status='in-use',
created_at='2015-08-27T09:49:58-05:00',
availability_zone='my-zone',
)
expected = dict(
name=vol['display_name'],
display_name=vol['display_name'],
description=vol['display_description'],
display_description=vol['display_description'],
bootable=False,
multiattach=True,
)
retval = _utils.normalize_volumes([vol])
self.assertEqual([expected], retval)
vol['os-vol-tenant-attr:tenant_id'] = 'my-project'
expected = {
'attachments': [],
'can_multiattach': True,
'consistencygroup_id': None,
'created_at': vol['created_at'],
'description': vol['description'],
'host': None,
'id': '55db9e89-9cb4-4202-af88-d8c4a174998e',
'is_bootable': False,
'is_encrypted': False,
'location': {
'cloud': '_test_cloud_',
'project': {
'domain_id': None,
'domain_name': None,
'id': vol['os-vol-tenant-attr:tenant_id'],
'name': None},
'region_name': u'RegionOne',
'zone': vol['availability_zone']},
'metadata': {},
'migration_status': None,
'name': vol['name'],
'properties': {},
'replication_driver': None,
'replication_extended_status': None,
'replication_status': None,
'size': 0,
'snapshot_id': None,
'source_volume_id': None,
'status': vol['status'],
'updated_at': None,
'volume_type': None,
}
retval = self.strict_cloud._normalize_volume(vol)
self.assertEqual(expected, retval.toDict())