openstack/trove
Code Issues Proposed changes

Use common methods for cluster validation

Browse Source 
Use a common set of tested utility methods for validations
in the API.

Change-Id: Icd3ea6251ef2ea2dbdaa05ae07f78bc3bd965b16
This commit is contained in:
Petr Malik 2016-05-24 16:39:56 -04:00
parent 2e34449db3
commit 4573969970
12 changed files with 261 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
trove/cluster/models.py
View File

@ -339,8 +339,8 @@ def is_cluster_deleting(context, cluster_id):
cluster.db_info.task_status == ClusterTasks.SHRINKING_CLUSTER)
def get_flavors_from_instance_defs(context, instances,
volume_enabled, ephemeral_enabled):
def validate_instance_flavors(context, instances,
volume_enabled, ephemeral_enabled):
"""Load and validate flavors for given instance definitions."""
flavors = dict()
nova_client = remote.create_nova_client(context)
@ -382,7 +382,46 @@ def get_required_volume_size(instances, volume_enabled):
return None
def assert_homogeneous_cluster(instances, required_flavor=None,
required_volume_size=None):
"""Verify that all instances have the same flavor and volume size
(volume size = 0 if there should be no Trove volumes).
"""
assert_same_instance_flavors(instances, required_flavor=required_flavor)
assert_same_instance_volumes(instances, required_size=required_volume_size)
def assert_same_instance_flavors(instances, required_flavor=None):
"""Verify that all instances have the same flavor.
:param required_flavor The flavor all instances should have or
None if no specific flavor is required.
:type required_flavor flavor_id
"""
flavors = {instance['flavor_id'] for instance in instances}
if len(flavors) != 1 or (required_flavor is not None and
required_flavor not in flavors):
raise exception.ClusterFlavorsNotEqual()
def assert_same_instance_volumes(instances, required_size=None):
"""Verify that all instances have the same volume size (size = 0 if there
is not a Trove volume for the instance).
:param required_size Size in GB all instance's volumes should
have or 0 if there should be no attached
volumes.
None if no particular size is required.
:type required_size int
"""
sizes = {instance.get('volume_size', 0) for instance in instances}
if len(sizes) != 1 or (required_size is not None and
required_size not in sizes):
raise exception.ClusterVolumeSizesNotEqual()
def validate_volume_size(size):
"""Verify the volume size is within the maximum limit for Trove volumes."""
if size is None:
raise exception.VolumeSizeNotSpecified()
max_size = CONF.max_accepted_volume_size

4
trove/common/strategies/cluster/experimental/cassandra/api.py
View File

@ -114,8 +114,8 @@ class CassandraCluster(models.Cluster):
vol_enabled = cassandra_conf.volume_support
# Validate instance flavors.
models.get_flavors_from_instance_defs(context, instances,
vol_enabled, eph_enabled)
models.validate_instance_flavors(context, instances,
vol_enabled, eph_enabled)
# Compute the total volume allocation.
req_volume_size = models.get_required_volume_size(instances,

39
trove/common/strategies/cluster/experimental/galera_common/api.py
View File

@ -64,36 +64,16 @@ class GaleraCommonCluster(cluster_models.Cluster):
raise exception.ClusterNumInstancesNotLargeEnough(
num_instances=ds_conf.min_cluster_member_count)
# Checking flavors and get delta for quota check
flavor_ids = [instance['flavor_id'] for instance in instances]
if len(set(flavor_ids)) != 1:
raise exception.ClusterFlavorsNotEqual()
flavor_id = flavor_ids[0]
nova_client = remote.create_nova_client(context)
try:
flavor = nova_client.flavors.get(flavor_id)
except nova_exceptions.NotFound:
raise exception.FlavorNotFound(uuid=flavor_id)
deltas = {'instances': num_instances}
# Checking volumes and get delta for quota check
volume_sizes = [instance['volume_size'] for instance in instances
if instance.get('volume_size', None)]
volume_size = None
if ds_conf.volume_support:
if len(volume_sizes) != num_instances:
raise exception.ClusterVolumeSizeRequired()
if len(set(volume_sizes)) != 1:
raise exception.ClusterVolumeSizesNotEqual()
volume_size = volume_sizes[0]
cluster_models.validate_volume_size(volume_size)
deltas['volumes'] = volume_size * num_instances
else:
if len(volume_sizes) > 0:
raise exception.VolumeNotSupported()
ephemeral_support = ds_conf.device_path
if ephemeral_support and flavor.ephemeral == 0:
raise exception.LocalStorageNotSpecified(flavor=flavor_id)
cluster_models.validate_instance_flavors(
context, instances, ds_conf.volume_support, ds_conf.device_path)
req_volume_size = cluster_models.get_required_volume_size(
instances, ds_conf.volume_support)
cluster_models.assert_homogeneous_cluster(instances)
deltas = {'instances': num_instances, 'volumes': req_volume_size}
# quota check
check_quotas(context.tenant, deltas)
@ -110,6 +90,7 @@ class GaleraCommonCluster(cluster_models.Cluster):
return
instance_nic = instance_nics[0]
try:
nova_client = remote.create_nova_client(context)
nova_client.networks.get(instance_nic)
except nova_exceptions.NotFound:
raise exception.NetworkNotFound(uuid=instance_nic)

42
trove/common/strategies/cluster/experimental/mongodb/api.py
View File

@ -68,43 +68,26 @@ class MongoDbCluster(models.Cluster):
if num_instances != 3:
raise exception.ClusterNumInstancesNotSupported(num_instances=3)
flavor_ids = [instance['flavor_id'] for instance in instances]
if len(set(flavor_ids)) != 1:
raise exception.ClusterFlavorsNotEqual()
flavor_id = flavor_ids[0]
nova_client = remote.create_nova_client(context)
try:
flavor = nova_client.flavors.get(flavor_id)
except nova_exceptions.NotFound:
raise exception.FlavorNotFound(uuid=flavor_id)
mongo_conf = CONF.get(datastore_version.manager)
num_configsvr = mongo_conf.num_config_servers_per_cluster
num_mongos = mongo_conf.num_query_routers_per_cluster
delta_instances = num_instances + num_configsvr + num_mongos
deltas = {'instances': delta_instances}
volume_sizes = [instance['volume_size'] for instance in instances
if instance.get('volume_size', None)]
volume_size = None
if mongo_conf.volume_support:
if len(volume_sizes) != num_instances:
raise exception.ClusterVolumeSizeRequired()
if len(set(volume_sizes)) != 1:
raise exception.ClusterVolumeSizesNotEqual()
volume_size = volume_sizes[0]
models.validate_volume_size(volume_size)
# TODO(amcreynolds): for now, mongos+configsvr same flavor+disk
deltas['volumes'] = volume_size * delta_instances
else:
# TODO(amcreynolds): is ephemeral possible for mongodb clusters?
if len(volume_sizes) > 0:
raise exception.VolumeNotSupported()
ephemeral_support = mongo_conf.device_path
if ephemeral_support and flavor.ephemeral == 0:
raise exception.LocalStorageNotSpecified(flavor=flavor_id)
models.validate_instance_flavors(
context, instances, mongo_conf.volume_support,
mongo_conf.device_path)
models.assert_homogeneous_cluster(instances)
req_volume_size = models.get_required_volume_size(
instances, mongo_conf.volume_support)
deltas = {'instances': delta_instances, 'volumes': req_volume_size}
check_quotas(context.tenant, deltas)
flavor_id = instances[0]['flavor_id']
volume_size = instances[0].get('volume_size', None)
nics = [instance.get('nics', None) for instance in instances]
azs = [instance.get('availability_zone', None)
@ -632,6 +615,7 @@ class MongoDbCluster(models.Cluster):
class MongoDbClusterView(ClusterView):
def build_instances(self):
return self._build_instances(['query_router'], ['member'])

35
trove/common/strategies/cluster/experimental/redis/api.py
View File

@ -11,7 +11,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from novaclient import exceptions as nova_exceptions
from oslo_log import log as logging
from trove.cluster import models
@ -20,7 +19,6 @@ from trove.cluster.tasks import ClusterTasks
from trove.cluster.views import ClusterView
from trove.common import cfg
from trove.common import exception
from trove.common import remote
from trove.common import server_group as srv_grp
from trove.common.strategies.cluster import base
from trove.extensions.mgmt.clusters.views import MgmtClusterView
@ -51,38 +49,23 @@ class RedisCluster(models.Cluster):
@staticmethod
def _create_instances(context, db_info, datastore, datastore_version,
instances, extended_properties, locality):
Redis_conf = CONF.get(datastore_version.manager)
redis_conf = CONF.get(datastore_version.manager)
ephemeral_enabled = redis_conf.device_path
volume_enabled = redis_conf.volume_support
num_instances = len(instances)
total_volume_allocation = 0
# Validate and Cache flavors
nova_client = remote.create_nova_client(context)
unique_flavors = set(inst['flavor_id'] for inst in instances)
flavor_cache = {}
for fid in unique_flavors:
try:
flavor_cache.update({fid: nova_client.flavors.get(fid)})
except nova_exceptions.NotFound:
raise exception.FlavorNotFound(uuid=fid)
models.validate_instance_flavors(
context, instances, volume_enabled, ephemeral_enabled)
total_volume_allocation = models.get_required_volume_size(
instances, volume_enabled)
# Checking volumes
name_index = 1
for instance in instances:
if not instance.get('name'):
instance['name'] = "%s-member-%s" % (db_info.name, name_index)
name_index += 1
volume_size = instance.get('volume_size')
if Redis_conf.volume_support:
models.validate_volume_size(volume_size)
total_volume_allocation += volume_size
else:
if volume_size:
raise exception.VolumeNotSupported()
ephemeral_support = Redis_conf.device_path
flavor_id = instance['flavor_id']
flavor = flavor_cache[flavor_id]
if ephemeral_support and flavor.ephemeral == 0:
raise exception.LocalStorageNotSpecified(flavor=flavor_id)
# Check quotas
quota_request = {'instances': num_instances,

42
trove/common/strategies/cluster/experimental/vertica/api.py
View File

@ -11,7 +11,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from novaclient import exceptions as nova_exceptions
from oslo_log import log as logging
from trove.cluster import models
@ -19,7 +18,6 @@ from trove.cluster.tasks import ClusterTasks
from trove.cluster.views import ClusterView
from trove.common import cfg
from trove.common import exception
from trove.common import remote
from trove.common import server_group as srv_grp
from trove.common.strategies.cluster import base
from trove.common import utils
@ -85,39 +83,21 @@ class VerticaCluster(models.Cluster):
raise exception.ClusterNumInstancesNotSupported(
num_instances=vertica_conf.cluster_member_count)
# Checking flavors
flavor_ids = [instance['flavor_id'] for instance in instances]
if len(set(flavor_ids)) != 1:
raise exception.ClusterFlavorsNotEqual()
flavor_id = flavor_ids[0]
nova_client = remote.create_nova_client(context)
try:
flavor = nova_client.flavors.get(flavor_id)
except nova_exceptions.NotFound:
raise exception.FlavorNotFound(uuid=flavor_id)
deltas = {'instances': num_instances}
models.validate_instance_flavors(
context, instances, vertica_conf.volume_support,
vertica_conf.device_path)
# Checking volumes
volume_sizes = [instance['volume_size'] for instance in instances
if instance.get('volume_size', None)]
volume_size = None
if vertica_conf.volume_support:
if len(volume_sizes) != num_instances:
raise exception.ClusterVolumeSizeRequired()
if len(set(volume_sizes)) != 1:
raise exception.ClusterVolumeSizesNotEqual()
volume_size = volume_sizes[0]
models.validate_volume_size(volume_size)
deltas['volumes'] = volume_size * num_instances
else:
if len(volume_sizes) > 0:
raise exception.VolumeNotSupported()
ephemeral_support = vertica_conf.device_path
if ephemeral_support and flavor.ephemeral == 0:
raise exception.LocalStorageNotSpecified(flavor=flavor_id)
req_volume_size = models.get_required_volume_size(
instances, vertica_conf.volume_support)
models.assert_homogeneous_cluster(instances)
deltas = {'instances': num_instances, 'volumes': req_volume_size}
check_quotas(context.tenant, deltas)
flavor_id = instances[0]['flavor_id']
volume_size = instances[0].get('volume_size', None)
nics = [instance.get('nics', None) for instance in instances]
azs = [instance.get('availability_zone', None)

2
trove/tests/unittests/cluster/test_cassandra_cluster.py
View File

@ -39,7 +39,7 @@ class ClusterTest(trove_testtools.TestCase):
@patch.object(inst_models.Instance, 'create')
@patch.object(quota.QUOTAS, 'check_quotas')
@patch.object(models, 'get_flavors_from_instance_defs')
@patch.object(models, 'validate_instance_flavors')
@patch.object(models, 'get_required_volume_size', return_value=3)
def test_create_cluster_instances(self, get_vol_size, _, check_quotas,
inst_create):

3
trove/tests/unittests/cluster/test_cluster.py
View File

@ -84,7 +84,8 @@ class ClusterTest(trove_testtools.TestCase):
None, None
)
def test_create_unequal_flavors(self):
@patch.object(remote, 'create_nova_client')
def test_create_unequal_flavors(self, mock_client):
instances = self.instances
instances[0]['flavor_id'] = '4567'
self.assertRaises(exception.ClusterFlavorsNotEqual,

3
trove/tests/unittests/cluster/test_galera_cluster.py
View File

@ -87,7 +87,8 @@ class ClusterTest(trove_testtools.TestCase):
[], {}, None
)
def test_create_flavor_not_specified(self):
@patch.object(remote, 'create_nova_client')
def test_create_flavor_not_specified(self, mock_client):
instances = self.instances
instances[0]['flavor_id'] = None
self.assertRaises(exception.ClusterFlavorsNotEqual,

168
trove/tests/unittests/cluster/test_models.py Normal file
View File

@ -0,0 +1,168 @@
# Copyright 2016 Tesora Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from mock import ANY
from mock import call
from mock import DEFAULT
from mock import MagicMock
from mock import Mock
from mock import patch
from mock import PropertyMock
from trove.cluster import models
from trove.common import exception
from trove.common import remote
from trove.tests.unittests import trove_testtools
class TestModels(trove_testtools.TestCase):
@patch.object(remote, 'create_nova_client', return_value=MagicMock())
def test_validate_instance_flavors(self, create_nove_cli_mock):
patch.object(
create_nove_cli_mock.return_value, 'flavors',
new_callable=PropertyMock(return_value=Mock()))
mock_flv = create_nove_cli_mock.return_value.flavors.get.return_value
mock_flv.ephemeral = 0
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 1.5},
{'flavor_id': 2, 'volume_size': 3}]
models.validate_instance_flavors(Mock(), test_instances,
True, True)
create_nove_cli_mock.assert_called_once_with(ANY)
self.assertRaises(exception.LocalStorageNotSpecified,
models.validate_instance_flavors,
Mock(), test_instances, False, True)
mock_flv.ephemeral = 1
models.validate_instance_flavors(Mock(), test_instances,
False, True)
def test_validate_volume_size(self):
self.patch_conf_property('max_accepted_volume_size', 10)
models.validate_volume_size(9)
models.validate_volume_size(10)
self.assertRaises(exception.VolumeQuotaExceeded,
models.validate_volume_size, 11)
self.assertRaises(exception.VolumeSizeNotSpecified,
models.validate_volume_size, None)
@patch.object(models, 'validate_volume_size')
def test_get_required_volume_size(self, vol_size_validator_mock):
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 1.5},
{'flavor_id': 1, 'volume_size': 3}]
total_size = models.get_required_volume_size(test_instances, True)
self.assertEqual(14.5, total_size)
vol_size_validator_mock.assert_has_calls([call(10),
call(1.5),
call(3)], any_order=True)
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 1.5},
{'flavor_id': 1, 'volume_size': None}]
self.assertRaises(exception.ClusterVolumeSizeRequired,
models.get_required_volume_size,
test_instances, True)
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 1.5},
{'flavor_id': 1}]
self.assertRaises(exception.ClusterVolumeSizeRequired,
models.get_required_volume_size,
test_instances, True)
test_instances = [{'flavor_id': 1},
{'flavor_id': 1},
{'flavor_id': 1}]
total_size = models.get_required_volume_size(test_instances, False)
self.assertIsNone(total_size)
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 1.5}]
self.assertRaises(exception.VolumeNotSupported,
models.get_required_volume_size,
test_instances, False)
def test_assert_same_instance_volumes(self):
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10}]
models.assert_same_instance_volumes(test_instances)
test_instances = [{'flavor_id': 1, 'volume_size': 5},
{'flavor_id': 1, 'volume_size': 5},
{'flavor_id': 1, 'volume_size': 5}]
models.assert_same_instance_volumes(test_instances, required_size=5)
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 1.5},
{'flavor_id': 1, 'volume_size': 10}]
self.assertRaises(exception.ClusterVolumeSizesNotEqual,
models.assert_same_instance_volumes,
test_instances)
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10}]
self.assertRaises(exception.ClusterVolumeSizesNotEqual,
models.assert_same_instance_volumes,
test_instances, required_size=5)
def test_assert_same_instance_flavors(self):
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10}]
models.assert_same_instance_flavors(test_instances)
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10}]
models.assert_same_instance_flavors(test_instances, required_flavor=1)
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 2, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10}]
self.assertRaises(exception.ClusterFlavorsNotEqual,
models.assert_same_instance_flavors,
test_instances)
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10}]
self.assertRaises(exception.ClusterFlavorsNotEqual,
models.assert_same_instance_flavors,
test_instances, required_flavor=2)
@patch.multiple(models, assert_same_instance_flavors=DEFAULT,
assert_same_instance_volumes=DEFAULT)
def test_assert_homogeneous_cluster(self, assert_same_instance_flavors,
assert_same_instance_volumes):
test_instances = [{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10},
{'flavor_id': 1, 'volume_size': 10}]
required_flavor = Mock()
required_volume_size = Mock()
models.assert_homogeneous_cluster(
test_instances, required_flavor=required_flavor,
required_volume_size=required_volume_size)
assert_same_instance_flavors.assert_called_once_with(
test_instances, required_flavor=required_flavor)
assert_same_instance_volumes.assert_called_once_with(
test_instances, required_size=required_volume_size)

2
trove/tests/unittests/cluster/test_redis_cluster.py
View File

@ -101,7 +101,7 @@ class ClusterTest(trove_testtools.TestCase):
def test_create_volume_no_specified(self, mock_conf, mock_client):
mock_conf.get = Mock(
return_value=FakeOptGroup(volume_support=True))
self.assertRaises(exception.VolumeSizeNotSpecified,
self.assertRaises(exception.ClusterVolumeSizeRequired,
Cluster.create,
Mock(),
self.cluster_name,

1
trove/tests/unittests/cluster/test_vertica_cluster.py
View File

@ -87,6 +87,7 @@ class ClusterTest(trove_testtools.TestCase):
@patch.object(DBCluster, 'create')
@patch.object(inst_models.DBInstance, 'find_all')
@patch.object(remote, 'create_nova_client')
def test_create_flavor_not_specified(self, *args):
instances = self.instances
instances[0]['flavor_id'] = None