openstack/rally-openstack
Code Issues Proposed changes

[validation] Use jsonschema validator in Context plugins

Browse Source 
* Added custom validators for the following plugins:
 - api_versions
 - admin_cleanup
 - cleanup

Change-Id: Ibbd1702c9a83a1995f2eff86fb5b51ea1a6816b5
This commit is contained in:
Anton Studenov 2017-04-06 13:28:16 +03:00
parent c44bbb7ec2
commit 55a131d831
10 changed files with 205 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
rally/plugins/openstack/context/api_versions.py
View File

@ -12,13 +12,37 @@
import random
from rally.common.i18n import _, _LE
from rally.common.i18n import _
from rally.common import validation
from rally import consts
from rally import exceptions
from rally import osclients
from rally.task import context
@validation.configure("check_api_versions")
class CheckOpenStackAPIVersionsValidator(validation.Validator):
"""Additional validation for api_versions context"""
def validate(self, credentials, config, plugin_cls, plugin_cfg):
for client in plugin_cfg:
client_cls = osclients.OSClient.get(client)
try:
if ("service_type" in plugin_cfg[client] or
"service_name" in plugin_cfg[client]):
client_cls.is_service_type_configurable()
if "version" in plugin_cfg[client]:
client_cls.validate_version(plugin_cfg[client]["version"])
except exceptions.RallyException as e:
return self.fail(
"Invalid settings for '%(client)s': %(error)s" % {
"client": client,
"error": e.format_message()})
@validation.add("check_api_versions")
@context.configure(name="api_versions", order=150)
class OpenStackAPIVersions(context.Context):
"""Context for specifying OpenStack clients versions and service types.
@ -157,30 +181,49 @@ class OpenStackAPIVersions(context.Context):
}
"""
VERSION_SCHEMA = {
"anyOf": [
{"type": "string", "description": "a string-like version."},
{"type": "number", "description": "a number-like version."}
]
}
CONFIG_SCHEMA = {
"type": "object",
"$schema": consts.JSON_SCHEMA,
"patternProperties": {
"^[a-z]+$": {
"type": "object",
"properties": {
"version": {
"anyOf": [{"type": "string",
"description": "a string-like version."},
{"type": "number",
"description": "a number-like version."}]
"oneOf": [
{
"description": "version only",
"properties": {
"version": VERSION_SCHEMA,
},
"required": ["version"],
"additionalProperties": False
},
"service_name": {
"type": "string"
{
"description": "version and service_name",
"properties": {
"version": VERSION_SCHEMA,
"service_name": {"type": "string"}
},
"required": ["service_name"],
"additionalProperties": False
},
"service_type": {
"type": "string"
{
"description": "version and service_type",
"properties": {
"version": VERSION_SCHEMA,
"service_type": {"type": "string"}
},
"required": ["service_type"],
"additionalProperties": False
}
},
"additionalProperties": False
],
}
},
"minProperties": 1,
"additionalProperties": False
}
@ -219,27 +262,3 @@ class OpenStackAPIVersions(context.Context):
def cleanup(self):
# nothing to do here
pass
@classmethod
def validate(cls, config):
super(OpenStackAPIVersions, cls).validate(config)
for client in config:
client_cls = osclients.OSClient.get(client)
if ("service_type" in config[client] and
"service_name" in config[client]):
raise exceptions.ValidationError(_LE(
"Setting both 'service_type' and 'service_name' properties"
" is restricted."))
try:
if ("service_type" in config[client] or
"service_name" in config[client]):
client_cls.is_service_type_configurable()
if "version" in config[client]:
client_cls.validate_version(config[client]["version"])
except exceptions.RallyException as e:
raise exceptions.ValidationError(
_LE("Invalid settings for '%(client)s': %(error)s") % {
"client": client,
"error": e.format_message()})

14
rally/plugins/openstack/context/cleanup/admin.py
View File

@ -17,6 +17,7 @@ import sys
from rally.common.i18n import _
from rally.common import logging
from rally.common import validation
from rally.plugins.openstack.cleanup import manager
from rally.plugins.openstack.context.cleanup import base
from rally.plugins.openstack import scenario
@ -26,23 +27,12 @@ from rally.task import context
LOG = logging.getLogger(__name__)
@validation.add(name="check_cleanup_resources", admin_required=True)
# NOTE(amaretskiy): Set order to run this just before UserCleanup
@context.configure(name="admin_cleanup", order=(sys.maxsize - 1), hidden=True)
class AdminCleanup(base.CleanupMixin, context.Context):
"""Context class for admin resources cleanup."""
@classmethod
def validate(cls, config):
super(AdminCleanup, cls).validate(config)
missing = set(config)
missing -= manager.list_resource_names(admin_required=True)
missing = ", ".join(missing)
if missing:
LOG.info(_("Couldn't find cleanup resource managers: %s")
% missing)
raise base.NoSuchCleanupResources(missing)
@logging.log_task_wrapper(LOG.info, _("admin resources cleanup"))
def cleanup(self):
manager.cleanup(

22
rally/plugins/openstack/context/cleanup/base.py
View File

@ -13,13 +13,27 @@
# License for the specific language governing permissions and limitations
# under the License.
from rally.common.i18n import _
from rally.common import validation
from rally import consts
from rally import exceptions
from rally.plugins.openstack.cleanup import manager
class NoSuchCleanupResources(exceptions.RallyException):
msg_fmt = _("Missing cleanup resource managers: %(message)s")
@validation.configure("check_cleanup_resources")
class CheckCleanupResourcesValidator(validation.Validator):
"""Validates that openstack resource managers exist"""
def __init__(self, admin_required):
super(CheckCleanupResourcesValidator, self).__init__()
self.admin_required = admin_required
def validate(self, credentials, config, plugin_cls, plugin_cfg):
missing = set(plugin_cfg)
missing -= manager.list_resource_names(
admin_required=self.admin_required)
missing = ", ".join(missing)
if missing:
return self.fail(
"Couldn't find cleanup resource managers: %s" % missing)
class CleanupMixin(object):

14
rally/plugins/openstack/context/cleanup/user.py
View File

@ -17,6 +17,7 @@ import sys
from rally.common.i18n import _
from rally.common import logging
from rally.common import validation
from rally.plugins.openstack.cleanup import manager
from rally.plugins.openstack.context.cleanup import base
from rally.plugins.openstack import scenario
@ -26,23 +27,12 @@ from rally.task import context
LOG = logging.getLogger(__name__)
@validation.add(name="check_cleanup_resources", admin_required=False)
# NOTE(amaretskiy): Set maximum order to run this last
@context.configure(name="cleanup", order=sys.maxsize, hidden=True)
class UserCleanup(base.CleanupMixin, context.Context):
"""Context class for user resources cleanup."""
@classmethod
def validate(cls, config):
super(UserCleanup, cls).validate(config)
missing = set(config)
missing -= manager.list_resource_names(admin_required=False)
missing = ", ".join(missing)
if missing:
LOG.info(_("Couldn't find cleanup resource managers: %s")
% missing)
raise base.NoSuchCleanupResources(missing)
@logging.log_task_wrapper(LOG.info, _("user resources cleanup"))
def cleanup(self):
manager.cleanup(

17
tests/unit/plugins/openstack/context/cinder/test_volumes.py
View File

@ -15,10 +15,10 @@
import copy
import ddt
import jsonschema
import mock
from rally.plugins.openstack.context.cinder import volumes
from rally.task import context
from tests.unit import test
CTX = "rally.plugins.openstack.context"
@ -50,16 +50,15 @@ class VolumeGeneratorTestCase(test.ScenarioTestCase):
@ddt.data({"config": {"size": 1, "volumes_per_tenant": 5}},
{"config": {"size": 1, "type": None, "volumes_per_tenant": 5}},
{"config": {"size": 1, "type": -1, "volumes_per_tenant": 5},
"validation_raises": jsonschema.exceptions.ValidationError})
"valid": False})
@ddt.unpack
@mock.patch("%s.block.BlockStorage" % SERVICE)
def test_setup(self, mock_block_storage, config,
validation_raises=None):
try:
volumes.VolumeGenerator.validate(config)
except Exception as e:
if not isinstance(e, validation_raises):
raise
def test_setup(self, mock_block_storage, config, valid=True):
results = context.Context.validate("volumes", None, None, config)
if valid:
self.assertEqual([], results)
else:
self.assertEqual(1, len(results))
from rally.plugins.openstack.services.storage import block
created_volume = block.Volume(id="uuid", size=config["size"],

45
tests/unit/plugins/openstack/context/cleanup/test_admin.py
View File

@ -13,44 +13,41 @@
# License for the specific language governing permissions and limitations
# under the License.
import jsonschema
import ddt
import mock
from rally.common import utils
from rally.plugins.openstack.context.cleanup import admin
from rally.plugins.openstack.context.cleanup import base
from rally.plugins.openstack import scenario
from rally.task import context
from tests.unit import test
BASE = "rally.plugins.openstack.context.cleanup.admin"
ADMIN = "rally.plugins.openstack.context.cleanup.admin"
BASE = "rally.plugins.openstack.context.cleanup.base"
@ddt.ddt
class AdminCleanupTestCase(test.TestCase):
@mock.patch("%s.manager" % BASE)
def test_validate(self, mock_manager):
mock_manager.list_resource_names.return_value = set(["a", "b", "c"])
admin.AdminCleanup.validate(["a"])
mock_manager.list_resource_names.assert_called_once_with(
admin_required=True)
@mock.patch("%s.manager" % BASE)
def test_validate_no_such_cleanup(self, mock_manager):
mock_manager.list_resource_names.return_value = set(["a", "b", "c"])
self.assertRaises(base.NoSuchCleanupResources,
admin.AdminCleanup.validate, ["a", "d"])
mock_manager.list_resource_names.assert_called_once_with(
admin_required=True)
def test_validate_invalid_config(self):
self.assertRaises(jsonschema.ValidationError,
admin.AdminCleanup.validate, {})
@ddt.data((["a", "b"], True),
(["a", "e"], False),
(3, False))
@ddt.unpack
def test_validate(self, config, valid, mock_manager):
mock_manager.list_resource_names.return_value = {"a", "b", "c"}
results = context.Context.validate(
"admin_cleanup", None, None, config, allow_hidden=True)
if valid:
self.assertEqual([], results)
else:
self.assertGreater(len(results), 0)
@mock.patch("rally.common.plugin.discover.itersubclasses")
@mock.patch("%s.manager.find_resource_managers" % BASE,
@mock.patch("%s.manager.find_resource_managers" % ADMIN,
return_value=[mock.MagicMock(), mock.MagicMock()])
@mock.patch("%s.manager.SeekAndDestroy" % BASE)
@mock.patch("%s.manager.SeekAndDestroy" % ADMIN)
def test_cleanup(self, mock_seek_and_destroy, mock_find_resource_managers,
mock_itersubclasses):
class ResourceClass(utils.RandomNameGeneratorMixin):
@ -89,9 +86,9 @@ class AdminCleanupTestCase(test.TestCase):
])
@mock.patch("rally.common.plugin.discover.itersubclasses")
@mock.patch("%s.manager.find_resource_managers" % BASE,
@mock.patch("%s.manager.find_resource_managers" % ADMIN,
return_value=[mock.MagicMock(), mock.MagicMock()])
@mock.patch("%s.manager.SeekAndDestroy" % BASE)
@mock.patch("%s.manager.SeekAndDestroy" % ADMIN)
def test_cleanup_admin_with_api_versions(self,
mock_seek_and_destroy,
mock_find_resource_managers,

45
tests/unit/plugins/openstack/context/cleanup/test_user.py
View File

@ -13,44 +13,41 @@
# License for the specific language governing permissions and limitations
# under the License.
import jsonschema
import ddt
import mock
from rally.common import utils
from rally.plugins.openstack.context.cleanup import base
from rally.plugins.openstack.context.cleanup import user
from rally.plugins.openstack import scenario
from rally.task import context
from tests.unit import test
BASE = "rally.plugins.openstack.context.cleanup.user"
ADMIN = "rally.plugins.openstack.context.cleanup.admin"
BASE = "rally.plugins.openstack.context.cleanup.base"
@ddt.ddt
class UserCleanupTestCase(test.TestCase):
@mock.patch("%s.manager" % BASE)
def test_validate(self, mock_manager):
mock_manager.list_resource_names.return_value = set(["a", "b", "c"])
user.UserCleanup.validate(["a"])
mock_manager.list_resource_names.assert_called_once_with(
admin_required=False)
@mock.patch("%s.manager" % BASE)
def test_validate_no_such_cleanup(self, mock_manager):
mock_manager.list_resource_names.return_value = set(["a", "b", "c"])
self.assertRaises(base.NoSuchCleanupResources,
user.UserCleanup.validate, ["a", "b", "d"])
mock_manager.list_resource_names.assert_called_once_with(
admin_required=False)
def test_validate_invalid_config(self):
self.assertRaises(jsonschema.ValidationError,
user.UserCleanup.validate, {})
@ddt.data((["a", "b"], True),
(["a", "e"], False),
(3, False))
@ddt.unpack
def test_validate(self, config, valid, mock_manager):
mock_manager.list_resource_names.return_value = {"a", "b", "c"}
results = context.Context.validate(
"cleanup", None, None, config, allow_hidden=True)
if valid:
self.assertEqual([], results)
else:
self.assertGreater(len(results), 0)
@mock.patch("rally.common.plugin.discover.itersubclasses")
@mock.patch("%s.manager.find_resource_managers" % BASE,
@mock.patch("%s.manager.find_resource_managers" % ADMIN,
return_value=[mock.MagicMock(), mock.MagicMock()])
@mock.patch("%s.manager.SeekAndDestroy" % BASE)
@mock.patch("%s.manager.SeekAndDestroy" % ADMIN)
def test_cleanup(self, mock_seek_and_destroy, mock_find_resource_managers,
mock_itersubclasses):
@ -83,9 +80,9 @@ class UserCleanupTestCase(test.TestCase):
])
@mock.patch("rally.common.plugin.discover.itersubclasses")
@mock.patch("%s.manager.find_resource_managers" % BASE,
@mock.patch("%s.manager.find_resource_managers" % ADMIN,
return_value=[mock.MagicMock(), mock.MagicMock()])
@mock.patch("%s.manager.SeekAndDestroy" % BASE)
@mock.patch("%s.manager.SeekAndDestroy" % ADMIN)
def test_cleanup_user_with_api_versions(
self,
mock_seek_and_destroy,

11
tests/unit/plugins/openstack/context/glance/test_images.py
View File

@ -16,7 +16,6 @@
import copy
import ddt
import jsonschema
import mock
from rally.plugins.openstack.context.glance import images
@ -59,16 +58,6 @@ class ImageGeneratorTestCase(test.ScenarioTestCase):
tenants[str(id_)] = {"name": str(id_)}
return tenants
def test_init_validation(self):
self.context["config"] = {
"images": {
"image_url": "mock_url"
}
}
self.assertRaises(jsonschema.ValidationError,
images.ImageGenerator.validate, self.context)
@ddt.data(
{},
{"min_disk": 1, "min_ram": 2},

143
tests/unit/plugins/openstack/context/quotas/test_quotas.py
View File

@ -14,14 +14,13 @@
# under the License.
import copy
import random
import ddt
import jsonschema
import mock
from rally.common import logging
from rally.plugins.openstack.context.quotas import quotas
from rally.task import context
from tests.unit import test
QUOTAS_PATH = "rally.plugins.openstack.context.quotas"
@ -43,98 +42,56 @@ class QuotasTestCase(test.TestCase):
"task": mock.MagicMock()
}
def test_quotas_schemas(self):
ctx = copy.deepcopy(self.context)
ctx["config"]["quotas"] = {
"cinder": {
"volumes": self.unlimited,
"snapshots": self.unlimited,
"gigabytes": self.unlimited
},
"nova": {
"instances": self.unlimited,
"cores": self.unlimited,
"ram": self.unlimited,
"floating_ips": self.unlimited,
"fixed_ips": self.unlimited,
"metadata_items": self.unlimited,
"injected_files": self.unlimited,
"injected_file_content_bytes": self.unlimited,
"injected_file_path_bytes": self.unlimited,
"key_pairs": self.unlimited,
"security_groups": self.unlimited,
"security_group_rules": self.unlimited
},
"neutron": {
"network": self.unlimited,
"subnet": self.unlimited,
"port": self.unlimited,
"router": self.unlimited,
"floatingip": self.unlimited,
"security_group": self.unlimited,
"security_group_rule": self.unlimited
}
}
for service in ctx["config"]["quotas"]:
for key in ctx["config"]["quotas"][service]:
# Test invalid values
ctx["config"]["quotas"][service][key] = self.unlimited - 1
try:
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
pass
else:
self.fail("Invalid value %s must raise a validation error"
% ctx["config"]["quotas"][service][key])
ctx["config"]["quotas"][service][key] = 2.5
try:
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
pass
else:
self.fail("Invalid value %s must raise a validation error"
% ctx["config"]["quotas"][service][key])
ctx["config"]["quotas"][service][key] = "-1"
try:
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
pass
else:
self.fail("Invalid value %s must raise a validation error"
% ctx["config"]["quotas"][service][key])
# Test valid values
ctx["config"]["quotas"][service][key] = random.randint(0,
1000000)
try:
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
self.fail("Positive integers are valid quota values")
ctx["config"]["quotas"][service][key] = self.unlimited
try:
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
self.fail("%d is a valid quota value" % self.unlimited)
# Test additional keys are refused
ctx["config"]["quotas"][service]["additional"] = self.unlimited
try:
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
pass
@ddt.data(("cinder", "backup_gigabytes"),
("cinder", "backups"),
("cinder", "gigabytes"),
("cinder", "snapshots"),
("cinder", "volumes"),
("manila", "gigabytes"),
("manila", "share_networks"),
("manila", "shares"),
("manila", "snapshot_gigabytes"),
("manila", "snapshots"),
("neutron", "floatingip"),
("neutron", "health_monitor"),
("neutron", "network"),
("neutron", "pool"),
("neutron", "port"),
("neutron", "router"),
("neutron", "security_group"),
("neutron", "security_group_rule"),
("neutron", "subnet"),
("neutron", "vip"),
("nova", "cores"),
("nova", "fixed_ips"),
("nova", "floating_ips"),
("nova", "injected_file_content_bytes"),
("nova", "injected_file_path_bytes"),
("nova", "injected_files"),
("nova", "instances"),
("nova", "key_pairs"),
("nova", "metadata_items"),
("nova", "ram"),
("nova", "security_group_rules"),
("nova", "security_groups"),
("nova", "server_group_members"),
("nova", "server_groups"))
@ddt.unpack
def test_validate(self, group, parameter):
configs = [
({group: {parameter: self.unlimited}}, True),
({group: {parameter: 0}}, True),
({group: {parameter: 10000}}, True),
({group: {parameter: 2.5}}, False),
({group: {parameter: "-1"}}, False),
({group: {parameter: -2}}, False),
]
for config, valid in configs:
results = context.Context.validate("quotas", None, None, config)
if valid:
self.assertEqual([], results)
else:
self.fail("Additional keys must raise a validation error")
del ctx["config"]["quotas"][service]["additional"]
# Test valid keys are optional
ctx["config"]["quotas"][service] = {}
try:
quotas.Quotas.validate(ctx["config"]["quotas"])
except jsonschema.ValidationError:
self.fail("Valid quota keys are optional")
self.assertGreater(len(results), 0)
@mock.patch("%s.quotas.osclients.Clients" % QUOTAS_PATH)
@mock.patch("%s.cinder_quotas.CinderQuotas" % QUOTAS_PATH)

81
tests/unit/plugins/openstack/context/test_api_versions.py
View File

@ -10,15 +10,17 @@
# License for the specific language governing permissions and limitations
# under the License.
import jsonschema
import ddt
import mock
from rally.common import utils
from rally import exceptions
from rally.plugins.openstack.context import api_versions
from rally.task import context
from tests.unit import test
@ddt.ddt
class OpenStackServicesTestCase(test.TestCase):
def setUp(self):
@ -30,82 +32,51 @@ class OpenStackServicesTestCase(test.TestCase):
self.service_catalog.get_endpoints.return_value = []
self.mock_kc.services.list.return_value = []
def test_validate_correct_config(self):
api_versions.OpenStackAPIVersions.validate({
"nova": {"service_type": "compute", "version": 2},
"cinder": {"service_name": "cinderv2", "version": 2},
"neutron": {"service_type": "network"},
"glance": {"service_name": "glance"},
"heat": {"version": 1}
})
def test_validate_wrong_configs(self):
# Non-existing clients should be caught
self.assertRaises(
exceptions.PluginNotFound,
api_versions.OpenStackAPIVersions.validate,
{"invalid": {"service_type": "some_type"}})
# Additional properties should be restricted
self.assertRaises(
jsonschema.ValidationError,
api_versions.OpenStackAPIVersions.validate,
{"nova": {"some_key": "some_value"}})
# Setting service_type is allowed only
# for those clients, which support it
self.assertRaises(
exceptions.ValidationError,
api_versions.OpenStackAPIVersions.validate,
{"keystone": {"service_type": "identity"}})
# Setting service_name is allowed only
# for those clients, which support it
self.assertRaises(
exceptions.ValidationError,
api_versions.OpenStackAPIVersions.validate,
{"keystone": {"service_name": "keystone"}})
# Setting version is allowed only
# for those clients, which support it
self.assertRaises(
exceptions.ValidationError,
api_versions.OpenStackAPIVersions.validate,
{"keystone": {"version": 1}})
# Unsupported version should be caught
self.assertRaises(
exceptions.ValidationError,
api_versions.OpenStackAPIVersions.validate,
{"nova": {"version": 666}})
@ddt.data(({"nova": {"service_type": "compute", "version": 2},
"cinder": {"service_name": "cinderv2", "version": 2},
"neutron": {"service_type": "network"},
"glance": {"service_name": "glance"},
"heat": {"version": 1}}, True),
({"nova": {"service_type": "compute",
"service_name": "nova"}}, False),
({"keystone": {"service_type": "foo"}}, False),
({"nova": {"version": "foo"}}, False),
({}, False))
@ddt.unpack
def test_validate(self, config, valid):
results = context.Context.validate("api_versions", None, None, config)
if valid:
self.assertEqual([], results)
else:
self.assertGreater(len(results), 0)
def test_setup_with_wrong_service_name(self):
context = {
context_obj = {
"config": {api_versions.OpenStackAPIVersions.get_name(): {
"nova": {"service_name": "service_name"}}},
"admin": {"credential": mock.MagicMock()},
"users": [{"credential": mock.MagicMock()}]}
ctx = api_versions.OpenStackAPIVersions(context)
ctx = api_versions.OpenStackAPIVersions(context_obj)
self.assertRaises(exceptions.ValidationError, ctx.setup)
self.service_catalog.get_endpoints.assert_called_once_with()
self.mock_kc.services.list.assert_called_once_with()
def test_setup_with_wrong_service_name_and_without_admin(self):
context = {
context_obj = {
"config": {api_versions.OpenStackAPIVersions.get_name(): {
"nova": {"service_name": "service_name"}}},
"users": [{"credential": mock.MagicMock()}]}
ctx = api_versions.OpenStackAPIVersions(context)
ctx = api_versions.OpenStackAPIVersions(context_obj)
self.assertRaises(exceptions.BenchmarkSetupFailure, ctx.setup)
self.service_catalog.get_endpoints.assert_called_once_with()
self.assertFalse(self.mock_kc.services.list.called)
def test_setup_with_wrong_service_type(self):
context = {
context_obj = {
"config": {api_versions.OpenStackAPIVersions.get_name(): {
"nova": {"service_type": "service_type"}}},
"users": [{"credential": mock.MagicMock()}]}
ctx = api_versions.OpenStackAPIVersions(context)
ctx = api_versions.OpenStackAPIVersions(context_obj)
self.assertRaises(exceptions.ValidationError, ctx.setup)
self.service_catalog.get_endpoints.assert_called_once_with()