From 065a2d76f280f5676ef24dd082c071825e0ade47 Mon Sep 17 00:00:00 2001 From: Manali Latkar Date: Mon, 19 Aug 2013 15:34:59 +0530 Subject: [PATCH] added null/type checks on specific fields of exit --- stacktach/models.py | 8 +- tests/unit/test_glance_verifier.py | 149 ++++++++++++++++++- tests/unit/test_nova_verifier.py | 226 +++++++++++++++++++++++++++++ verifier/__init__.py | 12 ++ verifier/base_verifier.py | 29 ++++ verifier/glance_verifier.py | 19 ++- verifier/nova_verifier.py | 20 +++ 7 files changed, 457 insertions(+), 6 deletions(-) diff --git a/stacktach/models.py b/stacktach/models.py index a34f494..658c959 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -424,7 +424,7 @@ class ImageUsage(models.Model): uuid = models.CharField(max_length=50, db_index=True) created_at = models.DecimalField(max_digits=20, decimal_places=6, db_index=True) - owner = models.CharField(max_length=50, db_index=True) + owner = models.CharField(max_length=50, db_index=True, null=True, blank=True) size = models.BigIntegerField(max_length=20) last_raw = models.ForeignKey(GlanceRawData, null=True) @@ -456,7 +456,7 @@ class ImageExists(models.Model): (FAILED, 'Failed Verification'), ] - uuid = models.CharField(max_length=50, db_index=True) + uuid = models.CharField(max_length=50, db_index=True, null=True) created_at = models.DecimalField(max_digits=20, decimal_places=6, db_index=True, null=True) @@ -476,8 +476,8 @@ class ImageExists(models.Model): usage = models.ForeignKey(ImageUsage, related_name='+', null=True) delete = models.ForeignKey(ImageDeletes, related_name='+', null=True) send_status = models.IntegerField(default=0, db_index=True) - owner = models.CharField(max_length=255, db_index=True) - size = models.BigIntegerField(max_length=20) + owner = models.CharField(max_length=255, db_index=True, null=True, blank=True) + size = models.BigIntegerField(max_length=20, null=True) def update_status(self, new_status): self.status = new_status diff --git a/tests/unit/test_glance_verifier.py b/tests/unit/test_glance_verifier.py index 5a8b726..f96b375 100644 --- a/tests/unit/test_glance_verifier.py +++ b/tests/unit/test_glance_verifier.py @@ -32,6 +32,8 @@ from tests.unit import StacktachBaseTestCase from utils import IMAGE_UUID_1 from utils import make_verifier_config from verifier import glance_verifier +from verifier import NullFieldException +from verifier import WrongTypeException from verifier import FieldMismatch from verifier import NotFound from verifier import VerificationException @@ -261,12 +263,156 @@ class GlanceVerifierTestCase(StacktachBaseTestCase): self.assertEqual(fm.actual, decimal.Decimal('6.1')) self.mox.VerifyAll() + def test_should_verify_that_image_size_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = None + exist.created_at = decimal.Decimal('5.1') + exist.uuid = 'abcd1234' + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'image_size') + self.assertEqual(nf.reason, "image_size field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_created_at_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 'size' + exist.created_at = None + exist.uuid = 'abcd1234' + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'created_at') + self.assertEqual(nf.reason, "created_at field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_uuid_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 'size' + exist.created_at = decimal.Decimal('5.1') + exist.uuid = None + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'uuid') + self.assertEqual(nf.reason, "uuid field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_owner_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 1234 + exist.created_at = decimal.Decimal('5.1') + exist.uuid = 'abcd1234' + exist.owner = None + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'owner') + self.assertEqual(nf.reason, "owner field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_uuid_value_is_uuid_like(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 'size' + exist.created_at = decimal.Decimal('5.1') + exist.uuid = "asdfe-fgh" + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'uuid') + self.assertEqual(wt.reason, "{ uuid : asdfe-fgh } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_created_at_is_decimal(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 'size' + exist.created_at = "123.a" + exist.uuid = "58fb036d-5ef8-47a8-b503-7571276c400a" + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'created_at') + self.assertEqual(wt.reason, "{ created_at : 123.a } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_image_size_is_of_type_decimal(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 'size' + exist.created_at = decimal.Decimal('5.1') + exist.uuid = "58fb036d-5ef8-47a8-b503-7571276c400a" + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'size') + self.assertEqual(wt.reason, "{ size : size } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_owner_is_of_type_hex(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 1234 + exist.created_at = decimal.Decimal('5.1') + exist.uuid = "58fb036d-5ef8-47a8-b503-7571276c400a" + exist.owner = "3762854cd6f6435998188d5120e4c271,kl" + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'owner') + self.assertEqual(wt.reason, "{ owner : 3762854cd6f6435998188d5120e4c271,kl } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_correctly_for_all_non_null_and_valid_types(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 983040 + exist.created_at = decimal.Decimal('5.1') + exist.uuid = "58fb036d-5ef8-47a8-b503-7571276c400a" + exist.owner = "3762854cd6f6435998188d5120e4c271" + self.mox.ReplayAll() + + glance_verifier._verify_validity(exist) + self.mox.VerifyAll() + def test_verify_should_verify_exists_for_usage_and_delete(self): exist = self.mox.CreateMockAnything() self.mox.StubOutWithMock(glance_verifier, '_verify_for_usage') glance_verifier._verify_for_usage(exist) self.mox.StubOutWithMock(glance_verifier, '_verify_for_delete') glance_verifier._verify_for_delete(exist) + self.mox.StubOutWithMock(glance_verifier, '_verify_validity') + glance_verifier._verify_validity(exist) exist.mark_verified() self.mox.ReplayAll() @@ -413,4 +559,5 @@ class GlanceVerifierTestCase(StacktachBaseTestCase): self.glance_verifier.send_verified_notification(exist, exchange, connection) - self.mox.VerifyAll() \ No newline at end of file + self.mox.VerifyAll() + diff --git a/tests/unit/test_nova_verifier.py b/tests/unit/test_nova_verifier.py index a5452e1..b6b080b 100644 --- a/tests/unit/test_nova_verifier.py +++ b/tests/unit/test_nova_verifier.py @@ -45,6 +45,8 @@ from utils import TENANT_ID_1 from utils import TENANT_ID_2 from utils import INSTANCE_TYPE_ID_1 from verifier import nova_verifier +from verifier import NullFieldException +from verifier import WrongTypeException from verifier import AmbiguousResults from verifier import FieldMismatch from verifier import NotFound @@ -324,6 +326,228 @@ class NovaVerifierTestCase(StacktachBaseTestCase): self.mox.VerifyAll() + def test_should_verify_that_tenant_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.tenant = None + exist.id = 23 + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'tenant') + self.assertEqual(nf.reason, "tenant field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_launched_at_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.tenant = 'tenant' + exist.id = 23 + exist.launched_at = None + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'launched_at') + self.assertEqual(nf.reason, "launched_at field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_instance_type_id_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.tenant = 'tenant' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.instance_type_id = None + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'instance_type_id') + self.assertEqual(nf.reason, "instance_type_id field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_tenant_id_is_of_type_hex(self): + exist = self.mox.CreateMockAnything() + exist.tenant = 'tenant' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.instance_type_id = 2 + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'tenant') + self.assertEqual(wt.reason, "{ tenant : tenant } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_flavor_is_of_type_integer(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.instance_type_id = 'flavor' + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'instance_type_id') + self.assertEqual(wt.reason, "{ instance_type_id : flavor } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_launched_at_is_of_type_decimal(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = 111 + exist.instance_type_id = 4 + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'launched_at') + self.assertEqual(wt.reason, "{ launched_at : 111 } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_deleted_at_is_of_decimal_type_if_present(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.instance_type_id = 4 + exist.deleted_at = 20 + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'deleted_at') + self.assertEqual(wt.reason, "{ deleted_at : 20 } of incorrect type for exist id 23") + self.mox.VerifyAll() + + + def test_should_verify_rax_options_should_be_of_integer_type(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = 4 + exist.rax_options = 'a' + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'rax_options') + self.assertEqual(wt.reason, "{ rax_options : a } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_os_arch_should_be_alphanumeric(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_arch = 'x64,' + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'os_arch') + self.assertEqual(wt.reason, "{ os_arch : x64, } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_os_distro_should_be_alphanumeric(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_arch = 'x64' + exist.os_distro = 'com.microsoft.server,' + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'os_distro') + self.assertEqual(wt.reason, "{ os_distro : com.microsoft.server, } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_os_version_should_be_alphanumeric(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_arch = 'x64' + exist.os_distro = 'com.microsoft.server' + exist.os_version = '2008.2,' + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'os_version') + self.assertEqual(wt.reason, "{ os_version : 2008.2, } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_exist_fields_correctly(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_arch = 'x64' + exist.os_distro = 'com.microsoft.server' + exist.os_version = '2008.2' + self.mox.ReplayAll() + + nova_verifier._verify_validity(exist) + self.mox.VerifyAll() + + def test_should_verify_exist_fields_correctly_and_deleted_at_can_be_none(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = None + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_arch = 'x64' + exist.os_distro = 'com.microsoft.server' + exist.os_version = '2008.2' + self.mox.ReplayAll() + + nova_verifier._verify_validity(exist) + self.mox.VerifyAll() + def test_verify_for_delete(self): exist = self.mox.CreateMockAnything() exist.delete = self.mox.CreateMockAnything() @@ -571,9 +795,11 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.launched_at = decimal.Decimal('1.1') self.mox.StubOutWithMock(nova_verifier, '_verify_for_launch') self.mox.StubOutWithMock(nova_verifier, '_verify_for_delete') + self.mox.StubOutWithMock(nova_verifier, '_verify_validity') self.mox.StubOutWithMock(exist, 'mark_verified') nova_verifier._verify_for_launch(exist) nova_verifier._verify_for_delete(exist) + nova_verifier._verify_validity(exist) exist.mark_verified() self.mox.ReplayAll() result, exists = nova_verifier._verify(exist) diff --git a/verifier/__init__.py b/verifier/__init__.py index eb30360..0c542f6 100644 --- a/verifier/__init__.py +++ b/verifier/__init__.py @@ -51,3 +51,15 @@ class FieldMismatch(VerificationException): self.reason = "Expected %s to be '%s' got '%s'" % (self.field_name, self.expected, self.actual) + +class NullFieldException(VerificationException): + def __init__(self, field_name, exist_id): + self.field_name = field_name + self.reason = "%s field was null for exist id %s" %(field_name, exist_id) + +class WrongTypeException(VerificationException): + def __init__(self, field_name, value, exist_id): + self.field_name = field_name + self.reason = "{ %s : %s } of incorrect type for exist id %s"\ + %(field_name, value, exist_id) + diff --git a/verifier/base_verifier.py b/verifier/base_verifier.py index cea72b7..01f9209 100644 --- a/verifier/base_verifier.py +++ b/verifier/base_verifier.py @@ -19,7 +19,9 @@ # IN THE SOFTWARE. import datetime +import decimal import os +import re import sys import time import multiprocessing @@ -32,6 +34,8 @@ POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): sys.path.insert(0, POSSIBLE_TOPDIR) +from verifier import WrongTypeException + from stacktach import stacklog, message_service LOG = stacklog.get_logger('verifier') @@ -65,6 +69,31 @@ def _verify_date_field(d1, d2, same_second=False): return False +def _is_like_uuid(attr_name, attr_value, exist_id): + if not re.match("[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$", + attr_value): + raise WrongTypeException(attr_name, attr_value, exist_id) + + +def _is_like_date(attr_name, attr_value, exist_id): + if not isinstance(attr_value, decimal.Decimal): + raise WrongTypeException(attr_name, attr_value, exist_id) + + +def _is_int(attr_name, attr_value, exist_id): + if not isinstance(attr_value, int): + raise WrongTypeException(attr_name, attr_value, exist_id) + + +def _is_hex_owner_id(attr_name, attr_value, exist_id): + if not re.match("[0-9a-f]{32}$", attr_value): + raise WrongTypeException(attr_name, attr_value, exist_id) + +def _is_alphanumeric(attr_name, attr_value, exist_id): + if not re.match("[a-zA-Z0-9.]+$", attr_value): + raise WrongTypeException(attr_name, attr_value, exist_id) + + class Verifier(object): def __init__(self, config, pool=None, reconciler=None): self.config = config diff --git a/verifier/glance_verifier.py b/verifier/glance_verifier.py index 3947cfe..0ba611c 100644 --- a/verifier/glance_verifier.py +++ b/verifier/glance_verifier.py @@ -31,7 +31,10 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): sys.path.insert(0, POSSIBLE_TOPDIR) from stacktach import models -from verifier import FieldMismatch, VerificationException, base_verifier +from verifier import FieldMismatch +from verifier import VerificationException +from verifier import base_verifier +from verifier import NullFieldException from verifier import NotFound from stacktach import datetime_to_decimal as dt import datetime @@ -54,6 +57,19 @@ def _verify_field_mismatch(exists, usage): usage.size) +def _verify_validity(exist): + fields = {exist.size: 'image_size', exist.created_at: 'created_at', + exist.uuid: 'uuid', exist.owner: 'owner'} + for (field_value, field_name) in fields.items(): + if field_value is None: + raise NullFieldException(field_name, exist.id) + base_verifier._is_like_uuid('uuid', exist.uuid, exist.id) + base_verifier._is_like_date('created_at', exist.created_at, exist.id) + base_verifier._is_int('size', exist.size, exist.id) + base_verifier._is_hex_owner_id('owner', exist.owner, exist.id) + + + def _verify_for_usage(exist, usage=None): usage_type = "ImageUsage" if not usage and exist.usage: @@ -116,6 +132,7 @@ def _verify(exist): try: _verify_for_usage(exist) _verify_for_delete(exist) + _verify_validity(exist) verified = True exist.mark_verified() diff --git a/verifier/nova_verifier.py b/verifier/nova_verifier.py index 605d232..5a65e2f 100644 --- a/verifier/nova_verifier.py +++ b/verifier/nova_verifier.py @@ -32,6 +32,7 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): sys.path.insert(0, POSSIBLE_TOPDIR) from verifier import base_verifier +from verifier import NullFieldException from stacktach import models from stacktach import datetime_to_decimal as dt from verifier import FieldMismatch @@ -145,6 +146,24 @@ def _verify_for_delete(exist, delete=None, 'deleted_at', exist.deleted_at, delete.deleted_at) +def _verify_validity(exist): + fields = {exist.tenant: 'tenant', exist.launched_at: 'launched_at', + exist.instance_type_id: 'instance_type_id'} + for (field_value, field_name) in fields.items(): + if field_value is None: + raise NullFieldException(field_name, exist.id) + base_verifier._is_hex_owner_id('tenant', exist.tenant, exist.id) + base_verifier._is_int('instance_type_id', exist.instance_type_id, exist.id) + base_verifier._is_like_date('launched_at', exist.launched_at, exist.id) + if exist.deleted_at is not None: + base_verifier._is_like_date('deleted_at', exist.deleted_at, exist.id) + base_verifier._is_int('rax_options', exist.rax_options, exist.id) + base_verifier._is_alphanumeric('os_arch', exist.os_arch, exist.id) + base_verifier._is_alphanumeric('os_distro', exist.os_distro, exist.id) + base_verifier._is_alphanumeric('os_version', exist.os_version, exist.id) + + + def _verify_with_reconciled_data(exist): if not exist.launched_at: raise VerificationException("Exists without a launched_at") @@ -201,6 +220,7 @@ def _verify(exist): _verify_for_launch(exist) _verify_for_delete(exist) + _verify_validity(exist) verified = True exist.mark_verified()