Add support for datatypes

Add datatype into TOSCA schema
Support new built-in DataType like NetworkInfo and PortInfo and user
defined datatypes.

Implements: bp implement-toscadatatype-networkinfo-and-portinfo
Change-Id: If39c0f5d4c4768fceabe1bbdadbc506861157468

translator/toscalib/common/exception.py

@@ -60,7 +60,7 @@ class MissingRequiredFieldError(TOSCAException):
 
 class UnknownFieldError(TOSCAException):
     msg_fmt = _('%(what)s contain(s) unknown field: "%(field)s", '
-                'refer to the TOSCA specs to verify valid values.')
+                'refer to the definition to verify valid values.')
 
 
 class TypeMismatchError(TOSCAException):
@@ -71,6 +71,10 @@ class InvalidNodeTypeError(TOSCAException):
     msg_fmt = _('Node type "%(what)s" is not a valid type.')
 
 
+class InvalidTypeError(TOSCAException):
+    msg_fmt = _('Type "%(what)s" is not a valid type.')
+
+
 class InvalidSchemaError(TOSCAException):
     msg_fmt = _("%(message)s")
 

translator/toscalib/dataentity.py

@@ -0,0 +1,138 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from translator.toscalib.common.exception import MissingRequiredFieldError
+from translator.toscalib.common.exception import TypeMismatchError
+from translator.toscalib.common.exception import UnknownFieldError
+from translator.toscalib.elements.datatype import DataType
+from translator.toscalib.elements.constraints import Constraint, Schema
+
+
+class DataEntity(object):
+    '''A complex data value entity.'''
+
+    def __init__(self, datatypename, value_dict, custom_def=None):
+        self.custom_def = custom_def
+        self.datatype = DataType(datatypename, custom_def)
+        self.schema = self.datatype.all_properties
+        self.value = value_dict
+
+    def validate(self):
+        '''Validate the value by the definition of the datatype.'''
+
+        #A datatype can not have both 'type' and 'properties' definitions.
+        #If the datatype has 'type' definition
+        if self.datatype.value_type:
+            DataEntity.validate_datatype(self.datatype.value_type, self.value,
+                                         None, self.custom_def)
+            schema = Schema(None, self.datatype.defs)
+            for constraint in schema.constraints:
+                constraint.validate(self.value)
+        # If the datatype has 'properties' definition
+        else:
+            if not isinstance(self.value, dict):
+                raise TypeMismatchError(what=self.value,
+                                        type=self.datatype.type)
+            allowed_props = []
+            required_props = []
+            default_props = {}
+            for prop_def in self.schema:
+                allowed_props.append(prop_def.name)
+                if prop_def.required:
+                    required_props.append(prop_def.name)
+                if prop_def.default:
+                    default_props[prop_def.name] = prop_def.default
+
+            # check allowed field
+            for value_key in list(self.value.keys()):
+                if value_key not in allowed_props:
+                    raise UnknownFieldError(what='Data value of type %s'
+                                            % self.datatype.type,
+                                            field=value_key)
+
+            # check default field
+            for def_key, def_value in list(default_props.items()):
+                if def_key not in list(self.value.keys()):
+                    self.value[def_key] = def_value
+
+            # check missing field
+            missingprop = []
+            for req_key in required_props:
+                if req_key not in list(self.value.keys()):
+                    missingprop.append(req_key)
+            if missingprop:
+                raise MissingRequiredFieldError(what='Data value of type %s'
+                                                % self.datatype.type,
+                                                required=missingprop)
+
+            # check every field
+            for name, value in list(self.value.items()):
+                prop_schema = Schema(name, self._find_schema(name))
+                # check if field value meets type defined
+                DataEntity.validate_datatype(prop_schema.type, value,
+                                             prop_schema.entry_schema,
+                                             self.custom_def)
+                # check if field value meets constraints defined
+                if prop_schema.constraints:
+                    for constraint in prop_schema.constraints:
+                        constraint.validate(value)
+
+        return self.value
+
+    def _find_schema(self, name):
+        for prop_def in self.schema:
+            if prop_def.name == name:
+                return prop_def.schema
+
+    @staticmethod
+    def validate_datatype(type, value, entry_schema=None, custom_def=None):
+        '''Validate value with given type.
+
+        If type is list or map, validate its entry by entry_schema(if defined)
+        If type is a user-defined complex datatype, custom_def is required.
+        '''
+        if type == Schema.STRING:
+            return Constraint.validate_string(value)
+        elif type == Schema.INTEGER:
+            return Constraint.validate_integer(value)
+        elif type == Schema.NUMBER:
+            return Constraint.validate_number(value)
+        elif type == Schema.BOOLEAN:
+            return Constraint.validate_boolean(value)
+        elif type == Schema.LIST:
+            Constraint.validate_list(value)
+            if entry_schema:
+                DataEntity.validate_entry(value, entry_schema, custom_def)
+            return value
+        elif type == Schema.MAP:
+            Constraint.validate_map(value)
+            if entry_schema:
+                DataEntity.validate_entry(value, entry_schema, custom_def)
+            return value
+        else:
+            data = DataEntity(type, value, custom_def)
+            return data.validate()
+
+    @staticmethod
+    def validate_entry(value, entry_schema, custom_def=None):
+        '''Validate entries for map and list.'''
+        schema = Schema(None, entry_schema)
+        valuelist = value
+        if isinstance(value, dict):
+            valuelist = list(value.values())
+        for v in valuelist:
+            DataEntity.validate_datatype(schema.type, v, schema.entry_schema,
+                                         custom_def)
+            if schema.constraints:
+                for constraint in schema.constraints:
+                    constraint.validate(v)
+        return value

translator/toscalib/elements/TOSCA_definition.yaml

@@ -283,3 +283,77 @@ tosca.interfaces.relationship.Configure:
     description: Operation to add a target node.
   remove_target:
     description: Operation to remove a target node.
+
+##########################################################################
+ # Data Type.
+ # A Datatype is a complex data type declaration which contains other
+ # complex or simple data types.
+##########################################################################
+tosca.datatypes.network.NetworkInfo:
+  properties:
+    network_name:
+      type: string
+    network_id:
+      type: string
+    addresses:
+      type: list
+      entry_schema:
+        type: string
+
+tosca.datatypes.network.PortInfo:
+  properties:
+    port_name:
+      type: string
+    port_id:
+      type: string
+    network_id:
+      type: string
+    mac_address:
+      type: string
+    addresses:
+      type: list
+      entry_schema:
+        type: string
+
+tosca.datatypes.network.PortDef:
+  type: integer
+  constraints:
+    - in_range: [ 1, 65535 ]
+
+tosca.datatypes.network.PortSpec:
+  properties:
+    protocol:
+      type: string
+      required: true
+      default: tcp
+      constraints:
+        - valid_values: [ udp, tcp, igmp ]
+    target:
+      type: list
+      entry_schema:
+        type: PortDef
+    target_range:
+      type: range
+      constraints:
+        - in_range: [ 1, 65535 ]
+    source:
+      type: list
+      entry_schema:
+        type: PortDef
+    source_range:
+      type: range
+      constraints:
+        - in_range: [ 1, 65535 ]
+
+tosca.datatypes.network.Credential:
+  properties:
+    protocol:
+      type: string
+    token_type:
+      type: string
+    token:
+      type: string
+    keys:
+      type: map
+      entry_schema:
+        type: string

translator/toscalib/elements/constraints.py

@@ -429,7 +429,7 @@ class ValidValues(Constraint):
             raise InvalidSchemaError(message=_('valid_values must be a list.'))
 
     def _is_valid(self, value):
-        if isinstance(value, collections.Sequence):
+        if isinstance(value, list):
             return all(v in self.constraint_value for v in value)
         return value in self.constraint_value
 

translator/toscalib/elements/datatype.py

@@ -0,0 +1,46 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from translator.toscalib.elements.statefulentitytype import StatefulEntityType
+
+
+class DataType(StatefulEntityType):
+    '''TOSCA built-in and user defined complex data type.'''
+
+    def __init__(self, datatypename, custom_def=None):
+        super(DataType, self).__init__(datatypename, self.DATATYPE_PREFIX,
+                                       custom_def)
+        self.custom_def = custom_def
+
+    @property
+    def parent_type(self):
+        '''Return a datatype this datatype is derived from.'''
+        ptype = self.derived_from(self.defs)
+        if ptype:
+            return DataType(ptype, self.custom_def)
+        return None
+
+    @property
+    def value_type(self):
+        '''Return 'type' section in the datatype schema.'''
+        return self.entity_value(self.defs, 'type')
+
+    @property
+    def all_properties(self):
+        '''Return all properties defined in this type and its parent type.'''
+        props_def = self.properties_def
+        ptype = self.parent_type
+        while ptype:
+            props_def.extend(ptype.properties_def)
+            ptype = ptype.parent_type
+        return props_def

translator/toscalib/elements/entitytype.py

@@ -44,6 +44,9 @@ class EntityType(object):
     RELATIONSHIP_PREFIX = 'tosca.relationships.'
     CAPABILITY_PREFIX = 'tosca.capabilities.'
     INTERFACE_PREFIX = 'tosca.interfaces.'
+    #currently the data types are defined only for network
+    #but may have changes in the future.
+    DATATYPE_PREFIX = 'tosca.datatypes.network.'
     TOSCA = 'tosca'
 
     def derived_from(self, defs):

translator/toscalib/elements/statefulentitytype.py

@@ -10,7 +10,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from translator.toscalib.common.exception import InvalidNodeTypeError
+from translator.toscalib.common.exception import InvalidTypeError
 from translator.toscalib.elements.attribute_definition import AttributeDef
 from translator.toscalib.elements.entitytype import EntityType
 from translator.toscalib.elements.property_definition import PropertyDef
@@ -36,7 +36,7 @@ class StatefulEntityType(EntityType):
         elif custom_def and entitytype in list(custom_def.keys()):
             self.defs = custom_def[entitytype]
         else:
-            raise InvalidNodeTypeError(what=entitytype)
+            raise InvalidTypeError(what=entitytype)
         self.type = entitytype
 
     @property

translator/toscalib/entity_template.py

@@ -155,11 +155,11 @@ class EntityTemplate(object):
         for name, value in properties.items():
             for p in self.type_definition.properties_def:
                 if p.name == name:
-                    prop = Property(name, value, p.schema)
+                    prop = Property(name, value, p.schema, self.custom_def)
                     props.append(prop)
         for p in self.type_definition.properties_def:
             if p.default is not None and p.name not in properties.keys():
-                prop = Property(p.name, p.default, p.schema)
+                prop = Property(p.name, p.default, p.schema, self.custom_def)
                 props.append(prop)
         return props
 

translator/toscalib/properties.py

@@ -13,11 +13,9 @@
 import re
 
 from translator.toscalib.common.exception import InvalidPropertyValueError
-from translator.toscalib.common.exception import InvalidSchemaError
-from translator.toscalib.elements.constraints import Constraint
+from translator.toscalib.dataentity import DataEntity
 from translator.toscalib.elements.constraints import Schema
 from translator.toscalib.functions import is_function
-from translator.toscalib.utils.gettextutils import _
 
 
 class Property(object):
@@ -35,9 +33,10 @@ class Property(object):
         'type', 'properties'
     )
 
-    def __init__(self, property_name, value, schema_dict):
+    def __init__(self, property_name, value, schema_dict, custom_def=None):
         self.name = property_name
         self.value = self._convert_value(value)
+        self.custom_def = custom_def
         self.schema = Schema(property_name, schema_dict)
 
     @property
@@ -69,52 +68,10 @@ class Property(object):
         if not is_function(self.value):
             if self.type == Schema.STRING:
                 self.value = str(self.value)
-
-            self._validate_datatype()
+            DataEntity.validate_datatype(self.type, self.value,
+                                         self.entry_schema, self.custom_def)
             self._validate_constraints()
 
-            if self.type == Schema.LIST:
-                self._validate_children(enumerate(self.value))
-            elif self.type == Schema.MAP:
-                self._validate_children(self.value.items())
-
-    def _validate_datatype(self):
-            dtype = self.type
-            if dtype == Schema.STRING:
-                return Constraint.validate_string(self.value)
-            elif dtype == Schema.INTEGER:
-                return Constraint.validate_integer(self.value)
-            elif dtype == Schema.NUMBER:
-                return Constraint.validate_number(self.value)
-            elif dtype == Schema.LIST:
-                return Constraint.validate_list(self.value)
-            elif dtype == Schema.MAP:
-                return Constraint.validate_map(self.value)
-            elif dtype == Schema.BOOLEAN:
-                return Constraint.validate_boolean(self.value)
-            else:
-                msg = _('Type (%s) is not a valid data type.') % self.type
-                raise InvalidSchemaError(message=msg)
-
-    def _validate_children(self, child_values):
-        entry_schema = self.schema.entry_schema
-        if entry_schema is not None:
-            if entry_schema.get(self.ENTRYTYPE) is not None:
-                for value in dict(child_values).values():
-                    entry_prop = Property(_('Entry of %s') % self.name,
-                                          value, entry_schema)
-                    entry_prop.validate()
-                return child_values
-
-            properties_schema = entry_schema.get(self.ENTRYPROPERTIES)
-            if properties_schema is not None:
-                for entry_values in dict(child_values).values():
-                    for k in list(properties_schema):
-                        entry_of_prop = Property(k, entry_values.get(k),
-                                                 properties_schema.get(k))
-                        entry_of_prop.validate()
-        return child_values
-
     def _validate_constraints(self):
         if self.constraints:
             for constraint in self.constraints:

translator/toscalib/tests/data/datatypes/custom_datatype_def.yaml

@@ -0,0 +1,48 @@
+tosca_definitions_version: tosca_simple_1.0
+
+description: >
+  Custom type and node definition used to test custom datatypes.
+
+node_types:
+  tosca.nodes.my.SomeNode:
+    derived_from: tosca.nodes.Root
+    properties:
+      people:
+        type: tosca.my.datatypes.People
+
+datatype_definitions:
+  tosca.my.datatypes.PeopleBase:
+    properties:
+      name:
+        type: string
+        required: true
+        constraints:
+          - min_length: 2
+      gender:
+        type: string
+        default: unknown
+
+  tosca.my.datatypes.People:
+    derived_from: tosca.my.datatypes.PeopleBase
+    properties:
+      addresses:
+        type: map
+        entry_schema:
+          type: string
+      contacts:
+        type: list
+        entry_schema:
+          type: tosca.my.datatypes.ContactInfo
+
+  tosca.my.datatypes.ContactInfo:
+    description: simple contact information
+    properties:
+      contact_name:
+        type: string
+        required: true
+        constraints:
+          - min_length: 2
+      contact_email:
+        type: string
+      contact_phone:
+        type: string

translator/toscalib/tests/data/datatypes/test_custom_datatypes_nested_datatype_error.yaml

@@ -0,0 +1,24 @@
+tosca_definitions_version: tosca_simple_1.0
+
+description: >
+  TOSCA templates used to test custom datatypes.
+
+imports:
+  - custom_datatype_def.yaml
+
+node_templates:
+  # 123456789 is not a string
+  error in nested datatype:
+    type: tosca.nodes.my.SomeNode
+    properties:
+      people:
+        name: Mike
+        gender: male
+        addresses: {Home: 1 foo street, Office: 9 bar avenue}
+        contacts:
+          - {contact_name: Tom,
+            contact_email: tom@email.com,
+            contact_phone: 123456789}
+          - {contact_name: Jerry,
+            contact_email: jerry@email.com,
+            contact_phone: '321654987'}

translator/toscalib/tests/data/datatypes/test_custom_datatypes_positive.yaml

@@ -0,0 +1,23 @@
+tosca_definitions_version: tosca_simple_1.0
+
+description: >
+  TOSCA templates used to test custom datatypes.
+
+imports:
+  - custom_datatype_def.yaml
+
+node_templates:
+  positive:
+    type: tosca.nodes.my.SomeNode
+    properties:
+      people:
+        name: Mike
+        gender: male
+        addresses: {Home: 1 foo street, Office: 9 bar avenue}
+        contacts:
+          - {contact_name: Tom,
+            contact_email: tom@email.com,
+            contact_phone: '123456789'}
+          - {contact_name: Jerry,
+            contact_email: jerry@email.com,
+            contact_phone: '321654987'}

translator/toscalib/tests/data/datatypes/test_custom_datatypes_value_error.yaml

@@ -0,0 +1,17 @@
+tosca_definitions_version: tosca_simple_1.0
+
+description: >
+  TOSCA templates used to test custom datatypes.
+
+imports:
+  - custom_datatype_def.yaml
+
+node_templates:
+  # addresses is not a map
+  error in field value:
+    type: tosca.nodes.my.SomeNode
+    properties:
+      people:
+        name: Mike
+        gender: male
+        addresses: [1 foo street, 9 bar avenue]

translator/toscalib/tests/test_datatypes.py

@@ -0,0 +1,271 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from testtools.testcase import skip
+from translator.toscalib.common.exception import MissingRequiredFieldError
+from translator.toscalib.common.exception import TypeMismatchError
+from translator.toscalib.common.exception import UnknownFieldError
+from translator.toscalib.common.exception import ValidationError
+from translator.toscalib.dataentity import DataEntity
+from translator.toscalib.elements.datatype import DataType
+from translator.toscalib.tests.base import TestCase
+from translator.toscalib.tosca_template import ToscaTemplate
+from translator.toscalib.utils import yamlparser
+
+
+class DataTypeTest(TestCase):
+
+    custom_type_schema = '''
+    tosca.my.datatypes.PeopleBase:
+      properties:
+        name:
+          type: string
+          required: true
+          constraints:
+            - min_length: 2
+        gender:
+          type: string
+          default: unknown
+
+    tosca.my.datatypes.People:
+      derived_from: tosca.my.datatypes.PeopleBase
+      properties:
+        addresses:
+          type: map
+          entry_schema:
+            type: string
+        contacts:
+          type: list
+          entry_schema:
+            type: tosca.my.datatypes.ContactInfo
+
+    tosca.my.datatypes.ContactInfo:
+      description: simple contact information
+      properties:
+        contact_name:
+          type: string
+          required: true
+          constraints:
+            - min_length: 2
+        contact_email:
+          type: string
+        contact_phone:
+          type: string
+    '''
+    custom_type_def = yamlparser.simple_parse(custom_type_schema)
+
+    def test_built_in_datatype(self):
+        value_snippet = '''
+        private_network:
+          network_name: private
+          network_id: 3e54214f-5c09-1bc9-9999-44100326da1b
+          addresses: [ 10.111.128.10 ]
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('tosca.datatypes.network.NetworkInfo',
+                          value.get('private_network'))
+        self.assertIsNotNone(data.validate())
+
+    def test_built_in_datatype_with_short_name(self):
+        value_snippet = '''
+        ethernet_port:
+          port_name: port1
+          port_id: 2c0c7a37-691a-23a6-7709-2d10ad041467
+          network_id: 3e54214f-5c09-1bc9-9999-44100326da1b
+          mac_address: f1:18:3b:41:92:1e
+          addresses: [ 172.24.9.102 ]
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('PortInfo', value.get('ethernet_port'))
+        self.assertIsNotNone(data.validate())
+
+    def test_built_in_datatype_without_properties(self):
+        value_snippet = '''
+        2
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        datatype = DataType('PortDef')
+        self.assertEqual('integer', datatype.value_type)
+        data = DataEntity('PortDef', value)
+        self.assertIsNotNone(data.validate())
+
+    @skip('The example in TOSCA spec may have some problem.')
+    def test_built_in_nested_datatype(self):
+        value_snippet = '''
+        user_port:
+          protocol: tcp
+          target: [50000]
+          source: [9000]
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('PortSpec', value.get('user_port'))
+        self.assertIsNotNone(data.validate())
+
+    def test_custom_datatype(self):
+        value_snippet = '''
+        name: Mike
+        gender: male
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('tosca.my.datatypes.PeopleBase', value,
+                          DataTypeTest.custom_type_def)
+        self.assertIsNotNone(data.validate())
+
+    def test_custom_datatype_with_parent(self):
+        value_snippet = '''
+        name: Mike
+        gender: male
+        contacts:
+          - {contact_name: Tom,
+            contact_email: tom@email.com,
+            contact_phone: '123456789'}
+          - {contact_name: Jerry,
+            contact_email: jerry@email.com,
+            contact_phone: '321654987'}
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('tosca.my.datatypes.People', value,
+                          DataTypeTest.custom_type_def)
+        self.assertIsNotNone(data.validate())
+
+    # [Tom, Jerry] is not a dict, it can't be a value of datatype PeopleBase
+    def test_non_dict_value_for_datatype(self):
+        value_snippet = '''
+        [Tom, Jerry]
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('tosca.my.datatypes.PeopleBase', value,
+                          DataTypeTest.custom_type_def)
+        error = self.assertRaises(TypeMismatchError, data.validate)
+        self.assertEqual('[\'Tom\', \'Jerry\'] must be of type: '
+                         '"tosca.my.datatypes.PeopleBase".', error.__str__())
+
+    # 'nema' is an invalid field name
+    def test_field_error_in_dataentity(self):
+        value_snippet = '''
+        nema: Mike
+        gender: male
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('tosca.my.datatypes.PeopleBase', value,
+                          DataTypeTest.custom_type_def)
+        error = self.assertRaises(UnknownFieldError, data.validate)
+        self.assertEqual('Data value of type tosca.my.datatypes.PeopleBase '
+                         'contain(s) unknown field: "nema", refer to the '
+                         'definition to verify valid values.',
+                         error.__str__())
+
+    def test_default_field_in_dataentity(self):
+        value_snippet = '''
+        name: Mike
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('tosca.my.datatypes.PeopleBase', value,
+                          DataTypeTest.custom_type_def)
+        data = data.validate()
+        self.assertEqual('unknown', data.get('gender'))
+
+    # required field 'name' is missing
+    def test_missing_field_in_dataentity(self):
+        value_snippet = '''
+        gender: male
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('tosca.my.datatypes.PeopleBase', value,
+                          DataTypeTest.custom_type_def)
+        error = self.assertRaises(MissingRequiredFieldError, data.validate)
+        self.assertEqual('Data value of type tosca.my.datatypes.PeopleBase '
+                         'is missing required field: "[\'name\']".',
+                         error.__str__())
+
+    # the value of name field is not a string
+    def test_type_error_in_dataentity(self):
+        value_snippet = '''
+        name: 123
+        gender: male
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('tosca.my.datatypes.PeopleBase', value,
+                          DataTypeTest.custom_type_def)
+        error = self.assertRaises(ValueError, data.validate)
+        self.assertEqual('"123" is not a string', error.__str__())
+
+    # the value of name doesn't meet the defined constraint
+    def test_value_error_in_dataentity(self):
+        value_snippet = '''
+        name: M
+        gender: male
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('tosca.my.datatypes.PeopleBase', value,
+                          DataTypeTest.custom_type_def)
+        error = self.assertRaises(ValidationError, data.validate)
+        self.assertEqual('length of name: M must be at least "2".',
+                         error.__str__())
+
+    # value of addresses doesn't fit the entry_schema
+    def test_validation_in_collection_entry(self):
+        value_snippet = '''
+        name: Mike
+        gender: male
+        addresses: {Home: 1, Office: 9 bar avenue}
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('tosca.my.datatypes.People', value,
+                          DataTypeTest.custom_type_def)
+        error = self.assertRaises(ValueError, data.validate)
+        self.assertEqual('"1" is not a string', error.__str__())
+
+    # contact_pone is an invalid field name in nested datatype
+    def test_validation_in_nested_datatype(self):
+        value_snippet = '''
+        name: Mike
+        gender: male
+        contacts:
+          - {contact_name: Tom,
+            contact_email: tom@email.com,
+            contact_pone: '123456789'}
+          - {contact_name: Jerry,
+            contact_email: jerry@email.com,
+            contact_phone: '321654987'}
+        '''
+        value = yamlparser.simple_parse(value_snippet)
+        data = DataEntity('tosca.my.datatypes.People', value,
+                          DataTypeTest.custom_type_def)
+        error = self.assertRaises(UnknownFieldError, data.validate)
+        self.assertEqual('Data value of type tosca.my.datatypes.ContactInfo '
+                         'contain(s) unknown field: "contact_pone", refer to '
+                         'the definition to verify valid values.',
+                         error.__str__())
+
+    def test_datatype_in_template_positive(self):
+        tpl_path = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            "data/datatypes/test_custom_datatypes_positive.yaml")
+        self.assertIsNotNone(ToscaTemplate(tpl_path))
+
+    def test_datatype_in_template_invalid_value(self):
+        tpl_path = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            "data/datatypes/test_custom_datatypes_value_error.yaml")
+        error = self.assertRaises(ValueError, ToscaTemplate, tpl_path)
+        self.assertEqual('"[\'1 foo street\', \'9 bar avenue\']" '
+                         'is not a map', error.__str__())
+
+    def test_datatype_in_template_nested_datatype_error(self):
+        tpl_path = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            "data/datatypes/test_custom_datatypes_nested_datatype_error.yaml")
+        error = self.assertRaises(ValueError, ToscaTemplate, tpl_path)
+        self.assertEqual('"123456789" is not a string', error.__str__())

translator/toscalib/tests/test_properties.py

@@ -1,6 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-#
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
 #    a copy of the License at
@@ -13,7 +10,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from translator.toscalib.common.exception import InvalidSchemaError
+from translator.toscalib.common.exception import InvalidTypeError
 from translator.toscalib.properties import Property
 from translator.toscalib.tests.base import TestCase
 from translator.toscalib.utils import yamlparser
@@ -31,9 +28,10 @@ class PropertyTest(TestCase):
         test_property_schema = {'type': 'Fish'}
         propertyInstance = Property('test_property', 'Hughes',
                                     test_property_schema)
-        error = self.assertRaises(InvalidSchemaError,
+        error = self.assertRaises(InvalidTypeError,
                                   propertyInstance.validate)
-        self.assertEqual('Type (Fish) is not a valid data type.', str(error))
+        self.assertEqual('Type "tosca.datatypes.network.Fish" '
+                         'is not a valid type.', str(error))
 
     def test_list(self):
         test_property_schema = {'type': 'list'}
@@ -79,46 +77,6 @@ class PropertyTest(TestCase):
         self.assertEqual('"b" is not an integer',
                          str(error))
 
-    def test_list_entry_schema_properties(self):
-        schema_snippet = '''
-        type: list
-        entry_schema:
-          properties:
-            valid:
-              type: boolean
-            contact_name:
-              type: string
-              constraints:
-                - min_length: 2
-        '''
-        test_property_schema = yamlparser.simple_parse(schema_snippet)
-        propertyInstance = Property('test_property',
-                                    [{'valid': True, 'contact_name': 'hughes'},
-                                     {'valid': True, 'contact_name': 'olive'}],
-                                    test_property_schema)
-
-        self.assertIsNone(propertyInstance.validate())
-        self.assertEqual([{'valid': True, 'contact_name': 'hughes'},
-                          {'valid': True, 'contact_name': 'olive'}],
-                         propertyInstance.value)
-
-    def test_list_entry_schema_properties_invalid(self):
-        schema_snippet = '''
-        type: list
-        entry_schema:
-          properties:
-            valid:
-              type: boolean
-            contact_name:
-              type: string
-        '''
-        test_property_schema = yamlparser.simple_parse(schema_snippet)
-        propertyInstance = Property('test_property',
-                                    [{'valid': 123, 'contact_name': 'olive'}],
-                                    test_property_schema)
-        error = self.assertRaises(ValueError, propertyInstance.validate)
-        self.assertEqual('"123" is not a boolean', str(error))
-
     def test_map(self):
         test_property_schema = {'type': 'map'}
         propertyInstance = Property('test_property', {'a': 'b'},
@@ -152,50 +110,6 @@ class PropertyTest(TestCase):
         error = self.assertRaises(ValueError, propertyInstance.validate)
         self.assertEqual('"123" is not a boolean', str(error))
 
-    def test_map_entry_schema_properties(self):
-        schema_snippet = '''
-        type: map
-        entry_schema:
-          properties:
-            valid:
-              type: boolean
-            contact_name:
-              type: string
-        '''
-        test_property_schema = yamlparser.simple_parse(schema_snippet)
-
-        tpl_snippet = '''
-        my_map:
-          valid: True
-          contact_name: hughes
-        your_map:
-          valid: True
-          contact_name: olive
-        '''
-        test_property_value = yamlparser.simple_parse(tpl_snippet)
-
-        propertyInstance = Property('test_property', test_property_value,
-                                    test_property_schema)
-        self.assertIsNone(propertyInstance.validate())
-        self.assertEqual(test_property_value, propertyInstance.value)
-
-    def test_map_entry_schema_properties_invalid(self):
-        schema_snippet = '''
-        type: map
-        entry_schema:
-          properties:
-            valid:
-              type: boolean
-        '''
-        test_property_schema = yamlparser.simple_parse(schema_snippet)
-
-        propertyInstance = Property('test_property',
-                                    {'my_map': {'valid': 123}},
-                                    test_property_schema)
-
-        error = self.assertRaises(ValueError, propertyInstance.validate)
-        self.assertEqual('"123" is not a boolean', str(error))
-
     def test_boolean(self):
         test_property_schema = {'type': 'boolean'}
         propertyInstance = Property('test_property', 'true',

translator/toscalib/tests/test_toscadef.py

@@ -10,7 +10,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from translator.toscalib.common.exception import InvalidNodeTypeError
+from translator.toscalib.common.exception import InvalidTypeError
 from translator.toscalib.elements.nodetype import NodeType
 from translator.toscalib.tests.base import TestCase
 compute_type = NodeType('tosca.nodes.Compute')
@@ -20,7 +20,7 @@ component_type = NodeType('tosca.nodes.SoftwareComponent')
 class ToscaDefTest(TestCase):
     def test_type(self):
         self.assertEqual(compute_type.type, "tosca.nodes.Compute")
-        self.assertRaises(InvalidNodeTypeError, NodeType,
+        self.assertRaises(InvalidTypeError, NodeType,
                           'tosca.nodes.Invalid')
 
     def test_parent_type(self):

translator/toscalib/tests/test_toscatplvalidation.py

@@ -13,7 +13,7 @@
 import os
 import six
 
-from translator.toscalib.common.exception import InvalidNodeTypeError
+from translator.toscalib.common.exception import InvalidTypeError
 from translator.toscalib.common.exception import MissingRequiredFieldError
 from translator.toscalib.common.exception import TypeMismatchError
 from translator.toscalib.common.exception import UnknownFieldError
@@ -37,23 +37,18 @@ class ToscaTemplateValidationTest(TestCase):
         tpl_path = os.path.join(
             os.path.dirname(os.path.abspath(__file__)),
             "data/test_tosca_top_level_error1.yaml")
-        try:
-            ToscaTemplate(tpl_path)
-        except Exception as err:
-            self.assertTrue(isinstance(err, MissingRequiredFieldError))
-            self.assertEqual('Template is missing required field: '
-                             '"tosca_definitions_version".', err.__str__())
+        err = self.assertRaises(MissingRequiredFieldError, ToscaTemplate,
+                                tpl_path)
+        self.assertEqual('Template is missing required field: '
+                         '"tosca_definitions_version".', err.__str__())
 
         tpl_path = os.path.join(
             os.path.dirname(os.path.abspath(__file__)),
             "data/test_tosca_top_level_error2.yaml")
-        try:
-            ToscaTemplate(tpl_path)
-        except Exception as err:
-            self.assertTrue(isinstance(err, UnknownFieldError))
-            self.assertEqual('Template contain(s) unknown field: '
-                             '"node_template", refer to the TOSCA specs '
-                             'to verify valid values.', err.__str__())
+        err = self.assertRaises(UnknownFieldError, ToscaTemplate, tpl_path)
+        self.assertEqual('Template contain(s) unknown field: '
+                         '"node_template", refer to the definition '
+                         'to verify valid values.', err.__str__())
 
     def test_inputs(self):
         tpl_snippet = '''
@@ -73,7 +68,7 @@ class ToscaTemplateValidationTest(TestCase):
         except Exception as err:
             self.assertTrue(isinstance(err, UnknownFieldError))
             self.assertEqual('Input cpus contain(s) unknown field: '
-                             '"constraint", refer to the TOSCA specs to '
+                             '"constraint", refer to the definition to '
                              'verify valid values.', err.__str__())
 
     def test_outputs(self):
@@ -109,8 +104,8 @@ class ToscaTemplateValidationTest(TestCase):
         except Exception as err:
             self.assertTrue(isinstance(err, UnknownFieldError))
             self.assertEqual('Output server_address contain(s) unknown '
-                             'field: "descriptions", refer to the TOSCA '
-                             'specs to verify valid values.',
+                             'field: "descriptions", refer to the definition '
+                             'to verify valid values.',
                              err.__str__())
 
     def _custom_types(self):
@@ -185,7 +180,7 @@ class ToscaTemplateValidationTest(TestCase):
         '''
         expectedmessage = ('Second level of template mysql_dbms '
                            'contain(s) unknown field: "requirement", '
-                           'refer to the TOSCA specs to verify valid values.')
+                           'refer to the definition to verify valid values.')
         self._single_node_template_content_test(tpl_snippet,
                                                 UnknownFieldError,
                                                 expectedmessage)
@@ -209,10 +204,10 @@ class ToscaTemplateValidationTest(TestCase):
               tosca.interfaces.node.Lifecycle:
                  configure: mysql_database_configure.sh
         '''
-        expectedmessage = ('Node type "tosca.nodes.Databases" is not '
+        expectedmessage = ('Type "tosca.nodes.Databases" is not '
                            'a valid type.')
         self._single_node_template_content_test(tpl_snippet,
-                                                InvalidNodeTypeError,
+                                                InvalidTypeError,
                                                 expectedmessage)
 
     def test_node_template_requirements(self):
@@ -254,7 +249,7 @@ class ToscaTemplateValidationTest(TestCase):
         '''
         expectedmessage = ('Requirements of template mysql_database '
                            'contain(s) unknown field: "database_endpoint", '
-                           'refer to the TOSCA specs to verify valid values.')
+                           'refer to the definition to verify valid values.')
         self._single_node_template_content_test(tpl_snippet,
                                                 UnknownFieldError,
                                                 expectedmessage)
@@ -280,7 +275,7 @@ class ToscaTemplateValidationTest(TestCase):
         '''
         expectedmessage = ('Capabilities of template mysql_database '
                            'contain(s) unknown field: "http_endpoint", '
-                           'refer to the TOSCA specs to verify valid values.')
+                           'refer to the definition to verify valid values.')
         self._single_node_template_content_test(tpl_snippet,
                                                 UnknownFieldError,
                                                 expectedmessage)
@@ -323,8 +318,8 @@ class ToscaTemplateValidationTest(TestCase):
               os_image: F18_x86_64
         '''
         expectedmessage = ('Properties of template server contain(s) '
-                           'unknown field: "os_image", refer to the TOSCA '
-                           'specs to verify valid values.')
+                           'unknown field: "os_image", refer to the '
+                           'definition to verify valid values.')
         self._single_node_template_content_test(tpl_snippet,
                                                 UnknownFieldError,
                                                 expectedmessage)
@@ -353,7 +348,7 @@ class ToscaTemplateValidationTest(TestCase):
         expectedmessage = ('Interfaces of template wordpress '
                            'contain(s) unknown field: '
                            '"tosca.interfaces.node.Lifecycles", '
-                           'refer to the TOSCA specs to verify valid values.')
+                           'refer to the definition to verify valid values.')
         self._single_node_template_content_test(tpl_snippet,
                                                 UnknownFieldError,
                                                 expectedmessage)
@@ -379,7 +374,7 @@ class ToscaTemplateValidationTest(TestCase):
                      database_endpoint, port ] }
         '''
         expectedmessage = ('Interfaces of template wordpress contain(s) '
-                           'unknown field: "config", refer to the TOSCA specs'
+                           'unknown field: "config", refer to the definition'
                            ' to verify valid values.')
         self._single_node_template_content_test(tpl_snippet,
                                                 UnknownFieldError,
@@ -406,7 +401,7 @@ class ToscaTemplateValidationTest(TestCase):
                      database_endpoint, port ] }
         '''
         expectedmessage = ('Interfaces of template wordpress contain(s) '
-                           'unknown field: "inputs", refer to the TOSCA specs'
+                           'unknown field: "inputs", refer to the definition'
                            ' to verify valid values.')
         self._single_node_template_content_test(tpl_snippet,
                                                 UnknownFieldError,

translator/toscalib/tosca_template.py

@@ -30,13 +30,13 @@ SECTIONS = (DEFINITION_VERSION, DEFAULT_NAMESPACE, TEMPLATE_NAME,
             TEMPLATE_AUTHOR, TEMPLATE_VERSION, DESCRIPTION, IMPORTS,
             DSL_DEFINITIONS, INPUTS, NODE_TEMPLATES, RELATIONSHIP_TEMPLATES,
             NODE_TYPES, RELATIONSHIP_TYPES, CAPABILITY_TYPES, ARTIFACT_TYPES,
-            OUTPUTS, GROUPS) = \
+            OUTPUTS, GROUPS, DATATYPE_DEFINITIONS) = \
            ('tosca_definitions_version', 'tosca_default_namespace',
             'template_name', 'template_author', 'template_version',
             'description', 'imports', 'dsl_definitions', 'inputs',
             'node_templates', 'relationship_templates', 'node_types',
             'relationship_types', 'capability_types', 'artifact_types',
-            'outputs', 'groups')
+            'outputs', 'groups', 'datatype_definitions')
 
 log = logging.getLogger("tosca.model")
 
@@ -67,7 +67,7 @@ class ToscaTemplate(object):
         return inputs
 
     def _nodetemplates(self):
-        custom_types = {}
+        custom_defs = {}
         imports = self._tpl_imports()
         if imports:
             for definition in imports:
@@ -77,14 +77,20 @@ class ToscaTemplate(object):
                     tpl_dir = os.path.dirname(os.path.abspath(self.path))
                     def_file = os.path.join(tpl_dir, definition)
                 custom_type = YAML_LOADER(def_file)
-                node_types = custom_type['node_types']
-                for name in node_types:
-                    defintion = node_types[name]
-                    custom_types[name] = defintion
+                custom_types = {}
+                node_types = custom_type.get(NODE_TYPES)
+                data_types = custom_type.get(DATATYPE_DEFINITIONS)
+                if node_types:
+                    custom_types.update(node_types)
+                if data_types:
+                    custom_types.update(data_types)
+                for name in custom_types:
+                    defintion = custom_types[name]
+                    custom_defs[name] = defintion
         nodetemplates = []
         tpls = self._tpl_nodetemplates()
         for name in tpls:
-            tpl = NodeTemplate(name, tpls, custom_types,
+            tpl = NodeTemplate(name, tpls, custom_defs,
                                self.relationship_templates)
             tpl.validate(self)
             nodetemplates.append(tpl)