Drydock orchestration of MaaS networking

- Create a MaaS API client for managing API access/authentication
- Start MaaS object model for accessing API resources
- Add orchestration step for PrepareSite action
- Create maasdriver logic to handle CreateNetworkTemplate action
- Separate tests for unit and integration
- Fix YAML ingester to use a default of None for VLAN tag instead of 1

helm_drydock/config.py

@@ -21,21 +21,13 @@
 
 class DrydockConfig(object):
 
-    def __init__(self):
+    node_driver = {
-        self.server_driver_config = {
+        'maasdriver':   {
-            selected_driver = helm_drydock.drivers.server.maasdriver,
+            'api_key':  'KTMHgA42cNSMnfmJ82:cdg4yQUhp542aHsCTV:7Dc2KB9hQpWq3LfQAAAKAj6wdg22yWxZ',
-            params = {
+            'api_url':  'http://localhost:5240/MAAS/api/2.0/'
-                maas_api_key = "" 
+        },
-                maas_api_url = ""
     }
-        }
+
-        self.selected_network_driver = helm_drydock.drivers.network.noopdriver
+    ingester_config = {
-        self.control_config = {}
+        'plugins': ['helm_drydock.ingester.plugins.yaml']
-        self.ingester_config = {
-            plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester]
-        }
-        self.introspection_config = {}
-        self.orchestrator_config = {}
-        self.statemgmt_config = {
-            backend_driver = helm_drydock.drivers.statemgmt.etcd,
     }

helm_drydock/drivers/node/__init__.py

@@ -13,16 +13,44 @@
 # limitations under the License.
 #
 
+import helm_drydock.objects.fields as hd_fields
+import helm_drydock.error as errors
+
 from helm_drydock.drivers import ProviderDriver
 
 class NodeDriver(ProviderDriver):
 
-class NodeAction(Enum):
+    def __init__(self, **kwargs):
-    PrepareNode = 'prepare_node'
+        super(NodeDriver, self).__init__(**kwargs)
-    ApplyNetworkConfig = 'apply_network_config'
+
-    ApplyStorageConfig = 'apply_storage_config'
+        self.supported_actions = [hd_fields.OrchestratorAction.ValidateNodeServices,
-    InterrogateNode = 'interrogate_node'
+                                  hd_fields.OrchestratorAction.CreateNetworkTemplate,
-    DeployNode = 'deploy_node'
+                                  hd_fields.OrchestratorAction.CreateStorageTemplate,
+                                  hd_fields.OrchestratorAction.CreateBootMedia,
+                                  hd_fields.OrchestratorAction.PrepareHardwareConfig,
+                                  hd_fields.OrchestratorAction.ConfigureHardware,
+                                  hd_fields.OrchestratorAction.InterrogateNode,
+                                  hd_fields.OrchestratorAction.ApplyNodeNetworking,
+                                  hd_fields.OrchestratorAction.ApplyNodeStorage,
+                                  hd_fields.OrchestratorAction.ApplyNodePlatform,
+                                  hd_fields.OrchestratorAction.DeployNode,
+                                  hd_fields.OrchestratorAction.DestroyNode]
+
+        self.driver_name = "node_generic"
+        self.driver_key = "node_generic"
+        self.driver_desc = "Generic Node Driver"
+
+    def execute_task(self, task_id):
+        task = self.state_manager.get_task(task_id)
+        task_action = task.action
+
+        if task_action in self.supported_actions:
+            return
+        else:
+            raise DriverError("Unsupported action %s for driver %s" %
+                (task_action, self.driver_desc))
+
+
 
 
 

helm_drydock/drivers/node/maasdriver/__init__.py

@@ -11,10 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from helm_drydock.drivers.node import NodeDriver
-
-class MaasNodeDriver(NodeDriver):
-
-    def __init__(self, kwargs):
-        super(MaasNodeDriver, self).__init__(**kwargs)
-	

helm_drydock/drivers/node/maasdriver/api_client.py

@@ -0,0 +1,147 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from oauthlib import oauth1
+import requests
+import requests.auth as req_auth
+import base64
+
+class MaasOauth(req_auth.AuthBase):
+    def __init__(self, apikey):
+        self.consumer_key, self.token_key, self.token_secret = apikey.split(':')
+        self.consumer_secret = ""
+        self.realm = "OAuth"
+
+        self.oauth_client = oauth1.Client(self.consumer_key, self.consumer_secret,
+            self.token_key, self.token_secret, signature_method=oauth1.SIGNATURE_PLAINTEXT,
+            realm=self.realm)
+
+    def __call__(self, req):
+        headers = req.headers
+        url = req.url
+        method = req.method
+        body = None if req.body is None or len(req.body) == 0 else req.body
+
+        new_url, signed_headers, new_body = self.oauth_client.sign(url, method, body, headers)
+
+        req.headers['Authorization'] = signed_headers['Authorization']
+
+        return req
+
+class MaasRequestFactory(object):
+
+    def __init__(self, base_url, apikey):
+        self.base_url = base_url
+        self.apikey = apikey
+        self.signer = MaasOauth(apikey)
+        self.http_session = requests.Session()
+
+    def get(self, endpoint, **kwargs):
+        return self._send_request('GET', endpoint, **kwargs)
+
+    def post(self, endpoint, **kwargs):
+        return self._send_request('POST', endpoint, **kwargs)
+
+    def delete(self, endpoint, **kwargs):
+        return self._send_request('DELETE', endpoint, **kwargs)
+
+    def put(self, endpoint, **kwargs):
+        return self._send_request('PUT', endpoint, **kwargs)
+        
+    def test_connectivity(self):
+        try:
+            resp = self.get('version/')
+        except requests.Timeout(ex):
+            raise errors.TransientDriverError("Timeout connection to MaaS")
+
+        if resp.status_code in [500, 503]:
+            raise errors.TransientDriverError("Received 50x error from MaaS")
+
+        if resp.status_code != 200:
+            raise errors.PersistentDriverError("Received unexpected error from MaaS")
+        
+        return True
+        
+    def test_authentication(self):
+        try:
+            resp = self.get('account/', op='list_authorisation_tokens')
+        except requests.Timeout(ex):
+            raise errors.TransientDriverError("Timeout connection to MaaS")
+        except:
+            raise errors.PersistentDriverError("Error accessing MaaS")
+
+        if resp.status_code in [401, 403] :
+            raise errors.PersistentDriverError("MaaS API Authentication Failed")
+
+        if resp.status_code in [500, 503]:
+            raise errors.TransientDriverError("Received 50x error from MaaS")
+
+        if resp.status_code != 200:
+            raise errors.PersistentDriverError("Received unexpected error from MaaS")
+            
+        return True
+
+    def _send_request(self, method, endpoint, **kwargs):
+        # Delete auth mechanism if defined
+        kwargs.pop('auth', None)
+
+        headers = kwargs.pop('headers', {})
+
+        if 'Accept' not in headers.keys():
+            headers['Accept'] = 'application/json'
+
+        if 'files' in kwargs.keys():
+            files = kwargs.pop('files')
+
+            files_tuples = {}
+
+            for (k, v) in files.items():
+                if v is None:
+                    continue
+                files_tuples[k] = (None, base64.b64encode(str(v).encode('utf-8')).decode('utf-8'), 'text/plain; charset="utf-8"', {'Content-Transfer-Encoding': 'base64'})
+    #            elif isinstance(v, str):
+    #                files_tuples[k] = (None, base64.b64encode(v.encode('utf-8')).decode('utf-8'), 'text/plain; charset="utf-8"', {'Content-Transfer-Encoding': 'base64'})
+    #            elif isinstance(v, int) or isinstance(v, bool):
+    #                if isinstance(v, bool):
+    #                    v = int(v)
+    #                files_tuples[k] = (None, base64.b64encode(v.to_bytes(2, byteorder='big')), 'application/octet-stream', {'Content-Transfer-Encoding': 'base64'})
+
+
+            kwargs['files'] = files_tuples
+
+        params = kwargs.get('params', None)
+
+        if params is None and 'op' in kwargs.keys():
+            params = {'op': kwargs.pop('op')}
+        elif 'op' in kwargs.keys() and 'op' not in params.keys():
+            params['op'] = kwargs.pop('op')
+        elif 'op' in kwargs.keys():
+            kwargs.pop('op')
+
+        # TODO timeouts should be configurable
+        timeout = kwargs.pop('timeout', None)
+        if timeout is None:
+            timeout = (2, 30)
+
+        request = requests.Request(method=method, url=self.base_url + endpoint, auth=self.signer,
+                            headers=headers, params=params, **kwargs)
+
+        prepared_req = self.http_session.prepare_request(request)
+
+        resp = self.http_session.send(prepared_req, timeout=timeout)
+
+        if resp.status_code >= 400:
+            print("FAILED API CALL:\nURL: %s %s\nBODY:\n%s\nRESPONSE: %s\nBODY:\n%s" %
+                (prepared_req.method, prepared_req.url, str(prepared_req.body).replace('\\r\\n','\n'),
+                 resp.status_code, resp.text))
+        return resp

helm_drydock/drivers/node/maasdriver/driver.py

@@ -0,0 +1,306 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import helm_drydock.error as errors
+import helm_drydock.config as config
+import helm_drydock.drivers as drivers
+import helm_drydock.objects.fields as hd_fields
+import helm_drydock.objects.task as task_model
+
+from helm_drydock.drivers.node import NodeDriver
+from .api_client import MaasRequestFactory
+import helm_drydock.drivers.node.maasdriver.models.fabric as maas_fabric
+import helm_drydock.drivers.node.maasdriver.models.vlan as maas_vlan
+import helm_drydock.drivers.node.maasdriver.models.subnet as maas_subnet
+
+class MaasNodeDriver(NodeDriver):
+
+    def __init__(self, **kwargs):
+        super(MaasNodeDriver, self).__init__(**kwargs)
+	
+        self.driver_name = "maasdriver"
+        self.driver_key = "maasdriver"
+        self.driver_desc = "MaaS Node Provisioning Driver"
+
+        self.config = config.DrydockConfig.node_driver[self.driver_key]
+
+    def execute_task(self, task_id):
+        task = self.state_manager.get_task(task_id)
+
+        if task is None:
+            raise errors.DriverError("Invalid task %s" % (task_id))
+
+        if task.action not in self.supported_actions:
+            raise errors.DriverError("Driver %s doesn't support task action %s"
+                % (self.driver_desc, task.action))
+
+        if task.action == hd_fields.OrchestratorAction.ValidateNodeServices:
+            self.orchestrator.task_field_update(task.get_id(),
+                                status=hd_fields.TaskStatus.Running)
+            maas_client = MaasRequestFactory(self.config['api_url'], self.config['api_key']) 
+
+            try:
+                if maas_client.test_connectivity():
+                    if maas_client.test_authentication():
+                        self.orchestrator.task_field_update(task.get_id(),
+                            status=hd_fields.TaskStatus.Complete,
+                            result=hd_fields.ActionResult.Success)
+                        return
+            except errors.TransientDriverError(ex):
+                result = {
+                    'retry': True,
+                    'detail':  str(ex),
+                }
+                self.orchestrator.task_field_update(task.get_id(),
+                            status=hd_fields.TaskStatus.Complete,
+                            result=hd_fields.ActionResult.Failure,
+                            result_details=result)
+                return
+            except errors.PersistentDriverError(ex):
+                result = {
+                    'retry': False,
+                    'detail':  str(ex),
+                }
+                self.orchestrator.task_field_update(task.get_id(),
+                            status=hd_fields.TaskStatus.Complete,
+                            result=hd_fields.ActionResult.Failure,
+                            result_details=result)
+                return
+            except Exception(ex):
+                result = {
+                    'retry': False,
+                    'detail':  str(ex),
+                }
+                self.orchestrator.task_field_update(task.get_id(),
+                            status=hd_fields.TaskStatus.Complete,
+                            result=hd_fields.ActionResult.Failure,
+                            result_details=result)
+                return
+
+        design_id = getattr(task, 'design_id', None)
+
+        if design_id is None:
+            raise errors.DriverError("No design ID specified in task %s" %
+                                     (task_id))
+
+
+        if task.site_name is None:
+            raise errors.DriverError("No site specified for task %s." %
+                                    (task_id))
+
+        self.orchestrator.task_field_update(task.get_id(),
+                            status=hd_fields.TaskStatus.Running)
+
+        site_design = self.orchestrator.get_effective_site(design_id, task.site_name)
+
+        if task.action == hd_fields.OrchestratorAction.CreateNetworkTemplate:
+            subtask = self.orchestrator.create_task(task_model.DriverTask,
+                        parent_task_id=task.get_id(), design_id=design_id,
+                        action=task.action, site_name=task.site_name,
+                        task_scope={'site': task.site_name})
+            runner = MaasTaskRunner(state_manager=self.state_manager,
+                        orchestrator=self.orchestrator,
+                        task_id=subtask.get_id(),config=self.config)
+            runner.start()
+
+            runner.join(timeout=120)
+
+            if runner.is_alive():
+                result =  {
+                    'retry': False,
+                    'detail': 'MaaS Network creation timed-out'
+                }
+                self.orchestrator.task_field_update(task.get_id(),
+                            status=hd_fields.TaskStatus.Complete,
+                            result=hd_fields.ActionResult.Failure,
+                            result_detail=result)
+            else:
+                subtask = self.state_manager.get_task(subtask.get_id())
+                self.orchestrator.task_field_update(task.get_id(),
+                            status=hd_fields.TaskStatus.Complete,
+                            result=subtask.get_result())
+
+            return
+
+class MaasTaskRunner(drivers.DriverTaskRunner):
+
+    def __init__(self, config=None, **kwargs):
+        super(MaasTaskRunner, self).__init__(**kwargs)
+
+        self.driver_config = config
+
+    def execute_task(self):
+        task_action = self.task.action
+
+        self.orchestrator.task_field_update(self.task.get_id(),
+                            status=hd_fields.TaskStatus.Running,
+                            result=hd_fields.ActionResult.Incomplete)
+
+        self.maas_client = MaasRequestFactory(self.driver_config['api_url'],
+                                              self.driver_config['api_key'])
+
+        site_design = self.orchestrator.get_effective_site(self.task.design_id,
+                                                           self.task.site_name)
+
+        if task_action == hd_fields.OrchestratorAction.CreateNetworkTemplate:
+            # Try to true up MaaS definitions of fabrics/vlans/subnets
+            # with the networks defined in Drydock
+            design_networks = site_design.networks
+
+            subnets = maas_subnet.Subnets(self.maas_client)
+            subnets.refresh()
+
+            result_detail = {
+                'detail': []
+            }
+
+            for n in design_networks:
+                exists = subnets.query({'cidr': n.cidr})
+
+                subnet = None
+
+                if len(exists) > 0:
+                    subnet = exists[0]
+
+                    subnet.name = n.name
+                    subnet.dns_servers = n.dns_servers
+
+                    vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=subnet.fabric)
+                    vlan_list.refresh()
+
+                    vlan = vlan_list.select(subnet.vlan)
+
+                    if vlan is not None:
+                        if ((n.vlan_id is None and vlan.vid != 0) or 
+                           (n.vlan_id is not None and vlan.vid != n.vlan_id)):
+
+                            # if the VLAN name matches, assume this is the correct resource
+                            # and it needs to be updated
+                            if vlan.name == n.name:
+                                vlan.set_vid(n.vlan_id)
+                                vlan.mtu = n.mtu
+                                vlan.update()
+                            else:
+                                vlan_id = n.vlan_id if n.vlan_id is not None else 0
+                                target_vlan = vlan_list.query({'vid': vlan_id})
+                                if len(target_vlan) > 0:
+                                    subnet.vlan = target_vlan[0].resource_id
+                                else:
+                                    # This is a flag that after creating a fabric and
+                                    # VLAN below, update the subnet
+                                    subnet.vlan = None
+                    else:
+                        subnet.vlan = None
+            
+                    # Check if the routes have a default route
+                    subnet.gateway_ip = n.get_default_gateway()
+
+
+                    result_detail['detail'].append("Subnet %s found for network %s, updated attributes"
+                                                % (exists[0].resource_id, n.name))
+
+                # Need to create a Fabric/Vlan for this network
+                if (subnet is None or (subnet is not None and subnet.vlan is None)):
+                    fabric_list = maas_fabric.Fabrics(self.maas_client)
+                    fabric_list.refresh()
+                    matching_fabrics = fabric_list.query({'name': n.name})
+                    
+                    fabric = None
+                    vlan = None
+                    
+                    if len(matching_fabrics) > 0:
+                        # Fabric exists, update VLAN
+                        fabric = matching_fabrics[0]
+
+                        vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=fabric.resource_id)
+                        vlan_list.refresh()
+                        vlan_id = n.vlan_id if n.vlan_id is not None else 0
+                        matching_vlans = vlan_list.query({'vid': vlan_id})
+
+                        if len(matching_vlans) > 0:
+                            vlan = matching_vlans[0]
+
+                            vlan.name = n.name
+                            if getattr(n, 'mtu', None) is not None:
+                                vlan.mtu = n.mtu
+
+                            if subnet is not None:
+                                subnet.vlan = vlan.resource_id
+                                subnet.update()
+                            vlan.update()
+                        else:
+                            vlan = maas_vlan.Vlan(self.maas_client, name=n.name, vid=vlan_id,
+                                                mtu=getattr(n, 'mtu', None),fabric_id=fabric.resource_id)
+                            vlan = vlan_list.add(vlan)
+
+                            if subnet is not None:
+                                subnet.vlan = vlan.resource_id
+                                subnet.update()
+
+                    else:
+                        new_fabric = maas_fabric.Fabric(self.maas_client, name=n.name)
+                        new_fabric = fabric_list.add(new_fabric)
+                        new_fabric.refresh()
+                        fabric = new_fabric
+
+                        vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=new_fabric.resource_id)
+                        vlan_list.refresh()
+                        vlan = vlan_list.single()
+
+                        vlan.name = n.name
+                        vlan.vid = n.vlan_id if n.vlan_id is not None else 0
+                        if getattr(n, 'mtu', None) is not None:
+                            vlan.mtu = n.mtu
+
+                        vlan.update()
+
+                        if subnet is not None:
+                            subnet.vlan = vlan.resource_id
+                            subnet.update()
+
+                    if subnet is None:
+                        subnet = maas_subnet.Subnet(self.maas_client, name=n.name, cidr=n.cidr, fabric=fabric.resource_id,
+                                                vlan=vlan.resource_id, gateway_ip=n.get_default_gateway())
+
+                        subnet_list = maas_subnet.Subnets(self.maas_client)
+                        subnet = subnet_list.add(subnet)
+
+            subnet_list = maas_subnet.Subnets(self.maas_client)
+            subnet_list.refresh()
+
+            action_result = hd_fields.ActionResult.Incomplete
+
+            success_rate = 0
+
+            for n in design_networks:
+                exists = subnet_list.query({'cidr': n.cidr})
+                if len(exists) > 0:
+                    subnet = exists[0]
+                    if subnet.name == n.name:
+                        success_rate = success_rate + 1
+                    else:
+                        success_rate = success_rate + 1
+                else:
+                    success_rate = success_rate + 1
+
+            if success_rate == len(design_networks):
+                action_result = hd_fields.ActionResult.Success
+            elif success_rate == - (len(design_networks)):
+                action_result = hd_fields.ActionResult.Failure
+            else:
+                action_result = hd_fields.ActionResult.PartialSuccess
+
+            self.orchestrator.task_field_update(self.task.get_id(),
+                            status=hd_fields.TaskStatus.Complete,
+                            result=action_result,
+                            result_detail=result_detail)

helm_drydock/drivers/node/maasdriver/models/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

helm_drydock/drivers/node/maasdriver/models/base.py

@@ -0,0 +1,273 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import re
+
+import helm_drydock.error as errors
+"""
+A representation of a MaaS REST resource. Should be subclassed
+for different resources and augmented with operations specific
+to those resources
+"""
+class ResourceBase(object):
+
+    resource_url = '/{id}'
+    fields = ['resource_id']
+    json_fields = ['resource_id']
+
+    def __init__(self, api_client, **kwargs):
+        self.api_client = api_client
+
+        for f in self.fields:
+            if f in kwargs.keys():
+                setattr(self, f, kwargs.get(f))
+
+    """
+    Update resource attributes from MaaS
+    """
+    def refresh(self):
+        url = self.interpolate_url()
+        resp = self.api_client.get(url)
+
+        updated_fields = resp.json()
+
+        for f in self.json_fields:
+            if f in updated_fields.keys():
+                setattr(self, f, updated_fields.get(f))
+
+    """
+    Parse URL for placeholders and replace them with current
+    instance values
+    """
+    def interpolate_url(self):
+        pattern = '\{([a-z_]+)\}'
+        regex = re.compile(pattern)
+        start = 0
+        new_url = self.resource_url
+
+        while (start+1) < len(self.resource_url):
+            match = regex.search(self.resource_url, start)
+            if match is None:
+                return new_url
+
+            param = match.group(1)
+            val = getattr(self, param, None)
+            if val is None:
+                raise ValueError("Missing variable value")
+            new_url = new_url.replace('{' + param + '}', str(val))
+            start = match.end(1) + 1
+
+        return new_url
+
+    """
+    Update MaaS with current resource attributes
+    """
+    def update(self):
+        data_dict = self.to_dict()
+        url = self.interpolate_url()
+
+        resp = self.api_client.put(url, files=data_dict)
+
+        if resp.status_code == 200:
+            return True
+        
+        raise errors.DriverError("Failed updating MAAS url %s - return code %s\n%s"
+                % (url, resp.status_code, resp.text))
+
+    """
+    Set the resource_id for this instance
+    Should only be called when creating new instances and MAAS has assigned
+    an id
+    """
+    def set_resource_id(self, res_id):
+        self.resource_id = res_id
+
+    """
+    Serialize this resource instance into JSON matching the
+    MaaS respresentation of this resource
+    """
+    def to_json(self):
+        return json.dumps(self.to_dict())
+
+    """
+    Serialize this resource instance into a dict matching the
+    MAAS representation of the resource
+    """
+    def to_dict(self):
+        data_dict = {}
+
+        for f in self.json_fields:
+            if getattr(self, f, None) is not None:
+                if f == 'resource_id':
+                    data_dict['id'] = getattr(self, f)
+                else:
+                    data_dict[f] = getattr(self, f)
+
+        return data_dict
+
+    """
+    Create a instance of this resource class based on the MaaS
+    representation of this resource type
+    """
+    @classmethod
+    def from_json(cls, api_client, json_string):
+        parsed = json.loads(json_string)
+
+        if isinstance(parsed, dict):
+            return cls.from_dict(api_client, parsed)
+
+        raise errors.DriverError("Invalid JSON for class %s" % (cls.__name__))
+
+    """
+    Create a instance of this resource class based on a dict
+    of MaaS type attributes
+    """
+    @classmethod
+    def from_dict(cls, api_client, obj_dict):
+        refined_dict = {k: obj_dict.get(k, None) for k in cls.fields}
+        if 'id' in obj_dict.keys():
+            refined_dict['resource_id'] = obj_dict.get('id')
+
+        i = cls(api_client, **refined_dict)
+        return i
+
+
+"""
+A collection of MaaS resources.
+
+Rather than a simple list, we will key the collection on resource
+ID for more efficient access.
+"""
+class ResourceCollectionBase(object):
+
+    collection_url = ''
+    collection_resource = ResourceBase
+
+    def __init__(self, api_client):
+        self.api_client = api_client
+        self.resources = {}
+
+    """
+    Parse URL for placeholders and replace them with current
+    instance values
+    """
+    def interpolate_url(self):
+        pattern = '\{([a-z_]+)\}'
+        regex = re.compile(pattern)
+        start = 0
+        new_url = self.collection_url
+
+        while (start+1) < len(self.collection_url):
+            match = regex.search(self.collection_url, start)
+            if match is None:
+                return new_url
+
+            param = match.group(1)
+            val = getattr(self, param, None)
+            if val is None:
+                raise ValueError("Missing variable value")
+            new_url = new_url.replace('{' + param + '}', str(val))
+            start = match.end(1) + 1
+
+        return new_url
+
+    """
+    Create a new resource in this collection in MaaS
+    """
+    def add(self, res):
+        data_dict = res.to_dict()
+        url = self.interpolate_url()
+
+        resp = self.api_client.post(url, files=data_dict)
+
+        if resp.status_code == 200:
+            resp_json = resp.json()
+            res.set_resource_id(resp_json.get('id'))
+            return res
+        
+        raise errors.DriverError("Failed updating MAAS url %s - return code %s"
+                % (url, resp.status_code))
+
+    """
+    Append a resource instance to the list locally only
+    """
+    def append(self, res):
+        if isinstance(res, self.collection_resource):
+            self.resources[res.resource_id] = res
+
+    """
+    Initialize or refresh the collection list from MaaS
+    """
+    def refresh(self):
+        url = self.interpolate_url()
+        resp = self.api_client.get(url)
+
+        if resp.status_code == 200:
+            self.resource = {}
+            json_list = resp.json()
+
+            for o in json_list:
+                if isinstance(o, dict):
+                    i = self.collection_resource.from_dict(self.api_client, o)
+                    self.resources[i.resource_id] = i
+
+        return
+
+    """
+    Check if resource id is in this collection
+    """
+    def contains(self, res_id):
+        if res_id in self.resources.keys():
+            return True
+
+        return False
+
+    """
+    Select a resource based on ID or None if not found
+    """
+    def select(self, res_id):
+        return self.resources.get(res_id, None)
+
+    """
+    Query the collection based on a resource attribute other than primary id
+    """
+    def query(self, query):
+        result = list(self.resources.values())
+        for (k, v) in query.items():
+            result = [i for i in result
+                      if str(getattr(i, k, None)) == str(v)]
+
+        return result
+
+    """
+    If the collection has a single item, return it
+    """
+    def single(self):
+        if self.len() == 1:
+            for v in self.resources.values():
+                return v
+        else:
+            return None
+
+    """
+    Iterate over the resources in the collection
+    """
+    def __iter__(self):
+        return iter(self.resources.values())
+
+    """
+    Resource count
+    """
+    def len(self):
+        return len(self.resources)

helm_drydock/drivers/node/maasdriver/models/fabric.py

@@ -0,0 +1,53 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+
+import helm_drydock.drivers.node.maasdriver.models.base as model_base
+import helm_drydock.drivers.node.maasdriver.models.vlan as model_vlan
+
+class Fabric(model_base.ResourceBase):
+
+    resource_url = 'fabrics/{resource_id}/'
+    fields = ['resource_id', 'name', 'description']
+    json_fields = ['name', 'description']
+
+    def __init__(self, api_client, **kwargs):
+        super(Fabric, self).__init__(api_client, **kwargs)
+
+        if hasattr(self, 'resource_id'):
+            self.refresh_vlans()
+
+    def refresh(self):
+        super(Fabric, self).refresh()
+        
+        self.refresh_vlans()
+
+        return
+
+    def refresh_vlans(self):
+        self.vlans = model_vlan.Vlans(self.api_client, fabric_id=self.resource_id)
+        self.vlans.refresh()
+        
+
+    def set_resource_id(self, res_id):
+        self.resource_id = res_id
+        self.refresh_vlans()
+
+class Fabrics(model_base.ResourceCollectionBase):
+
+    collection_url = 'fabrics/'
+    collection_resource = Fabric
+
+    def __init__(self, api_client):
+        super(Fabrics, self).__init__(api_client)

helm_drydock/drivers/node/maasdriver/models/subnet.py

@@ -0,0 +1,55 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import helm_drydock.drivers.node.maasdriver.models.base as model_base
+
+class Subnet(model_base.ResourceBase):
+
+    resource_url = 'subnets/{resource_id}/'
+    fields = ['resource_id', 'name', 'description', 'fabric', 'vlan', 'vid', 'dhcp_on',
+              'space', 'cidr', 'gateway_ip', 'rdns_mode', 'allow_proxy', 'dns_servers']
+    json_fields = ['name', 'description','vlan', 'space', 'cidr', 'gateway_ip', 'rdns_mode',
+                   'allow_proxy', 'dns_servers']
+
+    def __init__(self, api_client, **kwargs):
+        super(Subnet, self).__init__(api_client, **kwargs)
+
+        # For now all subnets will be part of the default space
+        self.space = 0
+
+    """
+    Because MaaS decides to replace the VLAN id with the
+    representation of the VLAN, we must reverse it for a true
+    representation of the resource
+    """
+    @classmethod
+    def from_dict(cls, api_client, obj_dict):
+        refined_dict = {k: obj_dict.get(k, None) for k in cls.fields}
+        if 'id' in obj_dict.keys():
+            refined_dict['resource_id'] = obj_dict.get('id')
+
+        if isinstance(refined_dict.get('vlan', None), dict):
+            refined_dict['fabric'] = refined_dict['vlan']['fabric_id']
+            refined_dict['vlan'] = refined_dict['vlan']['id']
+            
+        i = cls(api_client, **refined_dict)
+        return i
+
+class Subnets(model_base.ResourceCollectionBase):
+
+    collection_url = 'subnets/'
+    collection_resource = Subnet
+
+    def __init__(self, api_client, **kwargs):
+        super(Subnets, self).__init__(api_client)

helm_drydock/drivers/node/maasdriver/models/vlan.py

@@ -0,0 +1,86 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+
+import helm_drydock.error as errors
+import helm_drydock.drivers.node.maasdriver.models.base as model_base
+
+class Vlan(model_base.ResourceBase):
+
+    resource_url = 'fabrics/{fabric_id}/vlans/{api_id}/'
+    fields = ['resource_id', 'name', 'description', 'vid', 'fabric_id', 'dhcp_on', 'mtu']
+    json_fields = ['name', 'description', 'vid', 'dhcp_on', 'mtu']
+
+    def __init__(self, api_client, **kwargs):
+        super(Vlan, self).__init__(api_client, **kwargs)
+
+        if self.vid is None:
+            self.vid = 0
+
+        # the MaaS API decided that the URL endpoint for VLANs should use
+        # the VLAN tag (vid) rather than the resource ID. So to update the 
+        # vid, we have to keep two copies so that the resource_url
+        # is accurate for updates
+        self.api_id = self.vid
+
+    def update(self):
+        super(Vlan, self).update()
+
+        self.api_id = self.vid
+
+    def set_vid(self, new_vid):
+        if new_vid is None:
+            self.vid = 0
+        else:
+            self.vid = int(new_vid)
+
+class Vlans(model_base.ResourceCollectionBase):
+
+    collection_url = 'fabrics/{fabric_id}/vlans/'
+    collection_resource = Vlan
+
+    def __init__(self, api_client, **kwargs):
+        super(Vlans, self).__init__(api_client)
+
+        self.fabric_id = kwargs.get('fabric_id', None)
+    """
+    Create a new resource in this collection in MaaS
+    def add(self, res):
+        #MAAS API doesn't support all attributes in POST, so create and
+        # then promptly update via PUT
+
+        min_fields = {
+            'name':   res.name,
+            'description':  getattr(res, 'description', None),
+        }
+         
+        if getattr(res, 'vid', None) is None:
+            min_fields['vid'] = 0
+        else:
+            min_fields['vid'] = res.vid
+
+        url = self.interpolate_url()
+        resp = self.api_client.post(url, files=min_fields)
+
+        # Check on initial POST creation
+        if resp.status_code == 200:
+            resp_json = resp.json()
+            res.id = resp_json.get('id')
+            # Submit PUT for additonal fields
+            res.update()
+            return res
+        
+        raise errors.DriverError("Failed updating MAAS url %s - return code %s\n%s"
+                % (url, resp.status_code, resp.text))
+    """

helm_drydock/drivers/node/maasdriver/readme.md

@@ -0,0 +1,46 @@
+# MaaS Node Driver #
+
+This driver will handle node provisioning using Ubuntu MaaS 2.1. It expects
+the Drydock config to hold a valid MaaS API URL (e.g. http://host:port/MAAS/api/2.0)
+and a valid API key for authentication.
+
+## Drydock Model to MaaS Model Relationship ##
+
+### Site ###
+
+Will provide some attributes used for configuring MaaS site-wide such
+as tag definitions and repositories.
+
+### Network Link ###
+
+Will provide attributes for configuring Node/Machine interfaces
+
+### Network ###
+
+MaaS will be configured with a single 'space'. Each Network in Drydock
+will translate to a unique MaaS fabric+vlan+subnet. Any network with
+an address range of type 'dhcp' will cause DHCP to be enabled in MaaS
+for that network. 
+
+### Hardware Profile ###
+
+A foundation to a Baremetal Node definition. Not directly used in MaaS
+
+### Host Profile ###
+
+A foundation to a Baremetal Node definition. Not directly used in MaaS
+
+### Baremetal Node ###
+
+Defines all the attributes required to commission and deploy nodes via MaaS
+
+* bootdisk fields and partitions list - Define local node storage configuration
+to be implemented by MaaS
+* addressing and interface list - Combined with referenced network links and networks, define
+interface (physical and virtual (bond / vlan)) configurations and network
+addressing
+* tags and owner data - Statically defined metadata that will propagate to
+MaaS
+* base_os - Select which stream a node will be deployed with
+* kernel and kernel params - Allow for custom kernel selection and parameter
+definition

helm_drydock/drivers/oob/__init__.py

@@ -12,13 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# OOB:
-# sync_hardware_clock
-# collect_chassis_sysinfo
-# enable_netboot
-# initiate_reboot
-# set_power_off
-# set_power_on
 import helm_drydock.objects.fields as hd_fields
 import helm_drydock.error as errors
 
@@ -29,12 +22,13 @@ class OobDriver(ProviderDriver):
     def __init__(self, **kwargs):
         super(OobDriver, self).__init__(**kwargs)
 
-        self.supported_actions = [hd_fields.OrchestratorAction.ConfigNodePxe,
+        self.supported_actions = [hd_fields.OrchestrationAction.ValidateOobServices,
+                                  hd_fields.OrchestratorAction.ConfigNodePxe,
                                   hd_fields.OrchestratorAction.SetNodeBoot,
                                   hd_fields.OrchestratorAction.PowerOffNode,
                                   hd_fields.OrchestratorAction.PowerOnNode,
                                   hd_fields.OrchestratorAction.PowerCycleNode,
-                                  hd_fields.OrchestratorAction.InterrogateNode]
+                                  hd_fields.OrchestratorAction.InterrogateOob]
 
         self.driver_name = "oob_generic"
         self.driver_key = "oob_generic"

helm_drydock/drivers/oob/pyghmi_driver/__init__.py

@@ -16,6 +16,7 @@ import time
 from pyghmi.ipmi.command import Command
 
 import helm_drydock.error as errors
+import helm_drydock.config as config
 
 import helm_drydock.objects.fields as hd_fields
 import helm_drydock.objects.task as task_model
@@ -33,6 +34,8 @@ class PyghmiDriver(oob.OobDriver):
         self.driver_key = "pyghmi_driver"
         self.driver_desc = "Pyghmi OOB Driver"
 
+        self.config = config.DrydockConfig.node_driver[self.driver_key]
+
     def execute_task(self, task_id):
         task = self.state_manager.get_task(task_id)
 
@@ -57,6 +60,12 @@ class PyghmiDriver(oob.OobDriver):
         self.orchestrator.task_field_update(task.get_id(),
                             status=hd_fields.TaskStatus.Running)
 
+        if task.action == hd_fields.OrchestratorAction.ValidateOobServices:
+            self.orchestrator.task_field_update(task.get_id(),
+                                status=hd_fields.TaskStatus.Complete,
+                                result=hd_fields.ActionResult.Success)
+            return
+            
         site_design = self.orchestrator.get_effective_site(design_id, task.site_name)
 
         target_nodes = []
@@ -284,7 +293,7 @@ class PyghmiTaskRunner(drivers.DriverTaskRunner):
                     result=hd_fields.ActionResult.Failure,
                     status=hd_fields.TaskStatus.Complete)
             return
-        elif task_action == hd_fields.OrchestratorAction.InterrogateNode:
+        elif task_action == hd_fields.OrchestratorAction.InterrogateOob:
             mci_id = ipmi_session.get_mci()
 
             self.orchestrator.task_field_update(self.task.get_id(),

helm_drydock/drivers/readme.md

@@ -2,7 +2,7 @@
 
 Drivers are downstream actors that Drydock will use to actually execute
 orchestration actions. It is intended to be a pluggable architecture
-so that various downstream automation can be used.
+so that various downstream automation can be used. A driver must implement all actions even if the implementation is effectively a no-op.
 
 ## oob ##
 
@@ -11,12 +11,35 @@ management system (e.g. Dell iDRAC, HP iLO, etc...). OOB management
 will be used for setting a system to use PXE boot and power cycling
 servers. 
 
+### Actions ###
+
+* ConfigNodePxe - Where available, configure PXE boot options (e.g. PXE interface)
+* SetNodeBoot - Set boot source (PXE, hard disk) of a node
+* PowerOffNode - Power down a node
+* PowerOnNode - Power up a node
+* PowerCycleNode - Power cycle a node
+* InterrogateOob - Interrogate a node's OOB interface. Resultant data is dependent on what functionality is implemented for a particular OOB interface
+
 ## node ##
 
 The node drivers will interface with an external bootstrapping system
 for loading the base OS on a server and configuring hardware, network,
 and storage.
 
+### Actions ###
+
+* CreateNetworkTemplate - Configure site-wide network information in bootstrapper
+* CreateStorageTemplate - Configure site-wide storage information in bootstrapper
+* CreateBootMedia - Ensure all needed boot media is available to the bootstrapper including external repositories
+* PrepareHardwareConfig - Prepare the bootstrapper to handle all hardware configuration actions (firmware updates, RAID configuration, driver installation)
+* ConfigureHardware - Update and validate all hardware configurations on a node prior to deploying the OS on it
+* InterrogateNode - Interrogate the bootstrapper about node information. Depending on the current state of the node, this interrogation will produce different information.
+* ApplyNodeNetworking - Configure networking for a node
+* ApplyNodeStorage - Configure storage for a node
+* ApplyNodePlatform - Configure stream and kernel options for a node
+* DeployNode - Deploy the OS to a node
+* DestroyNode - Take steps to bring a node back to a blank undeployed state
+
 ## network ##
 
 The network drivers will interface with switches for managing port
@@ -24,3 +47,9 @@ configuration to support the bootstrapping of physical nodes. This is not
 intended to be a network provisioner, but instead is a support driver
 for node bootstrapping where temporary changes to network configurations
 are required.
+
+### Actions ###
+
+* InterrogatePort - Request information about the current configuration of a network port
+* ConfigurePortProvisioning - Configure a network port in provisioning (PXE) mode
+* ConfigurePortProduction - Configure a network port in production (configuration post-deployment) mode

helm_drydock/error.py

@@ -21,5 +21,17 @@ class StateError(Exception):
 class OrchestratorError(Exception):
     pass
 
+class TransientOrchestratorError(OrchestratorError):
+    pass
+
+class PersistentOrchestratorError(OrchestratorError):
+    pass
+
 class DriverError(Exception):
     pass
+
+class TransientDriverError(DriverError):
+    pass
+
+class PersistentDriverError(DriverError):
+    pass

helm_drydock/ingester/plugins/yaml.py

@@ -161,7 +161,7 @@ class YamlIngester(IngesterPlugin):
 
                             model.cidr = spec.get('cidr', None)
                             model.allocation_strategy = spec.get('allocation', 'static')
-                            model.vlan_id = spec.get('vlan_id', 1)
+                            model.vlan_id = spec.get('vlan_id', None)
                             model.mtu = spec.get('mtu', None)
 
                             dns = spec.get('dns', {})
@@ -286,6 +286,7 @@ class YamlIngester(IngesterPlugin):
 
                             int_model.device_name = i.get('device_name', None)
                             int_model.network_link = i.get('device_link', None)
+                            int_model.primary_netowrk = i.get('primary', False)
 
                             int_model.hardware_slaves = []
                             slaves = i.get('slaves', [])

helm_drydock/objects/base.py

@@ -31,6 +31,13 @@ class DrydockObject(base.VersionedObject):
 
     OBJ_PROJECT_NAMESPACE = 'helm_drydock.objects'
 
+    # Return None for undefined attributes
+    def obj_load_attr(self, attrname):
+        if attrname in self.fields.keys():
+            setattr(self, attrname, None)
+        else:
+            raise ValueError("Unknown field %s" % (attrname))
+
 class DrydockPersistentObject(base.VersionedObject):
 
     fields = {

helm_drydock/objects/fields.py

@@ -30,17 +30,41 @@ class OrchestratorAction(BaseDrydockEnum):
     DestroyNode = 'destroy_node'
 
     # OOB driver actions
+    ValidateOobServices = 'validate_oob_services'
     ConfigNodePxe = 'config_node_pxe'
     SetNodeBoot = 'set_node_boot'
     PowerOffNode = 'power_off_node'
     PowerOnNode = 'power_on_node'
     PowerCycleNode = 'power_cycle_node'
+    InterrogateOob = 'interrogate_oob'
+
+    # Node driver actions
+    ValidateNodeServices = 'validate_node_services'
+    CreateNetworkTemplate = 'create_network_template'
+    CreateStorageTemplate = 'create_storage_template'
+    CreateBootMedia = 'create_boot_media'
+    PrepareHardwareConfig = 'prepare_hardware_config'
+    ConfigureHardware = 'configure_hardware'
     InterrogateNode = 'interrogate_node'
+    ApplyNodeNetworking = 'apply_node_networking'
+    ApplyNodeStorage = 'apply_node_storage'
+    ApplyNodePlatform = 'apply_node_platform'
+    DeployNode = 'deploy_node'
+    DestroyNode = 'destroy_node'
+
+    # Network driver actions
+    ValidateNetworkServices = 'validate_network_services'
+    InterrogatePort = 'interrogate_port'
+    ConfigurePortProvisioning = 'config_port_provisioning'
+    ConfigurePortProduction = 'config_port_production'
 
     ALL = (Noop, ValidateDesign, VerifySite, PrepareSite, VerifyNode,
            PrepareNode, DeployNode, DestroyNode, ConfigNodePxe,
            SetNodeBoot, PowerOffNode, PowerOnNode, PowerCycleNode,
-           InterrogateNode)
+           InterrogateOob, CreateNetworkTemplate, CreateStorageTemplate,
+           CreateBootMedia, PrepareHardwareConfig, ConfigureHardware,
+           InterrogateNode, ApplyNodeNetworking, ApplyNodeStorage,
+           ApplyNodePlatform, DeployNode, DestroyNode)
 
 class OrchestratorActionField(fields.BaseEnumField):
     AUTO_TYPE = OrchestratorAction()
@@ -52,7 +76,7 @@ class ActionResult(BaseDrydockEnum):
     Failure = 'failure'
     DependentFailure = 'dependent_failure'
 
-    ALL = (Incomplete, Success, PartialSuccess, Failure)
+    ALL = (Incomplete, Success, PartialSuccess, Failure, DependentFailure)
 
 class ActionResultField(fields.BaseEnumField):
     AUTO_TYPE = ActionResult()

helm_drydock/objects/hostprofile.py

@@ -189,7 +189,7 @@ class HostInterface(base.DrydockObject):
         if len(child_list) == 0 and len(parent_list) > 0:
             for p in parent_list:
                 pp = deepcopy(p)
-                pp.source = hd_obj_fields.ModelSource.Compiled
+                pp.source = hd_fields.ModelSource.Compiled
                 effective_list.append(pp)
         elif len(parent_list) == 0 and len(child_list) > 0:
             for i in child_list:
@@ -197,7 +197,7 @@ class HostInterface(base.DrydockObject):
                     continue
                 else:
                     ii = deepcopy(i)
-                    ii.source = hd_obj_fields.ModelSource.Compiled
+                    ii.source = hd_fields.ModelSource.Compiled
                     effective_list.append(ii)
         elif len(parent_list) > 0 and len(child_list) > 0:
             parent_interfaces = []
@@ -212,8 +212,8 @@ class HostInterface(base.DrydockObject):
                     elif j.get_name() == parent_name:
                         m = objects.HostInterface()
                         m.device_name = j.get_name()
-                        m.primary_network =
+                        m.primary_network = \
-                            objects.Util.apply_field_inheritance(
+                            objects.Utils.apply_field_inheritance(
                                 getattr(j, 'primary_network', None),
                                 getattr(i, 'primary_network', None))
                             
@@ -243,7 +243,7 @@ class HostInterface(base.DrydockObject):
                              if not x.startswith("!")])
 
                         m.networks = n
-                        m.source = hd_obj_fields.ModelSource.Compiled
+                        m.source = hd_fields.ModelSource.Compiled
 
                         effective_list.append(m)
                         add = False
@@ -251,14 +251,14 @@ class HostInterface(base.DrydockObject):
 
                 if add:
                     ii = deepcopy(i)
-                    ii.source = hd_obj_fields.ModelSource.Compiled
+                    ii.source = hd_fields.ModelSource.Compiled
                     effective_list.append(ii)
 
             for j in child_list:
                 if (j.device_name not in parent_interfaces
                     and not j.get_name().startswith("!")):
                     jj = deepcopy(j)
-                    jj.source = hd_obj_fields.ModelSource.Compiled
+                    jj.source = hd_fields.ModelSource.Compiled
                     effective_list.append(jj)
 
         return effective_list

helm_drydock/objects/network.py

@@ -34,11 +34,11 @@ class NetworkLink(base.DrydockPersistentObject, base.DrydockObject):
         'site':             ovo_fields.StringField(),
         'bonding_mode':     hd_fields.NetworkLinkBondingModeField(
                                         default=hd_fields.NetworkLinkBondingMode.Disabled),
-        'bonding_xmit_hash':    ovo_fields.StringField(nullable=True),
+        'bonding_xmit_hash':    ovo_fields.StringField(nullable=True, default='layer3+4'),
-        'bonding_peer_rate':    ovo_fields.StringField(nullable=True),
+        'bonding_peer_rate':    ovo_fields.StringField(nullable=True, default='slow'),
-        'bonding_mon_rate':     ovo_fields.IntegerField(nullable=True),
+        'bonding_mon_rate':     ovo_fields.IntegerField(nullable=True, default=100),
-        'bonding_up_delay':     ovo_fields.IntegerField(nullable=True),
+        'bonding_up_delay':     ovo_fields.IntegerField(nullable=True, default=200),
-        'bonding_down_delay':   ovo_fields.IntegerField(nullable=True),
+        'bonding_down_delay':   ovo_fields.IntegerField(nullable=True, default=200),
         'mtu':              ovo_fields.IntegerField(default=1500),
         'linkspeed':        ovo_fields.StringField(default='auto'),
         'trunk_mode':       hd_fields.NetworkLinkTrunkingModeField(
@@ -81,7 +81,9 @@ class Network(base.DrydockPersistentObject, base.DrydockObject):
         'mtu':      ovo_fields.IntegerField(nullable=True),
         'dns_domain':   ovo_fields.StringField(nullable=True),
         'dns_servers':  ovo_fields.StringField(nullable=True),
+        # Keys of ranges are 'type', 'start', 'end'
         'ranges':   ovo_fields.ListOfDictOfNullableStringsField(),
+        # Keys of routes are 'subnet', 'gateway', 'metric'
         'routes':   ovo_fields.ListOfDictOfNullableStringsField(),
     }
 
@@ -95,6 +97,14 @@ class Network(base.DrydockPersistentObject, base.DrydockObject):
     def get_name(self):
         return self.name
 
+    def get_default_gateway(self):
+        for r in getattr(self,'routes', []):
+            if r.get('subnet', '') == '0.0.0.0/0':
+                return r.get('gateway', None)
+
+        return None
+
+
 
 @base.DrydockObjectRegistry.register
 class NetworkList(base.DrydockObjectListBase, base.DrydockObject):

helm_drydock/objects/site.py

@@ -126,12 +126,7 @@ class SiteDesign(base.DrydockPersistentObject, base.DrydockObject):
     def __init__(self, **kwargs):
         super(SiteDesign, self).__init__(**kwargs)
 
-    # Initialize lists for blank instances
+
-    def obj_load_attr(self, attrname):
-        if attrname in self.fields.keys():
-            setattr(self, attrname, None)
-        else:
-            raise ValueError("Unknown field %s" % (attrname))
 
     # Assign UUID id
     def assign_id(self):

helm_drydock/objects/task.py

@@ -87,9 +87,6 @@ class OrchestratorTask(Task):
 
 
 class DriverTask(Task):
-    # subclasses implemented by each driver should override this with the list
-    # of actions that driver supports
-
     def __init__(self, task_scope={}, **kwargs):
         super(DriverTask, self).__init__(**kwargs)
 

helm_drydock/orchestrator/__init__.py

@@ -116,6 +116,53 @@ class Orchestrator(object):
             
             self.task_field_update(task_id, status=hd_fields.TaskStatus.Complete)
             return
+        elif task.action == hd_fields.OrchestratorAction.VerifySite:
+            self.task_field_update(task_id,
+                                   status=hd_fields.TaskStatus.Running)
+
+            node_driver = self.enabled_drivers['node']
+
+            if node_driver is not None:
+                node_driver_task = self.create_task(tasks.DriverTask,
+                                           parent_task_id=task.get_id(),
+                                           design_id=design_id,
+                                           action=hd_fields.OrchestratorAction.ValidateNodeServices)
+
+                node_driver.execute_task(node_driver_task.get_id())
+
+                node_driver_task = self.state_manager.get_task(node_driver_task.get_id())
+
+                self.task_field_update(task_id,
+                                   status=hd_fields.TaskStatus.Complete,
+                                   result=node_driver_task.get_result())
+            return
+        elif task.action == hd_fields.OrchestratorAction.PrepareSite:
+            driver = self.enabled_drivers['node']
+
+            if driver is None:
+                self.task_field_update(task_id,
+                        status=hd_fields.TaskStatus.Errored,
+                        result=hd_fields.ActionResult.Failure)
+                return
+
+            task_scope = {
+                'site': task.site
+            }
+
+            driver_task = self.create_task(tasks.DriverTask,
+                                           parent_task_id=task.get_id(),
+                                           design_id=design_id,
+                                           task_scope=task_scope,
+                                           action=hd_fields.OrchestratorAction.CreateNetworkTemplate)
+
+            driver.execute_task(driver_task.get_id())
+
+            driver_task = self.state_manager.get_task(driver_task.get_id())
+
+            self.task_field_update(task_id,
+                                   status=hd_fields.TaskStatus.Complete,
+                                   result=driver_task.get_result())
+            return
         elif task.action == hd_fields.OrchestratorAction.VerifyNode:
             self.task_field_update(task_id,
                                    status=hd_fields.TaskStatus.Running)

helm_drydock/orchestrator/readme.md

@@ -11,6 +11,7 @@ such that on failure the task can retried and only the
 steps needed will be executed.
 
 ## Drydock Tasks ##
+
 Bullet points listed below are not exhaustive and will
 change as we move through testing
 
@@ -21,6 +22,14 @@ validate that the current state of design data represents
 a valid site design. No claim is made that the design data
 is compatible with the physical state of the site.
 
+#### Validations ####
+
+* All baremetal nodes have an address, either static or DHCP, for all networks they are attached to.
+* No static IP assignments are duplicated
+* No static IP assignments are outside of the network they are targetted for
+* No network MTU mismatches due to a network riding different links on different nodes
+* Boot drive is above minimum size
+
 ### VerifySite ###
 
 Verify site-wide resources are in a useful state
@@ -67,6 +76,9 @@ Prepare a node for bootstrapping
     - Hardware configuration (e.g. RAID)
 * Configure node networking
 * Configure node storage
+* Interrogate node
+    - lshw output
+    - lldp output
 
 ### DeployNode ###
 

setup.py

@@ -48,19 +48,18 @@ setup(name='helm_drydock',
                 'helm_drydock.control',
                 'helm_drydock.drivers',
                 'helm_drydock.drivers.oob',
-                'helm_drydock.drivers.oob.pyghmi_driver'],
+                'helm_drydock.drivers.oob.pyghmi_driver',
+                'helm_drydock.drivers.node',
+                'helm_drydock.drivers.node.maasdriver',
+                'helm_drydock.drivers.node.maasdriver.models'],
       install_requires=[
         'PyYAML',
-        'oauth',
-        'requests-oauthlib',
         'pyghmi>=1.0.18',
         'netaddr',
         'falcon',
-        'webob',
         'oslo.versionedobjects>=1.23.0',
-      ],
+        'requests',
-      dependency_link=[
+        'oauthlib',
-        'git+https://github.com/maas/python-libmaas.git'
       ]
      )
 

tests/integration/test_maasdriver_client.py

@@ -0,0 +1,30 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+
+import helm_drydock.config as config
+import helm_drydock.drivers.node.maasdriver.api_client as client
+
+class TestClass(object):
+
+    def test_client_authenticate(self):
+        client_config = config.DrydockConfig.node_driver['maasdriver']
+
+        maas_client = client.MaasRequestFactory(client_config['api_url'], client_config['api_key'])
+
+        resp = maas_client.get('account/', params={'op': 'list_authorisation_tokens'})
+
+        parsed = resp.json()
+        
+        assert len(parsed) > 0

tests/integration/test_maasdriver_network.py

@@ -0,0 +1,58 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import uuid
+
+import helm_drydock.config as config
+import helm_drydock.drivers.node.maasdriver.api_client as client
+import helm_drydock.drivers.node.maasdriver.models.fabric as maas_fabric
+import helm_drydock.drivers.node.maasdriver.models.subnet as maas_subnet
+
+class TestClass(object):
+
+    def test_maas_fabric(self):
+         client_config = config.DrydockConfig.node_driver['maasdriver']
+
+         maas_client = client.MaasRequestFactory(client_config['api_url'], client_config['api_key'])
+
+         fabric_name = str(uuid.uuid4())
+
+         fabric_list = maas_fabric.Fabrics(maas_client)
+         fabric_list.refresh()
+
+         test_fabric = maas_fabric.Fabric(maas_client, name=fabric_name, description='Test Fabric')
+         test_fabric = fabric_list.add(test_fabric)
+
+         assert test_fabric.name == fabric_name
+         assert test_fabric.resource_id is not None
+
+         query_fabric = maas_fabric.Fabric(maas_client, resource_id=test_fabric.resource_id)
+         query_fabric.refresh()
+
+         assert query_fabric.name == test_fabric.name
+
+    def test_maas_subnet(self):
+        client_config = config.DrydockConfig.node_driver['maasdriver']
+
+        maas_client = client.MaasRequestFactory(client_config['api_url'], client_config['api_key'])
+
+        subnet_list = maas_subnet.Subnets(maas_client)
+        subnet_list.refresh()
+
+        for s in subnet_list:
+            print(s.to_dict())
+        assert False
+
+
+

tests/integration/test_orch_node_networks.py

@@ -0,0 +1,94 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import pytest
+import shutil
+import os
+import uuid
+
+import helm_drydock.config as config
+import helm_drydock.drivers.node.maasdriver.api_client as client
+import helm_drydock.ingester.plugins.yaml
+import helm_drydock.statemgmt as statemgmt
+import helm_drydock.objects as objects
+import helm_drydock.orchestrator as orch
+import helm_drydock.objects.fields as hd_fields
+import helm_drydock.objects.task as task
+import helm_drydock.drivers as drivers
+from helm_drydock.ingester import Ingester
+
+class TestClass(object):
+
+    def test_client_verify(self):
+        design_state = statemgmt.DesignState()
+        orchestrator = orch.Orchestrator(state_manager=design_state,
+                                    enabled_drivers={'node': 'helm_drydock.drivers.node.maasdriver.driver.MaasNodeDriver'})
+
+        orch_task = orchestrator.create_task(task.OrchestratorTask,
+                                             site='sitename',
+                                             design_id=None,
+                                             action=hd_fields.OrchestratorAction.VerifySite)
+
+        orchestrator.execute_task(orch_task.get_id())
+
+        orch_task = design_state.get_task(orch_task.get_id())
+
+        assert orch_task.result == hd_fields.ActionResult.Success
+
+    def test_orch_preparesite(self, input_files):
+        objects.register_all()
+
+        input_file = input_files.join("fullsite.yaml")
+
+        design_state = statemgmt.DesignState()
+        design_data = objects.SiteDesign()
+        design_id = design_data.assign_id()
+        design_state.post_design(design_data)
+
+        ingester = Ingester()
+        ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester])
+        ingester.ingest_data(plugin_name='yaml', design_state=design_state,
+                             filenames=[str(input_file)], design_id=design_id)
+
+        design_data = design_state.get_design(design_id)
+
+        orchestrator = orch.Orchestrator(state_manager=design_state,
+                                    enabled_drivers={'node': 'helm_drydock.drivers.node.maasdriver.driver.MaasNodeDriver'})
+
+        orch_task = orchestrator.create_task(task.OrchestratorTask,
+                                             site='sitename',
+                                             design_id=design_id,
+                                             action=hd_fields.OrchestratorAction.PrepareSite)
+
+        orchestrator.execute_task(orch_task.get_id())
+
+        orch_task = design_state.get_task(orch_task.get_id())
+
+        assert orch_task.result == hd_fields.ActionResult.Success
+
+
+    
+
+    @pytest.fixture(scope='module')
+    def input_files(self, tmpdir_factory, request):
+        tmpdir = tmpdir_factory.mktemp('data')
+        samples_dir = os.path.dirname(str(request.fspath)) + "/../yaml_samples"
+        samples = os.listdir(samples_dir)
+
+        for f in samples:
+            src_file = samples_dir + "/" + f
+            dst_file = str(tmpdir) + "/" + f
+            shutil.copyfile(src_file, dst_file)
+
+        return tmpdir

tests/unit/test_design_inheritance.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from helm_drydock.ingester import Ingester
-from helm_drydock.statemgmt import DesignState, SiteDesign
+from helm_drydock.statemgmt import DesignState
 from helm_drydock.orchestrator import Orchestrator
 
 from copy import deepcopy
@@ -72,7 +72,7 @@ class TestClass(object):
     @pytest.fixture(scope='module')
     def input_files(self, tmpdir_factory, request):
         tmpdir = tmpdir_factory.mktemp('data')
-        samples_dir = os.path.dirname(str(request.fspath)) + "/yaml_samples"
+        samples_dir = os.path.dirname(str(request.fspath)) + "../yaml_samples"
         samples = os.listdir(samples_dir)
 
         for f in samples:

tests/unit/test_ingester.py

@@ -70,7 +70,7 @@ class TestClass(object):
     @pytest.fixture(scope='module')
     def input_files(self, tmpdir_factory, request):
         tmpdir = tmpdir_factory.mktemp('data')
-        samples_dir = os.path.dirname(str(request.fspath)) + "/yaml_samples"
+        samples_dir = os.path.dirname(str(request.fspath)) + "../yaml_samples"
         samples = os.listdir(samples_dir)
 
         for f in samples:

tests/unit/test_ingester_yaml.py

@@ -44,7 +44,7 @@ class TestClass(object):
     @pytest.fixture(scope='module')
     def input_files(self, tmpdir_factory, request):
         tmpdir = tmpdir_factory.mktemp('data')
-        samples_dir = os.path.dirname(str(request.fspath)) + "/yaml_samples"
+        samples_dir = os.path.dirname(str(request.fspath)) + "../yaml_samples"
         samples = os.listdir(samples_dir)
 
         for f in samples:

tests/unit/test_orch_oob.py

@@ -96,7 +96,7 @@ class TestClass(object):
     @pytest.fixture(scope='module')
     def input_files(self, tmpdir_factory, request):
         tmpdir = tmpdir_factory.mktemp('data')
-        samples_dir = os.path.dirname(str(request.fspath)) + "/yaml_samples"
+        samples_dir = os.path.dirname(str(request.fspath)) + "../yaml_samples"
         samples = os.listdir(samples_dir)
 
         for f in samples: