File parsing in AIC YAML ingester and corresponding

test cases

helm_drydock/config.py

@@ -22,7 +22,13 @@
 class DrydockConfig(object):
 
 	def __init__(self):
-		self.selected_server_driver = helm_drydock.drivers.server.maasdriver
+        self.server_driver_config = {
+            selected_driver = helm_drydock.drivers.server.maasdriver,
+            params = {
+                maas_api_key = "" 
+                maas_api_url = ""
+            }
+        }
 		self.selected_network_driver = helm_drydock.drivers.network.noopdriver
 		self.control_config = {}
 		self.ingester_config = {
@@ -31,5 +37,5 @@ class DrydockConfig(object):
 		self.introspection_config = {}
 		self.orchestrator_config = {}
 		self.statemgmt_config = {
-			backend_driver = 'helm_drydock.drivers.statemgmt.etcd',
+			backend_driver = helm_drydock.drivers.statemgmt.etcd,
 		}

helm_drydock/ingester/plugins/aicyaml.py

@@ -19,10 +19,20 @@
 import yaml
 import logging
 
-import helm_drydock.ingester.plugins.IngesterPlugin
+import helm_drydock.model as model
+from helm_drydock.ingester.plugins import IngesterPlugin
 
 class AicYamlIngester(IngesterPlugin):
 
+    kind_map = {
+        "Region": model.Site,
+        "NetworkLink": model.NetworkLink,
+        "HardwareProfile": model.HardwareProfile,
+        "Network": model.Network,
+        "HostProfile": model.HostProfile,
+        "BaremetalNode": model.BaremetalNode,
+    }
+
     def __init__(self):
         super(AicYamlIngester, self).__init__()
 
@@ -32,24 +42,58 @@ class AicYamlIngester(IngesterPlugin):
     """
     AIC YAML ingester params
 
-	filename - Absolute path to the YAML file to ingest
+    filenames - Array of absolute path to the YAML files to ingest
+
+    returns an array of objects from helm_drydock.model
+
     """
     def ingest_data(self, **kwargs):
-		if 'filename' in params:
+        if 'filenames' in kwargs:
-			input_string = read_input_file(params['filename'])
+            # TODO validate filenames is array
-			parsed_data = parse_input_data(input_string)
+            for f in kwargs.get('filenames'):
-			processed_data = compute_effective_data(parsed_data)
+                try:
-		else:
+                    file = open(f,'rt')
+                    contents = file.read()
+                    file.close()
+                except OSError as err:
+                    self.log.error(
+                        "Error opening input file %s for ingestion: %s" 
+                        % (filename, err))
+                    continue
         
-			raise Exception('Missing parameter')
+                try:
+                    parsed_data = yaml.load_all(contents)
+                except yaml.YAMLError as err:
+                    self.log.error("Error parsing YAML in %s: %s" % (f, err))
+                    continue
+
+                models = []
+                for d in parsed_data:
+                    kind = d.get('kind', '')
+                    if kind != '':
+                        if kind in AicYamlIngester.kind_map:
+                            try:
+                                model = AicYamlIngester.kind_map[kind](**d)
+                                models.append(model)
+                            except Exception as err:
+                                self.log.error("Error building model %s: %s" 
+                                    % (kind, str(err)))
+                        else:
+                            self.log.error(
+                                "Error processing document, unknown kind %s" 
+                                % (kind))
+                            continue
+                    else:
+                        self.log.error(
+                            "Error processing document in %s, no kind field"
+                            % (f))
+                        continue
+
+                return models
+        else:
+            raise ValueError('Missing parameter "filename"')
         
         return processed_data
         
-	def read_input_file(self, filename):
-		try:
-			file = open(filename,'rt')
-		except OSError as err:
-			self.log.error("Error opening input file %s for ingestion: %s" % (filename, err))
-			return {}
 
 

setup.py

@@ -41,7 +41,8 @@ setup(name='helm_drydock',
       license='Apache 2.0',
       packages=['helm_drydock',
                 'helm_drydock.model',
-                'helm_drydock.ingester'],
+                'helm_drydock.ingester',
+                'helm_drydock.ingester.plugins'],
       install_requires=[
         'PyYAML',
         'oauth',

tests/aicyaml_samples/invalid.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: 
+: Network
+metadata:
+  name: public
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample network definition
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  vlan_id: '102'
+    # How are addresses assigned?
+  allocation: static
+    # MTU size for the VLAN interface
+  mtu: 1500
+  cidr: 172.16.3.0/24
+    # Desribe IP address ranges
+  ranges: 
+  - type: static
+    start: 172.16.3.15
+    end: 172.16.3.254
+  routes:
+  - subnet: 0.0.0.0/0
+    gateway: 172.16.3.1
+    metric: 9
+  dns:
+    domain: sitename.example.com
+    servers: 8.8.8.8

tests/aicyaml_samples/multidoc.yaml

@@ -0,0 +1,77 @@
+---
+apiVersion: '1.0'
+kind: NetworkLink
+metadata:
+  name: oob
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample network link
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
+spec:
+  bonding:
+    mode: none
+  mtu: 1500
+  linkspeed: 100full
+  trunking:
+    mode: none
+    default_network: oob
+---
+# pxe is a bit of 'magic' indicating the link config used when PXE booting
+# a node. All other links indicate network configs applied when the node
+# is deployed.
+apiVersion: '1.0'
+kind: NetworkLink
+metadata:
+  name: pxe
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample network link
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
+spec:
+  bonding:
+    mode: none
+  mtu: 1500
+  linkspeed: auto
+  # Is this link supporting multiple layer 2 networks?
+  # none is a port-based VLAN identified by default_network
+  # tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr
+  trunking:
+    mode: none
+    # use name, will translate to VLAN ID
+    default_network: pxe
+---
+apiVersion: '1.0'
+kind: NetworkLink
+metadata:
+  name: gp
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample network link
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on
+  # pxe is a bit of 'magic' indicating the link config used when PXE booting
+  # a node. All other links indicate network configs applied when the node
+  # is deployed.
+spec:
+  # If this link is a bond of physical links, how is it configured
+  # 802.3ad
+  # active-backup
+  # balance-rr 
+  # Can add support for others down the road
+  bonding:
+    mode: 802.3ad
+    # For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4
+    hash: layer3+4
+    # 802.3ad specific options
+    peer_rate: slow
+    mon_rate: default
+    up_delay: default
+    down_delay: default
+  mtu: 9000
+  linkspeed: auto
+  # Is this link supporting multiple layer 2 networks?
+  trunking:
+    mode: tagged
+    default_network: mgmt

tests/aicyaml_samples/singledoc.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: '1.0'
+kind: HardwareProfile
+metadata:
+  name: HPGen8v3
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample hardware definition
+  author: Scott Hussey
+spec:
+  # Vendor of the server chassis
+  vendor: HP
+  # Generation of the chassis model
+  generation: '8'
+  # Version of the chassis model within its generation - not version of the hardware definition
+  hw_version: '3'
+  # The certified version of the chassis BIOS
+  bios_version: '2.2.3'
+  # Mode of the default boot of hardware - bios, uefi
+  boot_mode: bios
+  # Protocol of boot of the hardware - pxe, usb, hdd
+  bootstrap_protocol: pxe
+  # Which interface to use for network booting within the OOB manager, not OS device
+  pxe_interface: 0
+  # Map hardware addresses to aliases/roles to allow a mix of hardware configs
+  # in a site to result in a consistent configuration
+  device_aliases:
+    pci:
+    - address: pci@0000:00:03.0
+      alias: prim_nic01
+      # type could identify expected hardware - used for hardware manifest validation
+      type: '82540EM Gigabit Ethernet Controller'
+    - address: pci@0000:00:04.0
+      alias: prim_nic02
+      type: '82540EM Gigabit Ethernet Controller'
+    scsi:
+    - address: scsi@2:0.0.0
+      alias: primary_boot
+      type: 'VBOX HARDDISK'

tests/aicyaml_samples/unknown_kind.yaml

@@ -0,0 +1,60 @@
+---
+apiVersion: '1.0'
+kind: FooBar
+metadata:
+  name: default
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample network definition
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+  # No magic to this host_profile, it just provides a way to specify
+  # sitewide settings. If it is absent from a node's inheritance chain
+  # then these values will NOT be applied
+spec:
+  # OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such
+  # as IPMI over vender-specific when possible.
+  oob:
+    type: ipmi
+    # OOB networking should be preconfigured, but we can include a network
+    # definition for validation or enhancement (DNS registration)
+    network: oob
+    account: admin
+    credential: admin
+  # Specify storage layout of base OS. Ceph out of scope
+  storage:
+    # How storage should be carved up: lvm (logical volumes), flat 
+    # (single partition)
+    layout: lvm
+    # Info specific to the boot and root disk/partitions
+    bootdisk:
+      # Device will specify an alias defined in hwdefinition.yaml
+      device: primary_boot
+      # For LVM, the size of the partition added to VG as a PV
+      # For flat, the size of the partition formatted as ext4
+      root_size: 50g
+      # The /boot partition. If not specified, /boot will in root
+      boot_size: 2g
+    # Info for additional partitions. Need to balance between
+    # flexibility and complexity
+    partitions:
+    - name: logs
+      device: primary_boot
+      # Partition uuid if needed
+      part_uuid: 84db9664-f45e-11e6-823d-080027ef795a
+      size: 10g
+      # Optional, can carve up unformatted block devices
+      mountpoint: /var/log
+      fstype: ext4
+      mount_options: defaults
+      # Filesystem UUID or label can be specified. UUID recommended
+      fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e
+      fs_label: logs
+  # Platform (Operating System) settings
+  platform:
+    image: ubuntu_16.04_hwe
+    kernel_params: default
+  # Additional metadata to apply to a node
+  metadata:
+    # Base URL of the introspection service - may go in curtin data
+    introspection_url: http://172.16.1.10:9090

tests/test_ingester_aicyaml.py

@@ -0,0 +1,54 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from helm_drydock.ingester.plugins.aicyaml import AicYamlIngester
+import pytest
+import shutil
+import os
+
+class TestClass(object):
+
+    def setup_method(self, method):
+        print("Running test {0}".format(method.__name__))
+
+    def test_ingest_singledoc(self, input_files):
+        input_file = input_files.join("singledoc.yaml")
+
+        ingester = AicYamlIngester()
+
+        models = ingester.ingest_data(filenames=[str(input_file)])
+
+        assert len(models) == 1
+
+    def test_ingest_multidoc(self, input_files):
+        input_file = input_files.join("multidoc.yaml")
+ 
+        ingester = AicYamlIngester()
+
+        models = ingester.ingest_data(filenames=[str(input_file)])
+
+        assert len(models) == 3
+
+    @pytest.fixture(scope='module')
+    def input_files(self, tmpdir_factory, request):
+        tmpdir = tmpdir_factory.mktemp('data')
+        samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples"
+        samples = os.listdir(samples_dir)
+
+        for f in samples:
+            src_file = samples_dir + "/" + f
+            dst_file = str(tmpdir) + "/" + f
+            shutil.copyfile(src_file, dst_file)
+
+        return tmpdir