airship/drydock
Code Issues Proposed changes

File parsing in AIC YAML ingester and corresponding

Browse Source 
test cases
This commit is contained in:
Scott Hussey 2017-02-23 12:20:50 -06:00
parent 2356bbeb0f
commit 5051b8bf9e
9 changed files with 346 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
helm_drydock/config.py
View File

@ -22,7 +22,13 @@
class DrydockConfig(object):
def __init__(self):
self.selected_server_driver = helm_drydock.drivers.server.maasdriver
self.server_driver_config = {
selected_driver = helm_drydock.drivers.server.maasdriver,
params = {
maas_api_key = ""
maas_api_url = ""
}
}
self.selected_network_driver = helm_drydock.drivers.network.noopdriver
self.control_config = {}
self.ingester_config = {
@ -31,5 +37,5 @@ class DrydockConfig(object):
self.introspection_config = {}
self.orchestrator_config = {}
self.statemgmt_config = {
backend_driver = 'helm_drydock.drivers.statemgmt.etcd',
backend_driver = helm_drydock.drivers.statemgmt.etcd,
}

14
helm_drydock/ingester/plugins/ingester_plugin.py → helm_drydock/ingester/plugins/__init__.py
View File

@ -19,12 +19,12 @@ import logging
class IngesterPlugin(object):
def __init__(self):
self.log = logging.Logger('ingester')
return
def __init__(self):
self.log = logging.Logger('ingester')
return
def get_data(self):
return "ingester_skeleton"
def get_data(self):
return "ingester_skeleton"
def ingest_data(self, **kwargs):
return {}
def ingest_data(self, **kwargs):
return {}

96
helm_drydock/ingester/plugins/aicyaml.py
View File

@ -14,42 +14,86 @@
#
# AIC YAML Ingester - This data ingester will consume a AIC YAML design
# file
#
# file
#
import yaml
import logging
import helm_drydock.ingester.plugins.IngesterPlugin
import helm_drydock.model as model
from helm_drydock.ingester.plugins import IngesterPlugin
class AicYamlIngester(IngesterPlugin):
def __init__(self):
super(AicYamlIngester, self).__init__()
kind_map = {
"Region": model.Site,
"NetworkLink": model.NetworkLink,
"HardwareProfile": model.HardwareProfile,
"Network": model.Network,
"HostProfile": model.HostProfile,
"BaremetalNode": model.BaremetalNode,
}
def get_name(self):
return "aic_yaml"
def __init__(self):
super(AicYamlIngester, self).__init__()
"""
AIC YAML ingester params
def get_name(self):
return "aic_yaml"
filename - Absolute path to the YAML file to ingest
"""
def ingest_data(self, **kwargs):
if 'filename' in params:
input_string = read_input_file(params['filename'])
parsed_data = parse_input_data(input_string)
processed_data = compute_effective_data(parsed_data)
else:
"""
AIC YAML ingester params
raise Exception('Missing parameter')
return processed_data
filenames - Array of absolute path to the YAML files to ingest
def read_input_file(self, filename):
try:
file = open(filename,'rt')
except OSError as err:
self.log.error("Error opening input file %s for ingestion: %s" % (filename, err))
return {}
returns an array of objects from helm_drydock.model
"""
def ingest_data(self, **kwargs):
if 'filenames' in kwargs:
# TODO validate filenames is array
for f in kwargs.get('filenames'):
try:
file = open(f,'rt')
contents = file.read()
file.close()
except OSError as err:
self.log.error(
"Error opening input file %s for ingestion: %s"
% (filename, err))
continue
try:
parsed_data = yaml.load_all(contents)
except yaml.YAMLError as err:
self.log.error("Error parsing YAML in %s: %s" % (f, err))
continue
models = []
for d in parsed_data:
kind = d.get('kind', '')
if kind != '':
if kind in AicYamlIngester.kind_map:
try:
model = AicYamlIngester.kind_map[kind](**d)
models.append(model)
except Exception as err:
self.log.error("Error building model %s: %s"
% (kind, str(err)))
else:
self.log.error(
"Error processing document, unknown kind %s"
% (kind))
continue
else:
self.log.error(
"Error processing document in %s, no kind field"
% (f))
continue
return models
else:
raise ValueError('Missing parameter "filename"')
return processed_data

3
setup.py
View File

@ -41,7 +41,8 @@ setup(name='helm_drydock',
license='Apache 2.0',
packages=['helm_drydock',
'helm_drydock.model',
'helm_drydock.ingester'],
'helm_drydock.ingester',
'helm_drydock.ingester.plugins'],
install_requires=[
'PyYAML',
'oauth',

29
tests/aicyaml_samples/invalid.yaml Normal file
View File

@ -0,0 +1,29 @@
---
apiVersion:
: Network
metadata:
name: public
region: sitename
date: 17-FEB-2017
name: Sample network definition
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '102'
# How are addresses assigned?
allocation: static
# MTU size for the VLAN interface
mtu: 1500
cidr: 172.16.3.0/24
# Desribe IP address ranges
ranges:
- type: static
start: 172.16.3.15
end: 172.16.3.254
routes:
- subnet: 0.0.0.0/0
gateway: 172.16.3.1
metric: 9
dns:
domain: sitename.example.com
servers: 8.8.8.8

77
tests/aicyaml_samples/multidoc.yaml Normal file
View File

@ -0,0 +1,77 @@
---
apiVersion: '1.0'
kind: NetworkLink
metadata:
name: oob
region: sitename
date: 17-FEB-2017
name: Sample network link
author: sh8121@att.com
description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
spec:
bonding:
mode: none
mtu: 1500
linkspeed: 100full
trunking:
mode: none
default_network: oob
---
# pxe is a bit of 'magic' indicating the link config used when PXE booting
# a node. All other links indicate network configs applied when the node
# is deployed.
apiVersion: '1.0'
kind: NetworkLink
metadata:
name: pxe
region: sitename
date: 17-FEB-2017
name: Sample network link
author: sh8121@att.com
description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
spec:
bonding:
mode: none
mtu: 1500
linkspeed: auto
# Is this link supporting multiple layer 2 networks?
# none is a port-based VLAN identified by default_network
# tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr
trunking:
mode: none
# use name, will translate to VLAN ID
default_network: pxe
---
apiVersion: '1.0'
kind: NetworkLink
metadata:
name: gp
region: sitename
date: 17-FEB-2017
name: Sample network link
author: sh8121@att.com
description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on
# pxe is a bit of 'magic' indicating the link config used when PXE booting
# a node. All other links indicate network configs applied when the node
# is deployed.
spec:
# If this link is a bond of physical links, how is it configured
# 802.3ad
# active-backup
# balance-rr
# Can add support for others down the road
bonding:
mode: 802.3ad
# For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4
hash: layer3+4
# 802.3ad specific options
peer_rate: slow
mon_rate: default
up_delay: default
down_delay: default
mtu: 9000
linkspeed: auto
# Is this link supporting multiple layer 2 networks?
trunking:
mode: tagged
default_network: mgmt

39
tests/aicyaml_samples/singledoc.yaml Normal file
View File

@ -0,0 +1,39 @@
---
apiVersion: '1.0'
kind: HardwareProfile
metadata:
name: HPGen8v3
region: sitename
date: 17-FEB-2017
name: Sample hardware definition
author: Scott Hussey
spec:
# Vendor of the server chassis
vendor: HP
# Generation of the chassis model
generation: '8'
# Version of the chassis model within its generation - not version of the hardware definition
hw_version: '3'
# The certified version of the chassis BIOS
bios_version: '2.2.3'
# Mode of the default boot of hardware - bios, uefi
boot_mode: bios
# Protocol of boot of the hardware - pxe, usb, hdd
bootstrap_protocol: pxe
# Which interface to use for network booting within the OOB manager, not OS device
pxe_interface: 0
# Map hardware addresses to aliases/roles to allow a mix of hardware configs
# in a site to result in a consistent configuration
device_aliases:
pci:
- address: pci@0000:00:03.0
alias: prim_nic01
# type could identify expected hardware - used for hardware manifest validation
type: '82540EM Gigabit Ethernet Controller'
- address: pci@0000:00:04.0
alias: prim_nic02
type: '82540EM Gigabit Ethernet Controller'
scsi:
- address: scsi@2:0.0.0
alias: primary_boot
type: 'VBOX HARDDISK'

60
tests/aicyaml_samples/unknown_kind.yaml Normal file
View File

@ -0,0 +1,60 @@
---
apiVersion: '1.0'
kind: FooBar
metadata:
name: default
region: sitename
date: 17-FEB-2017
name: Sample network definition
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
# No magic to this host_profile, it just provides a way to specify
# sitewide settings. If it is absent from a node's inheritance chain
# then these values will NOT be applied
spec:
# OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such
# as IPMI over vender-specific when possible.
oob:
type: ipmi
# OOB networking should be preconfigured, but we can include a network
# definition for validation or enhancement (DNS registration)
network: oob
account: admin
credential: admin
# Specify storage layout of base OS. Ceph out of scope
storage:
# How storage should be carved up: lvm (logical volumes), flat
# (single partition)
layout: lvm
# Info specific to the boot and root disk/partitions
bootdisk:
# Device will specify an alias defined in hwdefinition.yaml
device: primary_boot
# For LVM, the size of the partition added to VG as a PV
# For flat, the size of the partition formatted as ext4
root_size: 50g
# The /boot partition. If not specified, /boot will in root
boot_size: 2g
# Info for additional partitions. Need to balance between
# flexibility and complexity
partitions:
- name: logs
device: primary_boot
# Partition uuid if needed
part_uuid: 84db9664-f45e-11e6-823d-080027ef795a
size: 10g
# Optional, can carve up unformatted block devices
mountpoint: /var/log
fstype: ext4
mount_options: defaults
# Filesystem UUID or label can be specified. UUID recommended
fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e
fs_label: logs
# Platform (Operating System) settings
platform:
image: ubuntu_16.04_hwe
kernel_params: default
# Additional metadata to apply to a node
metadata:
# Base URL of the introspection service - may go in curtin data
introspection_url: http://172.16.1.10:9090

54
tests/test_ingester_aicyaml.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from helm_drydock.ingester.plugins.aicyaml import AicYamlIngester
import pytest
import shutil
import os
class TestClass(object):
def setup_method(self, method):
print("Running test {0}".format(method.__name__))
def test_ingest_singledoc(self, input_files):
input_file = input_files.join("singledoc.yaml")
ingester = AicYamlIngester()
models = ingester.ingest_data(filenames=[str(input_file)])
assert len(models) == 1
def test_ingest_multidoc(self, input_files):
input_file = input_files.join("multidoc.yaml")
ingester = AicYamlIngester()
models = ingester.ingest_data(filenames=[str(input_file)])
assert len(models) == 3
@pytest.fixture(scope='module')
def input_files(self, tmpdir_factory, request):
tmpdir = tmpdir_factory.mktemp('data')
samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples"
samples = os.listdir(samples_dir)
for f in samples:
src_file = samples_dir + "/" + f
dst_file = str(tmpdir) + "/" + f
shutil.copyfile(src_file, dst_file)
return tmpdir