airship/drydock
Code Issues Proposed changes

Merge branch 'start_coding'

Browse Source
This commit is contained in:
Scott Hussey 2017-02-28 15:18:52 -06:00
commit 3a331f6c51
30 changed files with 2239 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
README.md
View File

@ -1,33 +1,33 @@
# drydock
# helm_drydock
A python REST orchestrator to translate a YAML host topology to a provisioned set of hosts and provide a set of cloud-init post-provisioning instructions.
## Modular service
### Design Consumer ###
aka smelter
aka ingester
Pluggable service to ingest a inventory/design specification, convert it to a standard
internal representaion, and persist it to the Design State API. Initial implementation
is the consumer of AIC YAML schema.
is the consumer of YAML schema.
### Design State API ###
aka tarot
aka statemgmt
API for querying and updating the current design specification and persisted orchestration status.
CRUD support of CIs that are not bootstrap-related, but can be used by other automation.
### Control API ###
aka cockpit
aka control
User-approachable API for initiating orchestration actions or accessing other internal
APIs
### Infrastructure Orchestrator ###
aka alchemist
aka orchestrator
Handle validation of complete design, ordering and managing downstream API calls for hardware
provisioning/bootstrapping
@ -38,13 +38,9 @@ aka maasdriver
Pluggable provisioner for server bootstrapping. Initial implementation is MaaS client.
### Network Driver ###
Pluggable provisioner for network provisioning. Initial implementation is Noop.
### Introspection API ###
aka jabberwocky
aka introspection
API for bootstrapping nodes to load self data. Possibly pluggable as this is basically an
authenticated bridge to the Design State API

3
examples/bootstrap_seed.yaml
View File

@ -397,7 +397,8 @@ spec:
- network: public
address: 172.16.3.20
metadata:
roles: os_ctl
tags:
- os_ctl
rack: rack01
---
apiVersion: 'v1.0'

13
helm_drydock/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

41
helm_drydock/config.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Read application configuration
#
# configuration map with defaults
class DrydockConfig(object):
def __init__(self):
self.server_driver_config = {
selected_driver = helm_drydock.drivers.server.maasdriver,
params = {
maas_api_key = ""
maas_api_url = ""
}
}
self.selected_network_driver = helm_drydock.drivers.network.noopdriver
self.control_config = {}
self.ingester_config = {
plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester]
}
self.introspection_config = {}
self.orchestrator_config = {}
self.statemgmt_config = {
backend_driver = helm_drydock.drivers.statemgmt.etcd,
}

13
helm_drydock/control/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

35
helm_drydock/control/root.py Normal file
View File

@ -0,0 +1,35 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pecan import expose
from webob.exc import status_map
class RootController(object):
@expose(generic=True, template='index.html')
def index(self):
return dict()
@index.when(method='POST')
def index_post(self, q):
redirect('https://pecan.readthedocs.io/en/latest/search.html?q=%s' % q)
@expose('error.html')
def error(self, status):
try:
status = int(status)
except ValueError:
status = 0
message = getattr(status_map.get(status), 'explanation', '')
return dict(status=status, message=message)

19
helm_drydock/drivers/__init__.py Normal file
View File

@ -0,0 +1,19 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class ProviderDriver(object):
__init__(self):
pass

18
helm_drydock/drivers/server/__init__.py Normal file
View File

@ -0,0 +1,18 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import helm_drydock.drivers.ProviderDriver
class ServerDriver(ProviderDriver):

17
helm_drydock/drivers/server/maasdriver/__init__.py Normal file
View File

@ -0,0 +1,17 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import helm_drydock.drivers.server.ServerDriver
class MaasServerDriver(object):

90
helm_drydock/ingester/__init__.py Normal file
View File

@ -0,0 +1,90 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ingester - Ingest host topologies to define site design and
# persist design to helm-drydock's statemgmt service
import logging
import yaml
import helm_drydock.model as model
from helm_drydock.statemgmt import DesignState
class Ingester(object):
def __init__(self):
logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s")
self.log = logging.Logger("ingester")
self.registered_plugins = {}
"""
enable_plugins
params: plugins - A list of class objects denoting the ingester plugins to be enabled
Enable plugins that can be used for ingest_data calls. Each plugin should use
helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one
enabled plugin successfully initializes, the call is considered successful. Otherwise
it will throw an exception
"""
def enable_plugins(self, plugins=[]):
if len(plugins) == 0:
self.log.error("Cannot have an empty plugin list.")
for plugin in plugins:
try:
new_plugin = plugin()
plugin_name = new_plugin.get_name()
self.registered_plugins[plugin_name] = new_plugin
except:
self.log.error("Could not enable plugin %s" % (plugin.__name__))
if len(self.registered_plugins) == 0:
self.log.error("Could not enable at least one plugin")
raise Exception("Could not enable at least one plugin")
"""
ingest_data
params: plugin_name - Which plugin should be used for ingestion
params: params - A map of parameters that will be passed to the plugin's ingest_data method
Execute a data ingestion using the named plugin (assuming it is enabled)
"""
def ingest_data(self, plugin_name='', design_state=None, **kwargs):
if design_state is None:
self.log.error("ingest_data called without valid DesignState handler")
raise Exception("Invalid design_state handler")
if plugin_name in self.registered_plugins:
design_data = self.registered_plugins[plugin_name].ingest_data(**kwargs)
# Need to persist data here, but we don't yet have the statemgmt service working
for m in design_data:
if type(m) is model.Site:
design_state.add_site(m)
elif type(m) is model.Network:
design_state.add_network(m)
elif type(m) is model.NetworkLink:
design_state.add_network_link(m)
elif type(m) is model.HostProfile:
design_state.add_host_profile(m)
elif type(m) is model.HardwareProfile:
design_state.add_hardware_profile(m)
elif type(m) is model.BaremetalNode:
design_state.add_baremetal_node(m)
else:
self.log.error("Could not find plugin %s to ingest data." % (plugin_name))
raise LookupError("Could not find plugin %s" % plugin_name)

30
helm_drydock/ingester/plugins/__init__.py Normal file
View File

@ -0,0 +1,30 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Plugins to parse incoming topology and translate it to helm-drydock's
# model representation
import logging
class IngesterPlugin(object):
def __init__(self):
self.log = logging.Logger('ingester')
return
def get_data(self):
return "ingester_skeleton"
def ingest_data(self, **kwargs):
return {}

99
helm_drydock/ingester/plugins/aicyaml.py Normal file
View File

@ -0,0 +1,99 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# AIC YAML Ingester - This data ingester will consume a AIC YAML design
# file
#
import yaml
import logging
import helm_drydock.model as model
from helm_drydock.ingester.plugins import IngesterPlugin
class AicYamlIngester(IngesterPlugin):
kind_map = {
"Region": model.Site,
"NetworkLink": model.NetworkLink,
"HardwareProfile": model.HardwareProfile,
"Network": model.Network,
"HostProfile": model.HostProfile,
"BaremetalNode": model.BaremetalNode,
}
def __init__(self):
super(AicYamlIngester, self).__init__()
def get_name(self):
return "aic_yaml"
"""
AIC YAML ingester params
filenames - Array of absolute path to the YAML files to ingest
returns an array of objects from helm_drydock.model
"""
def ingest_data(self, **kwargs):
if 'filenames' in kwargs:
# TODO validate filenames is array
for f in kwargs.get('filenames'):
try:
file = open(f,'rt')
contents = file.read()
file.close()
except OSError as err:
self.log.error(
"Error opening input file %s for ingestion: %s"
% (filename, err))
continue
try:
parsed_data = yaml.load_all(contents)
except yaml.YAMLError as err:
self.log.error("Error parsing YAML in %s: %s" % (f, err))
continue
models = []
for d in parsed_data:
kind = d.get('kind', '')
if kind != '':
if kind in AicYamlIngester.kind_map:
try:
model = AicYamlIngester.kind_map[kind](**d)
models.append(model)
except Exception as err:
self.log.error("Error building model %s: %s"
% (kind, str(err)))
else:
self.log.error(
"Error processing document, unknown kind %s"
% (kind))
continue
else:
self.log.error(
"Error processing document in %s, no kind field"
% (f))
continue
return models
else:
raise ValueError('Missing parameter "filename"')
return processed_data

671
helm_drydock/model/__init__.py Normal file
View File

@ -0,0 +1,671 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Models for helm_drydock
#
from copy import deepcopy
class HardwareProfile(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
# Need to add validation logic, we'll assume the input is
# valid for now
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.vendor = spec.get('vendor', None)
self.generation = spec.get('generation', None)
self.hw_version = spec.get('hw_version', None)
self.bios_version = spec.get('bios_version', None)
self.boot_mode = spec.get('boot_mode', None)
self.bootstrap_protocol = spec.get('bootstrap_protocol', None)
self.pxe_interface = spec.get('pxe_interface', None)
self.devices = []
device_aliases = spec.get('device_aliases', {})
pci_devices = device_aliases.get('pci', [])
scsi_devices = device_aliases.get('scsi', [])
for d in pci_devices:
d['bus_type'] = 'pci'
self.devices.append(
HardwareDeviceAlias(self.api_version, **d))
for d in scsi_devices:
d['bus_type'] = 'scsi'
self.devices.append(
HardwareDeviceAlias(self.api_version, **d))
else:
raise ValueError('Unknown API version of object')
return
def resolve_alias(self, alias_type, alias):
for d in self.devices:
if d.alias == alias and d.bus_type == alias_type:
return deepcopy(d)
return None
class HardwareDeviceAlias(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.bus_type = kwargs.get('bus_type', None)
self.address = kwargs.get('address', None)
self.alias = kwargs.get('alias', None)
self.type = kwargs.get('type', None)
else:
raise ValueError('Unknown API version of object')
class Site(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
# Need to add validation logic, we'll assume the input is
# valid for now
self.name = metadata.get('name', '')
self.networks = []
self.network_links = []
self.host_profiles = []
self.hardware_profiles = []
self.baremetal_nodes = []
else:
raise ValueError('Unknown API version of object')
def get_network(self, network_name):
for n in self.networks:
if n.name == network_name:
return n
return None
def get_network_link(self, link_name):
for l in self.network_links:
if l.name == link_name:
return l
return None
def get_host_profile(self, profile_name):
for p in self.host_profiles:
if p.name == profile_name:
return p
return None
def get_hardware_profile(self, profile_name):
for p in self.hardware_profiles:
if p.name == profile_name:
return p
return None
def get_baremetal_node(self, node_name):
for n in self.baremetal_nodes:
if n.name == node_name:
return n
return None
class NetworkLink(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
bonding = spec.get('bonding', {})
self.bonding_mode = bonding.get('mode', 'none')
# TODO How should we define defaults for CIs not in the input?
if self.bonding_mode == '802.3ad':
self.bonding_xmit_hash = bonding.get('hash', 'layer3+4')
self.bonding_peer_rate = bonding.get('peer_rate', 'fast')
self.bonding_mon_rate = bonding.get('mon_rate', '')
self.bonding_up_delay = bonding.get('up_delay', '')
self.bonding_down_delay = bonding.get('down_delay', '')
self.mtu = spec.get('mtu', 1500)
self.linkspeed = spec.get('linkspeed', 'auto')
trunking = spec.get('trunking', {})
self.trunk_mode = trunking.get('mode', 'none')
self.native_network = spec.get('default_network', '')
else:
raise ValueError('Unknown API version of object')
class Network(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.cidr = spec.get('cidr', None)
self.allocation_strategy = spec.get('allocation', 'static')
self.vlan_id = spec.get('vlan_id', 1)
self.mtu = spec.get('mtu', 0)
dns = spec.get('dns', {})
self.dns_domain = dns.get('domain', 'local')
self.dns_servers = dns.get('servers', None)
ranges = spec.get('ranges', [])
self.ranges = []
for r in ranges:
self.ranges.append(NetworkAddressRange(self.api_version, **r))
routes = spec.get('routes', [])
self.routes = []
for r in routes:
self.routes.append(NetworkRoute(self.api_version, **r))
else:
raise ValueError('Unknown API version of object')
class NetworkAddressRange(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.type = kwargs.get('type', None)
self.start = kwargs.get('start', None)
self.end = kwargs.get('end', None)
else:
raise ValueError('Unknown API version of object')
class NetworkRoute(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.type = kwargs.get('subnet', None)
self.start = kwargs.get('gateway', None)
self.end = kwargs.get('metric', 100)
else:
raise ValueError('Unknown API version of object')
class HostProfile(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.parent_profile = spec.get('host_profile', None)
self.hardware_profile = spec.get('hardware_profile', None)
oob = spec.get('oob', {})
self.oob_type = oob.get('type', None)
self.oob_network = oob.get('network', None)
self.oob_account = oob.get('account', None)
self.oob_credential = oob.get('credential', None)
storage = spec.get('storage', {})
self.storage_layout = storage.get('layout', 'lvm')
bootdisk = storage.get('bootdisk', {})
self.bootdisk_device = bootdisk.get('device', None)
self.bootdisk_root_size = bootdisk.get('root_size', None)
self.bootdisk_boot_size = bootdisk.get('boot_size', None)
partitions = storage.get('partitions', [])
self.partitions = []
for p in partitions:
self.partitions.append(HostPartition(self.api_version, **p))
interfaces = spec.get('interfaces', [])
self.interfaces = []
for i in interfaces:
self.interfaces.append(HostInterface(self.api_version, **i))
metadata = spec.get('metadata', {})
metadata_tags = metadata.get('tags', [])
self.tags = []
for t in metadata_tags:
self.tags.append(t)
owner_data = metadata.get('owner_data', {})
self.owner_data = {}
for k, v in owner_data.items():
self.owner_data[k] = v
self.rack = metadata.get('rack', None)
else:
raise ValueError('Unknown API version of object')
def apply_inheritance(self, site):
# We return a deep copy of the profile so as not to corrupt
# the original model
self_copy = deepcopy(self)
if self.parent_profile is None:
return self_copy
parent = site.get_host_profile(self.parent_profile)
if parent is None:
return self_copy
parent = parent.apply_inheritance(site)
# First compute inheritance for simple fields
inheritable_field_list = [
"hardware_profile", "oob_type", "oob_network",
"oob_credential", "oob_account", "storage_layout",
"bootdisk_device", "bootdisk_root_size", "bootdisk_boot_size",
"rack"]
for f in inheritable_field_list:
setattr(self_copy, f,
Utils.apply_field_inheritance(getattr(self, f, None),
getattr(parent, f, None)))
# Now compute inheritance for complex types
self_copy.tags = Utils.merge_lists(self.tags, parent.tags)
self_copy.owner_data = Utils.merge_dicts(
self.owner_data, parent.owner_data)
self_copy.interfaces = HostInterface.merge_lists(
self.interfaces, parent.interfaces)
self_copy.partitions = HostPartition.merge_lists(
self.partitions, parent.partitions)
return self_copy
class HostInterface(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.device_name = kwargs.get('device_name', None)
self.network_link = kwargs.get('device_link', None)
self.hardware_slaves = []
slaves = kwargs.get('slaves', [])
for s in slaves:
self.hardware_slaves.append(s)
self.networks = []
networks = kwargs.get('networks', [])
for n in networks:
self.networks.append(n)
else:
raise ValueError('Unknown API version of object')
# The device attribute may be hardware alias that translates to a
# physical device address. If the device attribute does not match an
# alias, we assume it directly identifies a OS device name. When the
# apply_hardware_profile method is called on the parent Node of this
# device, the selector will be decided and applied
def add_selector(self, sel_type, selector):
if getattr(self, 'selectors', None) is None:
self.selectors = []
new_selector = {}
new_selector['selector_type'] = sel_type
new_selector['selector'] = selector
self.selectors.append(new_selector)
"""
Merge two lists of HostInterface models with child_list taking
priority when conflicts. If a member of child_list has a device_name
beginning with '!' it indicates that HostInterface should be
removed from the merged list
"""
@staticmethod
def merge_lists(child_list, parent_list):
if len(child_list) == 0:
return deepcopy(parent_list)
effective_list = []
if len(parent_list) == 0:
for i in child_list:
if i.device_name.startswith('!'):
continue
else:
effective_list.append(deepcopy(i))
parent_interfaces = []
for i in parent_list:
parent_name = i.device_name
parent_interfaces.append(parent_name)
for j in child_list:
if j.device_name == ("!" + parent_name):
break
elif j.device_name == parent_name:
m = HostInterface(j.api_version)
m.device_name = j.device_name
m.network_link = \
Utils.apply_field_inheritance(j.network_link,
i.network_link)
s = filter(lambda x: ("!" + x) not in j.hardware_slaves,
i.hardware_slaves)
s = list(s)
s.extend(filter(lambda x: not x.startswith("!"),
j.hardware_slaves))
m.hardware_slaves = s
n = filter(lambda x: ("!" + x) not in j.networks,
i.networks)
n = list(n)
n.extend(filter(lambda x: not x.startswith("!"),
j.networks))
m.networks = n
effective_list.append(m)
for j in child_list:
if j.device_name not in parent_list:
effective_list.append(deepcopy(j))
return effective_list
class HostPartition(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.name = kwargs.get('name', None)
self.device = kwargs.get('device', None)
self.part_uuid = kwargs.get('part_uuid', None)
self.size = kwargs.get('size', None)
self.mountpoint = kwargs.get('mountpoint', None)
self.fstype = kwargs.get('fstype', 'ext4')
self.mount_options = kwargs.get('mount_options', 'defaults')
self.fs_uuid = kwargs.get('fs_uuid', None)
self.fs_label = kwargs.get('fs_label', None)
else:
raise ValueError('Unknown API version of object')
# The device attribute may be hardware alias that translates to a
# physical device address. If the device attribute does not match an
# alias, we assume it directly identifies a OS device name. When the
# apply_hardware_profile method is called on the parent Node of this
# device, the selector will be decided and applied
def set_selector(self, sel_type, selector):
self.selector_type = sel_type
self.selector = selector
"""
Merge two lists of HostPartition models with child_list taking
priority when conflicts. If a member of child_list has a name
beginning with '!' it indicates that HostPartition should be
removed from the merged list
"""
@staticmethod
def merge_lists(child_list, parent_list):
if len(child_list) == 0:
return deepcopy(parent_list)
effective_list = []
if len(parent_list) == 0:
for i in child_list:
if i.name.startswith('!'):
continue
else:
effective_list.append(deepcopy(i))
inherit_field_list = ["device", "part_uuid", "size",
"mountpoint", "fstype", "mount_options",
"fs_uuid", "fs_label"]
parent_partitions = []
for i in parent_list:
parent_name = i.name
parent_partitions.append(parent_name)
for j in child_list:
if j.name == ("!" + parent_name):
break
elif j.name == parent_name:
p = HostPartition(j.api_version)
p.name = j.name
for f in inherit_field_list:
setattr(p, Utils.apply_field_inheritance(getattr(j, f),
getattr(i, f))
)
effective_list.append(p)
for j in child_list:
if j.name not in parent_list:
effective_list.append(deepcopy(j))
return effective_list
# A BaremetalNode is really nothing more than a physical
# instantiation of a HostProfile, so they both represent
# the same set of CIs
class BaremetalNode(HostProfile):
def __init__(self, **kwargs):
super(BaremetalNode, self).__init__(**kwargs)
def apply_host_profile(self, site):
return self.apply_inheritance(site)
# Translate device alises to physical selectors and copy
# other hardware attributes into this object
def apply_hardware_profile(self, site):
self_copy = deepcopy(self)
if self.hardware_profile is None:
raise ValueError("Hardware profile not set")
hw_profile = site.get_hardware_profile(self.hardware_profile)
for i in self_copy.interfaces:
for s in i.hardware_slaves:
selector = hw_profile.resolve_alias("pci", s)
if selector is None:
i.add_selector("name", s)
else:
i.add_selector("address", selector)
for p in self_copy.partitions:
selector = hw_profile.resolve_alias("scsi", p.device)
if selector is None:
p.set_selector("name", p.device)
else:
p.set_selector("address", selector)
hardware = {"vendor": getattr(hw_profile, 'vendor', None),
"generation": getattr(hw_profile, 'generation', None),
"hw_version": getattr(hw_profile, 'hw_version', None),
"bios_version": getattr(hw_profile, 'bios_version', None),
"boot_mode": getattr(hw_profile, 'boot_mode', None),
"bootstrap_protocol": getattr(hw_profile,
'bootstrap_protocol',
None),
"pxe_interface": getattr(hw_profile, 'pxe_interface', None)
}
self_copy.hardware = hardware
return self_copy
# Utility class for calculating inheritance
class Utils(object):
"""
apply_field_inheritance - apply inheritance rules to a single field value
param child_field - value of the child field, or the field receiving
the inheritance
param parent_field - value of the parent field, or the field supplying
the inheritance
return the correct value for child_field based on the inheritance algorithm
Inheritance algorithm
1. If child_field is not None, '!' for string vals or -1 for numeric
vals retain the value
of child_field
2. If child_field is '!' return None to unset the field value
3. If child_field is -1 return None to unset the field value
4. If child_field is None return parent_field
"""
@staticmethod
def apply_field_inheritance(child_field, parent_field):
if child_field is not None:
if child_field != '!' and child_field != -1:
return child_field
else:
return None
else:
return parent_field
"""
merge_lists - apply inheritance rules to a list of simple values
param child_list - list of values from the child
param parent_list - list of values from the parent
return a merged list with child values taking prority
1. All members in the child list not starting with '!'
2. If a member in the parent list has a corresponding member in the
chid list prefixed with '!' it is removed
3. All remaining members of the parent list
"""
@staticmethod
def merge_lists(child_list, parent_list):
if type(child_list) is not list or type(parent_list) is not list:
raise ValueError("One parameter is not a list")
effective_list = []
# Probably should handle non-string values
effective_list.extend(
filter(lambda x: not x.startswith("!"), child_list))
effective_list.extend(
filter(lambda x: ("!" + x) not in child_list,
filter(lambda x: x not in effective_list, parent_list)))
return effective_list
"""
merge_dicts - apply inheritance rules to a dict
param child_dict - dict of k:v from child
param parent_dict - dict of k:v from the parent
return a merged dict with child values taking prority
1. All members in the child dict with a key not starting with '!'
2. If a member in the parent dict has a corresponding member in the
chid dict where the key is prefixed with '!' it is removed
3. All remaining members of the parent dict
"""
@staticmethod
def merge_dicts(child_dict, parent_dict):
if type(child_dict) is not dict or type(parent_dict) is not dict:
raise ValueError("One parameter is not a dict")
effective_dict = {}
# Probably should handle non-string keys
use_keys = filter(lambda x: ("!" + x) not in child_dict.keys(),
parent_dict)
for k in use_keys:
effective_dict[k] = deepcopy(parent_dict[k])
use_keys = filter(lambda x: not x.startswith("!"), child_dict)
for k in use_keys:
effective_dict[k] = deepcopy(child_dict[k])
return effective_dict

13
helm_drydock/orchestrator/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

100
helm_drydock/orchestrator/designdata.py Normal file
View File

@ -0,0 +1,100 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from copy import deepcopy
class DesignStateClient(object):
def __init__(self):
self.log = logging.Logger('orchestrator')
"""
load_design_data - Pull all the defined models in statemgmt and assemble
them into a representation of the site. Does not compute inheritance.
Throws an exception if multiple Site models are found.
param design_state - Instance of statemgmt.DesignState to load data from
return a Site model populated with all components from the design state
"""
def load_design_data(self, design_state=None):
if len(design_state.get_sites()) != 1:
self.log.error("Invalid design state, should only have 1 Site model")
raise Exception("Invalid design state, should only have 1 Site model")
site = design_state.get_sites()[0]
site_name = site.name
networks = design_state.get_networks()
for n in networks:
if n.site == site_name:
site.networks.append(n)
network_links = design_state.get_network_links()
for l in network_links:
if l.site == site_name:
site.network_links.append(l)
host_profiles = design_state.get_host_profiles()
for p in host_profiles:
if p.site == site_name:
site.host_profiles.append(p)
hardware_profiles = design_state.get_hardware_profiles()
for p in hardware_profiles:
if p.site == site_name:
site.hardware_profiles.append(p)
baremetal_nodes = design_state.get_baremetal_nodes()
for n in baremetal_nodes:
if n.site == site_name:
site.baremetal_nodes.append(n)
return site
"""
compute_model_inheritance - given a fully populated Site model, compute the effecitve
design by applying inheritance and references
return a Site model reflecting the effective design for the site
"""
def compute_model_inheritance(self, site_root):
# For now the only thing that really incorporates inheritance is
# host profiles and baremetal nodes. So we'll just resolve it for
# the baremetal nodes which recursively resolves it for host profiles
# assigned to those nodes
site_copy = deepcopy(site_root)
effective_nodes = []
for n in site_copy.baremetal_nodes:
resolved = n.apply_host_profile(site_copy)
resolved = resolved.apply_hardware_profile(site_copy)
effective_nodes.append(resolved)
site_copy.baremetal_nodes = effective_nodes
return site_copy

122
helm_drydock/statemgmt/__init__.py Normal file
View File

@ -0,0 +1,122 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import helm_drydock.model as model
class DesignState(object):
def __init__(self):
self.sites = []
self.networks = []
self.network_links = []
self.host_profiles = []
self.hardware_profiles = []
self.baremetal_nodes = []
return
def add_site(self, new_site):
if new_site is None or not isinstance(new_site, model.Site):
raise Exception("Invalid Site model")
self.sites.append(new_site)
def get_sites(self):
return self.sites
def get_site(self, site_name):
for s in self.sites:
if s.name == site_name:
return s
raise NameError("Site %s not found in design state" % site_name)
def add_network(self, new_network):
if new_network is None or not isinstance(new_network, model.Network):
raise Exception("Invalid Network model")
self.networks.append(new_network)
def get_networks(self):
return self.networks
def get_network(self, network_name):
for n in self.networks:
if n.name == network_name:
return n
raise NameError("Network %s not found in design state" % network_name)
def add_network_link(self, new_network_link):
if new_network_link is None or not isinstance(new_network_link, model.NetworkLink):
raise Exception("Invalid NetworkLink model")
self.network_links.append(new_network_link)
def get_network_links(self):
return self.network_links
def get_network_link(self, link_name):
for l in self.network_links:
if l.name == link_name:
return l
raise NameError("NetworkLink %s not found in design state" % link_name)
def add_host_profile(self, new_host_profile):
if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile):
raise Exception("Invalid HostProfile model")
self.host_profiles.append(new_host_profile)
def get_host_profiles(self):
return self.host_profiles
def get_host_profile(self, profile_name):
for p in self.host_profiles:
if p.name == profile_name:
return p
raise NameError("HostProfile %s not found in design state" % profile_name)
def add_hardware_profile(self, new_hardware_profile):
if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile):
raise Exception("Invalid HardwareProfile model")
self.hardware_profiles.append(new_hardware_profile)
def get_hardware_profiles(self):
return self.hardware_profiles
def get_hardware_profile(self, profile_name):
for p in self.hardware_profiles:
if p.name == profile_name:
return p
raise NameError("HardwareProfile %s not found in design state" % profile_name)
def add_baremetal_node(self, new_baremetal_node):
if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode):
raise Exception("Invalid BaremetalNode model")
self.baremetal_nodes.append(new_baremetal_node)
def get_baremetal_nodes(self):
return self.baremetal_nodes
def get_baremetal_node(self, node_name):
for n in self.baremetal_nodes:
if n.name == node_name:
return n
raise NameError("BaremetalNode %s not found in design state" % node_name)

11
helm_drydock/tox.ini Normal file
View File

@ -0,0 +1,11 @@
[tox]
envlist = py35
[testenv]
deps=
-rrequirements.txt
setenv=
PYTHONWARNING=all
[flake8]
ignore=E302,H306

7
requirements.txt Normal file
View File

@ -0,0 +1,7 @@
PyYAML
oauth
requests-oauthlib
pyipmi
netaddr
pecan
python-libmaas

8
setup.py
View File

@ -43,16 +43,20 @@ setup(name='helm_drydock',
'helm_drydock.model',
'helm_drydock.ingester',
'helm_drydock.ingester.plugins',
'helm_drydock.statemgmt'],
'helm_drydock.statemgmt',
'helm_drydock.orchestrator',
'helm_drydock.control'],
install_requires=[
'PyYAML',
'oauth',
'requests-oauthlib',
'pyipmi',
'netaddr',
'pecan'
'pecan',
'webob'
],
dependency_link=[
'git+https://github.com/maas/python-libmaas.git'
]
)

2
testrequirements.txt Normal file
View File

@ -0,0 +1,2 @@
pytest
tox

457
tests/aicyaml_samples/fullsite.yaml Normal file
View File

@ -0,0 +1,457 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
####################
#
# bootstrap_seed.yaml - Site server design definition for physical layer
#
####################
# version the schema in this file so consumers can rationally parse it
---
apiVersion: 'v1.0'
kind: Region
metadata:
name: sitename
date: 17-FEB-2017
description: Sample site design
author: sh8121@att.com
spec:
# Not sure if we have site wide data that doesn't fall into another 'Kind'
---
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: oob
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
spec:
bonding:
mode: none
mtu: 1500
linkspeed: 100full
trunking:
mode: none
default_network: oob
---
# pxe is a bit of 'magic' indicating the link config used when PXE booting
# a node. All other links indicate network configs applied when the node
# is deployed.
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: pxe
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
spec:
bonding:
mode: none
mtu: 1500
linkspeed: auto
# Is this link supporting multiple layer 2 networks?
# none is a port-based VLAN identified by default_network
# tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr
trunking:
mode: none
# use name, will translate to VLAN ID
default_network: pxe
---
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: gp
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on
# pxe is a bit of 'magic' indicating the link config used when PXE booting
# a node. All other links indicate network configs applied when the node
# is deployed.
spec:
# If this link is a bond of physical links, how is it configured
# 802.3ad
# active-backup
# balance-rr
# Can add support for others down the road
bonding:
mode: 802.3ad
# For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4
hash: layer3+4
# 802.3ad specific options
peer_rate: slow
mon_rate: default
up_delay: default
down_delay: default
mtu: 9000
linkspeed: auto
# Is this link supporting multiple layer 2 networks?
trunking:
mode: tagged
default_network: mgmt
---
apiVersion: 'v1.0'
kind: Network
metadata:
name: oob
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
allocation: static
cidr: 172.16.100.0/24
ranges:
- type: static
start: 172.16.100.15
end: 172.16.100.254
dns:
domain: ilo.sitename.att.com
servers: 172.16.100.10
---
apiVersion: 'v1.0'
kind: Network
metadata:
name: pxe
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
# Layer 2 VLAN segment id, could support other segmentations. Optional
vlan_id: '99'
# How are addresses assigned?
allocation: dhcp
# MTU for this VLAN interface, if not specified it will be inherited from the link
mtu: 1500
# Network address
cidr: 172.16.0.0/24
# Desribe IP address ranges
ranges:
- type: dhcp
start: 172.16.0.5
end: 172.16.0.254
# DNS settings for this network
dns:
# Domain addresses on this network will be registered under
domain: admin.sitename.att.com
# DNS servers that a server using this network as its default gateway should use
servers: 172.16.0.10
---
apiVersion: 'v1.0'
kind: Network
metadata:
name: mgmt
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '100'
# How are addresses assigned?
allocation: static
# Allow MTU to be inherited from link the network rides on
mtu: 1500
# Network address
cidr: 172.16.1.0/24
# Desribe IP address ranges
ranges:
- type: static
start: 172.16.1.15
end: 172.16.1.254
# Static routes to be added for this network
routes:
- subnet: 0.0.0.0/0
# A blank gateway would leave to a static route specifying
# only the interface as a source
gateway: 172.16.1.1
metric: 10
# DNS settings for this network
dns:
# Domain addresses on this network will be registered under
domain: mgmt.sitename.example.com
# DNS servers that a server using this network as its default gateway should use
servers: 172.16.1.9,172.16.1.10
---
apiVersion: 'v1.0'
kind: Network
metadata:
name: private
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '101'
allocation: static
mtu: 9000
cidr: 172.16.2.0/24
# Desribe IP address ranges
ranges:
# Type can be reserved (not used for baremetal), static (all explicit
# assignments should fall here), dhcp (will be used by a DHCP server on this network)
- type: static
start: 172.16.2.15
end: 172.16.2.254
dns:
domain: priv.sitename.example.com
servers: 172.16.2.9,172.16.2.10
---
apiVersion: 'v1.0'
kind: Network
metadata:
name: public
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '102'
# How are addresses assigned?
allocation: static
# MTU size for the VLAN interface
mtu: 1500
cidr: 172.16.3.0/24
# Desribe IP address ranges
ranges:
- type: static
start: 172.16.3.15
end: 172.16.3.254
routes:
- subnet: 0.0.0.0/0
gateway: 172.16.3.1
metric: 9
dns:
domain: sitename.example.com
servers: 8.8.8.8
---
apiVersion: 'v1.0'
kind: HostProfile
metadata:
name: defaults
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
# No magic to this host_profile, it just provides a way to specify
# sitewide settings. If it is absent from a node's inheritance chain
# then these values will NOT be applied
spec:
# OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such
# as IPMI over vender-specific when possible.
oob:
type: ipmi
# OOB networking should be preconfigured, but we can include a network
# definition for validation or enhancement (DNS registration)
network: oob
account: admin
credential: admin
# Specify storage layout of base OS. Ceph out of scope
storage:
# How storage should be carved up: lvm (logical volumes), flat
# (single partition)
layout: lvm
# Info specific to the boot and root disk/partitions
bootdisk:
# Device will specify an alias defined in hwdefinition.yaml
device: primary_boot
# For LVM, the size of the partition added to VG as a PV
# For flat, the size of the partition formatted as ext4
root_size: 50g
# The /boot partition. If not specified, /boot will in root
boot_size: 2g
# Info for additional partitions. Need to balance between
# flexibility and complexity
partitions:
- name: logs
device: primary_boot
# Partition uuid if needed
part_uuid: 84db9664-f45e-11e6-823d-080027ef795a
size: 10g
# Optional, can carve up unformatted block devices
mountpoint: /var/log
fstype: ext4
mount_options: defaults
# Filesystem UUID or label can be specified. UUID recommended
fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e
fs_label: logs
# Platform (Operating System) settings
platform:
image: ubuntu_16.04_hwe
kernel_params: default
# Additional metadata to apply to a node
metadata:
# Base URL of the introspection service - may go in curtin data
introspection_url: http://172.16.1.10:9090
---
apiVersion: 'v1.0'
kind: HostProfile
metadata:
name: k8-node
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
# host_profile inheritance allows for deduplication of common CIs
# Inheritance is additive for CIs that are lists of multiple items
# To remove an inherited list member, prefix the primary key value
# with '!'.
host_profile: defaults
# Hardware profile will map hardware specific details to the abstract
# names uses in the host profile as well as specify hardware specific
# configs. A viable model should be to build a host profile without a
# hardware_profile and then for each node inherit the host profile and
# specify a hardware_profile to map that node's hardware to the abstract
# settings of the host_profile
hardware_profile: HPGen9v3
# Network interfaces.
interfaces:
# Keyed on device_name
# pxe is a special marker indicating which device should be used for pxe boot
- device_name: pxe
# The network link attached to this
network_link: pxe
# Slaves will specify aliases from hwdefinition.yaml
slaves:
- prim_nic01
# Which networks will be configured on this interface
networks:
- pxe
- device_name: bond0
network_link: gp
# If multiple slaves are specified, but no bonding config
# is applied to the link, design validation will fail
slaves:
- prim_nic01
- prim_nic02
# If multiple networks are specified, but no trunking
# config is applied to the link, design validation will fail
networks:
- mgmt
- private
metadata:
# Explicit tag assignment
tags:
- 'test'
# MaaS supports key/value pairs. Not sure of the use yet
owner_data:
foo: bar
---
apiVersion: 'v1.0'
kind: HostProfile
metadata:
name: k8-node-public
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
host_profile: k8-node
interfaces:
- device_name: bond0
networks:
# This is additive, so adds a network to those defined in the host_profile
# inheritance chain
- public
---
apiVersion: 'v1.0'
kind: BaremetalNode
metadata:
name: controller01
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
host_profile: k8-node-public
# the hostname for a server, could be used in multiple DNS domains to
# represent different interfaces
interfaces:
- device_name: bond0
networks:
# '!' prefix for the value of the primary key indicates a record should be removed
- '!private'
# Addresses assigned to network interfaces
addressing:
# Which network the address applies to. If a network appears in addressing
# that isn't assigned to an interface, design validation will fail
- network: pxe
# The address assigned. Either a explicit IPv4 or IPv6 address
# or dhcp or slaac
address: dhcp
- network: mgmt
address: 172.16.1.20
- network: public
address: 172.16.3.20
metadata:
roles: os_ctl
rack: rack01
---
apiVersion: 'v1.0'
kind: BaremetalNode
metadata:
name: compute01
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
host_profile: k8-node
addressing:
- network: pxe
address: dhcp
- network: mgmt
address: 172.16.1.21
- network: private
address: 172.16.2.21
---
apiVersion: 'v1.0'
kind: HardwareProfile
metadata:
name: HPGen9v3
region: sitename
date: 17-FEB-2017
author: Scott Hussey
spec:
# Vendor of the server chassis
vendor: HP
# Generation of the chassis model
generation: '8'
# Version of the chassis model within its generation - not version of the hardware definition
hw_version: '3'
# The certified version of the chassis BIOS
bios_version: '2.2.3'
# Mode of the default boot of hardware - bios, uefi
boot_mode: bios
# Protocol of boot of the hardware - pxe, usb, hdd
bootstrap_protocol: pxe
# Which interface to use for network booting within the OOB manager, not OS device
pxe_interface: 0
# Map hardware addresses to aliases/roles to allow a mix of hardware configs
# in a site to result in a consistent configuration
device_aliases:
pci:
- address: pci@0000:00:03.0
alias: prim_nic01
# type could identify expected hardware - used for hardware manifest validation
type: '82540EM Gigabit Ethernet Controller'
- address: pci@0000:00:04.0
alias: prim_nic02
type: '82540EM Gigabit Ethernet Controller'
scsi:
- address: scsi@2:0.0.0
alias: primary_boot
type: 'VBOX HARDDISK'

29
tests/aicyaml_samples/invalid.yaml Normal file
View File

@ -0,0 +1,29 @@
---
apiVersion:
: Network
metadata:
name: public
region: sitename
date: 17-FEB-2017
name: Sample network definition
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '102'
# How are addresses assigned?
allocation: static
# MTU size for the VLAN interface
mtu: 1500
cidr: 172.16.3.0/24
# Desribe IP address ranges
ranges:
- type: static
start: 172.16.3.15
end: 172.16.3.254
routes:
- subnet: 0.0.0.0/0
gateway: 172.16.3.1
metric: 9
dns:
domain: sitename.example.com
servers: 8.8.8.8

77
tests/aicyaml_samples/multidoc.yaml Normal file
View File

@ -0,0 +1,77 @@
---
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: oob
region: sitename
date: 17-FEB-2017
name: Sample network link
author: sh8121@att.com
description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
spec:
bonding:
mode: none
mtu: 1500
linkspeed: 100full
trunking:
mode: none
default_network: oob
---
# pxe is a bit of 'magic' indicating the link config used when PXE booting
# a node. All other links indicate network configs applied when the node
# is deployed.
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: pxe
region: sitename
date: 17-FEB-2017
name: Sample network link
author: sh8121@att.com
description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
spec:
bonding:
mode: none
mtu: 1500
linkspeed: auto
# Is this link supporting multiple layer 2 networks?
# none is a port-based VLAN identified by default_network
# tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr
trunking:
mode: none
# use name, will translate to VLAN ID
default_network: pxe
---
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: gp
region: sitename
date: 17-FEB-2017
name: Sample network link
author: sh8121@att.com
description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on
# pxe is a bit of 'magic' indicating the link config used when PXE booting
# a node. All other links indicate network configs applied when the node
# is deployed.
spec:
# If this link is a bond of physical links, how is it configured
# 802.3ad
# active-backup
# balance-rr
# Can add support for others down the road
bonding:
mode: 802.3ad
# For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4
hash: layer3+4
# 802.3ad specific options
peer_rate: slow
mon_rate: default
up_delay: default
down_delay: default
mtu: 9000
linkspeed: auto
# Is this link supporting multiple layer 2 networks?
trunking:
mode: tagged
default_network: mgmt

39
tests/aicyaml_samples/singledoc.yaml Normal file
View File

@ -0,0 +1,39 @@
---
apiVersion: 'v1.0'
kind: HardwareProfile
metadata:
name: HPGen8v3
region: sitename
date: 17-FEB-2017
name: Sample hardware definition
author: Scott Hussey
spec:
# Vendor of the server chassis
vendor: HP
# Generation of the chassis model
generation: '8'
# Version of the chassis model within its generation - not version of the hardware definition
hw_version: '3'
# The certified version of the chassis BIOS
bios_version: '2.2.3'
# Mode of the default boot of hardware - bios, uefi
boot_mode: bios
# Protocol of boot of the hardware - pxe, usb, hdd
bootstrap_protocol: pxe
# Which interface to use for network booting within the OOB manager, not OS device
pxe_interface: 0
# Map hardware addresses to aliases/roles to allow a mix of hardware configs
# in a site to result in a consistent configuration
device_aliases:
pci:
- address: pci@0000:00:03.0
alias: prim_nic01
# type could identify expected hardware - used for hardware manifest validation
type: '82540EM Gigabit Ethernet Controller'
- address: pci@0000:00:04.0
alias: prim_nic02
type: '82540EM Gigabit Ethernet Controller'
scsi:
- address: scsi@2:0.0.0
alias: primary_boot
type: 'VBOX HARDDISK'

60
tests/aicyaml_samples/unknown_kind.yaml Normal file
View File

@ -0,0 +1,60 @@
---
apiVersion: 'v1.0'
kind: FooBar
metadata:
name: default
region: sitename
date: 17-FEB-2017
name: Sample network definition
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
# No magic to this host_profile, it just provides a way to specify
# sitewide settings. If it is absent from a node's inheritance chain
# then these values will NOT be applied
spec:
# OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such
# as IPMI over vender-specific when possible.
oob:
type: ipmi
# OOB networking should be preconfigured, but we can include a network
# definition for validation or enhancement (DNS registration)
network: oob
account: admin
credential: admin
# Specify storage layout of base OS. Ceph out of scope
storage:
# How storage should be carved up: lvm (logical volumes), flat
# (single partition)
layout: lvm
# Info specific to the boot and root disk/partitions
bootdisk:
# Device will specify an alias defined in hwdefinition.yaml
device: primary_boot
# For LVM, the size of the partition added to VG as a PV
# For flat, the size of the partition formatted as ext4
root_size: 50g
# The /boot partition. If not specified, /boot will in root
boot_size: 2g
# Info for additional partitions. Need to balance between
# flexibility and complexity
partitions:
- name: logs
device: primary_boot
# Partition uuid if needed
part_uuid: 84db9664-f45e-11e6-823d-080027ef795a
size: 10g
# Optional, can carve up unformatted block devices
mountpoint: /var/log
fstype: ext4
mount_options: defaults
# Filesystem UUID or label can be specified. UUID recommended
fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e
fs_label: logs
# Platform (Operating System) settings
platform:
image: ubuntu_16.04_hwe
kernel_params: default
# Additional metadata to apply to a node
metadata:
# Base URL of the introspection service - may go in curtin data
introspection_url: http://172.16.1.10:9090

70
tests/test_design_inheritance.py Normal file
View File

@ -0,0 +1,70 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from helm_drydock.ingester import Ingester
from helm_drydock.statemgmt import DesignState
from helm_drydock.orchestrator.designdata import DesignStateClient
from copy import deepcopy
import pytest
import shutil
import os
import helm_drydock.ingester.plugins.aicyaml
import yaml
class TestClass(object):
def setup_method(self, method):
print("Running test {0}".format(method.__name__))
def test_design_inheritance(self, loaded_design):
client = DesignStateClient()
design_data = client.load_design_data(design_state=loaded_design)
design_data = client.compute_model_inheritance(design_data)
print(yaml.dump(design_data, default_flow_style=False))
node = design_data.get_baremetal_node("controller01")
assert node.hardware_profile == 'HPGen9v3'
@pytest.fixture(scope='module')
def loaded_design(self, input_files):
input_file = input_files.join("fullsite.yaml")
module_design_state = DesignState()
ingester = Ingester()
ingester.enable_plugins([helm_drydock.ingester.plugins.aicyaml.AicYamlIngester])
ingester.ingest_data(plugin_name='aic_yaml', design_state=module_design_state, filenames=[str(input_file)])
return module_design_state
@pytest.fixture(scope='module')
def input_files(self, tmpdir_factory, request):
tmpdir = tmpdir_factory.mktemp('data')
samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples"
samples = os.listdir(samples_dir)
for f in samples:
src_file = samples_dir + "/" + f
dst_file = str(tmpdir) + "/" + f
shutil.copyfile(src_file, dst_file)
return tmpdir

53
tests/test_ingester.py Normal file
View File

@ -0,0 +1,53 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from helm_drydock.ingester import Ingester
from helm_drydock.statemgmt import DesignState
import pytest
import shutil
import os
import helm_drydock.ingester.plugins.aicyaml
class TestClass(object):
def setup_method(self, method):
print("Running test {0}".format(method.__name__))
def test_ingest_full_site(self, design_state,input_files):
input_file = input_files.join("fullsite.yaml")
ingester = Ingester()
ingester.enable_plugins([helm_drydock.ingester.plugins.aicyaml.AicYamlIngester])
ingester.ingest_data(plugin_name='aic_yaml', design_state=design_state, filenames=[str(input_file)])
assert len(design_state.host_profiles) == 3
@pytest.fixture(scope='module')
def input_files(self, tmpdir_factory, request):
tmpdir = tmpdir_factory.mktemp('data')
samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples"
samples = os.listdir(samples_dir)
for f in samples:
src_file = samples_dir + "/" + f
dst_file = str(tmpdir) + "/" + f
shutil.copyfile(src_file, dst_file)
return tmpdir
@pytest.fixture(scope='session')
def design_state(self):
design_state = DesignState()
return design_state

54
tests/test_ingester_aicyaml.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from helm_drydock.ingester.plugins.aicyaml import AicYamlIngester
import pytest
import shutil
import os
class TestClass(object):
def setup_method(self, method):
print("Running test {0}".format(method.__name__))
def test_ingest_singledoc(self, input_files):
input_file = input_files.join("singledoc.yaml")
ingester = AicYamlIngester()
models = ingester.ingest_data(filenames=[str(input_file)])
assert len(models) == 1
def test_ingest_multidoc(self, input_files):
input_file = input_files.join("multidoc.yaml")
ingester = AicYamlIngester()
models = ingester.ingest_data(filenames=[str(input_file)])
assert len(models) == 3
@pytest.fixture(scope='module')
def input_files(self, tmpdir_factory, request):
tmpdir = tmpdir_factory.mktemp('data')
samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples"
samples = os.listdir(samples_dir)
for f in samples:
src_file = samples_dir + "/" + f
dst_file = str(tmpdir) + "/" + f
shutil.copyfile(src_file, dst_file)
return tmpdir

69
tests/test_models.py Normal file
View File

@ -0,0 +1,69 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
import yaml
from helm_drydock.model import HardwareProfile
class TestClass(object):
def setup_method(self, method):
print("Running test {0}".format(method.__name__))
def test_hardwareprofile(self):
yaml_snippet = ("---\n"
"apiVersion: 'v1.0'\n"
"kind: HardwareProfile\n"
"metadata:\n"
" name: HPGen8v3\n"
" region: sitename\n"
" date: 17-FEB-2017\n"
" name: Sample hardware definition\n"
" author: Scott Hussey\n"
"spec:\n"
" # Vendor of the server chassis\n"
" vendor: HP\n"
" # Generation of the chassis model\n"
" generation: '8'\n"
" # Version of the chassis model within its generation - not version of the hardware definition\n"
" hw_version: '3'\n"
" # The certified version of the chassis BIOS\n"
" bios_version: '2.2.3'\n"
" # Mode of the default boot of hardware - bios, uefi\n"
" boot_mode: bios\n"
" # Protocol of boot of the hardware - pxe, usb, hdd\n"
" bootstrap_protocol: pxe\n"
" # Which interface to use for network booting within the OOB manager, not OS device\n"
" pxe_interface: 0\n"
" # Map hardware addresses to aliases/roles to allow a mix of hardware configs\n"
" # in a site to result in a consistent configuration\n"
" device_aliases:\n"
" pci:\n"
" - address: pci@0000:00:03.0\n"
" alias: prim_nic01\n"
" # type could identify expected hardware - used for hardware manifest validation\n"
" type: '82540EM Gigabit Ethernet Controller'\n"
" - address: pci@0000:00:04.0\n"
" alias: prim_nic02\n"
" type: '82540EM Gigabit Ethernet Controller'\n"
" scsi:\n"
" - address: scsi@2:0.0.0\n"
" alias: primary_boot\n"
" type: 'VBOX HARDDISK'\n")
hw_profile = yaml.load(yaml_snippet)
hw_profile_model = HardwareProfile(**hw_profile)
assert hasattr(hw_profile_model, 'bootstrap_protocol')

15
tox.ini Normal file
View File

@ -0,0 +1,15 @@
[tox]
envlist = py35
[testenv]
deps=
-rrequirements.txt
-rtestrequirements.txt
setenv=
PYTHONWARNING=all
commands=
py.test \
{posargs}
[flake8]
ignore=E302,H306