From 07cb34e82d30efd501e3fe513827acb527e41749 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Wed, 22 Feb 2017 17:32:55 -0600 Subject: [PATCH 1/9] Initial Python skeleton and the model classes to match the YAML schema --- helm_drydock/__init__.py | 13 + helm_drydock/config.py | 35 +++ helm_drydock/drivers/__init__.py | 19 ++ helm_drydock/drivers/server/__init__.py | 18 ++ .../drivers/server/maasdriver/__init__.py | 17 ++ helm_drydock/ingester/ingester.py | 72 +++++ helm_drydock/ingester/plugins/aicyaml.py | 55 ++++ .../ingester/plugins/ingester_plugin.py | 30 ++ helm_drydock/model/__init__.py | 286 ++++++++++++++++++ helm_drydock/model/__init__.pyc | Bin 0 -> 8583 bytes helm_drydock/tox.ini | 11 + requirements.txt | 7 + setup.py | 56 ++++ testrequirements.txt | 2 + .../test_models.cpython-35-PYTEST.pyc | Bin 0 -> 2697 bytes tests/test_models.py | 69 +++++ tests/test_models.pyc | Bin 0 -> 1985 bytes tox.ini | 15 + 18 files changed, 705 insertions(+) create mode 100644 helm_drydock/__init__.py create mode 100644 helm_drydock/config.py create mode 100644 helm_drydock/drivers/__init__.py create mode 100644 helm_drydock/drivers/server/__init__.py create mode 100644 helm_drydock/drivers/server/maasdriver/__init__.py create mode 100644 helm_drydock/ingester/ingester.py create mode 100644 helm_drydock/ingester/plugins/aicyaml.py create mode 100644 helm_drydock/ingester/plugins/ingester_plugin.py create mode 100644 helm_drydock/model/__init__.py create mode 100644 helm_drydock/model/__init__.pyc create mode 100644 helm_drydock/tox.ini create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 testrequirements.txt create mode 100644 tests/__pycache__/test_models.cpython-35-PYTEST.pyc create mode 100644 tests/test_models.py create mode 100644 tests/test_models.pyc create mode 100644 tox.ini diff --git a/helm_drydock/__init__.py b/helm_drydock/__init__.py new file mode 100644 index 00000000..2a385a45 --- /dev/null +++ b/helm_drydock/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/helm_drydock/config.py b/helm_drydock/config.py new file mode 100644 index 00000000..bf364d1f --- /dev/null +++ b/helm_drydock/config.py @@ -0,0 +1,35 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Read application configuration +# + +# configuration map with defaults + +class DrydockConfig(object): + + def __init__(self): + self.selected_server_driver = helm_drydock.drivers.server.maasdriver + self.selected_network_driver = helm_drydock.drivers.network.noopdriver + self.control_config = {} + self.ingester_config = { + plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester] + } + self.introspection_config = {} + self.orchestrator_config = {} + self.statemgmt_config = { + backend_driver = 'helm_drydock.drivers.statemgmt.etcd', + } diff --git a/helm_drydock/drivers/__init__.py b/helm_drydock/drivers/__init__.py new file mode 100644 index 00000000..5c73a1fa --- /dev/null +++ b/helm_drydock/drivers/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class ProviderDriver(object): + + __init__(self): + pass + diff --git a/helm_drydock/drivers/server/__init__.py b/helm_drydock/drivers/server/__init__.py new file mode 100644 index 00000000..febcdad1 --- /dev/null +++ b/helm_drydock/drivers/server/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import helm_drydock.drivers.ProviderDriver + +class ServerDriver(ProviderDriver): + \ No newline at end of file diff --git a/helm_drydock/drivers/server/maasdriver/__init__.py b/helm_drydock/drivers/server/maasdriver/__init__.py new file mode 100644 index 00000000..21cbcd26 --- /dev/null +++ b/helm_drydock/drivers/server/maasdriver/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import helm_drydock.drivers.server.ServerDriver + +class MaasServerDriver(object): + \ No newline at end of file diff --git a/helm_drydock/ingester/ingester.py b/helm_drydock/ingester/ingester.py new file mode 100644 index 00000000..ffdf2761 --- /dev/null +++ b/helm_drydock/ingester/ingester.py @@ -0,0 +1,72 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ingester - Ingest host topologies to define site design and +# persist design to helm-drydock's statemgmt service + +import logging +import yaml + +class Ingester(object): + + registered_plugins = {} + + def __init__(self): + logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s") + self.log = logging.Logger("ingester") + + """ + enable_plugins + + params: plugins - A list of class objects denoting the ingester plugins to be enabled + + Enable plugins that can be used for ingest_data calls. Each plugin should use + helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one + enabled plugin successfully initializes, the call is considered successful. Otherwise + it will throw an exception + """ + def enable_plugins(self, plugins=[]): + if len(plugin) == 0: + self.log.error("Cannot have an empty plugin list.") + + for plugin in plugins: + try: + new_plugin = plugin() + plugin_name = new_plugin.get_name() + registered_plugins[plugin_name] = new_plugin + except: + self.log.error("Could not enable plugin %s" % (plugin.__name__)) + + if len(registered_plugins) == 0: + self.log.error("Could not enable at least one plugin") + raise Exception("Could not enable at least one plugin") + + """ + ingest_data + + params: plugin_name - Which plugin should be used for ingestion + params: params - A map of parameters that will be passed to the plugin's ingest_data method + + Execute a data ingestion using the named plugin (assuming it is enabled) + """ + def ingest_data(self, plugin_name, params={}): + if plugin_name in registered_plugins: + design_data = registered_plugins[plugin_name].ingest_data(params) + # Need to persist data here, but we don't yet have the statemgmt service working + yaml.dump(design_data) + else + self.log.error("Could not find plugin %s to ingest data." % (plugin_name)) + raise LookupError("Could not find plugin %s" % plugin_name) + + diff --git a/helm_drydock/ingester/plugins/aicyaml.py b/helm_drydock/ingester/plugins/aicyaml.py new file mode 100644 index 00000000..c54c6eed --- /dev/null +++ b/helm_drydock/ingester/plugins/aicyaml.py @@ -0,0 +1,55 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# AIC YAML Ingester - This data ingester will consume a AIC YAML design +# file +# +import yaml +import logging + +import helm_drydock.ingester.plugins.IngesterPlugin + +class AicYamlIngester(IngesterPlugin): + + def __init__(self): + super(AicYamlIngester, self).__init__() + + def get_name(self): + return "aic_yaml" + + """ + AIC YAML ingester params + + filename - Absolute path to the YAML file to ingest + """ + def ingest_data(self, **kwargs): + if 'filename' in params: + input_string = read_input_file(params['filename']) + parsed_data = parse_input_data(input_string) + processed_data = compute_effective_data(parsed_data) + else: + + raise Exception('Missing parameter') + + return processed_data + + def read_input_file(self, filename): + try: + file = open(filename,'rt') + except OSError as err: + self.log.error("Error opening input file %s for ingestion: %s" % (filename, err)) + return {} + + diff --git a/helm_drydock/ingester/plugins/ingester_plugin.py b/helm_drydock/ingester/plugins/ingester_plugin.py new file mode 100644 index 00000000..8d4dc955 --- /dev/null +++ b/helm_drydock/ingester/plugins/ingester_plugin.py @@ -0,0 +1,30 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Plugins to parse incoming topology and translate it to helm-drydock's +# intermediate representation + +import logging + +class IngesterPlugin(object): + + def __init__(self): + self.log = logging.Logger('ingester') + return + + def get_data(self): + return "ingester_skeleton" + + def ingest_data(self, **kwargs): + return {} diff --git a/helm_drydock/model/__init__.py b/helm_drydock/model/__init__.py new file mode 100644 index 00000000..1be9d722 --- /dev/null +++ b/helm_drydock/model/__init__.py @@ -0,0 +1,286 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Models for helm_drydock +# + + +class HardwareProfile(object): + + def __init__(self, **kwargs): + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + # Need to add validation logic, we'll assume the input is + # valid for now + self.name = metadata.get('name', '') + self.region = metadata.get('region', '') + self.vendor = spec.get('vendor', '') + self.generation = spec.get('generation', '') + self.hw_version = spec.get('hw_version', '') + self.bios_version = spec.get('bios_version', '') + self.boot_mode = spec.get('boot_mode', '') + self.bootstrap_protocol = spec.get('bootstrap_protocol', '') + self.pxe_interface = spec.get('pxe_interface', '') + self.devices = [] + + device_aliases = spec.get('device_aliases', {}) + + pci_devices = device_aliases.get('pci', []) + scsi_devices = device_aliases.get('scsi', []) + + for d in pci_devices: + d['bus_type'] = 'pci' + self.devices.append( + HardwareDeviceAlias(self.api_version, **d)) + + for d in scsi_devices: + d['bus_type'] = 'scsi' + self.devices.append( + HardwareDeviceAlias(self.api_version, **d)) + else: + raise ValueError('Unknown API version of object') + + return + + +class HardwareDeviceAlias(object): + + def __init__(self, api_version, **kwargs): + self.api_version = api_version + + if self.api_version == "1.0": + self.bus_type = kwargs.get('bus_type', '') + self.address = kwargs.get('address', '') + self.alias = kwargs.get('alias', '') + self.type = kwargs.get('type', '') + else: + raise ValueError('Unknown API version of object') + + return + + +class Site(object): + + def __init__(self, **kwargs): + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "1.0": + metadata = kwargs.get('metadata', {}) + + # Need to add validation logic, we'll assume the input is + # valid for now + self.name = metadata.get('name', '') + + self.networks = [] + self.network_links = [] + self.host_profiles = [] + self.hardware_profiles = [] + self.baremetal_nodes = [] + + else: + raise ValueError('Unknown API version of object') + + +class NetworkLink(object): + + def __init__(self, **kwargs): + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + self.name = metadata.get('name', '') + self.region = metadata.get('region', '') + + bonding = spec.get('bonding', {}) + self.bonding_mode = bonding.get('mode', 'none') + + # TODO How should we define defaults for CIs not in the input? + if self.bonding_mode == '802.3ad': + self.bonding_xmit_hash = bonding.get('hash', 'layer3+4') + self.bonding_peer_rate = bonding.get('peer_rate', 'fast') + self.bonding_mon_rate = bonding.get('mon_rate', '') + self.bonding_up_delay = bonding.get('up_delay', '') + self.bonding_down_delay = bonding.get('down_delay', '') + + self.mtu = spec.get('mtu', 1500) + self.linkspeed = spec.get('linkspeed', 'auto') + + trunking = spec.get('trunking', {}) + self.trunk_mode = trunking.get('mode', 'none') + + self.native_network = spec.get('default_network', '') + else: + raise ValueError('Unknown API version of object') + + +class Network(object): + + def __init__(self, **kwargs): + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + self.name = metadata.get('name', '') + self.region = metadata.get('region', '') + self.cidr = spec.get('cidr', '') + self.allocation_strategy = spec.get('allocation', 'static') + self.vlan_id = spec.get('vlan_id', 1) + self.mtu = spec.get('mtu', 0) + + dns = spec.get('dns', {}) + self.dns_domain = dns.get('domain', 'local') + self.dns_servers = dns.get('servers', '') + + ranges = spec.get('ranges', []) + self.ranges = [] + + for r in ranges: + self.ranges.append(NetworkAddressRange(self.api_version, **r)) + + routes = spec.get('routes', []) + self.routes = [] + + for r in routes: + self.routes.append(NetworkRoute(self.api_version, **r)) + else: + raise ValueError('Unknown API version of object') + + +class NetworkAddressRange(object): + + def __init__(self, api_version, **kwargs): + self.api_version = api_version + + if self.api_version == "1.0": + self.type = kwargs.get('type', 'static') + self.start = kwargs.get('start', '') + self.end = kwargs.get('end', '') + else: + raise ValueError('Unknown API version of object') + + +class NetworkRoute(object): + + def __init__(self, api_version, **kwargs): + self.api_version = api_version + + if self.api_version == "1.0": + self.type = kwargs.get('subnet', '') + self.start = kwargs.get('gateway', '') + self.end = kwargs.get('metric', 100) + else: + raise ValueError('Unknown API version of object') + + +class HostProfile(object): + + def __init__(self, **kwargs): + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + self.name = metadata.get('name', '') + self.region = metadata.get('region', '') + + oob = spec.get('oob', {}) + self.oob_type = oob.get('type', 'ipmi') + self.oob_network = oob.get('network', 'oob') + self.oob_account = oob.get('account', '') + self.oob_credential = oob.get('credential', '') + + storage = spec.get('storage', {}) + self.storage_layout = storage.get('layout', 'lvm') + + bootdisk = storage.get('bootdisk', {}) + self.bootdisk_device = bootdisk.get('device', '') + self.bootdisk_root_size = bootdisk.get('root_size', '') + self.bootdisk_boot_size = bootdisk.get('boot_size', '') + + partitions = storage.get('partitions', []) + self.partitions = [] + + for p in partitions: + self.partitions.append(HostPartition(self.api_version, **p)) + + interfaces = spec.get('interfaces', []) + self.interfaces = [] + + for i in interfaces: + self.interfaces.append(HostInterface(self.api_version, **i)) + + else: + raise ValueError('Unknown API version of object') + + +class HostInterface(object): + + def __init__(self, api_version, **kwargs): + self.api_version = api_version + + if self.api_version == "1.0": + self.device_name = kwargs.get('device_name', '') + self.network_link = kwargs.get('device_link', '') + + self.hardware_slaves = [] + slaves = kwargs.get('slaves', []) + + for s in slaves: + self.hardware_slaves.append(s) + + self.networks = [] + networks = kwargs.get('networks', []) + + for n in networks: + self.networks.append(n) + + else: + raise ValueError('Unknown API version of object') + + +class HostPartition(object): + + def __init__(self, api_version, **kwargs): + self.api_version = api_version + + if self.api_version == "1.0": + self.name = kwargs.get('name', '') + self.device = kwargs.get('device', '') + self.part_uuid = kwargs.get('part_uuid', '') + self.size = kwargs.get('size', '') + self.mountpoint = kwargs.get('mountpoint', '') + self.fstype = kwargs.get('fstype', 'ext4') + self.mount_options = kwargs.get('mount_options', 'defaults') + self.fs_uuid = kwargs.get('fs_uuid', '') + self.fs_label = kwargs.get('fs_label', '') + else: + raise ValueError('Unknown API version of object') + + +# A BaremetalNode is really nothing more than a physical +# instantiation of a HostProfile, so they both represent +# the same set of CIs +class BaremetalNode(HostProfile): + + def __init__(self, **kwargs): + super(BaremetalNode, self).__init__() diff --git a/helm_drydock/model/__init__.pyc b/helm_drydock/model/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76853f3aee2eec228fcea1efaf93be5298859a61 GIT binary patch literal 8583 zcmc&(O>7)V6|Nq8#-6bq$BsQo7VTn^J$SQAw2%mav_P_(l{N&-QV|J6M7{0lw$t{w zd(ze8IJ=G(HXsg&BX=%H2x*TiBqS~fj@&tLLqZ%7mz6m4eXqJ|CW+0AEh21nP50~S z*Y)$>_rCXf;?ud-tAGFadmUB%P2l$m4)bp`u~Lar8-a=g^<$+TEA{<5xTrblgqPM- zJn5u$FP%_v!%3&SbW+9BPTKU+x{6y)I^(4c70){9oR?0i_=Jmo`fq@8mtf( zMN!M0n$kB{dAq}A8b=++4IJiAkn9I)3(G2N__(I_l?ns4TaZks?V2h+393GmD#eP| z)FX7WPt~1dLhaW?4_2qfKGcKO3b8m|h@KPD6KmszlTL`G@xr+K*!@i1@~_RxA)#s`wjot!dQ=e6x`0B5W_!3Js-(7 z-&|e7cCM8zv{5g;l@un;vl3Uz(0jgc@%54$zmt?v9F|In+2mt9 z-A%GMFGS&Hk|jk{@(%Uidl2qc`e$*up6132niS-D8Sdn9QZ80krYxdf*emif@8n&q zg;Kq}Buuk1DK?@`g2})DlijqFgi$w*Okzroy4OkVKszRt;ji~iSU&6}=0)87ZnmA} z53-f3?_OW22DFlItmNxknAR#?MU-qNWjTWjEQ=earu}QqQctnr%4iNh_bjiy|*pso2gS>2AofZo_pqjppn$_ds*% zGCoW;bg$vd4JPdcbli9qjgrQ2h#?j*EyM6)@1ZP?yI#gU^Ck8TmTU9DVz4;596bHs ztS#1Xr#2U~f@W~K`hB4`TU+JWDupzRFqBz^+|qs*mzs+|7|VeJ(tJG^$<)v^YPyY| z4i5R_Y(_y9pW{e)2vmKD6MzoARD9K97U3xulo9*_U_Ajn5CYk~(1H|%0Cq2ghXbS_ z1af&{UD3%Pfxix%Y#L)&a}K2mw3_1-6M}HeoSi>Gvs<{`i#}Q!n4n@ zImd$q!9vQN;2HCQ`1s4=|D+>A9% zBbD9t`^`ZMbc_K%f+n(;vQab&@u@5+ALPZh5fH2{!fu*Xl6$!+36zZNrlf)0tHS+2 zN@QCH0U(xl!wf)co;D8Jg<#AG_94i;~)V05Tm*5f&f2-F*K)#Tvd2p3 z$<1j;ZZM>}kelJT%(`dK483ShJOv{^JM1YhU~3Nb6bWxW(h5ed=UJR)n}Qp{5ZEWn zv&7KnzWn+ZFJ6jbxqB}%_YC2-8$C>lORruo=~2BTDMC<@MEYzX;EGf2njQV9R%F3djZF7+HvpCs^`rR^gZaW_DupsUo zUzkQIT*=PgvLkwTwIJc%4k9_{EI#j~gXt2$oz%~ESmrN^H3T1!j^zm#hL(dfPF6c( ziN_HG(kplgLB-*XK)1!4BV!GXhvq6+^Q*D2<`>TkYX)u)WU=l&?4mv~LJlH#plOJxRi5*5 zaJ$m8#JCGFa->SnoZ9;evz}{ry`DMIbK+S&U7PsKt9W@BgUCt7#UR2>Cyi}d6Lq_J z$EGzjc7v=a?FdZmcB3pzI?pm$j2`4k?!%^Or*a=8l8F#;NXijL+rG(JNi z47h(ngaAboS;-2n_x0*K8* z>dx7G7J~G`3nwMOg311ughTRxw=e>+I~Z(H+Wdo-iF|zw6BAgM$Ry-cG=2w9Yt81E zImU1BtfAQqa*X4_Irrz+RA%X;CO`eZ?U&y~zhBrLfi)uF82cs8Z~E&X4}&yu6Hfa8 z?1UQu24AGWBhGj8?lTIPxvnA>at{b#9gt|YO4pc(q z0zM-~E?|2u4GED8ScS{a#N=6t?Nv<1ixJOf)z%zKj#X@zc}fkl@Ugw^^fX;;pO@I) zw6Pt7fS!5LgYu~52$DXDp5NKnzM$gM!?B(GbKKZY49xR&O-PgUAnk!vPvNQ}?{Xe> zI(a`UB~*5bBu=t2bCj^q$v4bEU}G zxrWs|nd-RtRW(Onyycw3!6Z2%-__4n6QwP;V9it<-_2bU-=y9_@pV`9wZVqtJ&;!1 z=8Ye)LAg(fejyk=tBzbF9YHhxh(3ln&l5Dm(u;~>NIr59-8feD;}%}^2fnllUm4Di z9KL9=wB;LA^k*~&7d?MJif#;ruZnB~zigpX3SmUF3AOhU^NVYD+X8QJOMsc6m$+rD zZdcC&Zd{QR5t!y==uk-)u@zErp!>KKT3dOE6x<}g_tiCH^^+aTORRNrFPJp&XJhewsro&AEPJDU{gbLId)vYJk?oDP=I!ktxH#C}{^AVa zFnT*=Z{ExXMA+NjlSTHbCyc~D?kx>v*`|ws2$R(Vs&Js!=coz|(?+O4eEo35h0hJyByjF`SiMc#-x)l!@kJOfa* z1uBH`BMjNk;A{317njfUdr6^xgfsj7gL($xXoOe4=%%28tY_70bV?)BN8F4gWKyyuHB7;H8*@2%5->~2EzD8yt_5b#T_Q`g; ReXf1JJ%|4b?X&Ine*pbW#A*Nl literal 0 HcmV?d00001 diff --git a/helm_drydock/tox.ini b/helm_drydock/tox.ini new file mode 100644 index 00000000..ae414018 --- /dev/null +++ b/helm_drydock/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = py35 + +[testenv] +deps= + -rrequirements.txt +setenv= + PYTHONWARNING=all + +[flake8] +ignore=E302,H306 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..9c39a3da --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +PyYAML +oauth +requests-oauthlib +pyipmi +netaddr +pecan +python-libmaas \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..870b0bb0 --- /dev/null +++ b/setup.py @@ -0,0 +1,56 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# helm_drydock - A tool to consume a host topology and orchestrate +# and monitor the provisioning of those hosts and execution of bootstrap +# scripts +# +# Modular services: +# smelter - A service to consume the host topology, will support multiple +# input formats. Initially supports a YAML schema as demonstrated +# in the examples folder +# tarot - A service for persisting the host topology and orchestration state +# and making the data available via API +# cockpit - The entrypoint API for users to control helm-drydock and query +# current state +# alchemist - The core orchestrator +# drivers - A tree with all of the plugins that alchemist uses to execute +# orchestrated tasks +# jabberwocky - An introspection API that newly provisioned nodes can use to +# ingest self-data and bootstrap their application deployment process + +from setuptools import setup + +setup(name='helm_drydock', + version='0.1a1', + description='Bootstrapper for Kubernetes infrastructure', + url='http://github.com/att-comdev/drydock', + author='Scott Hussey - AT&T', + author_email='sh8121@att.com', + license='Apache 2.0', + packages=['helm_drydock', + 'helm_drydock.model', + 'helm_drydock.ingester'], + install_requires=[ + 'PyYAML', + 'oauth', + 'requests-oauthlib', + 'pyipmi', + 'netaddr', + 'pecan' + ], + dependency_link=[ + 'git+https://github.com/maas/python-libmaas.git' + ] + ) \ No newline at end of file diff --git a/testrequirements.txt b/testrequirements.txt new file mode 100644 index 00000000..642edbdd --- /dev/null +++ b/testrequirements.txt @@ -0,0 +1,2 @@ +pytest +tox \ No newline at end of file diff --git a/tests/__pycache__/test_models.cpython-35-PYTEST.pyc b/tests/__pycache__/test_models.cpython-35-PYTEST.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c65a57e221401247636b14740eef8cc2781f4ff0 GIT binary patch literal 2697 zcmb7G&2HO95MELeZP7}cI!&7PP%ID>wgBtbaS{k^;W%-VpoksBK^o}7uE~|PHbt_# zD?4)ROWhuO>!GjEOJAUG&`034r<{5$&{JoYq$4Npp%l5=ot^n+=G$2=uV1?4-mL!i zvUQ1&KgrZphW;8n>RV_6xGfSA(zlAfP4-G;uT1)tLgJ9VQ}ky@f5!A}63&uQmGrB` zdKi>QP$oYBSebB_oR&y`4uQApl{espY}T!G{t=VGA(Q;6j0Ym*K28*u4V^W3)Dkqj z2&Zo&NMIv!A6yh(`ehPS0NhEd&$-g~L#C9E5P0E#37-4#s868LCdWOC99v}1CZ`tO z7G%C%w>OWjJR3)mh<2$4B>K92QZJ>IM2bkK&LEZ}rqfx^i`a;JUfoX1iid;L8F4*~ zgI|fPf`flqo$-E|Wcw!Zfh%6vK)c!m7^_?U210a!Sa)L)4E(VHOcV zU;bR^n_&blXGu^cfqUNS_uL9XL90J0h>cMn!T zAo;EVP(pK@1QFFL)HhlEwq7hny-d54fcM5DHl+P&hEt7@EoQ#1O;gC46sMr#?KR$ybX~B zf#gs-3f-|#FzB{q92SBwKBSC}#H(}U_;EB4yI4p!Az)CzdPW4mLN<*Il%Uw4&Na>S z$61MiRTiL^`@@QbSbdRo8Z-|IsPV;G8?GMSD@}|aFJlL@QjEMv`0Y+Eze)8ZfeMMo zVL(Lyp2c87`778J8tP?gU4YoYFncBg4#fap(FcGIdotT)=nXNk=)0@8+8a;kL$S+t zgr*z7zJ!IL_v1(dVC)`x4(cY=j${jN2~DmfXzogTS_LYb>+U!JM;@9YS+~gN;-A5 zYy>NHRd-MNrDU>#%_=sX^a^ZjD1JPmv!e>d?Q{m-O-yT9pG(UVHVV^n7_&g)fh8|P zle%lk1W!4S$+YSzHsF#cGOdBw+Zl^ci%6wT82c<#=?zZ}u?jrY@pi*_hlS>lO)uXw zNx<@MNEyrYs+XO@9+XK4`v@1lJ|po2?AI_5Peme0xbC}e1DEj81D_!1I^(1}J-C$( zGThtd2+tZkQqvDj5=GtROx%DOwEzvVoTYiIW;{jU0XY_OQOlbhBQO4g$L# z@;L@ty|m_e-;G(AeOAahn3m{bDi-{(kQk6GL30%z1qU6e*|n09)NAQ%L94{+^kXku zfM%95@YIz2P{Phnt7oEgCYQ+%(8kG3`TN^7Y}F5T#a*+p2EU&pS~ctb?V>O4=BUB5<6zaZor8tiVluSVJLGIQ#pLH>s$uI{tU?_{SJB{A!{Q z;$$(eiIdvg9*L8>$ReRvL=IzJ{4T_`5Wk$<s+yG*C_=anx(_a>@e(F|nt3e2f9`zt#i^aaI#oHF1sCNKhoMqA;%us*9_@ z?K%?vdqFGWw}9PiGR{l?Cm>s`Rzr=}h0oTb){lF;t!`)Q(IT#=hND8uL6tye zIya>u0Ki3!X<%I}d#NRo`(y9*wBbiO<-F_iSeKb~(&mx|T6*m+w3F#Td2c)%F2XWX z3*O+Tonqv9MCB@nybti{FFazoU1bPTyf;8j3=K>v4f$J2k%p3{Y`wg-EGe>xT@xap{5!aBu_0kL8k?eRtVpe zW5~@|SWj4nIM@Rh>^0fc792N^XPs|hj4m}G{OP6~WLX8{$*EB?s)Az)p8)ssz@!6d zO42S@sg`6Bsb!80EHzE6I}ce**`Eu`$qo;9+)=BCdjv<|{>;7Vo$V6mFP1}Bsm zDxBoZe>^KLM9l~$RX(m&a81o@(vlS^NRyhK4z8Gg4{+Gn5mnDTvFgk)DNWkhs_g5M zrX$pc9T%B288VxED)lAK1EE@O?F)p>g0c%6G&DInjl2LlPUGx+aW=+`<;HgRyN8{< z1Nq$a)u|!52lX8`Mn1JAL9s%+jlknB{txfu->vxn-%LLB?`p=zoo?J|<9=5!<)#VO z>T<^8-NQF>|H;v_XD@m`Z8Yc$bUgN2F4g8en9&!7n5tnBCWTcQ-G^m=x&P4zu%XW= zB1fNM{sc9KQ)|mT=d25TJ^*-K#PkFj=>?5>PQ zjQJ%VA7NOHnydID9>50Nj@V4!U>N9Pm}G96+4Q^}8f*J)@mP=BZUM)ho}r9Gp8uv3 TzUYV9H$3M3@l0i{hCBKf_g^yN literal 0 HcmV?d00001 diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..c92d6f00 --- /dev/null +++ b/tox.ini @@ -0,0 +1,15 @@ +[tox] +envlist = py35 + +[testenv] +deps= + -rrequirements.txt + -rtestrequirements.txt +setenv= + PYTHONWARNING=all +commands= + py.test \ + {posargs} + +[flake8] +ignore=E302,H306 \ No newline at end of file From 2356bbeb0f698fecc7b7752e3d371ff7ac770ed5 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Wed, 22 Feb 2017 17:35:46 -0600 Subject: [PATCH 2/9] Remove compiled python --- helm_drydock/model/__init__.pyc | Bin 8583 -> 0 bytes .../test_models.cpython-35-PYTEST.pyc | Bin 2697 -> 0 bytes tests/test_models.pyc | Bin 1985 -> 0 bytes 3 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 helm_drydock/model/__init__.pyc delete mode 100644 tests/__pycache__/test_models.cpython-35-PYTEST.pyc delete mode 100644 tests/test_models.pyc diff --git a/helm_drydock/model/__init__.pyc b/helm_drydock/model/__init__.pyc deleted file mode 100644 index 76853f3aee2eec228fcea1efaf93be5298859a61..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8583 zcmc&(O>7)V6|Nq8#-6bq$BsQo7VTn^J$SQAw2%mav_P_(l{N&-QV|J6M7{0lw$t{w zd(ze8IJ=G(HXsg&BX=%H2x*TiBqS~fj@&tLLqZ%7mz6m4eXqJ|CW+0AEh21nP50~S z*Y)$>_rCXf;?ud-tAGFadmUB%P2l$m4)bp`u~Lar8-a=g^<$+TEA{<5xTrblgqPM- zJn5u$FP%_v!%3&SbW+9BPTKU+x{6y)I^(4c70){9oR?0i_=Jmo`fq@8mtf( zMN!M0n$kB{dAq}A8b=++4IJiAkn9I)3(G2N__(I_l?ns4TaZks?V2h+393GmD#eP| z)FX7WPt~1dLhaW?4_2qfKGcKO3b8m|h@KPD6KmszlTL`G@xr+K*!@i1@~_RxA)#s`wjot!dQ=e6x`0B5W_!3Js-(7 z-&|e7cCM8zv{5g;l@un;vl3Uz(0jgc@%54$zmt?v9F|In+2mt9 z-A%GMFGS&Hk|jk{@(%Uidl2qc`e$*up6132niS-D8Sdn9QZ80krYxdf*emif@8n&q zg;Kq}Buuk1DK?@`g2})DlijqFgi$w*Okzroy4OkVKszRt;ji~iSU&6}=0)87ZnmA} z53-f3?_OW22DFlItmNxknAR#?MU-qNWjTWjEQ=earu}QqQctnr%4iNh_bjiy|*pso2gS>2AofZo_pqjppn$_ds*% zGCoW;bg$vd4JPdcbli9qjgrQ2h#?j*EyM6)@1ZP?yI#gU^Ck8TmTU9DVz4;596bHs ztS#1Xr#2U~f@W~K`hB4`TU+JWDupzRFqBz^+|qs*mzs+|7|VeJ(tJG^$<)v^YPyY| z4i5R_Y(_y9pW{e)2vmKD6MzoARD9K97U3xulo9*_U_Ajn5CYk~(1H|%0Cq2ghXbS_ z1af&{UD3%Pfxix%Y#L)&a}K2mw3_1-6M}HeoSi>Gvs<{`i#}Q!n4n@ zImd$q!9vQN;2HCQ`1s4=|D+>A9% zBbD9t`^`ZMbc_K%f+n(;vQab&@u@5+ALPZh5fH2{!fu*Xl6$!+36zZNrlf)0tHS+2 zN@QCH0U(xl!wf)co;D8Jg<#AG_94i;~)V05Tm*5f&f2-F*K)#Tvd2p3 z$<1j;ZZM>}kelJT%(`dK483ShJOv{^JM1YhU~3Nb6bWxW(h5ed=UJR)n}Qp{5ZEWn zv&7KnzWn+ZFJ6jbxqB}%_YC2-8$C>lORruo=~2BTDMC<@MEYzX;EGf2njQV9R%F3djZF7+HvpCs^`rR^gZaW_DupsUo zUzkQIT*=PgvLkwTwIJc%4k9_{EI#j~gXt2$oz%~ESmrN^H3T1!j^zm#hL(dfPF6c( ziN_HG(kplgLB-*XK)1!4BV!GXhvq6+^Q*D2<`>TkYX)u)WU=l&?4mv~LJlH#plOJxRi5*5 zaJ$m8#JCGFa->SnoZ9;evz}{ry`DMIbK+S&U7PsKt9W@BgUCt7#UR2>Cyi}d6Lq_J z$EGzjc7v=a?FdZmcB3pzI?pm$j2`4k?!%^Or*a=8l8F#;NXijL+rG(JNi z47h(ngaAboS;-2n_x0*K8* z>dx7G7J~G`3nwMOg311ughTRxw=e>+I~Z(H+Wdo-iF|zw6BAgM$Ry-cG=2w9Yt81E zImU1BtfAQqa*X4_Irrz+RA%X;CO`eZ?U&y~zhBrLfi)uF82cs8Z~E&X4}&yu6Hfa8 z?1UQu24AGWBhGj8?lTIPxvnA>at{b#9gt|YO4pc(q z0zM-~E?|2u4GED8ScS{a#N=6t?Nv<1ixJOf)z%zKj#X@zc}fkl@Ugw^^fX;;pO@I) zw6Pt7fS!5LgYu~52$DXDp5NKnzM$gM!?B(GbKKZY49xR&O-PgUAnk!vPvNQ}?{Xe> zI(a`UB~*5bBu=t2bCj^q$v4bEU}G zxrWs|nd-RtRW(Onyycw3!6Z2%-__4n6QwP;V9it<-_2bU-=y9_@pV`9wZVqtJ&;!1 z=8Ye)LAg(fejyk=tBzbF9YHhxh(3ln&l5Dm(u;~>NIr59-8feD;}%}^2fnllUm4Di z9KL9=wB;LA^k*~&7d?MJif#;ruZnB~zigpX3SmUF3AOhU^NVYD+X8QJOMsc6m$+rD zZdcC&Zd{QR5t!y==uk-)u@zErp!>KKT3dOE6x<}g_tiCH^^+aTORRNrFPJp&XJhewsro&AEPJDU{gbLId)vYJk?oDP=I!ktxH#C}{^AVa zFnT*=Z{ExXMA+NjlSTHbCyc~D?kx>v*`|ws2$R(Vs&Js!=coz|(?+O4eEo35h0hJyByjF`SiMc#-x)l!@kJOfa* z1uBH`BMjNk;A{317njfUdr6^xgfsj7gL($xXoOe4=%%28tY_70bV?)BN8F4gWKyyuHB7;H8*@2%5->~2EzD8yt_5b#T_Q`g; ReXf1JJ%|4b?X&Ine*pbW#A*Nl diff --git a/tests/__pycache__/test_models.cpython-35-PYTEST.pyc b/tests/__pycache__/test_models.cpython-35-PYTEST.pyc deleted file mode 100644 index c65a57e221401247636b14740eef8cc2781f4ff0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2697 zcmb7G&2HO95MELeZP7}cI!&7PP%ID>wgBtbaS{k^;W%-VpoksBK^o}7uE~|PHbt_# zD?4)ROWhuO>!GjEOJAUG&`034r<{5$&{JoYq$4Npp%l5=ot^n+=G$2=uV1?4-mL!i zvUQ1&KgrZphW;8n>RV_6xGfSA(zlAfP4-G;uT1)tLgJ9VQ}ky@f5!A}63&uQmGrB` zdKi>QP$oYBSebB_oR&y`4uQApl{espY}T!G{t=VGA(Q;6j0Ym*K28*u4V^W3)Dkqj z2&Zo&NMIv!A6yh(`ehPS0NhEd&$-g~L#C9E5P0E#37-4#s868LCdWOC99v}1CZ`tO z7G%C%w>OWjJR3)mh<2$4B>K92QZJ>IM2bkK&LEZ}rqfx^i`a;JUfoX1iid;L8F4*~ zgI|fPf`flqo$-E|Wcw!Zfh%6vK)c!m7^_?U210a!Sa)L)4E(VHOcV zU;bR^n_&blXGu^cfqUNS_uL9XL90J0h>cMn!T zAo;EVP(pK@1QFFL)HhlEwq7hny-d54fcM5DHl+P&hEt7@EoQ#1O;gC46sMr#?KR$ybX~B zf#gs-3f-|#FzB{q92SBwKBSC}#H(}U_;EB4yI4p!Az)CzdPW4mLN<*Il%Uw4&Na>S z$61MiRTiL^`@@QbSbdRo8Z-|IsPV;G8?GMSD@}|aFJlL@QjEMv`0Y+Eze)8ZfeMMo zVL(Lyp2c87`778J8tP?gU4YoYFncBg4#fap(FcGIdotT)=nXNk=)0@8+8a;kL$S+t zgr*z7zJ!IL_v1(dVC)`x4(cY=j${jN2~DmfXzogTS_LYb>+U!JM;@9YS+~gN;-A5 zYy>NHRd-MNrDU>#%_=sX^a^ZjD1JPmv!e>d?Q{m-O-yT9pG(UVHVV^n7_&g)fh8|P zle%lk1W!4S$+YSzHsF#cGOdBw+Zl^ci%6wT82c<#=?zZ}u?jrY@pi*_hlS>lO)uXw zNx<@MNEyrYs+XO@9+XK4`v@1lJ|po2?AI_5Peme0xbC}e1DEj81D_!1I^(1}J-C$( zGThtd2+tZkQqvDj5=GtROx%DOwEzvVoTYiIW;{jU0XY_OQOlbhBQO4g$L# z@;L@ty|m_e-;G(AeOAahn3m{bDi-{(kQk6GL30%z1qU6e*|n09)NAQ%L94{+^kXku zfM%95@YIz2P{Phnt7oEgCYQ+%(8kG3`TN^7Y}F5T#a*+p2EU&pS~ctb?V>O4=BUB5<6zaZor8tiVluSVJLGIQ#pLH>s$uI{tU?_{SJB{A!{Q z;$$(eiIdvg9*L8>$ReRvL=IzJ{4T_`5Wk$<s+yG*C_=anx(_a>@e(F|nt3e2f9`zt#i^aaI#oHF1sCNKhoMqA;%us*9_@ z?K%?vdqFGWw}9PiGR{l?Cm>s`Rzr=}h0oTb){lF;t!`)Q(IT#=hND8uL6tye zIya>u0Ki3!X<%I}d#NRo`(y9*wBbiO<-F_iSeKb~(&mx|T6*m+w3F#Td2c)%F2XWX z3*O+Tonqv9MCB@nybti{FFazoU1bPTyf;8j3=K>v4f$J2k%p3{Y`wg-EGe>xT@xap{5!aBu_0kL8k?eRtVpe zW5~@|SWj4nIM@Rh>^0fc792N^XPs|hj4m}G{OP6~WLX8{$*EB?s)Az)p8)ssz@!6d zO42S@sg`6Bsb!80EHzE6I}ce**`Eu`$qo;9+)=BCdjv<|{>;7Vo$V6mFP1}Bsm zDxBoZe>^KLM9l~$RX(m&a81o@(vlS^NRyhK4z8Gg4{+Gn5mnDTvFgk)DNWkhs_g5M zrX$pc9T%B288VxED)lAK1EE@O?F)p>g0c%6G&DInjl2LlPUGx+aW=+`<;HgRyN8{< z1Nq$a)u|!52lX8`Mn1JAL9s%+jlknB{txfu->vxn-%LLB?`p=zoo?J|<9=5!<)#VO z>T<^8-NQF>|H;v_XD@m`Z8Yc$bUgN2F4g8en9&!7n5tnBCWTcQ-G^m=x&P4zu%XW= zB1fNM{sc9KQ)|mT=d25TJ^*-K#PkFj=>?5>PQ zjQJ%VA7NOHnydID9>50Nj@V4!U>N9Pm}G96+4Q^}8f*J)@mP=BZUM)ho}r9Gp8uv3 TzUYV9H$3M3@l0i{hCBKf_g^yN From 5051b8bf9e2cfe10ad5dc1c12872f5b2b563895f Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Thu, 23 Feb 2017 12:20:50 -0600 Subject: [PATCH 3/9] File parsing in AIC YAML ingester and corresponding test cases --- helm_drydock/config.py | 10 +- .../{ingester_plugin.py => __init__.py} | 14 +-- helm_drydock/ingester/plugins/aicyaml.py | 96 ++++++++++++++----- setup.py | 3 +- tests/aicyaml_samples/invalid.yaml | 29 ++++++ tests/aicyaml_samples/multidoc.yaml | 77 +++++++++++++++ tests/aicyaml_samples/singledoc.yaml | 39 ++++++++ tests/aicyaml_samples/unknown_kind.yaml | 60 ++++++++++++ tests/test_ingester_aicyaml.py | 54 +++++++++++ 9 files changed, 346 insertions(+), 36 deletions(-) rename helm_drydock/ingester/plugins/{ingester_plugin.py => __init__.py} (79%) create mode 100644 tests/aicyaml_samples/invalid.yaml create mode 100644 tests/aicyaml_samples/multidoc.yaml create mode 100644 tests/aicyaml_samples/singledoc.yaml create mode 100644 tests/aicyaml_samples/unknown_kind.yaml create mode 100644 tests/test_ingester_aicyaml.py diff --git a/helm_drydock/config.py b/helm_drydock/config.py index bf364d1f..8bd363bc 100644 --- a/helm_drydock/config.py +++ b/helm_drydock/config.py @@ -22,7 +22,13 @@ class DrydockConfig(object): def __init__(self): - self.selected_server_driver = helm_drydock.drivers.server.maasdriver + self.server_driver_config = { + selected_driver = helm_drydock.drivers.server.maasdriver, + params = { + maas_api_key = "" + maas_api_url = "" + } + } self.selected_network_driver = helm_drydock.drivers.network.noopdriver self.control_config = {} self.ingester_config = { @@ -31,5 +37,5 @@ class DrydockConfig(object): self.introspection_config = {} self.orchestrator_config = {} self.statemgmt_config = { - backend_driver = 'helm_drydock.drivers.statemgmt.etcd', + backend_driver = helm_drydock.drivers.statemgmt.etcd, } diff --git a/helm_drydock/ingester/plugins/ingester_plugin.py b/helm_drydock/ingester/plugins/__init__.py similarity index 79% rename from helm_drydock/ingester/plugins/ingester_plugin.py rename to helm_drydock/ingester/plugins/__init__.py index 8d4dc955..7281f22e 100644 --- a/helm_drydock/ingester/plugins/ingester_plugin.py +++ b/helm_drydock/ingester/plugins/__init__.py @@ -19,12 +19,12 @@ import logging class IngesterPlugin(object): - def __init__(self): - self.log = logging.Logger('ingester') - return + def __init__(self): + self.log = logging.Logger('ingester') + return - def get_data(self): - return "ingester_skeleton" + def get_data(self): + return "ingester_skeleton" - def ingest_data(self, **kwargs): - return {} + def ingest_data(self, **kwargs): + return {} diff --git a/helm_drydock/ingester/plugins/aicyaml.py b/helm_drydock/ingester/plugins/aicyaml.py index c54c6eed..2da7bba2 100644 --- a/helm_drydock/ingester/plugins/aicyaml.py +++ b/helm_drydock/ingester/plugins/aicyaml.py @@ -14,42 +14,86 @@ # # AIC YAML Ingester - This data ingester will consume a AIC YAML design -# file -# +# file +# import yaml import logging -import helm_drydock.ingester.plugins.IngesterPlugin +import helm_drydock.model as model +from helm_drydock.ingester.plugins import IngesterPlugin class AicYamlIngester(IngesterPlugin): - def __init__(self): - super(AicYamlIngester, self).__init__() + kind_map = { + "Region": model.Site, + "NetworkLink": model.NetworkLink, + "HardwareProfile": model.HardwareProfile, + "Network": model.Network, + "HostProfile": model.HostProfile, + "BaremetalNode": model.BaremetalNode, + } - def get_name(self): - return "aic_yaml" + def __init__(self): + super(AicYamlIngester, self).__init__() - """ - AIC YAML ingester params + def get_name(self): + return "aic_yaml" - filename - Absolute path to the YAML file to ingest - """ - def ingest_data(self, **kwargs): - if 'filename' in params: - input_string = read_input_file(params['filename']) - parsed_data = parse_input_data(input_string) - processed_data = compute_effective_data(parsed_data) - else: + """ + AIC YAML ingester params - raise Exception('Missing parameter') - - return processed_data + filenames - Array of absolute path to the YAML files to ingest - def read_input_file(self, filename): - try: - file = open(filename,'rt') - except OSError as err: - self.log.error("Error opening input file %s for ingestion: %s" % (filename, err)) - return {} + returns an array of objects from helm_drydock.model + + """ + def ingest_data(self, **kwargs): + if 'filenames' in kwargs: + # TODO validate filenames is array + for f in kwargs.get('filenames'): + try: + file = open(f,'rt') + contents = file.read() + file.close() + except OSError as err: + self.log.error( + "Error opening input file %s for ingestion: %s" + % (filename, err)) + continue + + try: + parsed_data = yaml.load_all(contents) + except yaml.YAMLError as err: + self.log.error("Error parsing YAML in %s: %s" % (f, err)) + continue + + models = [] + for d in parsed_data: + kind = d.get('kind', '') + if kind != '': + if kind in AicYamlIngester.kind_map: + try: + model = AicYamlIngester.kind_map[kind](**d) + models.append(model) + except Exception as err: + self.log.error("Error building model %s: %s" + % (kind, str(err))) + else: + self.log.error( + "Error processing document, unknown kind %s" + % (kind)) + continue + else: + self.log.error( + "Error processing document in %s, no kind field" + % (f)) + continue + + return models + else: + raise ValueError('Missing parameter "filename"') + + return processed_data + diff --git a/setup.py b/setup.py index 870b0bb0..5107af4e 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,8 @@ setup(name='helm_drydock', license='Apache 2.0', packages=['helm_drydock', 'helm_drydock.model', - 'helm_drydock.ingester'], + 'helm_drydock.ingester', + 'helm_drydock.ingester.plugins'], install_requires=[ 'PyYAML', 'oauth', diff --git a/tests/aicyaml_samples/invalid.yaml b/tests/aicyaml_samples/invalid.yaml new file mode 100644 index 00000000..ef525833 --- /dev/null +++ b/tests/aicyaml_samples/invalid.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: +: Network +metadata: + name: public + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + vlan_id: '102' + # How are addresses assigned? + allocation: static + # MTU size for the VLAN interface + mtu: 1500 + cidr: 172.16.3.0/24 + # Desribe IP address ranges + ranges: + - type: static + start: 172.16.3.15 + end: 172.16.3.254 + routes: + - subnet: 0.0.0.0/0 + gateway: 172.16.3.1 + metric: 9 + dns: + domain: sitename.example.com + servers: 8.8.8.8 \ No newline at end of file diff --git a/tests/aicyaml_samples/multidoc.yaml b/tests/aicyaml_samples/multidoc.yaml new file mode 100644 index 00000000..693afd1d --- /dev/null +++ b/tests/aicyaml_samples/multidoc.yaml @@ -0,0 +1,77 @@ +--- +apiVersion: '1.0' +kind: NetworkLink +metadata: + name: oob + region: sitename + date: 17-FEB-2017 + name: Sample network link + author: sh8121@att.com + description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on +spec: + bonding: + mode: none + mtu: 1500 + linkspeed: 100full + trunking: + mode: none + default_network: oob +--- +# pxe is a bit of 'magic' indicating the link config used when PXE booting +# a node. All other links indicate network configs applied when the node +# is deployed. +apiVersion: '1.0' +kind: NetworkLink +metadata: + name: pxe + region: sitename + date: 17-FEB-2017 + name: Sample network link + author: sh8121@att.com + description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on +spec: + bonding: + mode: none + mtu: 1500 + linkspeed: auto + # Is this link supporting multiple layer 2 networks? + # none is a port-based VLAN identified by default_network + # tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr + trunking: + mode: none + # use name, will translate to VLAN ID + default_network: pxe +--- +apiVersion: '1.0' +kind: NetworkLink +metadata: + name: gp + region: sitename + date: 17-FEB-2017 + name: Sample network link + author: sh8121@att.com + description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on + # pxe is a bit of 'magic' indicating the link config used when PXE booting + # a node. All other links indicate network configs applied when the node + # is deployed. +spec: + # If this link is a bond of physical links, how is it configured + # 802.3ad + # active-backup + # balance-rr + # Can add support for others down the road + bonding: + mode: 802.3ad + # For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4 + hash: layer3+4 + # 802.3ad specific options + peer_rate: slow + mon_rate: default + up_delay: default + down_delay: default + mtu: 9000 + linkspeed: auto + # Is this link supporting multiple layer 2 networks? + trunking: + mode: tagged + default_network: mgmt \ No newline at end of file diff --git a/tests/aicyaml_samples/singledoc.yaml b/tests/aicyaml_samples/singledoc.yaml new file mode 100644 index 00000000..8ad16094 --- /dev/null +++ b/tests/aicyaml_samples/singledoc.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: '1.0' +kind: HardwareProfile +metadata: + name: HPGen8v3 + region: sitename + date: 17-FEB-2017 + name: Sample hardware definition + author: Scott Hussey +spec: + # Vendor of the server chassis + vendor: HP + # Generation of the chassis model + generation: '8' + # Version of the chassis model within its generation - not version of the hardware definition + hw_version: '3' + # The certified version of the chassis BIOS + bios_version: '2.2.3' + # Mode of the default boot of hardware - bios, uefi + boot_mode: bios + # Protocol of boot of the hardware - pxe, usb, hdd + bootstrap_protocol: pxe + # Which interface to use for network booting within the OOB manager, not OS device + pxe_interface: 0 + # Map hardware addresses to aliases/roles to allow a mix of hardware configs + # in a site to result in a consistent configuration + device_aliases: + pci: + - address: pci@0000:00:03.0 + alias: prim_nic01 + # type could identify expected hardware - used for hardware manifest validation + type: '82540EM Gigabit Ethernet Controller' + - address: pci@0000:00:04.0 + alias: prim_nic02 + type: '82540EM Gigabit Ethernet Controller' + scsi: + - address: scsi@2:0.0.0 + alias: primary_boot + type: 'VBOX HARDDISK' \ No newline at end of file diff --git a/tests/aicyaml_samples/unknown_kind.yaml b/tests/aicyaml_samples/unknown_kind.yaml new file mode 100644 index 00000000..bd39bfc0 --- /dev/null +++ b/tests/aicyaml_samples/unknown_kind.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: '1.0' +kind: FooBar +metadata: + name: default + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces + # No magic to this host_profile, it just provides a way to specify + # sitewide settings. If it is absent from a node's inheritance chain + # then these values will NOT be applied +spec: + # OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such + # as IPMI over vender-specific when possible. + oob: + type: ipmi + # OOB networking should be preconfigured, but we can include a network + # definition for validation or enhancement (DNS registration) + network: oob + account: admin + credential: admin + # Specify storage layout of base OS. Ceph out of scope + storage: + # How storage should be carved up: lvm (logical volumes), flat + # (single partition) + layout: lvm + # Info specific to the boot and root disk/partitions + bootdisk: + # Device will specify an alias defined in hwdefinition.yaml + device: primary_boot + # For LVM, the size of the partition added to VG as a PV + # For flat, the size of the partition formatted as ext4 + root_size: 50g + # The /boot partition. If not specified, /boot will in root + boot_size: 2g + # Info for additional partitions. Need to balance between + # flexibility and complexity + partitions: + - name: logs + device: primary_boot + # Partition uuid if needed + part_uuid: 84db9664-f45e-11e6-823d-080027ef795a + size: 10g + # Optional, can carve up unformatted block devices + mountpoint: /var/log + fstype: ext4 + mount_options: defaults + # Filesystem UUID or label can be specified. UUID recommended + fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e + fs_label: logs + # Platform (Operating System) settings + platform: + image: ubuntu_16.04_hwe + kernel_params: default + # Additional metadata to apply to a node + metadata: + # Base URL of the introspection service - may go in curtin data + introspection_url: http://172.16.1.10:9090 \ No newline at end of file diff --git a/tests/test_ingester_aicyaml.py b/tests/test_ingester_aicyaml.py new file mode 100644 index 00000000..0827bc8b --- /dev/null +++ b/tests/test_ingester_aicyaml.py @@ -0,0 +1,54 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from helm_drydock.ingester.plugins.aicyaml import AicYamlIngester +import pytest +import shutil +import os + +class TestClass(object): + + def setup_method(self, method): + print("Running test {0}".format(method.__name__)) + + def test_ingest_singledoc(self, input_files): + input_file = input_files.join("singledoc.yaml") + + ingester = AicYamlIngester() + + models = ingester.ingest_data(filenames=[str(input_file)]) + + assert len(models) == 1 + + def test_ingest_multidoc(self, input_files): + input_file = input_files.join("multidoc.yaml") + + ingester = AicYamlIngester() + + models = ingester.ingest_data(filenames=[str(input_file)]) + + assert len(models) == 3 + + @pytest.fixture(scope='module') + def input_files(self, tmpdir_factory, request): + tmpdir = tmpdir_factory.mktemp('data') + samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples" + samples = os.listdir(samples_dir) + + for f in samples: + src_file = samples_dir + "/" + f + dst_file = str(tmpdir) + "/" + f + shutil.copyfile(src_file, dst_file) + + return tmpdir \ No newline at end of file From 5a3e9c78c013e06968f160471c496ade93f0cd93 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Thu, 23 Feb 2017 13:45:29 -0600 Subject: [PATCH 4/9] Quote numerical strings for correct deserialization typing --- examples/bootstrap_hwdefinition.yaml | 8 +++---- examples/bootstrap_seed.yaml | 36 ++++++++++++++-------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/examples/bootstrap_hwdefinition.yaml b/examples/bootstrap_hwdefinition.yaml index 65ba5d82..aaff67be 100644 --- a/examples/bootstrap_hwdefinition.yaml +++ b/examples/bootstrap_hwdefinition.yaml @@ -18,7 +18,7 @@ ############################################################################# # version the schema in this file so consumers can rationally parse it --- -apiVersion: 1.0 +apiVersion: '1.0' kind: HardwareProfile metadata: name: HPGen8v3 @@ -30,11 +30,11 @@ spec: # Vendor of the server chassis vendor: HP # Generation of the chassis model - generation: 8 + generation: '8' # Version of the chassis model within its generation - not version of the hardware definition - hw_version: 3 + hw_version: '3' # The certified version of the chassis BIOS - bios_version: 2.2.3 + bios_version: '2.2.3' # Mode of the default boot of hardware - bios, uefi boot_mode: bios # Protocol of boot of the hardware - pxe, usb, hdd diff --git a/examples/bootstrap_seed.yaml b/examples/bootstrap_seed.yaml index 4d7fc257..9c97a040 100644 --- a/examples/bootstrap_seed.yaml +++ b/examples/bootstrap_seed.yaml @@ -18,7 +18,7 @@ #################### # version the schema in this file so consumers can rationally parse it --- -apiVersion: 1.0 +apiVersion: '1.0' kind: Region metadata: name: sitename @@ -28,7 +28,7 @@ metadata: spec: # Not sure if we have site wide data that doesn't fall into another 'Kind' --- -apiVersion: 1.0 +apiVersion: '1.0' kind: NetworkLink metadata: name: oob @@ -49,7 +49,7 @@ spec: # pxe is a bit of 'magic' indicating the link config used when PXE booting # a node. All other links indicate network configs applied when the node # is deployed. -apiVersion: 1.0 +apiVersion: '1.0' kind: NetworkLink metadata: name: pxe @@ -71,7 +71,7 @@ spec: # use name, will translate to VLAN ID default_network: pxe --- -apiVersion: 1.0 +apiVersion: '1.0' kind: NetworkLink metadata: name: gp @@ -105,7 +105,7 @@ spec: mode: tagged default_network: mgmt --- -apiVersion: v1 +apiVersion: '1.0' kind: Network metadata: name: oob @@ -125,7 +125,7 @@ spec: domain: ilo.sitename.att.com servers: 172.16.100.10 --- -apiVersion: v1 +apiVersion: '1.0' kind: Network metadata: name: pxe @@ -136,7 +136,7 @@ metadata: description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: # Layer 2 VLAN segment id, could support other segmentations. Optional - vlan_id: 99 + vlan_id: '99' # How are addresses assigned? allocation: dhcp # MTU for this VLAN interface, if not specified it will be inherited from the link @@ -155,7 +155,7 @@ spec: # DNS servers that a server using this network as its default gateway should use servers: 172.16.0.10 --- -apiVersion: 1.0 +apiVersion: '1.0' kind: Network metadata: name: mgmt @@ -165,7 +165,7 @@ metadata: author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: - vlan_id: 100 + vlan_id: '100' # How are addresses assigned? allocation: static # Allow MTU to be inherited from link the network rides on @@ -191,7 +191,7 @@ spec: # DNS servers that a server using this network as its default gateway should use servers: 172.16.1.9,172.16.1.10 --- -apiVersion: 1.0 +apiVersion: '1.0' kind: Network metadata: name: private @@ -201,7 +201,7 @@ metadata: author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: - vlan_id: 101 + vlan_id: '101' allocation: static mtu: 9000 cidr: 172.16.2.0/24 @@ -216,7 +216,7 @@ spec: domain: priv.sitename.example.com servers: 172.16.2.9,172.16.2.10 --- -apiVersion: 1.0 +apiVersion: '1.0' kind: Network metadata: name: public @@ -226,7 +226,7 @@ metadata: author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: - vlan_id: 102 + vlan_id: '102' # How are addresses assigned? allocation: static # MTU size for the VLAN interface @@ -245,7 +245,7 @@ spec: domain: sitename.example.com servers: 8.8.8.8 --- -apiVersion: 1.0 +apiVersion: '1.0' kind: HostProfile metadata: name: default @@ -305,7 +305,7 @@ spec: # Base URL of the introspection service - may go in curtin data introspection_url: http://172.16.1.10:9090 --- -apiVersion: 1.0 +apiVersion: '1.0' kind: HostProfile metadata: name: k8-node @@ -360,7 +360,7 @@ spec: owner_data: foo: bar --- -apiVersion: 1.0 +apiVersion: '1.0' kind: HostProfile metadata: name: k8-node-public @@ -378,7 +378,7 @@ spec: # inheritance chain - name: public --- -apiVersion: 1.0 +apiVersion: '1.0' kind: BaremetalNode metadata: name: controller01 @@ -412,7 +412,7 @@ spec: roles: os_ctl rack: rack01 --- -apiVersion: 1.0 +apiVersion: '1.0' kind: BaremetalNode metadata: name: compute01 From edfd5c82014cbdd2a22fa4bcc0561f267d9e9166 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Thu, 23 Feb 2017 13:46:33 -0600 Subject: [PATCH 5/9] Begin Ingester service and provide skeleton statemgmt service --- helm_drydock/ingester/__init__.py | 92 ++++++ helm_drydock/statemgmt/__init__.py | 64 ++++ tests/aicyaml_samples/fullsite.yaml | 471 ++++++++++++++++++++++++++++ tests/test_ingester.py | 50 +++ 4 files changed, 677 insertions(+) create mode 100644 helm_drydock/ingester/__init__.py create mode 100644 helm_drydock/statemgmt/__init__.py create mode 100644 tests/aicyaml_samples/fullsite.yaml create mode 100644 tests/test_ingester.py diff --git a/helm_drydock/ingester/__init__.py b/helm_drydock/ingester/__init__.py new file mode 100644 index 00000000..44823078 --- /dev/null +++ b/helm_drydock/ingester/__init__.py @@ -0,0 +1,92 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ingester - Ingest host topologies to define site design and +# persist design to helm-drydock's statemgmt service + +import logging +import yaml +import helm_drydock.model as model + +from helm_drydock.statemgmt import DesignState + + +class Ingester(object): + + registered_plugins = {} + + def __init__(self): + logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s") + self.log = logging.Logger("ingester") + + """ + enable_plugins + + params: plugins - A list of class objects denoting the ingester plugins to be enabled + + Enable plugins that can be used for ingest_data calls. Each plugin should use + helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one + enabled plugin successfully initializes, the call is considered successful. Otherwise + it will throw an exception + """ + def enable_plugins(self, plugins=[]): + if len(plugin) == 0: + self.log.error("Cannot have an empty plugin list.") + + for plugin in plugins: + try: + new_plugin = plugin() + plugin_name = new_plugin.get_name() + registered_plugins[plugin_name] = new_plugin + except: + self.log.error("Could not enable plugin %s" % (plugin.__name__)) + + if len(registered_plugins) == 0: + self.log.error("Could not enable at least one plugin") + raise Exception("Could not enable at least one plugin") + + """ + ingest_data + + params: plugin_name - Which plugin should be used for ingestion + params: params - A map of parameters that will be passed to the plugin's ingest_data method + + Execute a data ingestion using the named plugin (assuming it is enabled) + """ + def ingest_data(self, plugin_name, design_state=None, **kwargs): + if design_state is None: + self.log.error("ingest_data called without valid DesignState handler") + raise Exception("Invalid design_state handler") + + if plugin_name in registered_plugins: + design_data = registered_plugins[plugin_name].ingest_data(kwargs) + # Need to persist data here, but we don't yet have the statemgmt service working + for m in design_data: + if type(m) is model.Site: + design_state.add_site(m) + elif type(m) is model.Network: + design_state.add_network(m) + elif type(m) is model.NetworkLink: + design_state.add_network_link(m) + elif type(m) is model.HostProfile: + design_state.add_host_profile(m) + elif type(m) is model.HardwareProfile: + design_state.add_hardware_profile(m) + elif type(m) is model.BaremetalNode: + design_state.add_baremetal_node(m) + else + self.log.error("Could not find plugin %s to ingest data." % (plugin_name)) + raise LookupError("Could not find plugin %s" % plugin_name) + + diff --git a/helm_drydock/statemgmt/__init__.py b/helm_drydock/statemgmt/__init__.py new file mode 100644 index 00000000..1fba0192 --- /dev/null +++ b/helm_drydock/statemgmt/__init__.py @@ -0,0 +1,64 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import helm_drydock.model as model + +class DesignState(object): + + def __init__(self): + self.sites = [] + self.networks = [] + self.network_links = [] + self.host_profiles = [] + self.hardware_profiles = [] + self.baremetal_nodes = [] + return + + def add_site(self, new_site): + if new_site is None or not isinstance(new_site, model.Site): + raise Exception("Invalid Site model") + + self.sites.append(new_site) + + def add_network(self, new_network): + if new_network is None or not isinstance(new_network, model.Network): + raise Exception("Invalid Network model") + + self.networks.append(new_network) + + def add_network_link(self, new_network_link): + if new_network_link is None or not isinstance(new_network_link, model.NetworkLink): + raise Exception("Invalid NetworkLink model") + + self.network_links.append(new_network_link) + + def add_host_profile(self, new_host_profile): + if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile): + raise Exception("Invalid HostProfile model") + + self.host_profiles.append(new_host_profile) + + def add_hardware_profile(self, new_hardware_profile): + if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile): + raise Exception("Invalid HardwareProfile model") + + self.hardware_profiles.append(new_hardware_profile) + + def add_baremetal_node(self, new_baremetal_node): + if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode): + raise Exception("Invalid BaremetalNode model") + + self.baremetal_nodes.append(new_baremetal_node) + + diff --git a/tests/aicyaml_samples/fullsite.yaml b/tests/aicyaml_samples/fullsite.yaml new file mode 100644 index 00000000..92e275ce --- /dev/null +++ b/tests/aicyaml_samples/fullsite.yaml @@ -0,0 +1,471 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#################### +# +# bootstrap_seed.yaml - Site server design definition for physical layer +# +#################### +# version the schema in this file so consumers can rationally parse it +--- +apiVersion: '1.0' +kind: Region +metadata: + name: sitename + date: 17-FEB-2017 + name: Sample site design + author: sh8121@att.com +spec: + # Not sure if we have site wide data that doesn't fall into another 'Kind' +--- +apiVersion: '1.0' +kind: NetworkLink +metadata: + name: oob + region: sitename + date: 17-FEB-2017 + name: Sample network link + author: sh8121@att.com + description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on +spec: + bonding: + mode: none + mtu: 1500 + linkspeed: 100full + trunking: + mode: none + default_network: oob +--- +# pxe is a bit of 'magic' indicating the link config used when PXE booting +# a node. All other links indicate network configs applied when the node +# is deployed. +apiVersion: '1.0' +kind: NetworkLink +metadata: + name: pxe + region: sitename + date: 17-FEB-2017 + name: Sample network link + author: sh8121@att.com + description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on +spec: + bonding: + mode: none + mtu: 1500 + linkspeed: auto + # Is this link supporting multiple layer 2 networks? + # none is a port-based VLAN identified by default_network + # tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr + trunking: + mode: none + # use name, will translate to VLAN ID + default_network: pxe +--- +apiVersion: '1.0' +kind: NetworkLink +metadata: + name: gp + region: sitename + date: 17-FEB-2017 + name: Sample network link + author: sh8121@att.com + description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on + # pxe is a bit of 'magic' indicating the link config used when PXE booting + # a node. All other links indicate network configs applied when the node + # is deployed. +spec: + # If this link is a bond of physical links, how is it configured + # 802.3ad + # active-backup + # balance-rr + # Can add support for others down the road + bonding: + mode: 802.3ad + # For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4 + hash: layer3+4 + # 802.3ad specific options + peer_rate: slow + mon_rate: default + up_delay: default + down_delay: default + mtu: 9000 + linkspeed: auto + # Is this link supporting multiple layer 2 networks? + trunking: + mode: tagged + default_network: mgmt +--- +apiVersion: '1.0' +kind: Network +metadata: + name: oob + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + allocation: static + cidr: 172.16.100.0/24 + ranges: + - type: static + start: 172.16.100.15 + end: 172.16.100.254 + dns: + domain: ilo.sitename.att.com + servers: 172.16.100.10 +--- +apiVersion: '1.0' +kind: Network +metadata: + name: pxe + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + # Layer 2 VLAN segment id, could support other segmentations. Optional + vlan_id: '99' + # How are addresses assigned? + allocation: dhcp + # MTU for this VLAN interface, if not specified it will be inherited from the link + mtu: 1500 + # Network address + cidr: 172.16.0.0/24 + # Desribe IP address ranges + ranges: + - type: dhcp + start: 172.16.0.5 + end: 172.16.0.254 + # DNS settings for this network + dns: + # Domain addresses on this network will be registered under + domain: admin.sitename.att.com + # DNS servers that a server using this network as its default gateway should use + servers: 172.16.0.10 +--- +apiVersion: '1.0' +kind: Network +metadata: + name: mgmt + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + vlan_id: '100' + # How are addresses assigned? + allocation: static + # Allow MTU to be inherited from link the network rides on + mtu: 1500 + # Network address + cidr: 172.16.1.0/24 + # Desribe IP address ranges + ranges: + - type: static + start: 172.16.1.15 + end: 172.16.1.254 + # Static routes to be added for this network + routes: + - subnet: 0.0.0.0/0 + # A blank gateway would leave to a static route specifying + # only the interface as a source + gateway: 172.16.1.1 + metric: 10 + # DNS settings for this network + dns: + # Domain addresses on this network will be registered under + domain: mgmt.sitename.example.com + # DNS servers that a server using this network as its default gateway should use + servers: 172.16.1.9,172.16.1.10 +--- +apiVersion: '1.0' +kind: Network +metadata: + name: private + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + vlan_id: '101' + allocation: static + mtu: 9000 + cidr: 172.16.2.0/24 + # Desribe IP address ranges + ranges: + # Type can be reserved (not used for baremetal), static (all explicit + # assignments should fall here), dhcp (will be used by a DHCP server on this network) + - type: static + start: 172.16.2.15 + end: 172.16.2.254 + dns: + domain: priv.sitename.example.com + servers: 172.16.2.9,172.16.2.10 +--- +apiVersion: '1.0' +kind: Network +metadata: + name: public + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + vlan_id: '102' + # How are addresses assigned? + allocation: static + # MTU size for the VLAN interface + mtu: 1500 + cidr: 172.16.3.0/24 + # Desribe IP address ranges + ranges: + - type: static + start: 172.16.3.15 + end: 172.16.3.254 + routes: + - subnet: 0.0.0.0/0 + gateway: 172.16.3.1 + metric: 9 + dns: + domain: sitename.example.com + servers: 8.8.8.8 +--- +apiVersion: '1.0' +kind: HostProfile +metadata: + name: default + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces + # No magic to this host_profile, it just provides a way to specify + # sitewide settings. If it is absent from a node's inheritance chain + # then these values will NOT be applied +spec: + # OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such + # as IPMI over vender-specific when possible. + oob: + type: ipmi + # OOB networking should be preconfigured, but we can include a network + # definition for validation or enhancement (DNS registration) + network: oob + account: admin + credential: admin + # Specify storage layout of base OS. Ceph out of scope + storage: + # How storage should be carved up: lvm (logical volumes), flat + # (single partition) + layout: lvm + # Info specific to the boot and root disk/partitions + bootdisk: + # Device will specify an alias defined in hwdefinition.yaml + device: primary_boot + # For LVM, the size of the partition added to VG as a PV + # For flat, the size of the partition formatted as ext4 + root_size: 50g + # The /boot partition. If not specified, /boot will in root + boot_size: 2g + # Info for additional partitions. Need to balance between + # flexibility and complexity + partitions: + - name: logs + device: primary_boot + # Partition uuid if needed + part_uuid: 84db9664-f45e-11e6-823d-080027ef795a + size: 10g + # Optional, can carve up unformatted block devices + mountpoint: /var/log + fstype: ext4 + mount_options: defaults + # Filesystem UUID or label can be specified. UUID recommended + fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e + fs_label: logs + # Platform (Operating System) settings + platform: + image: ubuntu_16.04_hwe + kernel_params: default + # Additional metadata to apply to a node + metadata: + # Base URL of the introspection service - may go in curtin data + introspection_url: http://172.16.1.10:9090 +--- +apiVersion: '1.0' +kind: HostProfile +metadata: + name: k8-node + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + # host_profile inheritance allows for deduplication of common CIs + # Inheritance is additive for CIs that are lists of multiple items + # To remove an inherited list member, prefix the primary key value + # with '!'. + host_profile: defaults + # Hardware profile will map hardware specific details to the abstract + # names uses in the host profile as well as specify hardware specific + # configs. A viable model should be to build a host profile without a + # hardware_profile and then for each node inherit the host profile and + # specify a hardware_profile to map that node's hardware to the abstract + # settings of the host_profile + hardware_profile: HPGen9v3 + # Network interfaces. + interfaces: + # Keyed on device_name + # pxe is a special marker indicating which device should be used for pxe boot + - device_name: pxe + # The network link attached to this + network_link: pxe + # Slaves will specify aliases from hwdefinition.yaml + slaves: + - prim_nic01 + # Which networks will be configured on this interface + networks: + - name: pxe + - device_name: bond0 + network_link: gp + # If multiple slaves are specified, but no bonding config + # is applied to the link, design validation will fail + slaves: + - prim_nic01 + - prim_nic02 + # If multiple networks are specified, but no trunking + # config is applied to the link, design validation will fail + networks: + - name: mgmt + - name: private + metadata: + # Explicit tag assignment + tags: + - 'test' + # MaaS supports key/value pairs. Not sure of the use yet + owner_data: + foo: bar +--- +apiVersion: '1.0' +kind: HostProfile +metadata: + name: k8-node-public + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + host_profile: k8-node + interfaces: + - device_name: bond0 + networks: + # This is additive, so adds a network to those defined in the host_profile + # inheritance chain + - name: public +--- +apiVersion: '1.0' +kind: BaremetalNode +metadata: + name: controller01 + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + host_profile: k8-node-public + # the hostname for a server, could be used in multiple DNS domains to + # represent different interfaces + interfaces: + - device_name: bond0 + networks: + # '!' prefix for the value of the primary key indicates a record should be removed + - name: '!private' + # Addresses assigned to network interfaces + addressing: + # Which network the address applies to. If a network appears in addressing + # that isn't assigned to an interface, design validation will fail + - network: pxe + # The address assigned. Either a explicit IPv4 or IPv6 address + # or dhcp or slaac + address: dhcp + - network: mgmt + address: 172.16.1.20 + - network: public + address: 172.16.3.20 + metadata: + roles: os_ctl + rack: rack01 +--- +apiVersion: '1.0' +kind: BaremetalNode +metadata: + name: compute01 + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + host_profile: k8-node + addressing: + - network: pxe + address: dhcp + - network: mgmt + address: 172.16.1.21 + - network: private + address: 172.16.2.21 +--- +apiVersion: '1.0' +kind: HardwareProfile +metadata: + name: HPGen8v3 + region: sitename + date: 17-FEB-2017 + name: Sample hardware definition + author: Scott Hussey +spec: + # Vendor of the server chassis + vendor: HP + # Generation of the chassis model + generation: '8' + # Version of the chassis model within its generation - not version of the hardware definition + hw_version: '3' + # The certified version of the chassis BIOS + bios_version: '2.2.3' + # Mode of the default boot of hardware - bios, uefi + boot_mode: bios + # Protocol of boot of the hardware - pxe, usb, hdd + bootstrap_protocol: pxe + # Which interface to use for network booting within the OOB manager, not OS device + pxe_interface: 0 + # Map hardware addresses to aliases/roles to allow a mix of hardware configs + # in a site to result in a consistent configuration + device_aliases: + pci: + - address: pci@0000:00:03.0 + alias: prim_nic01 + # type could identify expected hardware - used for hardware manifest validation + type: '82540EM Gigabit Ethernet Controller' + - address: pci@0000:00:04.0 + alias: prim_nic02 + type: '82540EM Gigabit Ethernet Controller' + scsi: + - address: scsi@2:0.0.0 + alias: primary_boot + type: 'VBOX HARDDISK' \ No newline at end of file diff --git a/tests/test_ingester.py b/tests/test_ingester.py new file mode 100644 index 00000000..fc5c3ed0 --- /dev/null +++ b/tests/test_ingester.py @@ -0,0 +1,50 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from helm_drydock.ingester import Ingester +from helm_drydock.statemgmt import DesignState + +import pytest +import shutil +import os + +class TestClass(object): + + def setup_method(self, method): + print("Running test {0}".format(method.__name__)) + + def test_ingest_full_site(self, design_state,input_files): + input_file = input_files.join("fullsite.yaml") + + ingester = Ingester() + ingester.enable_plugins([helm_drydock.ingester.plugins.aicyaml.AicYamlPlugin]) + ingester.ingest_data('aic_yaml', design_state=design_state, filenames=) + + @pytest.fixture(scope='module') + def input_files(self, tmpdir_factory, request): + tmpdir = tmpdir_factory.mktemp('data') + samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples" + samples = os.listdir(samples_dir) + + for f in samples: + src_file = samples_dir + "/" + f + dst_file = str(tmpdir) + "/" + f + shutil.copyfile(src_file, dst_file) + + return tmpdir + + @pytest.fixture(scope='session') + def design_state(self): + design_state = DesignState() + return design_state \ No newline at end of file From c63f9fcf0eb0520eb43983bf1d31cf2279dab50d Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Thu, 23 Feb 2017 13:47:30 -0600 Subject: [PATCH 6/9] Add packages to setup.py --- helm_drydock/ingester/ingester.py | 72 ------------------------------- setup.py | 3 +- 2 files changed, 2 insertions(+), 73 deletions(-) delete mode 100644 helm_drydock/ingester/ingester.py diff --git a/helm_drydock/ingester/ingester.py b/helm_drydock/ingester/ingester.py deleted file mode 100644 index ffdf2761..00000000 --- a/helm_drydock/ingester/ingester.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ingester - Ingest host topologies to define site design and -# persist design to helm-drydock's statemgmt service - -import logging -import yaml - -class Ingester(object): - - registered_plugins = {} - - def __init__(self): - logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s") - self.log = logging.Logger("ingester") - - """ - enable_plugins - - params: plugins - A list of class objects denoting the ingester plugins to be enabled - - Enable plugins that can be used for ingest_data calls. Each plugin should use - helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one - enabled plugin successfully initializes, the call is considered successful. Otherwise - it will throw an exception - """ - def enable_plugins(self, plugins=[]): - if len(plugin) == 0: - self.log.error("Cannot have an empty plugin list.") - - for plugin in plugins: - try: - new_plugin = plugin() - plugin_name = new_plugin.get_name() - registered_plugins[plugin_name] = new_plugin - except: - self.log.error("Could not enable plugin %s" % (plugin.__name__)) - - if len(registered_plugins) == 0: - self.log.error("Could not enable at least one plugin") - raise Exception("Could not enable at least one plugin") - - """ - ingest_data - - params: plugin_name - Which plugin should be used for ingestion - params: params - A map of parameters that will be passed to the plugin's ingest_data method - - Execute a data ingestion using the named plugin (assuming it is enabled) - """ - def ingest_data(self, plugin_name, params={}): - if plugin_name in registered_plugins: - design_data = registered_plugins[plugin_name].ingest_data(params) - # Need to persist data here, but we don't yet have the statemgmt service working - yaml.dump(design_data) - else - self.log.error("Could not find plugin %s to ingest data." % (plugin_name)) - raise LookupError("Could not find plugin %s" % plugin_name) - - diff --git a/setup.py b/setup.py index 5107af4e..caa39b96 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,8 @@ setup(name='helm_drydock', packages=['helm_drydock', 'helm_drydock.model', 'helm_drydock.ingester', - 'helm_drydock.ingester.plugins'], + 'helm_drydock.ingester.plugins', + 'helm_drydock.statemgmt'], install_requires=[ 'PyYAML', 'oauth', From f3c2a7e2de0abf3d2d065c0d1553c4cd89769ce8 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Thu, 23 Feb 2017 16:08:00 -0600 Subject: [PATCH 7/9] Complete Ingester basic test cases Begin orchestrator site design data management --- examples/bootstrap_seed.yaml | 3 +- helm_drydock/ingester/__init__.py | 18 +++-- helm_drydock/ingester/plugins/__init__.py | 2 +- helm_drydock/model/__init__.py | 27 ++++++-- helm_drydock/orchestrator/designdata.py | 80 +++++++++++++++++++++++ helm_drydock/statemgmt/__init__.py | 58 ++++++++++++++++ tests/aicyaml_samples/fullsite.yaml | 12 ++-- tests/test_ingester.py | 7 +- 8 files changed, 183 insertions(+), 24 deletions(-) create mode 100644 helm_drydock/orchestrator/designdata.py diff --git a/examples/bootstrap_seed.yaml b/examples/bootstrap_seed.yaml index 9c97a040..f5c20c14 100644 --- a/examples/bootstrap_seed.yaml +++ b/examples/bootstrap_seed.yaml @@ -409,7 +409,8 @@ spec: - network: public address: 172.16.3.20 metadata: - roles: os_ctl + tags: + - os_ctl rack: rack01 --- apiVersion: '1.0' diff --git a/helm_drydock/ingester/__init__.py b/helm_drydock/ingester/__init__.py index 44823078..beae66a3 100644 --- a/helm_drydock/ingester/__init__.py +++ b/helm_drydock/ingester/__init__.py @@ -21,14 +21,12 @@ import helm_drydock.model as model from helm_drydock.statemgmt import DesignState - class Ingester(object): - registered_plugins = {} - def __init__(self): logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s") self.log = logging.Logger("ingester") + self.registered_plugins = {} """ enable_plugins @@ -41,18 +39,18 @@ class Ingester(object): it will throw an exception """ def enable_plugins(self, plugins=[]): - if len(plugin) == 0: + if len(plugins) == 0: self.log.error("Cannot have an empty plugin list.") for plugin in plugins: try: new_plugin = plugin() plugin_name = new_plugin.get_name() - registered_plugins[plugin_name] = new_plugin + self.registered_plugins[plugin_name] = new_plugin except: self.log.error("Could not enable plugin %s" % (plugin.__name__)) - if len(registered_plugins) == 0: + if len(self.registered_plugins) == 0: self.log.error("Could not enable at least one plugin") raise Exception("Could not enable at least one plugin") @@ -64,13 +62,13 @@ class Ingester(object): Execute a data ingestion using the named plugin (assuming it is enabled) """ - def ingest_data(self, plugin_name, design_state=None, **kwargs): + def ingest_data(self, plugin_name='', design_state=None, **kwargs): if design_state is None: self.log.error("ingest_data called without valid DesignState handler") raise Exception("Invalid design_state handler") - if plugin_name in registered_plugins: - design_data = registered_plugins[plugin_name].ingest_data(kwargs) + if plugin_name in self.registered_plugins: + design_data = self.registered_plugins[plugin_name].ingest_data(**kwargs) # Need to persist data here, but we don't yet have the statemgmt service working for m in design_data: if type(m) is model.Site: @@ -85,7 +83,7 @@ class Ingester(object): design_state.add_hardware_profile(m) elif type(m) is model.BaremetalNode: design_state.add_baremetal_node(m) - else + else: self.log.error("Could not find plugin %s to ingest data." % (plugin_name)) raise LookupError("Could not find plugin %s" % plugin_name) diff --git a/helm_drydock/ingester/plugins/__init__.py b/helm_drydock/ingester/plugins/__init__.py index 7281f22e..cf92a0e9 100644 --- a/helm_drydock/ingester/plugins/__init__.py +++ b/helm_drydock/ingester/plugins/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. # # Plugins to parse incoming topology and translate it to helm-drydock's -# intermediate representation +# model representation import logging diff --git a/helm_drydock/model/__init__.py b/helm_drydock/model/__init__.py index 1be9d722..a6d32570 100644 --- a/helm_drydock/model/__init__.py +++ b/helm_drydock/model/__init__.py @@ -28,7 +28,7 @@ class HardwareProfile(object): # Need to add validation logic, we'll assume the input is # valid for now self.name = metadata.get('name', '') - self.region = metadata.get('region', '') + self.site = metadata.get('region', '') self.vendor = spec.get('vendor', '') self.generation = spec.get('generation', '') self.hw_version = spec.get('hw_version', '') @@ -106,7 +106,7 @@ class NetworkLink(object): spec = kwargs.get('spec', {}) self.name = metadata.get('name', '') - self.region = metadata.get('region', '') + self.site = metadata.get('region', '') bonding = spec.get('bonding', {}) self.bonding_mode = bonding.get('mode', 'none') @@ -140,7 +140,7 @@ class Network(object): spec = kwargs.get('spec', {}) self.name = metadata.get('name', '') - self.region = metadata.get('region', '') + self.site = metadata.get('region', '') self.cidr = spec.get('cidr', '') self.allocation_strategy = spec.get('allocation', 'static') self.vlan_id = spec.get('vlan_id', 1) @@ -201,7 +201,7 @@ class HostProfile(object): spec = kwargs.get('spec', {}) self.name = metadata.get('name', '') - self.region = metadata.get('region', '') + self.site = metadata.get('region', '') oob = spec.get('oob', {}) self.oob_type = oob.get('type', 'ipmi') @@ -229,9 +229,28 @@ class HostProfile(object): for i in interfaces: self.interfaces.append(HostInterface(self.api_version, **i)) + metadata = spec.get('metadata', {}) + + metadata_tags = metadata.get('tags', []) + self.tags = [] + + for t in metadata_tags: + self.tags.append(t) + + owner_data = metadata.get('owner_data', {}) + self.owner_data = {} + + for k, v in owner_data.items(): + self.owner_data[k] = v + + self.rack = metadata.get('rack', '') + else: raise ValueError('Unknown API version of object') + def inherit_parent(self, site): + + def apply_hardware_profile(self, site): class HostInterface(object): diff --git a/helm_drydock/orchestrator/designdata.py b/helm_drydock/orchestrator/designdata.py new file mode 100644 index 00000000..fe7a268a --- /dev/null +++ b/helm_drydock/orchestrator/designdata.py @@ -0,0 +1,80 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +class DesignStateClient(object): + + def __init__(self): + self.log = logging.Logger('orchestrator') + + """ + load_design_data - Pull all the defined models in statemgmt and assemble + them into a representation of the site. Does not compute inheritance. + Throws an exception if multiple Site models are found. + + param design_state - Instance of statemgmt.DesignState to load data from + + return a Site model populated with all components from the design state + """ + + def load_design_data(self, design_state=None): + if len(design_state.get_sites()) != 1: + self.log.error("Invalid design state, should only have 1 Site model") + raise Exception("Invalid design state, should only have 1 Site model") + + site = design_state.get_sites()[0] + site_name = site.name + + networks = design_state.get_networks() + + for n in networks: + if n.site == site_name: + site.networks.append(n) + + network_links = design_state.get_network_links() + + for l in network_links: + if l.site == site_name: + site.network_links.append(l) + + host_profiles = design_state.get_host_profiles() + + for p in host_profiles: + if p.site == site_name: + site.host_profiles.append(p) + + hardware_profiles = design_state.get_hardware_profiles() + + for p in hardware_profiles: + if p.site == site_name: + site.hardware_profiles.append(p) + + baremetal_nodes = design_state.get_baremetal_nodes() + + for n in baremetal_nodes: + if n.site == site_name: + site.baremetal_nodes.append(n) + + + return site + + """ + compute_model_inheritance - given a fully populated Site model, compute the effecitve + design by applying inheritance and references + + return a Site model reflecting the effective design for the site + """ + def compute_model_inheritance(self, site_root): + \ No newline at end of file diff --git a/helm_drydock/statemgmt/__init__.py b/helm_drydock/statemgmt/__init__.py index 1fba0192..110f974d 100644 --- a/helm_drydock/statemgmt/__init__.py +++ b/helm_drydock/statemgmt/__init__.py @@ -31,34 +31,92 @@ class DesignState(object): self.sites.append(new_site) + def get_sites(self): + return self.sites + + def get_site(self, site_name): + for s in self.sites: + if s.name == site_name: + return s + + raise NameError("Site %s not found in design state" % site_name) + def add_network(self, new_network): if new_network is None or not isinstance(new_network, model.Network): raise Exception("Invalid Network model") self.networks.append(new_network) + def get_networks(self): + return self.networks + + def get_network(self, network_name): + for n in self.networks: + if n.name == network_name: + return n + + raise NameError("Network %s not found in design state" % network_name) + def add_network_link(self, new_network_link): if new_network_link is None or not isinstance(new_network_link, model.NetworkLink): raise Exception("Invalid NetworkLink model") self.network_links.append(new_network_link) + def get_network_links(self): + return self.network_links + + def get_network_link(self, link_name): + for l in self.network_links: + if l.name == link_name: + return l + + raise NameError("NetworkLink %s not found in design state" % link_name) + def add_host_profile(self, new_host_profile): if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile): raise Exception("Invalid HostProfile model") self.host_profiles.append(new_host_profile) + def get_host_profiles(self): + return self.host_profiles + + def get_host_profile(self, profile_name): + for p in self.host_profiles: + if p.name == profile_name: + return p + + raise NameError("HostProfile %s not found in design state" % profile_name) + def add_hardware_profile(self, new_hardware_profile): if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile): raise Exception("Invalid HardwareProfile model") self.hardware_profiles.append(new_hardware_profile) + def get_hardware_profiles(self): + return self.hardware_profiles + + def get_hardware_profile(self, profile_name): + for p in self.hardware_profiles: + if p.name == profile_name: + return p + + raise NameError("HardwareProfile %s not found in design state" % profile_name) + def add_baremetal_node(self, new_baremetal_node): if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode): raise Exception("Invalid BaremetalNode model") self.baremetal_nodes.append(new_baremetal_node) + def get_baremetal_nodes(self): + return self.baremetal_nodes + def get_baremetal_node(self, node_name): + for n in self.baremetal_nodes: + if n.name == node_name: + return n + + raise NameError("BaremetalNode %s not found in design state" % node_name) diff --git a/tests/aicyaml_samples/fullsite.yaml b/tests/aicyaml_samples/fullsite.yaml index 92e275ce..08ebf4de 100644 --- a/tests/aicyaml_samples/fullsite.yaml +++ b/tests/aicyaml_samples/fullsite.yaml @@ -118,12 +118,12 @@ spec: allocation: static cidr: 172.16.100.0/24 ranges: - - type: static - start: 172.16.100.15 - end: 172.16.100.254 - dns: - domain: ilo.sitename.att.com - servers: 172.16.100.10 + - type: static + start: 172.16.100.15 + end: 172.16.100.254 + dns: + domain: ilo.sitename.att.com + servers: 172.16.100.10 --- apiVersion: '1.0' kind: Network diff --git a/tests/test_ingester.py b/tests/test_ingester.py index fc5c3ed0..edece137 100644 --- a/tests/test_ingester.py +++ b/tests/test_ingester.py @@ -18,6 +18,7 @@ from helm_drydock.statemgmt import DesignState import pytest import shutil import os +import helm_drydock.ingester.plugins.aicyaml class TestClass(object): @@ -28,8 +29,10 @@ class TestClass(object): input_file = input_files.join("fullsite.yaml") ingester = Ingester() - ingester.enable_plugins([helm_drydock.ingester.plugins.aicyaml.AicYamlPlugin]) - ingester.ingest_data('aic_yaml', design_state=design_state, filenames=) + ingester.enable_plugins([helm_drydock.ingester.plugins.aicyaml.AicYamlIngester]) + ingester.ingest_data(plugin_name='aic_yaml', design_state=design_state, filenames=[str(input_file)]) + + assert len(design_state.host_profiles) == 3 @pytest.fixture(scope='module') def input_files(self, tmpdir_factory, request): From 238eba89665b527fe763cbee28e7c7dac20af804 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Tue, 28 Feb 2017 08:52:52 -0600 Subject: [PATCH 8/9] Add methods to the model for computing inheritance Skeleton design state client for the orchestrator Tests for design inheritance --- README.md | 10 +- examples/bootstrap_hwdefinition.yaml | 2 +- examples/bootstrap_seed.yaml | 28 +- helm_drydock/model/__init__.py | 466 +++++++++++++++++++++--- helm_drydock/orchestrator/__init__.py | 13 + helm_drydock/orchestrator/designdata.py | 24 +- helm_drydock/statemgmt/__init__.py | 64 ++-- setup.py | 3 +- tests/aicyaml_samples/fullsite.yaml | 60 ++- tests/aicyaml_samples/multidoc.yaml | 6 +- tests/aicyaml_samples/singledoc.yaml | 2 +- tests/aicyaml_samples/unknown_kind.yaml | 2 +- tests/test_design_inheritance.py | 70 ++++ tests/test_models.py | 2 +- 14 files changed, 604 insertions(+), 148 deletions(-) create mode 100644 helm_drydock/orchestrator/__init__.py create mode 100644 tests/test_design_inheritance.py diff --git a/README.md b/README.md index 67e88f85..321649bb 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A python REST orchestrator to translate a YAML host topology to a provisioned se ### Design Consumer ### -aka smelter +aka ingester Pluggable service to ingest a inventory/design specification, convert it to a standard internal representaion, and persist it to the Design State API. Initial implementation @@ -13,21 +13,21 @@ is the consumer of AIC YAML schema. ### Design State API ### -aka tarot +aka statemgmt API for querying and updating the current design specification and persisted orchestration status. CRUD support of CIs that are not bootstrap-related, but can be used by other automation. ### Control API ### -aka cockpit +aka control User-approachable API for initiating orchestration actions or accessing other internal APIs ### Infrastructure Orchestrator ### -aka alchemist +aka orchestrator Handle validation of complete design, ordering and managing downstream API calls for hardware provisioning/bootstrapping @@ -44,7 +44,7 @@ Pluggable provisioner for network provisioning. Initial implementation is Noop. ### Introspection API ### -aka jabberwocky +aka introspection API for bootstrapping nodes to load self data. Possibly pluggable as this is basically an authenticated bridge to the Design State API \ No newline at end of file diff --git a/examples/bootstrap_hwdefinition.yaml b/examples/bootstrap_hwdefinition.yaml index aaff67be..8476ff6d 100644 --- a/examples/bootstrap_hwdefinition.yaml +++ b/examples/bootstrap_hwdefinition.yaml @@ -18,7 +18,7 @@ ############################################################################# # version the schema in this file so consumers can rationally parse it --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HardwareProfile metadata: name: HPGen8v3 diff --git a/examples/bootstrap_seed.yaml b/examples/bootstrap_seed.yaml index f5c20c14..e6d7d7b0 100644 --- a/examples/bootstrap_seed.yaml +++ b/examples/bootstrap_seed.yaml @@ -18,7 +18,7 @@ #################### # version the schema in this file so consumers can rationally parse it --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Region metadata: name: sitename @@ -28,7 +28,7 @@ metadata: spec: # Not sure if we have site wide data that doesn't fall into another 'Kind' --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: oob @@ -49,7 +49,7 @@ spec: # pxe is a bit of 'magic' indicating the link config used when PXE booting # a node. All other links indicate network configs applied when the node # is deployed. -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: pxe @@ -71,7 +71,7 @@ spec: # use name, will translate to VLAN ID default_network: pxe --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: gp @@ -105,7 +105,7 @@ spec: mode: tagged default_network: mgmt --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: oob @@ -125,7 +125,7 @@ spec: domain: ilo.sitename.att.com servers: 172.16.100.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: pxe @@ -155,7 +155,7 @@ spec: # DNS servers that a server using this network as its default gateway should use servers: 172.16.0.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: mgmt @@ -191,7 +191,7 @@ spec: # DNS servers that a server using this network as its default gateway should use servers: 172.16.1.9,172.16.1.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: private @@ -216,7 +216,7 @@ spec: domain: priv.sitename.example.com servers: 172.16.2.9,172.16.2.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: public @@ -245,7 +245,7 @@ spec: domain: sitename.example.com servers: 8.8.8.8 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: name: default @@ -305,7 +305,7 @@ spec: # Base URL of the introspection service - may go in curtin data introspection_url: http://172.16.1.10:9090 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: name: k8-node @@ -360,7 +360,7 @@ spec: owner_data: foo: bar --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: name: k8-node-public @@ -378,7 +378,7 @@ spec: # inheritance chain - name: public --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: BaremetalNode metadata: name: controller01 @@ -413,7 +413,7 @@ spec: - os_ctl rack: rack01 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: BaremetalNode metadata: name: compute01 diff --git a/helm_drydock/model/__init__.py b/helm_drydock/model/__init__.py index a6d32570..e5cec4bf 100644 --- a/helm_drydock/model/__init__.py +++ b/helm_drydock/model/__init__.py @@ -14,6 +14,7 @@ # # Models for helm_drydock # +from copy import deepcopy class HardwareProfile(object): @@ -21,7 +22,7 @@ class HardwareProfile(object): def __init__(self, **kwargs): self.api_version = kwargs.get('apiVersion', '') - if self.api_version == "1.0": + if self.api_version == "v1.0": metadata = kwargs.get('metadata', {}) spec = kwargs.get('spec', {}) @@ -29,13 +30,14 @@ class HardwareProfile(object): # valid for now self.name = metadata.get('name', '') self.site = metadata.get('region', '') - self.vendor = spec.get('vendor', '') - self.generation = spec.get('generation', '') - self.hw_version = spec.get('hw_version', '') - self.bios_version = spec.get('bios_version', '') - self.boot_mode = spec.get('boot_mode', '') - self.bootstrap_protocol = spec.get('bootstrap_protocol', '') - self.pxe_interface = spec.get('pxe_interface', '') + + self.vendor = spec.get('vendor', None) + self.generation = spec.get('generation', None) + self.hw_version = spec.get('hw_version', None) + self.bios_version = spec.get('bios_version', None) + self.boot_mode = spec.get('boot_mode', None) + self.bootstrap_protocol = spec.get('bootstrap_protocol', None) + self.pxe_interface = spec.get('pxe_interface', None) self.devices = [] device_aliases = spec.get('device_aliases', {}) @@ -57,29 +59,34 @@ class HardwareProfile(object): return + def resolve_alias(self, alias_type, alias): + for d in self.devices: + if d.alias == alias and d.bus_type == alias_type: + return deepcopy(d) + + return None + class HardwareDeviceAlias(object): def __init__(self, api_version, **kwargs): self.api_version = api_version - if self.api_version == "1.0": - self.bus_type = kwargs.get('bus_type', '') - self.address = kwargs.get('address', '') - self.alias = kwargs.get('alias', '') - self.type = kwargs.get('type', '') + if self.api_version == "v1.0": + self.bus_type = kwargs.get('bus_type', None) + self.address = kwargs.get('address', None) + self.alias = kwargs.get('alias', None) + self.type = kwargs.get('type', None) else: raise ValueError('Unknown API version of object') - return - class Site(object): def __init__(self, **kwargs): self.api_version = kwargs.get('apiVersion', '') - if self.api_version == "1.0": + if self.api_version == "v1.0": metadata = kwargs.get('metadata', {}) # Need to add validation logic, we'll assume the input is @@ -95,13 +102,47 @@ class Site(object): else: raise ValueError('Unknown API version of object') + def get_network(self, network_name): + for n in self.networks: + if n.name == network_name: + return n + + return None + + def get_network_link(self, link_name): + for l in self.network_links: + if l.name == link_name: + return l + + return None + + def get_host_profile(self, profile_name): + for p in self.host_profiles: + if p.name == profile_name: + return p + + return None + + def get_hardware_profile(self, profile_name): + for p in self.hardware_profiles: + if p.name == profile_name: + return p + + return None + + def get_baremetal_node(self, node_name): + for n in self.baremetal_nodes: + if n.name == node_name: + return n + + return None class NetworkLink(object): def __init__(self, **kwargs): self.api_version = kwargs.get('apiVersion', '') - if self.api_version == "1.0": + if self.api_version == "v1.0": metadata = kwargs.get('metadata', {}) spec = kwargs.get('spec', {}) @@ -135,20 +176,21 @@ class Network(object): def __init__(self, **kwargs): self.api_version = kwargs.get('apiVersion', '') - if self.api_version == "1.0": + if self.api_version == "v1.0": metadata = kwargs.get('metadata', {}) spec = kwargs.get('spec', {}) self.name = metadata.get('name', '') self.site = metadata.get('region', '') - self.cidr = spec.get('cidr', '') + + self.cidr = spec.get('cidr', None) self.allocation_strategy = spec.get('allocation', 'static') self.vlan_id = spec.get('vlan_id', 1) self.mtu = spec.get('mtu', 0) dns = spec.get('dns', {}) self.dns_domain = dns.get('domain', 'local') - self.dns_servers = dns.get('servers', '') + self.dns_servers = dns.get('servers', None) ranges = spec.get('ranges', []) self.ranges = [] @@ -170,10 +212,10 @@ class NetworkAddressRange(object): def __init__(self, api_version, **kwargs): self.api_version = api_version - if self.api_version == "1.0": - self.type = kwargs.get('type', 'static') - self.start = kwargs.get('start', '') - self.end = kwargs.get('end', '') + if self.api_version == "v1.0": + self.type = kwargs.get('type', None) + self.start = kwargs.get('start', None) + self.end = kwargs.get('end', None) else: raise ValueError('Unknown API version of object') @@ -183,9 +225,9 @@ class NetworkRoute(object): def __init__(self, api_version, **kwargs): self.api_version = api_version - if self.api_version == "1.0": - self.type = kwargs.get('subnet', '') - self.start = kwargs.get('gateway', '') + if self.api_version == "v1.0": + self.type = kwargs.get('subnet', None) + self.start = kwargs.get('gateway', None) self.end = kwargs.get('metric', 100) else: raise ValueError('Unknown API version of object') @@ -196,26 +238,29 @@ class HostProfile(object): def __init__(self, **kwargs): self.api_version = kwargs.get('apiVersion', '') - if self.api_version == "1.0": + if self.api_version == "v1.0": metadata = kwargs.get('metadata', {}) spec = kwargs.get('spec', {}) self.name = metadata.get('name', '') self.site = metadata.get('region', '') + self.parent_profile = spec.get('host_profile', None) + self.hardware_profile = spec.get('hardware_profile', None) + oob = spec.get('oob', {}) - self.oob_type = oob.get('type', 'ipmi') - self.oob_network = oob.get('network', 'oob') - self.oob_account = oob.get('account', '') - self.oob_credential = oob.get('credential', '') + self.oob_type = oob.get('type', None) + self.oob_network = oob.get('network', None) + self.oob_account = oob.get('account', None) + self.oob_credential = oob.get('credential', None) storage = spec.get('storage', {}) self.storage_layout = storage.get('layout', 'lvm') bootdisk = storage.get('bootdisk', {}) - self.bootdisk_device = bootdisk.get('device', '') - self.bootdisk_root_size = bootdisk.get('root_size', '') - self.bootdisk_boot_size = bootdisk.get('boot_size', '') + self.bootdisk_device = bootdisk.get('device', None) + self.bootdisk_root_size = bootdisk.get('root_size', None) + self.bootdisk_boot_size = bootdisk.get('boot_size', None) partitions = storage.get('partitions', []) self.partitions = [] @@ -243,23 +288,61 @@ class HostProfile(object): for k, v in owner_data.items(): self.owner_data[k] = v - self.rack = metadata.get('rack', '') + self.rack = metadata.get('rack', None) else: raise ValueError('Unknown API version of object') - def inherit_parent(self, site): + def apply_inheritance(self, site): + # We return a deep copy of the profile so as not to corrupt + # the original model + self_copy = deepcopy(self) + + if self.parent_profile is None: + return self_copy + + parent = site.get_host_profile(self.parent_profile) + + if parent is None: + return self_copy + + parent = parent.apply_inheritance(site) + + # First compute inheritance for simple fields + inheritable_field_list = [ + "hardware_profile", "oob_type", "oob_network", + "oob_credential", "oob_account", "storage_layout", + "bootdisk_device", "bootdisk_root_size", "bootdisk_boot_size", + "rack"] + + for f in inheritable_field_list: + setattr(self_copy, f, + Utils.apply_field_inheritance(getattr(self, f, None), + getattr(parent, f, None))) + + # Now compute inheritance for complex types + self_copy.tags = Utils.merge_lists(self.tags, parent.tags) + + self_copy.owner_data = Utils.merge_dicts( + self.owner_data, parent.owner_data) + + self_copy.interfaces = HostInterface.merge_lists( + self.interfaces, parent.interfaces) + + self_copy.partitions = HostPartition.merge_lists( + self.partitions, parent.partitions) + + return self_copy - def apply_hardware_profile(self, site): class HostInterface(object): def __init__(self, api_version, **kwargs): self.api_version = api_version - if self.api_version == "1.0": - self.device_name = kwargs.get('device_name', '') - self.network_link = kwargs.get('device_link', '') + if self.api_version == "v1.0": + self.device_name = kwargs.get('device_name', None) + self.network_link = kwargs.get('device_link', None) self.hardware_slaves = [] slaves = kwargs.get('slaves', []) @@ -276,25 +359,155 @@ class HostInterface(object): else: raise ValueError('Unknown API version of object') + # The device attribute may be hardware alias that translates to a + # physical device address. If the device attribute does not match an + # alias, we assume it directly identifies a OS device name. When the + # apply_hardware_profile method is called on the parent Node of this + # device, the selector will be decided and applied + + def add_selector(self, sel_type, selector): + if getattr(self, 'selectors', None) is None: + self.selectors = [] + + new_selector = {} + new_selector['selector_type'] = sel_type + new_selector['selector'] = selector + + self.selectors.append(new_selector) + + """ + Merge two lists of HostInterface models with child_list taking + priority when conflicts. If a member of child_list has a device_name + beginning with '!' it indicates that HostInterface should be + removed from the merged list + """ + + @staticmethod + def merge_lists(child_list, parent_list): + if len(child_list) == 0: + return deepcopy(parent_list) + + effective_list = [] + if len(parent_list) == 0: + for i in child_list: + if i.device_name.startswith('!'): + continue + else: + effective_list.append(deepcopy(i)) + + parent_interfaces = [] + for i in parent_list: + parent_name = i.device_name + parent_interfaces.append(parent_name) + for j in child_list: + if j.device_name == ("!" + parent_name): + break + elif j.device_name == parent_name: + m = HostInterface(j.api_version) + m.device_name = j.device_name + m.network_link = \ + Utils.apply_field_inheritance(j.network_link, + i.network_link) + s = filter(lambda x: ("!" + x) not in j.hardware_slaves, + i.hardware_slaves) + s = list(s) + + s.extend(filter(lambda x: not x.startswith("!"), + j.hardware_slaves)) + m.hardware_slaves = s + + n = filter(lambda x: ("!" + x) not in j.networks, + i.networks) + n = list(n) + + n.extend(filter(lambda x: not x.startswith("!"), + j.networks)) + m.networks = n + + effective_list.append(m) + + for j in child_list: + if j.device_name not in parent_list: + effective_list.append(deepcopy(j)) + + return effective_list + class HostPartition(object): def __init__(self, api_version, **kwargs): self.api_version = api_version - if self.api_version == "1.0": - self.name = kwargs.get('name', '') - self.device = kwargs.get('device', '') - self.part_uuid = kwargs.get('part_uuid', '') - self.size = kwargs.get('size', '') - self.mountpoint = kwargs.get('mountpoint', '') + if self.api_version == "v1.0": + self.name = kwargs.get('name', None) + self.device = kwargs.get('device', None) + self.part_uuid = kwargs.get('part_uuid', None) + self.size = kwargs.get('size', None) + self.mountpoint = kwargs.get('mountpoint', None) self.fstype = kwargs.get('fstype', 'ext4') self.mount_options = kwargs.get('mount_options', 'defaults') - self.fs_uuid = kwargs.get('fs_uuid', '') - self.fs_label = kwargs.get('fs_label', '') + self.fs_uuid = kwargs.get('fs_uuid', None) + self.fs_label = kwargs.get('fs_label', None) else: raise ValueError('Unknown API version of object') + # The device attribute may be hardware alias that translates to a + # physical device address. If the device attribute does not match an + # alias, we assume it directly identifies a OS device name. When the + # apply_hardware_profile method is called on the parent Node of this + # device, the selector will be decided and applied + + def set_selector(self, sel_type, selector): + self.selector_type = sel_type + self.selector = selector + + """ + Merge two lists of HostPartition models with child_list taking + priority when conflicts. If a member of child_list has a name + beginning with '!' it indicates that HostPartition should be + removed from the merged list + """ + + @staticmethod + def merge_lists(child_list, parent_list): + if len(child_list) == 0: + return deepcopy(parent_list) + + effective_list = [] + if len(parent_list) == 0: + for i in child_list: + if i.name.startswith('!'): + continue + else: + effective_list.append(deepcopy(i)) + + inherit_field_list = ["device", "part_uuid", "size", + "mountpoint", "fstype", "mount_options", + "fs_uuid", "fs_label"] + + parent_partitions = [] + for i in parent_list: + parent_name = i.name + parent_partitions.append(parent_name) + for j in child_list: + if j.name == ("!" + parent_name): + break + elif j.name == parent_name: + p = HostPartition(j.api_version) + p.name = j.name + + for f in inherit_field_list: + setattr(p, Utils.apply_field_inheritance(getattr(j, f), + getattr(i, f)) + ) + + effective_list.append(p) + + for j in child_list: + if j.name not in parent_list: + effective_list.append(deepcopy(j)) + + return effective_list # A BaremetalNode is really nothing more than a physical # instantiation of a HostProfile, so they both represent @@ -302,4 +515,157 @@ class HostPartition(object): class BaremetalNode(HostProfile): def __init__(self, **kwargs): - super(BaremetalNode, self).__init__() + super(BaremetalNode, self).__init__(**kwargs) + + def apply_host_profile(self, site): + return self.apply_inheritance(site) + + # Translate device alises to physical selectors and copy + # other hardware attributes into this object + def apply_hardware_profile(self, site): + self_copy = deepcopy(self) + + if self.hardware_profile is None: + raise ValueError("Hardware profile not set") + + hw_profile = site.get_hardware_profile(self.hardware_profile) + + for i in self_copy.interfaces: + for s in i.hardware_slaves: + selector = hw_profile.resolve_alias("pci", s) + if selector is None: + i.add_selector("name", s) + else: + i.add_selector("address", selector) + + for p in self_copy.partitions: + selector = hw_profile.resolve_alias("scsi", p.device) + if selector is None: + p.set_selector("name", p.device) + else: + p.set_selector("address", selector) + + + hardware = {"vendor": getattr(hw_profile, 'vendor', None), + "generation": getattr(hw_profile, 'generation', None), + "hw_version": getattr(hw_profile, 'hw_version', None), + "bios_version": getattr(hw_profile, 'bios_version', None), + "boot_mode": getattr(hw_profile, 'boot_mode', None), + "bootstrap_protocol": getattr(hw_profile, + 'bootstrap_protocol', + None), + "pxe_interface": getattr(hw_profile, 'pxe_interface', None) + } + + self_copy.hardware = hardware + + return self_copy + + +# Utility class for calculating inheritance + +class Utils(object): + + """ + apply_field_inheritance - apply inheritance rules to a single field value + + param child_field - value of the child field, or the field receiving + the inheritance + param parent_field - value of the parent field, or the field supplying + the inheritance + + return the correct value for child_field based on the inheritance algorithm + + Inheritance algorithm + + 1. If child_field is not None, '!' for string vals or -1 for numeric + vals retain the value + of child_field + + 2. If child_field is '!' return None to unset the field value + + 3. If child_field is -1 return None to unset the field value + + 4. If child_field is None return parent_field + """ + + @staticmethod + def apply_field_inheritance(child_field, parent_field): + + if child_field is not None: + if child_field != '!' and child_field != -1: + return child_field + else: + return None + else: + return parent_field + + """ + merge_lists - apply inheritance rules to a list of simple values + + param child_list - list of values from the child + param parent_list - list of values from the parent + + return a merged list with child values taking prority + + 1. All members in the child list not starting with '!' + + 2. If a member in the parent list has a corresponding member in the + chid list prefixed with '!' it is removed + + 3. All remaining members of the parent list + """ + @staticmethod + def merge_lists(child_list, parent_list): + + if type(child_list) is not list or type(parent_list) is not list: + raise ValueError("One parameter is not a list") + + effective_list = [] + + # Probably should handle non-string values + effective_list.extend( + filter(lambda x: not x.startswith("!"), child_list)) + + effective_list.extend( + filter(lambda x: ("!" + x) not in child_list, + filter(lambda x: x not in effective_list, parent_list))) + + return effective_list + + """ + merge_dicts - apply inheritance rules to a dict + + param child_dict - dict of k:v from child + param parent_dict - dict of k:v from the parent + + return a merged dict with child values taking prority + + 1. All members in the child dict with a key not starting with '!' + + 2. If a member in the parent dict has a corresponding member in the + chid dict where the key is prefixed with '!' it is removed + + 3. All remaining members of the parent dict + """ + @staticmethod + def merge_dicts(child_dict, parent_dict): + + if type(child_dict) is not dict or type(parent_dict) is not dict: + raise ValueError("One parameter is not a dict") + + effective_dict = {} + + # Probably should handle non-string keys + use_keys = filter(lambda x: ("!" + x) not in child_dict.keys(), + parent_dict) + + for k in use_keys: + effective_dict[k] = deepcopy(parent_dict[k]) + + use_keys = filter(lambda x: not x.startswith("!"), child_dict) + + for k in use_keys: + effective_dict[k] = deepcopy(child_dict[k]) + + return effective_dict diff --git a/helm_drydock/orchestrator/__init__.py b/helm_drydock/orchestrator/__init__.py new file mode 100644 index 00000000..2a385a45 --- /dev/null +++ b/helm_drydock/orchestrator/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/helm_drydock/orchestrator/designdata.py b/helm_drydock/orchestrator/designdata.py index fe7a268a..2ec2cf65 100644 --- a/helm_drydock/orchestrator/designdata.py +++ b/helm_drydock/orchestrator/designdata.py @@ -14,6 +14,8 @@ import logging +from copy import deepcopy + class DesignStateClient(object): def __init__(self): @@ -67,7 +69,6 @@ class DesignStateClient(object): if n.site == site_name: site.baremetal_nodes.append(n) - return site """ @@ -77,4 +78,23 @@ class DesignStateClient(object): return a Site model reflecting the effective design for the site """ def compute_model_inheritance(self, site_root): - \ No newline at end of file + + # For now the only thing that really incorporates inheritance is + # host profiles and baremetal nodes. So we'll just resolve it for + # the baremetal nodes which recursively resolves it for host profiles + # assigned to those nodes + + site_copy = deepcopy(site_root) + + effective_nodes = [] + + for n in site_copy.baremetal_nodes: + resolved = n.apply_host_profile(site_copy) + resolved = resolved.apply_hardware_profile(site_copy) + effective_nodes.append(resolved) + + site_copy.baremetal_nodes = effective_nodes + + return site_copy + + diff --git a/helm_drydock/statemgmt/__init__.py b/helm_drydock/statemgmt/__init__.py index 110f974d..2fb13a51 100644 --- a/helm_drydock/statemgmt/__init__.py +++ b/helm_drydock/statemgmt/__init__.py @@ -16,20 +16,20 @@ import helm_drydock.model as model class DesignState(object): - def __init__(self): - self.sites = [] - self.networks = [] - self.network_links = [] - self.host_profiles = [] - self.hardware_profiles = [] - self.baremetal_nodes = [] - return + def __init__(self): + self.sites = [] + self.networks = [] + self.network_links = [] + self.host_profiles = [] + self.hardware_profiles = [] + self.baremetal_nodes = [] + return - def add_site(self, new_site): - if new_site is None or not isinstance(new_site, model.Site): - raise Exception("Invalid Site model") + def add_site(self, new_site): + if new_site is None or not isinstance(new_site, model.Site): + raise Exception("Invalid Site model") - self.sites.append(new_site) + self.sites.append(new_site) def get_sites(self): return self.sites @@ -41,11 +41,11 @@ class DesignState(object): raise NameError("Site %s not found in design state" % site_name) - def add_network(self, new_network): - if new_network is None or not isinstance(new_network, model.Network): - raise Exception("Invalid Network model") + def add_network(self, new_network): + if new_network is None or not isinstance(new_network, model.Network): + raise Exception("Invalid Network model") - self.networks.append(new_network) + self.networks.append(new_network) def get_networks(self): return self.networks @@ -57,11 +57,11 @@ class DesignState(object): raise NameError("Network %s not found in design state" % network_name) - def add_network_link(self, new_network_link): - if new_network_link is None or not isinstance(new_network_link, model.NetworkLink): - raise Exception("Invalid NetworkLink model") + def add_network_link(self, new_network_link): + if new_network_link is None or not isinstance(new_network_link, model.NetworkLink): + raise Exception("Invalid NetworkLink model") - self.network_links.append(new_network_link) + self.network_links.append(new_network_link) def get_network_links(self): return self.network_links @@ -73,11 +73,11 @@ class DesignState(object): raise NameError("NetworkLink %s not found in design state" % link_name) - def add_host_profile(self, new_host_profile): - if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile): - raise Exception("Invalid HostProfile model") + def add_host_profile(self, new_host_profile): + if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile): + raise Exception("Invalid HostProfile model") - self.host_profiles.append(new_host_profile) + self.host_profiles.append(new_host_profile) def get_host_profiles(self): return self.host_profiles @@ -89,11 +89,11 @@ class DesignState(object): raise NameError("HostProfile %s not found in design state" % profile_name) - def add_hardware_profile(self, new_hardware_profile): - if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile): - raise Exception("Invalid HardwareProfile model") + def add_hardware_profile(self, new_hardware_profile): + if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile): + raise Exception("Invalid HardwareProfile model") - self.hardware_profiles.append(new_hardware_profile) + self.hardware_profiles.append(new_hardware_profile) def get_hardware_profiles(self): return self.hardware_profiles @@ -105,11 +105,11 @@ class DesignState(object): raise NameError("HardwareProfile %s not found in design state" % profile_name) - def add_baremetal_node(self, new_baremetal_node): - if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode): - raise Exception("Invalid BaremetalNode model") + def add_baremetal_node(self, new_baremetal_node): + if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode): + raise Exception("Invalid BaremetalNode model") - self.baremetal_nodes.append(new_baremetal_node) + self.baremetal_nodes.append(new_baremetal_node) def get_baremetal_nodes(self): return self.baremetal_nodes diff --git a/setup.py b/setup.py index caa39b96..adc96bd7 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,8 @@ setup(name='helm_drydock', 'helm_drydock.model', 'helm_drydock.ingester', 'helm_drydock.ingester.plugins', - 'helm_drydock.statemgmt'], + 'helm_drydock.statemgmt', + 'helm_drydock.orchestrator'], install_requires=[ 'PyYAML', 'oauth', diff --git a/tests/aicyaml_samples/fullsite.yaml b/tests/aicyaml_samples/fullsite.yaml index 08ebf4de..3a7f8ede 100644 --- a/tests/aicyaml_samples/fullsite.yaml +++ b/tests/aicyaml_samples/fullsite.yaml @@ -18,23 +18,22 @@ #################### # version the schema in this file so consumers can rationally parse it --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Region metadata: name: sitename date: 17-FEB-2017 - name: Sample site design + description: Sample site design author: sh8121@att.com spec: # Not sure if we have site wide data that doesn't fall into another 'Kind' --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: oob region: sitename date: 17-FEB-2017 - name: Sample network link author: sh8121@att.com description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on spec: @@ -49,13 +48,12 @@ spec: # pxe is a bit of 'magic' indicating the link config used when PXE booting # a node. All other links indicate network configs applied when the node # is deployed. -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: pxe region: sitename date: 17-FEB-2017 - name: Sample network link author: sh8121@att.com description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on spec: @@ -71,13 +69,12 @@ spec: # use name, will translate to VLAN ID default_network: pxe --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: gp region: sitename date: 17-FEB-2017 - name: Sample network link author: sh8121@att.com description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on # pxe is a bit of 'magic' indicating the link config used when PXE booting @@ -105,13 +102,12 @@ spec: mode: tagged default_network: mgmt --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: oob region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -125,13 +121,12 @@ spec: domain: ilo.sitename.att.com servers: 172.16.100.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: pxe region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -155,13 +150,12 @@ spec: # DNS servers that a server using this network as its default gateway should use servers: 172.16.0.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: mgmt region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -191,13 +185,12 @@ spec: # DNS servers that a server using this network as its default gateway should use servers: 172.16.1.9,172.16.1.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: private region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -216,13 +209,12 @@ spec: domain: priv.sitename.example.com servers: 172.16.2.9,172.16.2.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: public region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -245,13 +237,12 @@ spec: domain: sitename.example.com servers: 8.8.8.8 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: - name: default + name: defaults region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces # No magic to this host_profile, it just provides a way to specify @@ -305,13 +296,12 @@ spec: # Base URL of the introspection service - may go in curtin data introspection_url: http://172.16.1.10:9090 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: name: k8-node region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -339,7 +329,7 @@ spec: - prim_nic01 # Which networks will be configured on this interface networks: - - name: pxe + - pxe - device_name: bond0 network_link: gp # If multiple slaves are specified, but no bonding config @@ -350,8 +340,8 @@ spec: # If multiple networks are specified, but no trunking # config is applied to the link, design validation will fail networks: - - name: mgmt - - name: private + - mgmt + - private metadata: # Explicit tag assignment tags: @@ -360,13 +350,12 @@ spec: owner_data: foo: bar --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: name: k8-node-public region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -376,15 +365,14 @@ spec: networks: # This is additive, so adds a network to those defined in the host_profile # inheritance chain - - name: public + - public --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: BaremetalNode metadata: name: controller01 region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -395,7 +383,7 @@ spec: - device_name: bond0 networks: # '!' prefix for the value of the primary key indicates a record should be removed - - name: '!private' + - '!private' # Addresses assigned to network interfaces addressing: # Which network the address applies to. If a network appears in addressing @@ -412,13 +400,12 @@ spec: roles: os_ctl rack: rack01 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: BaremetalNode metadata: name: compute01 region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -431,13 +418,12 @@ spec: - network: private address: 172.16.2.21 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HardwareProfile metadata: - name: HPGen8v3 + name: HPGen9v3 region: sitename date: 17-FEB-2017 - name: Sample hardware definition author: Scott Hussey spec: # Vendor of the server chassis diff --git a/tests/aicyaml_samples/multidoc.yaml b/tests/aicyaml_samples/multidoc.yaml index 693afd1d..0bd35190 100644 --- a/tests/aicyaml_samples/multidoc.yaml +++ b/tests/aicyaml_samples/multidoc.yaml @@ -1,5 +1,5 @@ --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: oob @@ -20,7 +20,7 @@ spec: # pxe is a bit of 'magic' indicating the link config used when PXE booting # a node. All other links indicate network configs applied when the node # is deployed. -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: pxe @@ -42,7 +42,7 @@ spec: # use name, will translate to VLAN ID default_network: pxe --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: gp diff --git a/tests/aicyaml_samples/singledoc.yaml b/tests/aicyaml_samples/singledoc.yaml index 8ad16094..6d24c8c8 100644 --- a/tests/aicyaml_samples/singledoc.yaml +++ b/tests/aicyaml_samples/singledoc.yaml @@ -1,5 +1,5 @@ --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HardwareProfile metadata: name: HPGen8v3 diff --git a/tests/aicyaml_samples/unknown_kind.yaml b/tests/aicyaml_samples/unknown_kind.yaml index bd39bfc0..bfcd4c2b 100644 --- a/tests/aicyaml_samples/unknown_kind.yaml +++ b/tests/aicyaml_samples/unknown_kind.yaml @@ -1,5 +1,5 @@ --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: FooBar metadata: name: default diff --git a/tests/test_design_inheritance.py b/tests/test_design_inheritance.py new file mode 100644 index 00000000..991289bb --- /dev/null +++ b/tests/test_design_inheritance.py @@ -0,0 +1,70 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from helm_drydock.ingester import Ingester +from helm_drydock.statemgmt import DesignState +from helm_drydock.orchestrator.designdata import DesignStateClient + +from copy import deepcopy + +import pytest +import shutil +import os +import helm_drydock.ingester.plugins.aicyaml +import yaml + +class TestClass(object): + + def setup_method(self, method): + print("Running test {0}".format(method.__name__)) + + + def test_design_inheritance(self, loaded_design): + client = DesignStateClient() + + design_data = client.load_design_data(design_state=loaded_design) + design_data = client.compute_model_inheritance(design_data) + + print(yaml.dump(design_data, default_flow_style=False)) + + node = design_data.get_baremetal_node("controller01") + + assert node.hardware_profile == 'HPGen9v3' + + @pytest.fixture(scope='module') + def loaded_design(self, input_files): + input_file = input_files.join("fullsite.yaml") + + module_design_state = DesignState() + + ingester = Ingester() + ingester.enable_plugins([helm_drydock.ingester.plugins.aicyaml.AicYamlIngester]) + ingester.ingest_data(plugin_name='aic_yaml', design_state=module_design_state, filenames=[str(input_file)]) + + return module_design_state + + + + @pytest.fixture(scope='module') + def input_files(self, tmpdir_factory, request): + tmpdir = tmpdir_factory.mktemp('data') + samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples" + samples = os.listdir(samples_dir) + + for f in samples: + src_file = samples_dir + "/" + f + dst_file = str(tmpdir) + "/" + f + shutil.copyfile(src_file, dst_file) + + return tmpdir \ No newline at end of file diff --git a/tests/test_models.py b/tests/test_models.py index c8bfdfab..3e993c44 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -23,7 +23,7 @@ class TestClass(object): def test_hardwareprofile(self): yaml_snippet = ("---\n" - "apiVersion: '1.0'\n" + "apiVersion: 'v1.0'\n" "kind: HardwareProfile\n" "metadata:\n" " name: HPGen8v3\n" From f1e2a3f343570b8836cd89ca85aa493a61247da1 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Tue, 28 Feb 2017 15:14:09 -0600 Subject: [PATCH 9/9] Roll this into master after pull request was approved --- README.md | 8 ++------ helm_drydock/control/__init__.py | 13 ++++++++++++ helm_drydock/control/root.py | 35 ++++++++++++++++++++++++++++++++ setup.py | 6 ++++-- 4 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 helm_drydock/control/__init__.py create mode 100644 helm_drydock/control/root.py diff --git a/README.md b/README.md index 321649bb..b97df8b9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# drydock +# helm_drydock A python REST orchestrator to translate a YAML host topology to a provisioned set of hosts and provide a set of cloud-init post-provisioning instructions. ## Modular service @@ -9,7 +9,7 @@ aka ingester Pluggable service to ingest a inventory/design specification, convert it to a standard internal representaion, and persist it to the Design State API. Initial implementation -is the consumer of AIC YAML schema. +is the consumer of YAML schema. ### Design State API ### @@ -38,10 +38,6 @@ aka maasdriver Pluggable provisioner for server bootstrapping. Initial implementation is MaaS client. -### Network Driver ### - -Pluggable provisioner for network provisioning. Initial implementation is Noop. - ### Introspection API ### aka introspection diff --git a/helm_drydock/control/__init__.py b/helm_drydock/control/__init__.py new file mode 100644 index 00000000..2a385a45 --- /dev/null +++ b/helm_drydock/control/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/helm_drydock/control/root.py b/helm_drydock/control/root.py new file mode 100644 index 00000000..eb4bb8af --- /dev/null +++ b/helm_drydock/control/root.py @@ -0,0 +1,35 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pecan import expose +from webob.exc import status_map + +class RootController(object): + + @expose(generic=True, template='index.html') + def index(self): + return dict() + + @index.when(method='POST') + def index_post(self, q): + redirect('https://pecan.readthedocs.io/en/latest/search.html?q=%s' % q) + + @expose('error.html') + def error(self, status): + try: + status = int(status) + except ValueError: + status = 0 + message = getattr(status_map.get(status), 'explanation', '') + return dict(status=status, message=message) \ No newline at end of file diff --git a/setup.py b/setup.py index adc96bd7..c065e7eb 100644 --- a/setup.py +++ b/setup.py @@ -44,14 +44,16 @@ setup(name='helm_drydock', 'helm_drydock.ingester', 'helm_drydock.ingester.plugins', 'helm_drydock.statemgmt', - 'helm_drydock.orchestrator'], + 'helm_drydock.orchestrator', + 'helm_drydock.control'], install_requires=[ 'PyYAML', 'oauth', 'requests-oauthlib', 'pyipmi', 'netaddr', - 'pecan' + 'pecan', + 'webob' ], dependency_link=[ 'git+https://github.com/maas/python-libmaas.git'