airship/spyglass
Code Issues Proposed changes

Merge "Use data objects for document generation"

Browse Source
This commit is contained in:
Zuul 2019-07-01 13:51:37 +00:00 committed by Gerrit Code Review
commit ca9756de09
18 changed files with 711 additions and 558 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spyglass/data_extractor/custom_exceptions.py
View File

@ -59,3 +59,7 @@ class TokenGenerationError(BaseError):
class FormationConnectionError(BaseError):
pass
class InvalidIntermediary(BaseError):
pass

112
spyglass/data_extractor/models.py
View File

@ -12,9 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from copy import deepcopy
import ipaddress
import logging
from spyglass.data_extractor.custom_exceptions import InvalidIntermediary
DATA_DEFAULT = "#CHANGE_ME"
LOG = logging.getLogger(__name__)
@ -253,6 +256,19 @@ class Rack(object):
return host
return None
def get_host_by_type(self, host_type: str):
"""Gets host(s) on rack by role
:param host_type: Role of the host(s) to be retrieved
:return: list of hosts
:rtype: list
"""
matching_hosts = []
for host in self.hosts:
if host.type == host_type:
matching_hosts.append(host)
return matching_hosts
class VLANNetworkData(object):
"""Model for single entry of VLAN Network Data"""
@ -594,3 +610,99 @@ class SiteDocumentData(object):
if rack.name == name:
return rack
return None
def get_baremetal_host_by_type(self, *args):
"""Return baremetal host(s) with matching type
:param args: type(s) of the baremetal host
:return: Host object(s) matching the specified host_type
:rtype: list
"""
host_list = []
for rack in self.baremetal:
rack_hosts = []
for arg in args:
rack_hosts.extend(rack.get_host_by_type(arg))
host_list.extend(rack_hosts)
return host_list
def _validate_key_in_intermediary_dict(key: str, dictionary: dict):
if key not in dictionary:
raise InvalidIntermediary(
'%s is not defined in the given intermediary file.' % key)
def site_document_data_factory(intermediary_dict: dict) -> SiteDocumentData:
"""Uses intermediary file data to create a SiteDocumentData object
:param intermediary_dict: A loaded intermediary file dictionary
:return: all intermediary dictionary data returned as an object
"""
# Validate baremetal in intermediary
_validate_key_in_intermediary_dict('baremetal', intermediary_dict)
# Pull out baremetal data into Rack and Host objects
rack_list = []
for rack, hosts in intermediary_dict['baremetal'].items():
host_list = []
for host_name, host_data in hosts.items():
host_ip_list = IPList(**host_data['ip'])
host_kwargs = {
'rack_name': rack,
'host_profile': host_data['host_profile'],
'type': host_data['type'],
'ip': host_ip_list
}
new_host = Host(host_name, **host_kwargs)
host_list.append(new_host)
new_rack = Rack(rack, host_list)
rack_list.append(new_rack)
# Validate network in intermediary
_validate_key_in_intermediary_dict('network', intermediary_dict)
# Validate vlan_network_data in intermediary
_validate_key_in_intermediary_dict(
'vlan_network_data', intermediary_dict['network'])
# Validate bgp in intermediary
_validate_key_in_intermediary_dict('bgp', intermediary_dict['network'])
# Pull out network data into Network object
vlan_data_list = []
for network_type, network_data in \
intermediary_dict['network']['vlan_network_data'].items():
vlan_data_list.append(VLANNetworkData(network_type, **network_data))
network = Network(vlan_data_list, bgp=intermediary_dict['network']['bgp'])
# Validate site_info in intermediary
_validate_key_in_intermediary_dict('site_info', intermediary_dict)
# Validate dns in intermediary
_validate_key_in_intermediary_dict('dns', intermediary_dict['site_info'])
# Validate ntp in intermediary
_validate_key_in_intermediary_dict('ntp', intermediary_dict['site_info'])
# Validate region_name in intermediary
_validate_key_in_intermediary_dict('region_name', intermediary_dict)
# Pull out site_info into a SiteInfo object
dns_server_list = ServerList(
intermediary_dict['site_info']['dns']['servers'].split(','))
ntp_server_list = ServerList(
intermediary_dict['site_info']['ntp']['servers'].split(','))
site_info_dict = deepcopy(intermediary_dict['site_info'])
site_info_dict.pop('dns')
site_info_dict.pop('ntp')
site_info_dict['dns'] = dns_server_list
site_info_dict['ntp'] = ntp_server_list
site_info_dict['region_name'] = intermediary_dict['region_name']
site_info = SiteInfo(**site_info_dict)
# Validate storage in intermediary
_validate_key_in_intermediary_dict('storage', intermediary_dict)
# Create and return SiteDocumentData object
site_document_data = SiteDocumentData(
site_info=site_info,
network=network,
baremetal=rack_list,
storage=intermediary_dict['storage'])
return site_document_data

26
spyglass/examples/templates/baremetal/bootactions/promjoin.yaml.j2
View File

@ -1,26 +0,0 @@
---
schema: 'drydock/BootAction/v1'
metadata:
schema: 'metadata/Document/v1'
name: promjoin
storagePolicy: 'cleartext'
layeringDefinition:
abstract: false
layer: site
labels:
application: 'drydock'
data:
signaling: false
assets:
- path: /opt/promjoin.sh
type: file
permissions: '555'
{% raw %}
location: promenade+http://promenade-api.ucp.svc.cluster.local/api/v1.0/join-scripts?design_ref={{ action.design_ref | urlencode }}&hostname={{ node.hostname }}&ip={{ node.network.calico.ip }}{% endif %}{% for k, v in node.labels.items() %}&labels.dynamic={{ k }}={{ v }}{% endfor %}
{% endraw %}
location_pipeline:
- template
data_pipeline:
- utf8_decode
...

54
spyglass/examples/templates/baremetal/nodes.yaml.j2
View File

@ -1,50 +1,46 @@
{% set control_count = [1] %}
{% for rack in data['baremetal'].keys() %}
{% for host in data['baremetal'][rack].keys()%}
{% if data['baremetal'][rack][host]['type'] != 'genesis' %}
{% for rack in data.baremetal %}
{% for host in rack.hosts %}
{% if host.type != 'genesis' %}
---
schema: 'drydock/BaremetalNode/v1'
metadata:
schema: 'metadata/Document/v1'
name: {{ host }}
name: {{ host.name }}
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
oob:
account: 'root'
{% if data['baremetal'][rack][host]['host_profile'] == 'cp' %}
{% if control_count.append(control_count.pop()+1) %} {% endif %}
{% if control_count[0] < 4 %}
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-primary
{% else %}
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-secondary
{% endif %}
{% else %}
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}
{% endif %}
{% if host.host_profile == 'cp' %}
{% if loop.index - 1 < 4 %}
host_profile: nc-{{ host.host_profile }}-primary
{% else %}
host_profile: nc-{{ host.host_profile }}-secondary
{% endif %}
{% else %}
host_profile: nc-{{ host.host_profile }}
{% endif %}
addressing:
- network: oob
address: {{ data['baremetal'][rack][host]['ip']['oob'] }}
address: {{ host.ip.oob }}
- network: oam
address: {{ data['baremetal'][rack][host]['ip']['oam'] }}
address: {{ host.ip.oam }}
- network: pxe
address: {{ data['baremetal'][rack][host]['ip']['pxe'] }}
address: {{ host.ip.pxe }}
- network: storage
address: {{ data['baremetal'][rack][host]['ip']['storage'] }}
address: {{ host.ip.storage }}
- network: calico
address: {{ data['baremetal'][rack][host]['ip']['calico'] }}
address: {{ host.ip.calico }}
- network: overlay
address: {{ data['baremetal'][rack][host]['ip']['overlay'] }}
address: {{ host.ip.overlay }}
metadata:
rack: RACK{{rack[-2:] }}
rack: {{ rack.name }}
tags:
{% if data['baremetal'][rack][host]['type'] == 'compute' %}
- 'workers'
{% else %}
- 'masters'
{% endif %}
{% if host.type == 'compute' %}
- 'workers'
{% else %}
- 'masters'
{% endif %}
...
{% endif %}
{%endfor%}

29
spyglass/examples/templates/deployment/deployment-strategy.yaml.j2
View File

@ -11,7 +11,6 @@
schema: shipyard/DeploymentStrategy/v1
metadata:
schema: metadata/Document/v1
replacement: true
name: deployment-strategy
layeringDefinition:
abstract: false
@ -44,13 +43,13 @@ data:
selectors:
# NEWSITE-CHANGEME: The following should be a list of the computes in the site's first rack
- node_names:
{% for rack in data['baremetal'].keys() %}
{% for host in data['baremetal'][rack].keys()%}
{% if rack == 'rack03' or rack == 'rack04' %}
- {{ host }}
{% endif %}
{% endfor %}
{% endfor %}
{% for rack in data.baremetal %}
{% for host in rack.hosts %}
{% if rack.name == 'rack03' or rack.name == 'rack04' %}
- {{ host.name }}
{% endif %}
{% endfor %}
{% endfor %}
node_labels: []
node_tags: []
rack_names: []
@ -61,13 +60,13 @@ data:
selectors:
# NEWSITE-CHANGEME: The following should be a list of the computes in the site's second rack
- node_names:
{% for rack in data['baremetal'].keys() %}
{% for host in data['baremetal'][rack].keys()%}
{% if rack == 'rack05' or rack == 'rack06' %}
- {{ host }}
{% endif %}
{% endfor %}
{% endfor %}
{% for rack in data.baremetal %}
{% for host in rack.hosts %}
{% if rack.name == 'rack05' or rack.name == 'rack06' %}
- {{ host.name }}
{% endif %}
{% endfor %}
{% endfor %}
node_labels: []
node_tags: []
rack_names: []

95
spyglass/examples/templates/networks/common_addresses.yaml.j2
View File

@ -6,102 +6,69 @@ metadata:
layeringDefinition:
abstract: false
layer: site
parentSelector:
name: common-addresses-global
actions:
- method: replace
path: .dns.upstream_servers
- method: merge
path: .
storagePolicy: cleartext
replacement: true
data:
calico:
ip_autodetection_method: interface=bond1.{{ data['network']['vlan_network_data']['calico']['vlan']}}
ip_autodetection_method: interface=bond1.{{ data.network.get_vlan_data_by_name('calico').vlan }}
etcd:
service_ip: 10.96.232.136
ip_rule:
gateway: {{ data['network']['vlan_network_data']['calico']['gateway']}}
gateway: {{ data.network.get_vlan_data_by_name('calico').gateway }}
overlap_cidr: 10.96.0.0/15
bgp:
ipv4:
public_service_cidr: {{ data['network']['vlan_network_data']['ingress']['subnet'][0] }}
ingress_vip: {{ data['network']['bgp']['ingress_vip'] }}
public_service_cidr: {{ data.network.get_vlan_data_by_name('ingress').subnet[0] }}
ingress_vip: {{ data.network.bgp['ingress_vip'] }}
peers:
{% for peer in data['network']['bgp']['peers'] %}
{% for peer in data.network.bgp['peers'] %}
- {{ peer }}
{% endfor %}
{% endfor %}
dns:
cluster_domain: cluster.local
service_ip: 10.96.0.10
upstream_servers:
{% for server in (data['site_info']['dns']['servers']).split(',') %}
{% for server in data.site_info.dns.servers %}
- {{ server }}
{% endfor %}
upstream_servers_joined: {{ data['site_info']['dns']['servers']}}
ingress_domain: {{ data['site_info']['domain']|lower }}
{% endfor %}
upstream_servers_joined: {{ data.site_info.dns.__str__() }}
node_domain: {{ data.site_info.domain | lower }}
ingress_domain: {{ data.site_info.domain | lower }}
genesis:
hostname: {{ (data|get_role_wise_nodes)['genesis']['name'] }}
{% for rack in data['baremetal'] %}
{% for host in data['baremetal'][rack] %}
{% if data['baremetal'][rack][host]['type'] == 'genesis' %}
ip: {{ data['baremetal'][rack][host]['ip']['calico'] }}
{% endif %}
{% endfor %}
{% endfor %}
hostname: {{ data.get_baremetal_host_by_type('genesis')[0].name }}
ip: {{ data.get_baremetal_host_by_type('genesis')[0].ip.calico }}
bootstrap:
ip: {{ (data|get_role_wise_nodes)['genesis']['pxe'] }}
ip: {{ data.get_baremetal_host_by_type('genesis')[0].ip.pxe }}
kubernetes:
api_service_ip: 10.96.0.1
etcd_service_ip: 10.96.0.2
pod_cidr: 10.97.0.0/16
service_cidr: 10.96.0.0/16
# misc k8s port settings
apiserver_port: 6443
haproxy_port: 6553
service_node_port_range: 30000-32767
# etcd port settings
etcd:
container_port: 2379
haproxy_port: 2378
masters:
{% for host in (data|get_role_wise_nodes)['masters'] %}
- hostname: {{ host }}
{% endfor %}
# NEWSITE-CHANGEME: Environment proxy information.
# NOTE: Reference Airship sites do not deploy behind a proxy, so this proxy section
# should be commented out.
# However if you are in a lab that requires proxy, ensure that these proxy
# settings are correct and reachable in your environment; otherwise update
# them with the correct values for your environment.
proxy:
http: ""
https: ""
no_proxy: []
node_ports:
drydock_api: 30000
maas_api: 30001
maas_proxy: 31800 # hardcoded in MAAS
shipyard_api: 30003
airflow_web: 30004
ntp:
servers_joined: {{ data['site_info']['ntp']['servers'] }}
ldap:
base_url: {{ (data['site_info']['ldap']['url']|string).split('//')[1] }}
url: {{ data['site_info']['ldap']['url'] }}
auth_path: DC=test,DC=test,DC=com?sAMAccountName?sub?memberof=CN={{ data['site_info']['ldap']['common_name'] }},OU=Application,OU=Groups,DC=test,DC=test,DC=com
common_name: {{ data['site_info']['ldap']['common_name'] }}
subdomain: {{ data['site_info']['ldap']['subdomain'] }}
domain: {{ (data['site_info']['ldap']['url']|string).split('.')[1] }}
{% for host in data.get_baremetal_host_by_type('controller') %}
- hostname: {{ host.name }}
{% endfor %}
storage:
ceph:
public_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
cluster_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
public_cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
cluster_cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
neutron:
tunnel_device: 'bond1.{{ data['network']['vlan_network_data']['overlay']['vlan'] }}'
external_iface: 'bond1'
tunnel_device: "bond1.{{ data.network.get_vlan_data_by_name('overlay').vlan }}"
external_iface: "bond1"
openvswitch:
external_iface: 'bond1'
external_iface: "bond1"
...

216
spyglass/examples/templates/networks/physical/networks.yaml.j2
View File

@ -1,28 +1,4 @@
---
schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: oob
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
# MaaS doesnt own this network like it does the others, so the noconfig label
# is specified.
labels:
noconfig: enabled
bonding:
mode: disabled
mtu: 1500
linkspeed: auto
trunking:
mode: disabled
default_network: oob
allowed_networks:
- oob
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
@ -30,42 +6,33 @@ metadata:
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: oob
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['oob']['subnet'] }}
cidr: {{ data.network.get_vlan_data_by_name('oob').subnet[0] }}
routes:
- subnet: '0.0.0.0/0'
gateway: {{ data['network']['vlan_network_data']['oob']['gateway'] }}
gateway: {{ data.network.get_vlan_data_by_name('oob').gateway }}
metric: 100
ranges:
- type: reserved
start: {{ data.network.get_vlan_data_by_name('oob').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('oob').reserved_end }}
- type: static
start: {{ data['network']['vlan_network_data']['oob']['static_start'] }}
end: {{ data['network']['vlan_network_data']['oob']['static_end'] }}
start: {{ data.network.get_vlan_data_by_name('oob').static_start }}
end: {{ data.network.get_vlan_data_by_name('oob').static_end }}
- type: dhcp
start: {{ data.network.get_vlan_data_by_name('oob').dhcp_start }}
end: {{ data.network.get_vlan_data_by_name('oob').dhcp_end }}
...
---
schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: pxe
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
bonding:
mode: disabled
mtu: 1500
linkspeed: auto
trunking:
mode: disabled
default_network: pxe
allowed_networks:
- pxe
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: pxe
@ -80,56 +47,21 @@ metadata:
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['pxe']['subnet'] }}
cidr: {{ data.network.get_vlan_data_by_name('pxe').subnet[0] }}
routes:
{% for other_subnet in data['network']['vlan_network_data']['pxe']['routes'] %}
- subnet: {{ other_subnet }}
gateway: {{ data['network']['vlan_network_data']['pxe']['gateway'] }}
- subnet: '0.0.0.0/0'
gateway: {{ data.network.get_vlan_data_by_name('pxe').gateway }}
metric: 100
{% endfor %}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['pxe']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['pxe']['reserved_end'] }}
start: {{ data.network.get_vlan_data_by_name('pxe').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('pxe').reserved_end }}
- type: static
start: {{ data['network']['vlan_network_data']['pxe']['static_start'] }}
end: {{ data['network']['vlan_network_data']['pxe']['static_end'] }}
start: {{ data.network.get_vlan_data_by_name('pxe').static_start }}
end: {{ data.network.get_vlan_data_by_name('pxe').static_end }}
- type: dhcp
start: {{ data['network']['vlan_network_data']['pxe']['dhcp_start'] }}
end: {{ data['network']['vlan_network_data']['pxe']['dhcp_end'] }}
...
---
schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: data
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
bonding:
mode: 802.3ad
hash: layer3+4
peer_rate: fast
mon_rate: 100
up_delay: 1000
down_delay: 3000
# NEWSITE-CHANGEME: Ensure the network switches in the environment are
# configured for this MTU or greater. Even if switches are configured for or
# can support a slightly higher MTU, there is no need (and negliable benefit)
# to squeeze every last byte into the MTU (e.g., 9216 vs 9100). Leave MTU at
# 9100 for maximum compatibility.
mtu: 9100
linkspeed: auto
trunking:
mode: 802.1q
allowed_networks:
- oam
- storage
- overlay
- calico
start: {{ data.network.get_vlan_data_by_name('pxe').dhcp_start }}
end: {{ data.network.get_vlan_data_by_name('pxe').dhcp_end }}
...
---
@ -148,27 +80,24 @@ metadata:
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['oam']['subnet'] }}
{% set flag = [0] %}
{% for route in data['network']['vlan_network_data']['oam']['routes'] %}
{% if flag[0] == 0 %}
cidr: {{ data.network.get_vlan_data_by_name('oam').subnet[0] }}
{% if data.network.get_vlan_data_by_name('oam').routes %}
routes:
{% endif %}
{% if flag.append(flag.pop() + 1) %} {% endif %}
{% for route in data.network.get_vlan_data_by_name('oam').routes %}
- subnet: {{ route }}
gateway: {{ data['network']['vlan_network_data']['oam']['gateway'] }}
gateway: {{ data.network.get_vlan_data_by_name('oam').gateway }}
metric: 100
{% endfor %}
{% if flag[0] == 0 %}
routes:[]
{% endif %}
{% endfor %}
{% else %}
routes: []
{% endif %}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['oam']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['oam']['reserved_end'] }}
start: {{ data.network.get_vlan_data_by_name('oam').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('oam').reserved_end }}
- type: static
start: {{ data['network']['vlan_network_data']['oam']['static_start'] }}
end: {{ data['network']['vlan_network_data']['oam']['static_end'] }}
start: {{ data.network.get_vlan_data_by_name('oam').static_start }}
end: {{ data.network.get_vlan_data_by_name('oam').static_end }}
...
---
@ -187,40 +116,14 @@ metadata:
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['storage']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['storage']['reserved_end'] }}
start: {{ data.network.get_vlan_data_by_name('storage').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('storage').reserved_end }}
- type: static
start: {{ data['network']['vlan_network_data']['storage']['static_start'] }}
end: {{ data['network']['vlan_network_data']['storage']['static_end'] }}
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: overlay
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: os-overlay
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['overlay']['subnet'] }}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['overlay']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['overlay']['reserved_end'] }}
- type: static
start: {{ data['network']['vlan_network_data']['overlay']['static_start'] }}
end: {{ data['network']['vlan_network_data']['overlay']['static_end'] }}
start: {{ data.network.get_vlan_data_by_name('storage').static_start }}
end: {{ data.network.get_vlan_data_by_name('storage').static_end }}
...
---
@ -239,13 +142,38 @@ metadata:
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['calico']['subnet'] }}
cidr: {{ data.network.get_vlan_data_by_name('calico').subnet[0] }}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['calico']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['calico']['reserved_end'] }}
start: {{ data.network.get_vlan_data_by_name('calico').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('calico').reserved_end }}
- type: static
start: {{ data['network']['vlan_network_data']['calico']['static_start'] }}
end: {{ data['network']['vlan_network_data']['calico']['static_end'] }}
start: {{ data.network.get_vlan_data_by_name('calico').static_start }}
end: {{ data.network.get_vlan_data_by_name('calico').static_end }}
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: overlay
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: os-overlay
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data.network.get_vlan_data_by_name('overlay').subnet[0] }}
ranges:
- type: reserved
start: {{ data.network.get_vlan_data_by_name('overlay').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('overlay').reserved_end }}
- type: static
start: {{ data.network.get_vlan_data_by_name('overlay').static_start }}
end: {{ data.network.get_vlan_data_by_name('overlay').static_end }}
...

145
spyglass/examples/templates/pki/pki-catalogue.yaml.j2
View File

@ -21,33 +21,29 @@ data:
- 10.96.0.1
kubernetes_service_names:
- kubernetes.default.svc.cluster.local
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
{% for host in data.get_baremetal_host_by_type('genesis') %}
- document_name: kubelet-genesis
common_name: system:node:{{ host }}
common_name: system:node:{{ host.name }}
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
groups:
- system:nodes
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
- document_name: kubelet-{{ host }}
common_name: system:node:{{ host }}
{% endfor %}
{% for rack in data.baremetal %}
{% for host in rack.hosts %}
- document_name: kubelet-{{ host.name }}
common_name: system:node:{{ host.name }}
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
groups:
- system:nodes
{%endfor%}
{%endfor%}
{% endfor %}
{% endfor %}
- document_name: scheduler
description: Service certificate for Kubernetes scheduler
common_name: system:kube-scheduler
@ -62,6 +58,7 @@ data:
common_name: armada
groups:
- system:masters
kubernetes-etcd:
description: Certificates for Kubernetes's etcd servers
certificates:
@ -72,113 +69,93 @@ data:
- document_name: kubernetes-etcd-anchor
description: anchor
common_name: anchor
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
{% for host in data.get_baremetal_host_by_type('genesis') %}
- document_name: kubernetes-etcd-genesis
common_name: kubernetes-etcd-genesis
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis'%}
- document_name: kubernetes-etcd-{{ host }}
common_name: kubernetes-etcd-{{ host }}
{% endfor %}
{% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
- document_name: kubernetes-etcd-{{ host.name }}
common_name: kubernetes-etcd-{{ host.name }}
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
{% endfor %}
kubernetes-etcd-peer:
certificates:
{% for host in data.get_baremetal_host_by_type('genesis') %}
- document_name: kubernetes-etcd-genesis-peer
common_name: kubernetes-etcd-genesis-peer
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: kubernetes-etcd-{{ host }}-peer
common_name: kubernetes-etcd-{{ host }}-peer
{% endfor %}
{% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
- document_name: kubernetes-etcd-{{ host.name }}-peer
common_name: kubernetes-etcd-{{ host.name }}-peer
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}
{%endfor%}
{%endfor%}
ksn-etcd:
{% endfor %}
calico-etcd:
description: Certificates for Calico etcd client traffic
certificates:
- document_name: ksn-etcd-anchor
description: anchor
common_name: anchor
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: ksn-etcd-{{ host }}
common_name: ksn-etcd-{{ host }}
{% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
- document_name: ksn-etcd-{{ host.name }}
common_name: ksn-etcd-{{ host.name }}
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- 10.96.232.136
{% endif %}
{%endfor%}
{%endfor%}
{% endfor %}
- document_name: ksn-node
common_name: calcico-node
ksn-etcd-peer:
calico-etcd-peer:
description: Certificates for Calico etcd clients
certificates:
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: ksn-etcd-{{ host }}-peer
common_name: ksn-etcd-{{ host }}-peer
{% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
- document_name: ksn-etcd-{{ host.name }}-peer
common_name: ksn-etcd-{{ host.name }}-peer
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- 10.96.232.136
{% endif %}
{%endfor%}
{%endfor%}
{% endfor %}
- document_name: ksn-node-peer
common_name: calico-node-peer
keypairs:

4
spyglass/examples/templates/profile/region.yaml.j2
View File

@ -2,7 +2,7 @@
schema: 'drydock/Region/v1'
metadata:
schema: 'metadata/Document/v1'
name: {{ data['region_name'] }}
name: {{ data.site_info.region_name }}
layeringDefinition:
abstract: false
layer: site
@ -18,7 +18,7 @@ metadata:
path: .authorized_keys[1]
src:
schema: deckhand/PublicKey/v1
name: {{ data['region_name'] }}_ssh_public_key
name: {{ data.site_info.region_name }}_ssh_public_key
path: .
- dest:
path: .repositories.main_archive

4
spyglass/examples/templates/site-definition.yaml.j2
View File

@ -7,11 +7,11 @@ metadata:
abstract: false
layer: site
# NEWSITE-CHANGEME: Replace with the site name
name: {{ data['region_name'] }}
name: {{ data.site_info.region_name }}
storagePolicy: cleartext
data:
# The type layer this site will delpoy with. Type layer is found in the
# aic-clcp-manifests repo.
site_type: {{ data['site_info']['sitetype'] }}
site_type: {{ data.site_info.sitetype }}
...

55
spyglass/examples/templates/software/charts/kubernetes/etcd.yaml.j2
View File

@ -13,83 +13,70 @@ metadata:
path: .
storagePolicy: cleartext
substitutions:
{% set count = [0] %}
{% for rack in data.baremetal.keys() %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'controller' %}
{% for host in data.get_baremetal_host_by_type('controller') %}
- src:
schema: pegleg/CommonAddresses/v1
name: common-addresses
path: .masters[{{ count[0] }}].hostname
path: .masters[{{ loop.index - 1 }}].hostname
dest:
path: .values.nodes[{{ count[0] }}].name
path: .values.nodes[{{ loop.index - 1 }}].name
- src:
schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }}
name: calico-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }}
name: calico-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.key
path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src:
schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }}
name: calico-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }}-peer
name: calico-etcd-{{ host.name }}-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key
{% if count.append(count.pop() + 1) %}{% endif %} {# increment count by 1 #}
{% endif %}
{% endfor %}
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% endfor %}
{% for rack in data.baremetal.keys() %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'genesis' %}
{% for host in data.get_baremetal_host_by_type('genesis') %}
- src:
schema: pegleg/CommonAddresses/v1
name: common-addresses
path: .genesis.hostname
dest:
path: .values.nodes[{{ count[0] }}].name
path: .values.nodes[{{ loop.index - 1 }}].name
- src:
schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }}
name: calico-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }}
name: calico-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.key
path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src:
schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }}-peer
name: calico-etcd-{{ host.name }}-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }}-peer
name: calico-etcd-{{ host.name }}-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key
{% endif %}
{% endfor %}
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% endfor %}
data: {}

48
spyglass/examples/templates/software/charts/kubernetes/etcd/etcd.yaml.j2
View File

@ -13,80 +13,70 @@ metadata:
path: .
storagePolicy: cleartext
substitutions:
{% set count = [0] %}
{% for rack in data.baremetal.keys() %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'controller' %}
{% for host in data.get_baremetal_host_by_type('controller') %}
- src:
schema: pegleg/CommonAddresses/v1
name: common-addresses
path: .masters[{{ count[0] }}].hostname
path: .masters[{{ loop.index - 1 }}].hostname
dest:
path: .values.nodes[{{ count[0] }}].name
path: .values.nodes[{{ loop.index - 1 }}].name
- src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-{{ host }}
name: kubernetes-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-{{ host }}
name: kubernetes-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.key
path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-{{ host }}-peer
name: kubernetes-etcd-{{ host.name }}-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-{{ host }}-peer
name: kubernetes-etcd-{{ host.name }}-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key
{% if count.append(count.pop() + 1) %}{% endif %} {# increment count by 1 #}
{% endif %}
{% endfor %}
{% endfor %}
{% for rack in data.baremetal.keys() %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'genesis' %}
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% endfor %}
{% for host in data.get_baremetal_host_by_type('genesis') %}
- src:
schema: pegleg/CommonAddresses/v1
name: common-addresses
path: .genesis.hostname
dest:
path: .values.nodes[{{ count[0] }}].name
path: .values.nodes[{{ loop.index - 1 }}].name
- src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-genesis
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-genesis
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.key
path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-genesis-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-genesis-peer
path: $
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key
{% endif %}
{% endfor %}
{% endfor %}
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% endfor %}
data: {}
...

2
spyglass/examples/templates/software/charts/osh/openstack-tenant-ceph/ceph-client.yaml.j2
View File

@ -18,5 +18,5 @@ data:
conf:
pool:
target:
osd: {{ data['storage']['ceph']['controller']['osd_count'] }}
osd: {{ data.storage['ceph']['controller']['osd_count'] }}
...

3
spyglass/examples/templates/software/config/common-software-config.yaml.j2
View File

@ -13,4 +13,5 @@ metadata:
data:
osh:
# NEWSITE-CHANGEME: Replace with the site name
region_name: {{ data['region_name'] }}
region_name: {{ data.site_info.region_name }}
...

19
spyglass/site_processors/base.py
View File

@ -20,22 +20,3 @@ class BaseProcessor(object):
def render_template(self, template):
pass
@staticmethod
def get_role_wise_nodes(yaml_data):
hosts = {"genesis": {}, "masters": [], "workers": []}
for rack in yaml_data["baremetal"]:
for host in yaml_data["baremetal"][rack]:
if yaml_data["baremetal"][rack][host]["type"] == "genesis":
hosts["genesis"] = {
"name": host,
"pxe": yaml_data["baremetal"][rack][host]["ip"]["pxe"],
"oam": yaml_data["baremetal"][rack][host]["ip"]["oam"],
}
elif yaml_data["baremetal"][rack][host]["type"] \
== "controller":
hosts["masters"].append(host)
else:
hosts["workers"].append(host)
return hosts

19
spyglass/site_processors/site_processor.py
View File

@ -18,6 +18,8 @@ import shutil
import jinja2
from spyglass.data_extractor.models import site_document_data_factory
from spyglass.data_extractor.models import SiteDocumentData
from spyglass.site_processors.base import BaseProcessor
LOG = logging.getLogger(__name__)
@ -25,9 +27,12 @@ LOG = logging.getLogger(__name__)
class SiteProcessor(BaseProcessor):
def __init__(self, intermediary_yaml, manifest_dir, force_write):
def __init__(self, site_data, manifest_dir, force_write):
super().__init__()
self.yaml_data = intermediary_yaml
if isinstance(site_data, SiteDocumentData):
self.site_data = site_data
else:
self.site_data = site_document_data_factory(site_data)
self.manifest_dir = manifest_dir
self.force_write = force_write
@ -38,7 +43,7 @@ class SiteProcessor(BaseProcessor):
calico) are generated in a single file. Rack specific
configs( pxe and oob) are generated per rack.
"""
# Check of manifest_dir exists
# Check if manifest_dir exists
if self.manifest_dir is not None:
site_manifest_dir = os.path.join(
self.manifest_dir, 'pegleg_manifests', 'site')
@ -66,16 +71,16 @@ class SiteProcessor(BaseProcessor):
autoescape=True,
loader=loader,
trim_blocks=True,
lstrip_blocks=True,
undefined=logging_undefined)
j2_env.filters["get_role_wise_nodes"] = \
self.get_role_wise_nodes
templatefile = os.path.join(dirpath, filename)
LOG.debug("Template file: %s", templatefile)
outdirs = dirpath.split(template_folder_name)[1].lstrip(os.sep)
LOG.debug("outdirs: %s", outdirs)
outfile_path = os.path.join(
site_manifest_dir, self.yaml_data["region_name"], outdirs)
site_manifest_dir, self.site_data.site_info.region_name,
outdirs)
LOG.debug("outfile path: %s", outfile_path)
outfile_yaml = os.path.split(templatefile)[1]
outfile_yaml = os.path.splitext(outfile_yaml)[0]
@ -90,7 +95,7 @@ class SiteProcessor(BaseProcessor):
out = open(outfile, "w")
created_file_list.append(outfile)
LOG.info("Rendering {}".format(outfile_yaml))
rendered = template_j2.render(data=self.yaml_data)
rendered = template_j2.render(data=self.site_data)
out.write(rendered)
out.close()
except IOError as ioe:

187
tests/unit/data_extractor/test_models.py
View File

@ -13,11 +13,18 @@
# limitations under the License.
from copy import copy
import os
import unittest
from unittest import mock
import yaml
from spyglass.data_extractor.custom_exceptions import InvalidIntermediary
from spyglass.data_extractor import models
FIXTURE_DIR = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'shared')
class TestParseIp(unittest.TestCase):
"""Tests the _parse_ip validator for Spyglass models"""
@ -340,11 +347,7 @@ class TestRack(unittest.TestCase):
"""Tests for the Rack model"""
RACK_NAME = 'test_rack1'
HOST_DATA = {
'rack_name': RACK_NAME,
'host_profile': 'host',
'type': 'compute'
}
HOST_DATA = {'rack_name': RACK_NAME, 'host_profile': 'host'}
@mock.patch('spyglass.data_extractor.models.IPList', autospec=True)
def setUp(self, MockIPList):
@ -353,9 +356,9 @@ class TestRack(unittest.TestCase):
self.HOST_DATA['ip'] = MockIPList()
self.HOST_DATA['ip'].dict_from_class.return_value = 'success'
self.hosts = [
models.Host('test_host1', **self.HOST_DATA),
models.Host('test_host2', **self.HOST_DATA),
models.Host('test_host3', **self.HOST_DATA),
models.Host('test_host1', **self.HOST_DATA, type='genesis'),
models.Host('test_host2', **self.HOST_DATA, type='compute'),
models.Host('test_host3', **self.HOST_DATA, type='controller'),
]
def test___init__(self):
@ -408,6 +411,15 @@ class TestRack(unittest.TestCase):
self.assertEqual(
self.hosts[1], result.get_host_by_name(self.hosts[1].name))
def test_get_host_by_type(self):
"""Tests retrieval of a Rack's host(s) by type"""
result = models.Rack(self.RACK_NAME, self.hosts)
self.assertEqual(self.hosts[0], result.get_host_by_type('genesis')[0])
self.assertEqual(self.hosts[1], result.get_host_by_type('compute')[0])
self.assertEqual(
self.hosts[2],
result.get_host_by_type('controller')[0])
class TestVLANNetworkData(unittest.TestCase):
"""Tests for the VLANNetworkData model"""
@ -820,3 +832,162 @@ class TestSiteDocumentData(unittest.TestCase):
type(Rack()).name = mock.PropertyMock(side_effect=['rack1', 'rack3'])
result = models.SiteDocumentData(site_info, network, baremetal)
self.assertIsNone(result.get_baremetal_rack_by_name('rack2'))
@mock.patch('spyglass.data_extractor.models.SiteInfo')
@mock.patch('spyglass.data_extractor.models.Network')
@mock.patch('spyglass.data_extractor.models.Rack')
@mock.patch('spyglass.data_extractor.models.Host')
def test_get_baremetal_rack_by_name_multiple(
self, Host, Rack, Network, SiteInfo):
"""Tests retrieval of baremetal host(s) by type"""
site_info = SiteInfo()
network = Network()
baremetal = [Rack(), Rack()]
Rack().get_host_by_type.return_value = [Host()]
result = models.SiteDocumentData(site_info, network, baremetal)
self.assertEqual(2, len(result.get_baremetal_host_by_type('genesis')))
self.assertEqual(2, len(result.get_baremetal_host_by_type('computer')))
self.assertEqual(
2, len(result.get_baremetal_host_by_type('controller')))
class TestValidateKeyInIntermediaryDict(unittest.TestCase):
"""Tests the _validate_key_in_intermediary_dict function"""
def test__validate_key_in_intermediary_dict(self):
test_dictionary = {'test_key': 'value'}
key = 'test_key'
self.assertIsNone(
models._validate_key_in_intermediary_dict(key, test_dictionary))
def test__validate_key_in_intermediary_dict_key_dne(self):
test_dictionary = {'test_key': 'value'}
key = 'not_test_key'
with self.assertRaises(InvalidIntermediary):
models._validate_key_in_intermediary_dict(key, test_dictionary)
class TestSiteDocumentDataFactory(unittest.TestCase):
"""Tests the site_document_data_factory function"""
def setUp(self) -> None:
test_intermediary_path = os.path.join(
FIXTURE_DIR, 'test_intermediary.yaml')
with open(test_intermediary_path, 'r') as f:
self.intermediary_dict = yaml.safe_load(f)
def test_site_document_data_factory(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check correct return type
self.assertIsInstance(site_document_data, models.SiteDocumentData)
def test_site_document_data_factory_saves_storage(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that storage was saved without changes in the SiteDocumentData
self.assertDictEqual(
self.intermediary_dict['storage'], site_document_data.storage)
def test_site_document_data_factory_saves_site_info(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that site info saved correctly in SiteInfo object
site_info_dict = self.intermediary_dict['site_info']
self.assertIsInstance(site_document_data.site_info, models.SiteInfo)
self.assertEqual(
site_info_dict['name'], site_document_data.site_info.name)
self.assertEqual(
self.intermediary_dict['region_name'],
site_document_data.site_info.region_name)
self.assertEqual(
site_info_dict['state'], site_document_data.site_info.state)
self.assertEqual(
site_info_dict['physical_location_id'],
site_document_data.site_info.physical_location_id)
self.assertEqual(
site_info_dict['country'], site_document_data.site_info.country)
self.assertEqual(
site_info_dict['corridor'], site_document_data.site_info.corridor)
self.assertEqual(
site_info_dict['sitetype'], site_document_data.site_info.sitetype)
self.assertEqual(
site_info_dict['domain'], site_document_data.site_info.domain)
self.assertDictEqual(
site_info_dict['ldap'], site_document_data.site_info.ldap)
self.assertEqual(
site_info_dict['dns']['servers'],
str(site_document_data.site_info.dns))
self.assertEqual(
site_info_dict['ntp']['servers'],
str(site_document_data.site_info.ntp))
def test_site_document_data_factory_saves_network_data(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that network data saved correctly into a Network object
network_dict = self.intermediary_dict['network']
self.assertIsInstance(site_document_data.network, models.Network)
self.assertDictEqual(
network_dict['bgp'], site_document_data.network.bgp)
for network_type, network_data \
in network_dict['vlan_network_data'].items():
vlan_network_data = \
site_document_data.network.get_vlan_data_by_name(network_type)
self.assertIsInstance(vlan_network_data, models.VLANNetworkData)
self.assertEqual(network_type, vlan_network_data.name)
self.assertEqual(network_type, vlan_network_data.role)
self.assertEqual(network_data['subnet'], vlan_network_data.subnet)
if 'routes' in network_data:
self.assertEqual(
network_data['routes'], vlan_network_data.routes)
if 'gateway' in network_data:
self.assertEqual(
network_data['gateway'], vlan_network_data.gateway)
if 'vlan' in network_data:
self.assertEqual(network_data['vlan'], vlan_network_data.vlan)
if 'dhcp_start' in network_data and 'dhcp_end' in network_data:
self.assertEqual(
network_data['dhcp_start'], vlan_network_data.dhcp_start)
self.assertEqual(
network_data['dhcp_end'], vlan_network_data.dhcp_end)
if 'static_start' in network_data and 'static_end' in network_data:
self.assertEqual(
network_data['static_start'],
vlan_network_data.static_start)
self.assertEqual(
network_data['static_end'], vlan_network_data.static_end)
if 'reserved_start' in network_data \
and 'reserved_end' in network_data:
self.assertEqual(
network_data['reserved_start'],
vlan_network_data.reserved_start)
self.assertEqual(
network_data['reserved_end'],
vlan_network_data.reserved_end)
def test_site_document_data_factory_saves_baremetal_data(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that baremetal racks saved correctly into Rack objects
for rack_name, hosts \
in self.intermediary_dict['baremetal'].items():
rack = site_document_data.get_baremetal_rack_by_name(rack_name)
for host_name, host_data in hosts.items():
host = rack.get_host_by_name(host_name)
self.assertEqual(host_name, host.name)
self.assertEqual(rack_name, host.rack_name)
self.assertEqual(host_data['type'], host.type)
self.assertEqual(host_data['host_profile'], host.host_profile)
self.assertEqual(host_data['ip']['oob'], host.ip.oob)
self.assertEqual(host_data['ip']['oam'], host.ip.oam)
self.assertEqual(host_data['ip']['calico'], host.ip.calico)
self.assertEqual(host_data['ip']['overlay'], host.ip.overlay)
self.assertEqual(host_data['ip']['pxe'], host.ip.pxe)
self.assertEqual(host_data['ip']['storage'], host.ip.storage)
self.assertEqual(rack_name, rack.name)

247
tests/unit/site_processors/test_site_processor.py
View File

@ -15,117 +15,178 @@
import logging
import os
from tempfile import mkdtemp
import textwrap
import unittest
from unittest import mock
from jinja2 import UndefinedError
import pytest
from spyglass.data_extractor import models
from spyglass.site_processors.site_processor import SiteProcessor
LOG = logging.getLogger(__name__)
LOG.level = logging.DEBUG
J2_TPL = """---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: {{ data['region_name'] }}
storagePolicy: cleartext
data:
site_type:{{ data['site_info']['sitetype'] }}
..."""
class TestSiteProcessor(unittest.TestCase):
def test_render_template():
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(J2_TPL)
LOG.debug("Writing test template to %s", _tpl_file)
_input_yaml = {
"region_name": "test",
"site_info": {
"sitetype": "test_type"
}
}
_out_dir = mkdtemp()
site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=False)
site_processor.render_template(_tpl_parent_dir)
J2_TPL = textwrap.dedent(
"""
---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: {{ data.site_info.region_name }}
storagePolicy: cleartext
data:
site_type:{{ data.site_info.sitetype }}
...""")
expected_output = """---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: test
storagePolicy: cleartext
data:
site_type:test_type
..."""
J2_TPL_UNDEFINED = textwrap.dedent(
"""
---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: {{ data.site_info.region_name }}
storagePolicy: cleartext
data:
site_type:{{ undefined_param }}
...""")
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", _input_yaml["region_name"],
os.path.split(_tpl_dir)[1], "test.yaml")
LOG.debug(output_file)
assert (os.path.exists(output_file))
with open(output_file, 'r') as f:
content = f.read()
assert (expected_output == content)
@mock.patch(
'spyglass.data_extractor.models.SiteDocumentData',
spec=models.SiteDocumentData)
@mock.patch('spyglass.data_extractor.models.SiteInfo')
@mock.patch('spyglass.data_extractor.models.ServerList')
def test_render_template(self, ServerList, SiteInfo, SiteDocumentData):
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(self.J2_TPL)
LOG.debug("Writing test template to %s", _tpl_file)
site_data = SiteDocumentData()
type(SiteDocumentData()).site_info = SiteInfo()
region_name = 'test'
type(SiteInfo()).region_name = mock.PropertyMock(
return_value=region_name)
site_type = 'test_type'
type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
def test_render_template_missing_data():
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(J2_TPL)
LOG.debug("Writing test template to %s", _tpl_file)
_input_yaml = {"region_name": "test", "site_info": {}}
_out_dir = mkdtemp()
site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=False)
with pytest.raises(UndefinedError):
_out_dir = mkdtemp()
site_processor = SiteProcessor(site_data, _out_dir, force_write=False)
site_processor.render_template(_tpl_parent_dir)
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", _input_yaml["region_name"],
os.path.split(_tpl_dir)[1], "test.yaml")
assert (not os.path.exists(output_file))
expected_output = textwrap.dedent(
"""
---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: test
storagePolicy: cleartext
data:
site_type:test_type
...""")
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", region_name,
os.path.split(_tpl_dir)[1], "test.yaml")
LOG.debug(output_file)
self.assertTrue(os.path.exists(output_file))
with open(output_file, 'r') as f:
content = f.read()
self.assertEqual(expected_output, content)
def test_render_template_missing_data_force():
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(J2_TPL)
LOG.debug("Writing test template to %s", _tpl_file)
_input_yaml = {"region_name": "test", "site_info": {}}
_out_dir = mkdtemp()
site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=True)
site_processor.render_template(_tpl_parent_dir)
@mock.patch(
'spyglass.data_extractor.models.SiteDocumentData',
spec=models.SiteDocumentData)
@mock.patch('spyglass.data_extractor.models.SiteInfo')
@mock.patch('spyglass.data_extractor.models.ServerList')
def test_render_template_missing_data(
self, ServerList, SiteInfo, SiteDocumentData):
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(self.J2_TPL_UNDEFINED)
LOG.debug("Writing test template to %s", _tpl_file)
expected_output = """---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: test
storagePolicy: cleartext
data:
site_type:
..."""
site_data = SiteDocumentData()
type(SiteDocumentData()).site_info = SiteInfo()
region_name = 'test'
type(SiteInfo()).region_name = mock.PropertyMock(
return_value=region_name)
site_type = 'test_type'
type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", _input_yaml["region_name"],
os.path.split(_tpl_dir)[1], "test.yaml")
assert (os.path.exists(output_file))
with open(output_file, 'r') as f:
content = f.read()
assert (expected_output == content)
_out_dir = mkdtemp()
site_processor = SiteProcessor(site_data, _out_dir, force_write=False)
with pytest.raises(UndefinedError):
site_processor.render_template(_tpl_parent_dir)
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", region_name,
os.path.split(_tpl_dir)[1], "test.yaml")
self.assertFalse(os.path.exists(output_file))
@mock.patch(
'spyglass.data_extractor.models.SiteDocumentData',
spec=models.SiteDocumentData)
@mock.patch('spyglass.data_extractor.models.SiteInfo')
@mock.patch('spyglass.data_extractor.models.ServerList')
def test_render_template_missing_data_force(
self, ServerList, SiteInfo, SiteDocumentData):
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(self.J2_TPL_UNDEFINED)
LOG.debug("Writing test template to %s", _tpl_file)
site_data = SiteDocumentData()
type(SiteDocumentData()).site_info = SiteInfo()
region_name = 'test'
type(SiteInfo()).region_name = mock.PropertyMock(
return_value=region_name)
site_type = 'test_type'
type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
_out_dir = mkdtemp()
site_processor = SiteProcessor(site_data, _out_dir, force_write=True)
site_processor.render_template(_tpl_parent_dir)
expected_output = textwrap.dedent(
"""
---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: test
storagePolicy: cleartext
data:
site_type:
...""")
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", region_name,
os.path.split(_tpl_dir)[1], "test.yaml")
self.assertTrue(os.path.exists(output_file))
with open(output_file, 'r') as f:
content = f.read()
self.assertEqual(expected_output, content)