Merge "Use data objects for document generation"

spyglass/data_extractor/custom_exceptions.py

@@ -59,3 +59,7 @@ class TokenGenerationError(BaseError):
 
 class FormationConnectionError(BaseError):
     pass
+
+
+class InvalidIntermediary(BaseError):
+    pass

spyglass/data_extractor/models.py

@@ -12,9 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from copy import deepcopy
 import ipaddress
 import logging
 
+from spyglass.data_extractor.custom_exceptions import InvalidIntermediary
+
 DATA_DEFAULT = "#CHANGE_ME"
 
 LOG = logging.getLogger(__name__)
@@ -253,6 +256,19 @@ class Rack(object):
                 return host
         return None
 
+    def get_host_by_type(self, host_type: str):
+        """Gets host(s) on rack by role
+
+        :param host_type: Role of the host(s) to be retrieved
+        :return: list of hosts
+        :rtype: list
+        """
+        matching_hosts = []
+        for host in self.hosts:
+            if host.type == host_type:
+                matching_hosts.append(host)
+        return matching_hosts
+
 
 class VLANNetworkData(object):
     """Model for single entry of VLAN Network Data"""
@@ -594,3 +610,99 @@ class SiteDocumentData(object):
             if rack.name == name:
                 return rack
         return None
+
+    def get_baremetal_host_by_type(self, *args):
+        """Return baremetal host(s) with matching type
+
+        :param args: type(s) of the baremetal host
+        :return: Host object(s) matching the specified host_type
+        :rtype: list
+        """
+        host_list = []
+        for rack in self.baremetal:
+            rack_hosts = []
+            for arg in args:
+                rack_hosts.extend(rack.get_host_by_type(arg))
+            host_list.extend(rack_hosts)
+        return host_list
+
+
+def _validate_key_in_intermediary_dict(key: str, dictionary: dict):
+    if key not in dictionary:
+        raise InvalidIntermediary(
+            '%s is not defined in the given intermediary file.' % key)
+
+
+def site_document_data_factory(intermediary_dict: dict) -> SiteDocumentData:
+    """Uses intermediary file data to create a SiteDocumentData object
+
+    :param intermediary_dict: A loaded intermediary file dictionary
+    :return: all intermediary dictionary data returned as an object
+    """
+    # Validate baremetal in intermediary
+    _validate_key_in_intermediary_dict('baremetal', intermediary_dict)
+
+    # Pull out baremetal data into Rack and Host objects
+    rack_list = []
+    for rack, hosts in intermediary_dict['baremetal'].items():
+        host_list = []
+        for host_name, host_data in hosts.items():
+            host_ip_list = IPList(**host_data['ip'])
+            host_kwargs = {
+                'rack_name': rack,
+                'host_profile': host_data['host_profile'],
+                'type': host_data['type'],
+                'ip': host_ip_list
+            }
+            new_host = Host(host_name, **host_kwargs)
+            host_list.append(new_host)
+        new_rack = Rack(rack, host_list)
+        rack_list.append(new_rack)
+
+    # Validate network in intermediary
+    _validate_key_in_intermediary_dict('network', intermediary_dict)
+    # Validate vlan_network_data in intermediary
+    _validate_key_in_intermediary_dict(
+        'vlan_network_data', intermediary_dict['network'])
+    # Validate bgp in intermediary
+    _validate_key_in_intermediary_dict('bgp', intermediary_dict['network'])
+
+    # Pull out network data into Network object
+    vlan_data_list = []
+    for network_type, network_data in \
+            intermediary_dict['network']['vlan_network_data'].items():
+        vlan_data_list.append(VLANNetworkData(network_type, **network_data))
+    network = Network(vlan_data_list, bgp=intermediary_dict['network']['bgp'])
+
+    # Validate site_info in intermediary
+    _validate_key_in_intermediary_dict('site_info', intermediary_dict)
+    # Validate dns in intermediary
+    _validate_key_in_intermediary_dict('dns', intermediary_dict['site_info'])
+    # Validate ntp in intermediary
+    _validate_key_in_intermediary_dict('ntp', intermediary_dict['site_info'])
+    # Validate region_name in intermediary
+    _validate_key_in_intermediary_dict('region_name', intermediary_dict)
+
+    # Pull out site_info into a SiteInfo object
+    dns_server_list = ServerList(
+        intermediary_dict['site_info']['dns']['servers'].split(','))
+    ntp_server_list = ServerList(
+        intermediary_dict['site_info']['ntp']['servers'].split(','))
+    site_info_dict = deepcopy(intermediary_dict['site_info'])
+    site_info_dict.pop('dns')
+    site_info_dict.pop('ntp')
+    site_info_dict['dns'] = dns_server_list
+    site_info_dict['ntp'] = ntp_server_list
+    site_info_dict['region_name'] = intermediary_dict['region_name']
+    site_info = SiteInfo(**site_info_dict)
+
+    # Validate storage in intermediary
+    _validate_key_in_intermediary_dict('storage', intermediary_dict)
+
+    # Create and return SiteDocumentData object
+    site_document_data = SiteDocumentData(
+        site_info=site_info,
+        network=network,
+        baremetal=rack_list,
+        storage=intermediary_dict['storage'])
+    return site_document_data

spyglass/examples/templates/baremetal/bootactions/promjoin.yaml.j2

@@ -1,26 +0,0 @@
----
-schema: 'drydock/BootAction/v1'
-metadata:
-  schema: 'metadata/Document/v1'
-  name: promjoin
-  storagePolicy: 'cleartext'
-  layeringDefinition:
-    abstract: false
-    layer: site
-  labels:
-    application: 'drydock'
-data:
-  signaling: false
-  assets:
-    - path: /opt/promjoin.sh
-      type: file
-      permissions: '555'
-{% raw %}
-      location: promenade+http://promenade-api.ucp.svc.cluster.local/api/v1.0/join-scripts?design_ref={{ action.design_ref | urlencode }}&hostname={{ node.hostname }}&ip={{ node.network.calico.ip }}{% endif %}{% for k, v in node.labels.items() %}&labels.dynamic={{ k }}={{ v }}{% endfor %}
-
-{% endraw %}
-      location_pipeline:
-        - template
-      data_pipeline:
-        - utf8_decode
-...

spyglass/examples/templates/baremetal/nodes.yaml.j2

@@ -1,50 +1,46 @@
-{% set control_count = [1] %}
+{% for rack in data.baremetal %}
-{% for rack in data['baremetal'].keys() %}
+{% for host in rack.hosts %}
-{% for host in data['baremetal'][rack].keys()%}
+{% if host.type != 'genesis' %}
-{% if data['baremetal'][rack][host]['type'] != 'genesis' %}
 ---
 schema: 'drydock/BaremetalNode/v1'
 metadata:
   schema: 'metadata/Document/v1'
-  name: {{ host }}
+  name: {{ host.name }}
   layeringDefinition:
     abstract: false
     layer: site
   storagePolicy: cleartext
 data:
-  oob:
+  {% if host.host_profile == 'cp' %}
-    account: 'root'
+  {% if loop.index - 1 < 4 %}
-{% if data['baremetal'][rack][host]['host_profile'] == 'cp' %}
+  host_profile: nc-{{ host.host_profile }}-primary
-{% if control_count.append(control_count.pop()+1) %} {% endif %}
+  {% else %}
-{% if control_count[0] < 4 %}
+  host_profile: nc-{{ host.host_profile }}-secondary
-  host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-primary
+  {% endif %}
-{% else %}
+  {% else %}
-  host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-secondary
+  host_profile: nc-{{ host.host_profile }}
-{% endif %}
+  {% endif %}
-{% else %}
-  host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}
-{% endif %}
   addressing:
     - network: oob
-      address: {{ data['baremetal'][rack][host]['ip']['oob'] }}
+      address: {{ host.ip.oob }}
     - network: oam
-      address: {{ data['baremetal'][rack][host]['ip']['oam'] }}
+      address: {{ host.ip.oam }}
     - network: pxe
-      address: {{ data['baremetal'][rack][host]['ip']['pxe'] }}
+      address: {{ host.ip.pxe }}
     - network: storage
-      address: {{ data['baremetal'][rack][host]['ip']['storage'] }}
+      address: {{ host.ip.storage }}
     - network: calico
-      address: {{ data['baremetal'][rack][host]['ip']['calico'] }}
+      address: {{ host.ip.calico }}
     - network: overlay
-      address: {{ data['baremetal'][rack][host]['ip']['overlay'] }}
+      address: {{ host.ip.overlay }}
   metadata:
-    rack: RACK{{rack[-2:] }}
+    rack: {{ rack.name }}
     tags:
-{% if data['baremetal'][rack][host]['type'] == 'compute' %}
+    {% if host.type == 'compute' %}
       - 'workers'
-{% else %}
+    {% else %}
       - 'masters'
-{% endif %}
+    {% endif %}
 ...
 {% endif %}
 {%endfor%}

spyglass/examples/templates/deployment/deployment-strategy.yaml.j2

@@ -11,7 +11,6 @@
 schema: shipyard/DeploymentStrategy/v1
 metadata:
   schema: metadata/Document/v1
-  replacement: true
   name: deployment-strategy
   layeringDefinition:
     abstract: false
@@ -44,13 +43,13 @@ data:
       selectors:
         # NEWSITE-CHANGEME: The following should be a list of the computes in the site's first rack
         - node_names:
-{% for rack in data['baremetal'].keys() %}
+        {% for rack in data.baremetal %}
-{% for host in data['baremetal'][rack].keys()%}
+          {% for host in rack.hosts %}
-{% if rack == 'rack03' or rack == 'rack04' %}
+            {% if rack.name == 'rack03' or rack.name == 'rack04' %}
-          - {{ host }}
+          - {{ host.name }}
-{% endif %}
+            {% endif %}
-{% endfor %}
+          {% endfor %}
-{% endfor %}
+        {% endfor %}
           node_labels: []
           node_tags: []
           rack_names: []
@@ -61,13 +60,13 @@ data:
       selectors:
         # NEWSITE-CHANGEME: The following should be a list of the computes in the site's second rack
         - node_names:
-{% for rack in data['baremetal'].keys() %}
+        {% for rack in data.baremetal %}
-{% for host in data['baremetal'][rack].keys()%}
+          {% for host in rack.hosts %}
-{% if rack == 'rack05' or rack == 'rack06' %}
+            {% if rack.name == 'rack05' or rack.name == 'rack06' %}
-          - {{ host }}
+          - {{ host.name }}
-{% endif %}
+            {% endif %}
-{% endfor %}
+          {% endfor %}
-{% endfor %}
+        {% endfor %}
           node_labels: []
           node_tags: []
           rack_names: []

spyglass/examples/templates/networks/common_addresses.yaml.j2

@@ -6,102 +6,69 @@ metadata:
   layeringDefinition:
     abstract: false
     layer: site
+    parentSelector:
+      name: common-addresses-global
+    actions:
+      - method: replace
+        path: .dns.upstream_servers
+      - method: merge
+        path: .
   storagePolicy: cleartext
+  replacement: true
 data:
   calico:
-    ip_autodetection_method: interface=bond1.{{ data['network']['vlan_network_data']['calico']['vlan']}}
+    ip_autodetection_method: interface=bond1.{{ data.network.get_vlan_data_by_name('calico').vlan }}
     etcd:
       service_ip: 10.96.232.136
     ip_rule:
-      gateway: {{ data['network']['vlan_network_data']['calico']['gateway']}}
+      gateway: {{ data.network.get_vlan_data_by_name('calico').gateway }}
       overlap_cidr: 10.96.0.0/15
     bgp:
       ipv4:
-        public_service_cidr: {{ data['network']['vlan_network_data']['ingress']['subnet'][0] }}
+        public_service_cidr: {{ data.network.get_vlan_data_by_name('ingress').subnet[0] }}
-        ingress_vip: {{ data['network']['bgp']['ingress_vip'] }}
+        ingress_vip: {{ data.network.bgp['ingress_vip'] }}
         peers:
-{% for peer in data['network']['bgp']['peers'] %}
+          {% for peer in data.network.bgp['peers'] %}
           - {{ peer }}
-{% endfor %}
+          {% endfor %}
   dns:
-    cluster_domain: cluster.local
     service_ip: 10.96.0.10
     upstream_servers:
-{% for server in (data['site_info']['dns']['servers']).split(',') %}
+    {% for server in data.site_info.dns.servers %}
       - {{ server }}
-{% endfor %}
+    {% endfor %}
-    upstream_servers_joined: {{ data['site_info']['dns']['servers']}}
+    upstream_servers_joined: {{ data.site_info.dns.__str__() }}
-    ingress_domain: {{ data['site_info']['domain']|lower }}
+    node_domain: {{ data.site_info.domain | lower }}
+    ingress_domain: {{ data.site_info.domain | lower }}
 
   genesis:
-    hostname: {{ (data|get_role_wise_nodes)['genesis']['name'] }}
+    hostname: {{ data.get_baremetal_host_by_type('genesis')[0].name }}
-{% for rack in data['baremetal'] %}
+    ip: {{ data.get_baremetal_host_by_type('genesis')[0].ip.calico }}
-{% for host  in data['baremetal'][rack] %}
+
-{% if data['baremetal'][rack][host]['type'] == 'genesis' %}
-    ip: {{ data['baremetal'][rack][host]['ip']['calico'] }}
-{% endif %}
-{% endfor %}
-{% endfor %}
   bootstrap:
-    ip: {{ (data|get_role_wise_nodes)['genesis']['pxe'] }}
+    ip: {{ data.get_baremetal_host_by_type('genesis')[0].ip.pxe }}
 
   kubernetes:
     api_service_ip: 10.96.0.1
     etcd_service_ip: 10.96.0.2
     pod_cidr: 10.97.0.0/16
     service_cidr: 10.96.0.0/16
-    # misc k8s port settings
-    apiserver_port: 6443
-    haproxy_port: 6553
-    service_node_port_range: 30000-32767
-
-  # etcd port settings
-  etcd:
-    container_port: 2379
-    haproxy_port: 2378
 
   masters:
-{% for host in (data|get_role_wise_nodes)['masters'] %}
+  {% for host in data.get_baremetal_host_by_type('controller') %}
-    - hostname: {{ host }}
+    - hostname: {{ host.name }}
-{% endfor %}
+  {% endfor %}
-  # NEWSITE-CHANGEME: Environment proxy information.
-  # NOTE: Reference Airship sites do not deploy behind a proxy, so this proxy section
-  # should be commented out.
-  # However if you are in a lab that requires proxy, ensure that these proxy
-  # settings are correct and reachable in your environment; otherwise update
-  # them with the correct values for your environment.
-  proxy:
-    http: ""
-    https: ""
-    no_proxy: []
-
-  node_ports:
-    drydock_api: 30000
-    maas_api: 30001
-    maas_proxy: 31800  # hardcoded in MAAS
-    shipyard_api: 30003
-    airflow_web: 30004
-  ntp:
-    servers_joined: {{ data['site_info']['ntp']['servers'] }}
-
-  ldap:
-    base_url: {{ (data['site_info']['ldap']['url']|string).split('//')[1] }}
-    url: {{ data['site_info']['ldap']['url'] }}
-    auth_path: DC=test,DC=test,DC=com?sAMAccountName?sub?memberof=CN={{ data['site_info']['ldap']['common_name'] }},OU=Application,OU=Groups,DC=test,DC=test,DC=com
-    common_name: {{ data['site_info']['ldap']['common_name'] }}
-    subdomain: {{ data['site_info']['ldap']['subdomain'] }}
-    domain: {{ (data['site_info']['ldap']['url']|string).split('.')[1] }}
 
   storage:
     ceph:
-      public_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
+      public_cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
-      cluster_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
+      cluster_cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
 
   neutron:
-    tunnel_device: 'bond1.{{ data['network']['vlan_network_data']['overlay']['vlan'] }}'
+    tunnel_device: "bond1.{{ data.network.get_vlan_data_by_name('overlay').vlan }}"
-    external_iface: 'bond1'
+    external_iface: "bond1"
 
   openvswitch:
-    external_iface: 'bond1'
+    external_iface: "bond1"
 ...
 

spyglass/examples/templates/networks/physical/networks.yaml.j2

@@ -1,28 +1,4 @@
 ---
-schema: 'drydock/NetworkLink/v1'
-metadata:
-  schema: 'metadata/Document/v1'
-  name: oob
-  layeringDefinition:
-    abstract: false
-    layer: site
-  storagePolicy: cleartext
-data:
-  # MaaS doesnt own this network like it does the others, so the noconfig label
-  # is specified.
-  labels:
-    noconfig: enabled
-  bonding:
-    mode: disabled
-  mtu: 1500
-  linkspeed: auto
-  trunking:
-    mode: disabled
-    default_network: oob
-  allowed_networks:
-    - oob
-...
----
 schema: 'drydock/Network/v1'
 metadata:
   schema: 'metadata/Document/v1'
@@ -30,42 +6,33 @@ metadata:
   layeringDefinition:
     abstract: false
     layer: site
+    parentSelector:
+      network_role: oob
+      topology: cruiser
+    actions:
+      - method: merge
+        path: .
   storagePolicy: cleartext
 data:
-  cidr: {{ data['network']['vlan_network_data']['oob']['subnet'] }}
+  cidr: {{ data.network.get_vlan_data_by_name('oob').subnet[0] }}
   routes:
     - subnet: '0.0.0.0/0'
-      gateway: {{ data['network']['vlan_network_data']['oob']['gateway'] }}
+      gateway: {{ data.network.get_vlan_data_by_name('oob').gateway }}
       metric: 100
   ranges:
+    - type: reserved
+      start: {{ data.network.get_vlan_data_by_name('oob').reserved_start }}
+      end: {{ data.network.get_vlan_data_by_name('oob').reserved_end }}
     - type: static
-      start: {{ data['network']['vlan_network_data']['oob']['static_start'] }}
+      start: {{ data.network.get_vlan_data_by_name('oob').static_start }}
-      end: {{ data['network']['vlan_network_data']['oob']['static_end'] }}
+      end: {{ data.network.get_vlan_data_by_name('oob').static_end }}
+    - type: dhcp
+      start: {{ data.network.get_vlan_data_by_name('oob').dhcp_start }}
+      end: {{ data.network.get_vlan_data_by_name('oob').dhcp_end }}
 ...
 
 ---
 schema: 'drydock/NetworkLink/v1'
-metadata:
-  schema: 'metadata/Document/v1'
-  name: pxe
-  layeringDefinition:
-    abstract: false
-    layer: site
-  storagePolicy: cleartext
-data:
-  bonding:
-    mode: disabled
-  mtu: 1500
-  linkspeed: auto
-  trunking:
-    mode: disabled
-    default_network: pxe
-  allowed_networks:
-    - pxe
-...
-
----
-schema: 'drydock/Network/v1'
 metadata:
   schema: 'metadata/Document/v1'
   name: pxe
@@ -80,56 +47,21 @@ metadata:
         path: .
   storagePolicy: cleartext
 data:
-  cidr: {{ data['network']['vlan_network_data']['pxe']['subnet'] }}
+  cidr: {{ data.network.get_vlan_data_by_name('pxe').subnet[0] }}
   routes:
-{% for other_subnet in data['network']['vlan_network_data']['pxe']['routes'] %}
+    - subnet: '0.0.0.0/0'
-    - subnet: {{ other_subnet }}
+      gateway: {{ data.network.get_vlan_data_by_name('pxe').gateway }}
-      gateway: {{ data['network']['vlan_network_data']['pxe']['gateway'] }}
       metric: 100
-{% endfor %}
   ranges:
     - type: reserved
-      start: {{ data['network']['vlan_network_data']['pxe']['reserved_start'] }}
+      start: {{ data.network.get_vlan_data_by_name('pxe').reserved_start }}
-      end: {{ data['network']['vlan_network_data']['pxe']['reserved_end'] }}
+      end: {{ data.network.get_vlan_data_by_name('pxe').reserved_end }}
     - type: static
-      start: {{ data['network']['vlan_network_data']['pxe']['static_start'] }}
+      start: {{ data.network.get_vlan_data_by_name('pxe').static_start }}
-      end: {{ data['network']['vlan_network_data']['pxe']['static_end'] }}
+      end: {{ data.network.get_vlan_data_by_name('pxe').static_end }}
     - type: dhcp
-      start: {{ data['network']['vlan_network_data']['pxe']['dhcp_start'] }}
+      start: {{ data.network.get_vlan_data_by_name('pxe').dhcp_start }}
-      end: {{ data['network']['vlan_network_data']['pxe']['dhcp_end'] }}
+      end: {{ data.network.get_vlan_data_by_name('pxe').dhcp_end }}
-...
-
----
-schema: 'drydock/NetworkLink/v1'
-metadata:
-  schema: 'metadata/Document/v1'
-  name: data
-  layeringDefinition:
-    abstract: false
-    layer: site
-  storagePolicy: cleartext
-data:
-  bonding:
-    mode: 802.3ad
-    hash: layer3+4
-    peer_rate: fast
-    mon_rate: 100
-    up_delay: 1000
-    down_delay: 3000
-  # NEWSITE-CHANGEME: Ensure the network switches in the environment are
-  # configured for this MTU or greater. Even if switches are configured for or
-  # can support a slightly higher MTU, there is no need (and negliable benefit)
-  # to squeeze every last byte into the MTU (e.g., 9216 vs 9100). Leave MTU at
-  # 9100 for maximum compatibility.
-  mtu: 9100
-  linkspeed: auto
-  trunking:
-    mode: 802.1q
-  allowed_networks:
-    - oam
-    - storage
-    - overlay
-    - calico
 ...
 
 ---
@@ -148,27 +80,24 @@ metadata:
         path: .
   storagePolicy: cleartext
 data:
-  cidr: {{ data['network']['vlan_network_data']['oam']['subnet'] }}
+  cidr: {{ data.network.get_vlan_data_by_name('oam').subnet[0] }}
-{% set flag = [0] %}
+  {% if data.network.get_vlan_data_by_name('oam').routes %}
-{% for route in data['network']['vlan_network_data']['oam']['routes'] %}
-{% if flag[0] == 0 %}
   routes:
-{% endif %}
+  {% for route in data.network.get_vlan_data_by_name('oam').routes %}
-{% if flag.append(flag.pop() + 1) %} {% endif %}
     - subnet: {{ route }}
-      gateway: {{ data['network']['vlan_network_data']['oam']['gateway'] }}
+      gateway: {{ data.network.get_vlan_data_by_name('oam').gateway }}
       metric: 100
-{% endfor %}
+  {% endfor %}
-{% if flag[0] == 0 %}
+  {% else %}
-  routes:[]
+  routes: []
-{% endif %}
+  {% endif %}
   ranges:
     - type: reserved
-      start: {{ data['network']['vlan_network_data']['oam']['reserved_start'] }}
+      start: {{ data.network.get_vlan_data_by_name('oam').reserved_start }}
-      end: {{ data['network']['vlan_network_data']['oam']['reserved_end'] }}
+      end: {{ data.network.get_vlan_data_by_name('oam').reserved_end }}
     - type: static
-      start: {{ data['network']['vlan_network_data']['oam']['static_start'] }}
+      start: {{ data.network.get_vlan_data_by_name('oam').static_start }}
-      end: {{ data['network']['vlan_network_data']['oam']['static_end'] }}
+      end: {{ data.network.get_vlan_data_by_name('oam').static_end }}
 ...
 
 ---
@@ -187,40 +116,14 @@ metadata:
         path: .
   storagePolicy: cleartext
 data:
-  cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
+  cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
   ranges:
     - type: reserved
-      start: {{ data['network']['vlan_network_data']['storage']['reserved_start'] }}
+      start: {{ data.network.get_vlan_data_by_name('storage').reserved_start }}
-      end: {{ data['network']['vlan_network_data']['storage']['reserved_end'] }}
+      end: {{ data.network.get_vlan_data_by_name('storage').reserved_end }}
     - type: static
-      start: {{ data['network']['vlan_network_data']['storage']['static_start'] }}
+      start: {{ data.network.get_vlan_data_by_name('storage').static_start }}
-      end: {{ data['network']['vlan_network_data']['storage']['static_end'] }}
+      end: {{ data.network.get_vlan_data_by_name('storage').static_end }}
-...
-
----
-schema: 'drydock/Network/v1'
-metadata:
-  schema: 'metadata/Document/v1'
-  name: overlay
-  layeringDefinition:
-    abstract: false
-    layer: site
-    parentSelector:
-      network_role: os-overlay
-      topology: cruiser
-    actions:
-      - method: merge
-        path: .
-  storagePolicy: cleartext
-data:
-  cidr: {{ data['network']['vlan_network_data']['overlay']['subnet'] }}
-  ranges:
-    - type: reserved
-      start: {{ data['network']['vlan_network_data']['overlay']['reserved_start'] }}
-      end: {{ data['network']['vlan_network_data']['overlay']['reserved_end'] }}
-    - type: static
-      start: {{ data['network']['vlan_network_data']['overlay']['static_start'] }}
-      end: {{ data['network']['vlan_network_data']['overlay']['static_end'] }}
 ...
 
 ---
@@ -239,13 +142,38 @@ metadata:
         path: .
   storagePolicy: cleartext
 data:
-  cidr: {{ data['network']['vlan_network_data']['calico']['subnet'] }}
+  cidr: {{ data.network.get_vlan_data_by_name('calico').subnet[0] }}
   ranges:
     - type: reserved
-      start: {{ data['network']['vlan_network_data']['calico']['reserved_start'] }}
+      start: {{ data.network.get_vlan_data_by_name('calico').reserved_start }}
-      end: {{ data['network']['vlan_network_data']['calico']['reserved_end'] }}
+      end: {{ data.network.get_vlan_data_by_name('calico').reserved_end }}
     - type: static
-      start: {{ data['network']['vlan_network_data']['calico']['static_start'] }}
+      start: {{ data.network.get_vlan_data_by_name('calico').static_start }}
-      end: {{ data['network']['vlan_network_data']['calico']['static_end'] }}
+      end: {{ data.network.get_vlan_data_by_name('calico').static_end }}
 ...
 
+---
+schema: 'drydock/Network/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: overlay
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      network_role: os-overlay
+      topology: cruiser
+    actions:
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  cidr: {{ data.network.get_vlan_data_by_name('overlay').subnet[0] }}
+  ranges:
+    - type: reserved
+      start: {{ data.network.get_vlan_data_by_name('overlay').reserved_start }}
+      end: {{ data.network.get_vlan_data_by_name('overlay').reserved_end }}
+    - type: static
+      start: {{ data.network.get_vlan_data_by_name('overlay').static_start }}
+      end: {{ data.network.get_vlan_data_by_name('overlay').static_end }}
+...

spyglass/examples/templates/pki/pki-catalogue.yaml.j2

@@ -21,33 +21,29 @@ data:
             - 10.96.0.1
           kubernetes_service_names:
             - kubernetes.default.svc.cluster.local
-{% for racks in data['baremetal'].keys()%}
+        {% for host in data.get_baremetal_host_by_type('genesis') %}
-{% for host in data['baremetal'][racks].keys()%}
-{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
-
         - document_name: kubelet-genesis
-          common_name: system:node:{{ host }}
+          common_name: system:node:{{ host.name }}
           hosts:
-            - {{ host }}
+            - {{ host.name }}
-            - {{ data['baremetal'][racks][host]['ip']['oam'] }}
+            - {{ host.ip.oam }}
-            - {{ data['baremetal'][racks][host]['ip']['calico']}}
+            - {{ host.ip.calico }}
           groups:
             - system:nodes
-{% endif %}
+        {% endfor %}
-{%endfor%}
+
-{%endfor%}
+        {% for rack in data.baremetal %}
-{% for racks in data['baremetal'].keys()%}
+          {% for host in rack.hosts %}
-{% for host in data['baremetal'][racks].keys()%}
+        - document_name: kubelet-{{ host.name }}
-        - document_name: kubelet-{{ host }}
+          common_name: system:node:{{ host.name }}
-          common_name: system:node:{{ host }}
           hosts:
-            - {{ host }}
+            - {{ host.name }}
-            - {{ data['baremetal'][racks][host]['ip']['oam'] }}
+            - {{ host.ip.oam }}
-            - {{ data['baremetal'][racks][host]['ip']['calico']}}
+            - {{ host.ip.calico }}
           groups:
             - system:nodes
-{%endfor%}
+          {% endfor %}
-{%endfor%}
+        {% endfor %}
         - document_name: scheduler
           description: Service certificate for Kubernetes scheduler
           common_name: system:kube-scheduler
@@ -62,6 +58,7 @@ data:
           common_name: armada
           groups:
             - system:masters
+
     kubernetes-etcd:
       description: Certificates for Kubernetes's etcd servers
       certificates:
@@ -72,113 +69,93 @@ data:
         - document_name: kubernetes-etcd-anchor
           description: anchor
           common_name: anchor
-{% for racks in data['baremetal'].keys()%}
+        {% for host in data.get_baremetal_host_by_type('genesis') %}
-{% for host in data['baremetal'][racks].keys()%}
-{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
         - document_name: kubernetes-etcd-genesis
           common_name: kubernetes-etcd-genesis
           hosts:
-            - {{ host }}
+            - {{ host.name }}
-            - {{ data['baremetal'][racks][host]['ip']['oam'] }}
+            - {{ host.ip.oam }}
-            - {{ data['baremetal'][racks][host]['ip']['calico']}}
+            - {{ host.ip.calico }}
             - 127.0.0.1
             - localhost
             - kubernetes-etcd.kube-system.svc.cluster.local
             - 10.96.0.2
-{% endif %}
+        {% endfor %}
-{%endfor%}
+
-{%endfor%}
+        {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
-{% for racks in data['baremetal'].keys()%}
+        - document_name: kubernetes-etcd-{{ host.name }}
-{% for host in data['baremetal'][racks].keys()%}
+          common_name: kubernetes-etcd-{{ host.name }}
-{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis'%}
-        - document_name: kubernetes-etcd-{{ host }}
-          common_name: kubernetes-etcd-{{ host }}
           hosts:
-            - {{ host }}
+            - {{ host.name }}
-            - {{ data['baremetal'][racks][host]['ip']['oam'] }}
+            - {{ host.ip.oam }}
-            - {{ data['baremetal'][racks][host]['ip']['calico']}}
+            - {{ host.ip.calico }}
             - 127.0.0.1
             - localhost
             - kubernetes-etcd.kube-system.svc.cluster.local
             - 10.96.0.2
-{% endif %}
+        {% endfor %}
-{%endfor%}
+
-{%endfor%}
-{% for racks in data['baremetal'].keys()%}
-{% for host in data['baremetal'][racks].keys()%}
-{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
     kubernetes-etcd-peer:
       certificates:
+        {% for host in data.get_baremetal_host_by_type('genesis') %}
         - document_name: kubernetes-etcd-genesis-peer
           common_name: kubernetes-etcd-genesis-peer
           hosts:
-            - {{ host }}
+            - {{ host.name }}
-            - {{ data['baremetal'][racks][host]['ip']['oam'] }}
+            - {{ host.ip.oam }}
-            - {{ data['baremetal'][racks][host]['ip']['calico']}}
+            - {{ host.ip.calico }}
             - 127.0.0.1
             - localhost
             - kubernetes-etcd.kube-system.svc.cluster.local
             - 10.96.0.2
-{% endif %}
+        {% endfor %}
-{%endfor%}
+
-{%endfor%}
+        {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
-{% for racks in data['baremetal'].keys()%}
+        - document_name: kubernetes-etcd-{{ host.name }}-peer
-{% for host in data['baremetal'][racks].keys()%}
+          common_name: kubernetes-etcd-{{ host.name }}-peer
-{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %}
-        - document_name: kubernetes-etcd-{{ host }}-peer
-          common_name: kubernetes-etcd-{{ host }}-peer
           hosts:
-            - {{ host }}
+            - {{ host.name }}
-            - {{ data['baremetal'][racks][host]['ip']['oam'] }}
+            - {{ host.ip.oam }}
-            - {{ data['baremetal'][racks][host]['ip']['calico']}}
+            - {{ host.ip.calico }}
             - 127.0.0.1
             - localhost
             - kubernetes-etcd.kube-system.svc.cluster.local
             - 10.96.0.2
-{% endif %}
+        {% endfor %}
-{%endfor%}
+
-{%endfor%}
+    calico-etcd:
-    ksn-etcd:
       description: Certificates for Calico etcd client traffic
       certificates:
         - document_name: ksn-etcd-anchor
           description: anchor
           common_name: anchor
-{% for racks in data['baremetal'].keys()%}
+        {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
-{% for host in data['baremetal'][racks].keys()%}
+        - document_name: ksn-etcd-{{ host.name }}
-{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %}
+          common_name: ksn-etcd-{{ host.name }}
-        - document_name: ksn-etcd-{{ host }}
-          common_name: ksn-etcd-{{ host }}
           hosts:
-            - {{ host }}
+            - {{ host.name }}
-            - {{ data['baremetal'][racks][host]['ip']['oam'] }}
+            - {{ host.ip.oam }}
-            - {{ data['baremetal'][racks][host]['ip']['calico']}}
+            - {{ host.ip.calico }}
             - 127.0.0.1
             - localhost
             - 10.96.232.136
-{% endif %}
+        {% endfor %}
-{%endfor%}
-{%endfor%}
         - document_name: ksn-node
           common_name: calcico-node
-    ksn-etcd-peer:
+    calico-etcd-peer:
       description: Certificates for Calico etcd clients
       certificates:
-{% for racks in data['baremetal'].keys()%}
+        {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
-{% for host in data['baremetal'][racks].keys()%}
+        - document_name: ksn-etcd-{{ host.name }}-peer
-{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %}
+          common_name: ksn-etcd-{{ host.name }}-peer
-        - document_name: ksn-etcd-{{ host }}-peer
-          common_name: ksn-etcd-{{ host }}-peer
           hosts:
-            - {{ host }}
+            - {{ host.name }}
-            - {{ data['baremetal'][racks][host]['ip']['oam'] }}
+            - {{ host.ip.oam }}
-            - {{ data['baremetal'][racks][host]['ip']['calico']}}
+            - {{ host.ip.calico }}
             - 127.0.0.1
             - localhost
             - 10.96.232.136
-{% endif %}
+        {% endfor %}
-{%endfor%}
-{%endfor%}
         - document_name: ksn-node-peer
           common_name: calico-node-peer
   keypairs:

spyglass/examples/templates/profile/region.yaml.j2

@@ -2,7 +2,7 @@
 schema: 'drydock/Region/v1'
 metadata:
   schema: 'metadata/Document/v1'
-  name: {{ data['region_name'] }}
+  name: {{ data.site_info.region_name }}
   layeringDefinition:
     abstract: false
     layer: site
@@ -18,7 +18,7 @@ metadata:
         path: .authorized_keys[1]
       src:
         schema: deckhand/PublicKey/v1
-        name: {{ data['region_name'] }}_ssh_public_key
+        name: {{ data.site_info.region_name }}_ssh_public_key
         path: .
     - dest:
         path: .repositories.main_archive

spyglass/examples/templates/site-definition.yaml.j2

@@ -7,11 +7,11 @@ metadata:
     abstract: false
     layer: site
   # NEWSITE-CHANGEME: Replace with the site name
-  name: {{ data['region_name'] }}
+  name: {{ data.site_info.region_name }}
   storagePolicy: cleartext
 data:
   # The type layer this site will delpoy with. Type layer is found in the
   # aic-clcp-manifests repo.
-  site_type: {{ data['site_info']['sitetype'] }}
+  site_type: {{ data.site_info.sitetype }}
 ...
 

spyglass/examples/templates/software/charts/kubernetes/etcd.yaml.j2

@@ -13,83 +13,70 @@ metadata:
         path: .
   storagePolicy: cleartext
   substitutions:
-{% set count = [0] %}
+{% for host in data.get_baremetal_host_by_type('controller') %}
-{% for rack in data.baremetal.keys() %}
-{% for host in data["baremetal"][rack] %}
-{% if data["baremetal"][rack][host]["type"]  == 'controller' %}
     - src:
         schema: pegleg/CommonAddresses/v1
         name: common-addresses
-        path: .masters[{{ count[0] }}].hostname
+        path: .masters[{{ loop.index - 1 }}].hostname
       dest:
-        path: .values.nodes[{{ count[0] }}].name
+        path: .values.nodes[{{ loop.index - 1 }}].name
     - src:
         schema: deckhand/Certificate/v1
-        name: calico-etcd-{{ host }}
+        name: calico-etcd-{{ host.name }}
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.client.cert
+        path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
     - src:
         schema: deckhand/CertificateKey/v1
-        name: calico-etcd-{{ host }}
+        name: calico-etcd-{{ host.name }}
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.client.key
+        path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
     - src:
         schema: deckhand/Certificate/v1
-        name: calico-etcd-{{ host }}
+        name: calico-etcd-{{ host.name }}
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.peer.cert
+        path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
     - src:
         schema: deckhand/CertificateKey/v1
-        name: calico-etcd-{{ host }}-peer
+        name: calico-etcd-{{ host.name }}-peer
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.peer.key
+        path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
-
-    {% if count.append(count.pop() + 1) %}{% endif %} {# increment count by 1 #}
-
-{% endif %}
-{% endfor %}
 {% endfor %}
 
-{% for rack in data.baremetal.keys() %}
+{% for host in data.get_baremetal_host_by_type('genesis') %}
-{% for host in data["baremetal"][rack] %}
-{% if data["baremetal"][rack][host]["type"]  == 'genesis' %}
     - src:
         schema: pegleg/CommonAddresses/v1
         name: common-addresses
         path: .genesis.hostname
       dest:
-        path: .values.nodes[{{ count[0] }}].name
+        path: .values.nodes[{{ loop.index - 1 }}].name
     - src:
         schema: deckhand/Certificate/v1
-        name: calico-etcd-{{ host }}
+        name: calico-etcd-{{ host.name }}
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.client.cert
+        path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
     - src:
         schema: deckhand/CertificateKey/v1
-        name: calico-etcd-{{ host }}
+        name: calico-etcd-{{ host.name }}
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.client.key
+        path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
     - src:
         schema: deckhand/Certificate/v1
-        name: calico-etcd-{{ host }}-peer
+        name: calico-etcd-{{ host.name }}-peer
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.peer.cert
+        path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
     - src:
         schema: deckhand/CertificateKey/v1
-        name: calico-etcd-{{ host }}-peer
+        name: calico-etcd-{{ host.name }}-peer
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.peer.key
+        path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
-
-{% endif %}
-{% endfor %}
 {% endfor %}
 
 data: {}

spyglass/examples/templates/software/charts/kubernetes/etcd/etcd.yaml.j2

@@ -13,80 +13,70 @@ metadata:
         path: .
   storagePolicy: cleartext
   substitutions:
-{% set count = [0] %}
+  {% for host in data.get_baremetal_host_by_type('controller') %}
-{% for rack in data.baremetal.keys() %}
-{% for host in data["baremetal"][rack] %}
-{% if data["baremetal"][rack][host]["type"]  == 'controller' %}
     - src:
         schema: pegleg/CommonAddresses/v1
         name: common-addresses
-        path: .masters[{{ count[0] }}].hostname
+        path: .masters[{{ loop.index - 1 }}].hostname
       dest:
-        path: .values.nodes[{{ count[0] }}].name
+        path: .values.nodes[{{ loop.index - 1 }}].name
     - src:
         schema: deckhand/Certificate/v1
-        name: kubernetes-etcd-{{ host }}
+        name: kubernetes-etcd-{{ host.name }}
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.client.cert
+        path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
     - src:
         schema: deckhand/CertificateKey/v1
-        name: kubernetes-etcd-{{ host }}
+        name: kubernetes-etcd-{{ host.name }}
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.client.key
+        path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
     - src:
         schema: deckhand/Certificate/v1
-        name: kubernetes-etcd-{{ host }}-peer
+        name: kubernetes-etcd-{{ host.name }}-peer
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.peer.cert
+        path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
     - src:
         schema: deckhand/CertificateKey/v1
-        name: kubernetes-etcd-{{ host }}-peer
+        name: kubernetes-etcd-{{ host.name }}-peer
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.peer.key
+        path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
-    {% if count.append(count.pop() + 1) %}{% endif %} {# increment count by 1 #}
+  {% endfor %}
-{% endif %}
+  {% for host in data.get_baremetal_host_by_type('genesis') %}
-{% endfor %}
-{% endfor %}
-{% for rack in data.baremetal.keys() %}
-{% for host in data["baremetal"][rack] %}
-{% if data["baremetal"][rack][host]["type"]  == 'genesis' %}
     - src:
         schema: pegleg/CommonAddresses/v1
         name: common-addresses
         path: .genesis.hostname
       dest:
-        path: .values.nodes[{{ count[0] }}].name
+        path: .values.nodes[{{ loop.index - 1 }}].name
     - src:
         schema: deckhand/Certificate/v1
         name: kubernetes-etcd-genesis
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.client.cert
+        path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
     - src:
         schema: deckhand/CertificateKey/v1
         name: kubernetes-etcd-genesis
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.client.key
+        path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
     - src:
         schema: deckhand/Certificate/v1
         name: kubernetes-etcd-genesis-peer
         path: .
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.peer.cert
+        path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
     - src:
         schema: deckhand/CertificateKey/v1
         name: kubernetes-etcd-genesis-peer
         path: $
       dest:
-        path: .values.nodes[{{ count[0] }}].tls.peer.key
+        path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
-{% endif %}
+  {% endfor %}
-{% endfor %}
-{% endfor %}
 
 data: {}
 ...

spyglass/examples/templates/software/charts/osh/openstack-tenant-ceph/ceph-client.yaml.j2

@@ -18,5 +18,5 @@ data:
     conf:
       pool:
         target:
-          osd: {{ data['storage']['ceph']['controller']['osd_count'] }}
+          osd: {{ data.storage['ceph']['controller']['osd_count'] }}
 ...

spyglass/examples/templates/software/config/common-software-config.yaml.j2

@@ -13,4 +13,5 @@ metadata:
 data:
   osh:
     # NEWSITE-CHANGEME: Replace with the site name
-    region_name: {{ data['region_name'] }}
+    region_name: {{ data.site_info.region_name }}
+...

spyglass/site_processors/base.py

@@ -20,22 +20,3 @@ class BaseProcessor(object):
 
     def render_template(self, template):
         pass
-
-    @staticmethod
-    def get_role_wise_nodes(yaml_data):
-        hosts = {"genesis": {}, "masters": [], "workers": []}
-
-        for rack in yaml_data["baremetal"]:
-            for host in yaml_data["baremetal"][rack]:
-                if yaml_data["baremetal"][rack][host]["type"] == "genesis":
-                    hosts["genesis"] = {
-                        "name": host,
-                        "pxe": yaml_data["baremetal"][rack][host]["ip"]["pxe"],
-                        "oam": yaml_data["baremetal"][rack][host]["ip"]["oam"],
-                    }
-                elif yaml_data["baremetal"][rack][host]["type"] \
-                        == "controller":
-                    hosts["masters"].append(host)
-                else:
-                    hosts["workers"].append(host)
-        return hosts

spyglass/site_processors/site_processor.py

@@ -18,6 +18,8 @@ import shutil
 
 import jinja2
 
+from spyglass.data_extractor.models import site_document_data_factory
+from spyglass.data_extractor.models import SiteDocumentData
 from spyglass.site_processors.base import BaseProcessor
 
 LOG = logging.getLogger(__name__)
@@ -25,9 +27,12 @@ LOG = logging.getLogger(__name__)
 
 class SiteProcessor(BaseProcessor):
 
-    def __init__(self, intermediary_yaml, manifest_dir, force_write):
+    def __init__(self, site_data, manifest_dir, force_write):
         super().__init__()
-        self.yaml_data = intermediary_yaml
+        if isinstance(site_data, SiteDocumentData):
+            self.site_data = site_data
+        else:
+            self.site_data = site_document_data_factory(site_data)
         self.manifest_dir = manifest_dir
         self.force_write = force_write
 
@@ -38,7 +43,7 @@ class SiteProcessor(BaseProcessor):
         calico) are generated in a single file. Rack specific
         configs( pxe and oob) are generated per rack.
         """
-        # Check of manifest_dir exists
+        # Check if manifest_dir exists
         if self.manifest_dir is not None:
             site_manifest_dir = os.path.join(
                 self.manifest_dir, 'pegleg_manifests', 'site')
@@ -66,16 +71,16 @@ class SiteProcessor(BaseProcessor):
                     autoescape=True,
                     loader=loader,
                     trim_blocks=True,
+                    lstrip_blocks=True,
                     undefined=logging_undefined)
-                j2_env.filters["get_role_wise_nodes"] = \
-                    self.get_role_wise_nodes
                 templatefile = os.path.join(dirpath, filename)
                 LOG.debug("Template file: %s", templatefile)
                 outdirs = dirpath.split(template_folder_name)[1].lstrip(os.sep)
                 LOG.debug("outdirs: %s", outdirs)
 
                 outfile_path = os.path.join(
-                    site_manifest_dir, self.yaml_data["region_name"], outdirs)
+                    site_manifest_dir, self.site_data.site_info.region_name,
+                    outdirs)
                 LOG.debug("outfile path: %s", outfile_path)
                 outfile_yaml = os.path.split(templatefile)[1]
                 outfile_yaml = os.path.splitext(outfile_yaml)[0]
@@ -90,7 +95,7 @@ class SiteProcessor(BaseProcessor):
                     out = open(outfile, "w")
                     created_file_list.append(outfile)
                     LOG.info("Rendering {}".format(outfile_yaml))
-                    rendered = template_j2.render(data=self.yaml_data)
+                    rendered = template_j2.render(data=self.site_data)
                     out.write(rendered)
                     out.close()
                 except IOError as ioe:

tests/unit/data_extractor/test_models.py

@@ -13,11 +13,18 @@
 # limitations under the License.
 
 from copy import copy
+import os
 import unittest
 from unittest import mock
 
+import yaml
+
+from spyglass.data_extractor.custom_exceptions import InvalidIntermediary
 from spyglass.data_extractor import models
 
+FIXTURE_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'shared')
+
 
 class TestParseIp(unittest.TestCase):
     """Tests the _parse_ip validator for Spyglass models"""
@@ -340,11 +347,7 @@ class TestRack(unittest.TestCase):
     """Tests for the Rack model"""
 
     RACK_NAME = 'test_rack1'
-    HOST_DATA = {
+    HOST_DATA = {'rack_name': RACK_NAME, 'host_profile': 'host'}
-        'rack_name': RACK_NAME,
-        'host_profile': 'host',
-        'type': 'compute'
-    }
 
     @mock.patch('spyglass.data_extractor.models.IPList', autospec=True)
     def setUp(self, MockIPList):
@@ -353,9 +356,9 @@ class TestRack(unittest.TestCase):
         self.HOST_DATA['ip'] = MockIPList()
         self.HOST_DATA['ip'].dict_from_class.return_value = 'success'
         self.hosts = [
-            models.Host('test_host1', **self.HOST_DATA),
+            models.Host('test_host1', **self.HOST_DATA, type='genesis'),
-            models.Host('test_host2', **self.HOST_DATA),
+            models.Host('test_host2', **self.HOST_DATA, type='compute'),
-            models.Host('test_host3', **self.HOST_DATA),
+            models.Host('test_host3', **self.HOST_DATA, type='controller'),
         ]
 
     def test___init__(self):
@@ -408,6 +411,15 @@ class TestRack(unittest.TestCase):
         self.assertEqual(
             self.hosts[1], result.get_host_by_name(self.hosts[1].name))
 
+    def test_get_host_by_type(self):
+        """Tests retrieval of a Rack's host(s) by type"""
+        result = models.Rack(self.RACK_NAME, self.hosts)
+        self.assertEqual(self.hosts[0], result.get_host_by_type('genesis')[0])
+        self.assertEqual(self.hosts[1], result.get_host_by_type('compute')[0])
+        self.assertEqual(
+            self.hosts[2],
+            result.get_host_by_type('controller')[0])
+
 
 class TestVLANNetworkData(unittest.TestCase):
     """Tests for the VLANNetworkData model"""
@@ -820,3 +832,162 @@ class TestSiteDocumentData(unittest.TestCase):
         type(Rack()).name = mock.PropertyMock(side_effect=['rack1', 'rack3'])
         result = models.SiteDocumentData(site_info, network, baremetal)
         self.assertIsNone(result.get_baremetal_rack_by_name('rack2'))
+
+    @mock.patch('spyglass.data_extractor.models.SiteInfo')
+    @mock.patch('spyglass.data_extractor.models.Network')
+    @mock.patch('spyglass.data_extractor.models.Rack')
+    @mock.patch('spyglass.data_extractor.models.Host')
+    def test_get_baremetal_rack_by_name_multiple(
+            self, Host, Rack, Network, SiteInfo):
+        """Tests retrieval of baremetal host(s) by type"""
+        site_info = SiteInfo()
+        network = Network()
+        baremetal = [Rack(), Rack()]
+        Rack().get_host_by_type.return_value = [Host()]
+        result = models.SiteDocumentData(site_info, network, baremetal)
+        self.assertEqual(2, len(result.get_baremetal_host_by_type('genesis')))
+        self.assertEqual(2, len(result.get_baremetal_host_by_type('computer')))
+        self.assertEqual(
+            2, len(result.get_baremetal_host_by_type('controller')))
+
+
+class TestValidateKeyInIntermediaryDict(unittest.TestCase):
+    """Tests the _validate_key_in_intermediary_dict function"""
+
+    def test__validate_key_in_intermediary_dict(self):
+        test_dictionary = {'test_key': 'value'}
+        key = 'test_key'
+        self.assertIsNone(
+            models._validate_key_in_intermediary_dict(key, test_dictionary))
+
+    def test__validate_key_in_intermediary_dict_key_dne(self):
+        test_dictionary = {'test_key': 'value'}
+        key = 'not_test_key'
+        with self.assertRaises(InvalidIntermediary):
+            models._validate_key_in_intermediary_dict(key, test_dictionary)
+
+
+class TestSiteDocumentDataFactory(unittest.TestCase):
+    """Tests the site_document_data_factory function"""
+
+    def setUp(self) -> None:
+        test_intermediary_path = os.path.join(
+            FIXTURE_DIR, 'test_intermediary.yaml')
+        with open(test_intermediary_path, 'r') as f:
+            self.intermediary_dict = yaml.safe_load(f)
+
+    def test_site_document_data_factory(self):
+        site_document_data = models.site_document_data_factory(
+            self.intermediary_dict)
+
+        # Check correct return type
+        self.assertIsInstance(site_document_data, models.SiteDocumentData)
+
+    def test_site_document_data_factory_saves_storage(self):
+        site_document_data = models.site_document_data_factory(
+            self.intermediary_dict)
+
+        # Check that storage was saved without changes in the SiteDocumentData
+        self.assertDictEqual(
+            self.intermediary_dict['storage'], site_document_data.storage)
+
+    def test_site_document_data_factory_saves_site_info(self):
+        site_document_data = models.site_document_data_factory(
+            self.intermediary_dict)
+
+        # Check that site info saved correctly in SiteInfo object
+        site_info_dict = self.intermediary_dict['site_info']
+        self.assertIsInstance(site_document_data.site_info, models.SiteInfo)
+        self.assertEqual(
+            site_info_dict['name'], site_document_data.site_info.name)
+        self.assertEqual(
+            self.intermediary_dict['region_name'],
+            site_document_data.site_info.region_name)
+        self.assertEqual(
+            site_info_dict['state'], site_document_data.site_info.state)
+        self.assertEqual(
+            site_info_dict['physical_location_id'],
+            site_document_data.site_info.physical_location_id)
+        self.assertEqual(
+            site_info_dict['country'], site_document_data.site_info.country)
+        self.assertEqual(
+            site_info_dict['corridor'], site_document_data.site_info.corridor)
+        self.assertEqual(
+            site_info_dict['sitetype'], site_document_data.site_info.sitetype)
+        self.assertEqual(
+            site_info_dict['domain'], site_document_data.site_info.domain)
+        self.assertDictEqual(
+            site_info_dict['ldap'], site_document_data.site_info.ldap)
+        self.assertEqual(
+            site_info_dict['dns']['servers'],
+            str(site_document_data.site_info.dns))
+        self.assertEqual(
+            site_info_dict['ntp']['servers'],
+            str(site_document_data.site_info.ntp))
+
+    def test_site_document_data_factory_saves_network_data(self):
+        site_document_data = models.site_document_data_factory(
+            self.intermediary_dict)
+
+        # Check that network data saved correctly into a Network object
+        network_dict = self.intermediary_dict['network']
+        self.assertIsInstance(site_document_data.network, models.Network)
+        self.assertDictEqual(
+            network_dict['bgp'], site_document_data.network.bgp)
+        for network_type, network_data \
+                in network_dict['vlan_network_data'].items():
+            vlan_network_data = \
+                site_document_data.network.get_vlan_data_by_name(network_type)
+            self.assertIsInstance(vlan_network_data, models.VLANNetworkData)
+            self.assertEqual(network_type, vlan_network_data.name)
+            self.assertEqual(network_type, vlan_network_data.role)
+            self.assertEqual(network_data['subnet'], vlan_network_data.subnet)
+            if 'routes' in network_data:
+                self.assertEqual(
+                    network_data['routes'], vlan_network_data.routes)
+            if 'gateway' in network_data:
+                self.assertEqual(
+                    network_data['gateway'], vlan_network_data.gateway)
+            if 'vlan' in network_data:
+                self.assertEqual(network_data['vlan'], vlan_network_data.vlan)
+            if 'dhcp_start' in network_data and 'dhcp_end' in network_data:
+                self.assertEqual(
+                    network_data['dhcp_start'], vlan_network_data.dhcp_start)
+                self.assertEqual(
+                    network_data['dhcp_end'], vlan_network_data.dhcp_end)
+            if 'static_start' in network_data and 'static_end' in network_data:
+                self.assertEqual(
+                    network_data['static_start'],
+                    vlan_network_data.static_start)
+                self.assertEqual(
+                    network_data['static_end'], vlan_network_data.static_end)
+            if 'reserved_start' in network_data \
+                    and 'reserved_end' in network_data:
+                self.assertEqual(
+                    network_data['reserved_start'],
+                    vlan_network_data.reserved_start)
+                self.assertEqual(
+                    network_data['reserved_end'],
+                    vlan_network_data.reserved_end)
+
+    def test_site_document_data_factory_saves_baremetal_data(self):
+        site_document_data = models.site_document_data_factory(
+            self.intermediary_dict)
+
+        # Check that baremetal racks saved correctly into Rack objects
+        for rack_name, hosts \
+                in self.intermediary_dict['baremetal'].items():
+            rack = site_document_data.get_baremetal_rack_by_name(rack_name)
+            for host_name, host_data in hosts.items():
+                host = rack.get_host_by_name(host_name)
+                self.assertEqual(host_name, host.name)
+                self.assertEqual(rack_name, host.rack_name)
+                self.assertEqual(host_data['type'], host.type)
+                self.assertEqual(host_data['host_profile'], host.host_profile)
+                self.assertEqual(host_data['ip']['oob'], host.ip.oob)
+                self.assertEqual(host_data['ip']['oam'], host.ip.oam)
+                self.assertEqual(host_data['ip']['calico'], host.ip.calico)
+                self.assertEqual(host_data['ip']['overlay'], host.ip.overlay)
+                self.assertEqual(host_data['ip']['pxe'], host.ip.pxe)
+                self.assertEqual(host_data['ip']['storage'], host.ip.storage)
+            self.assertEqual(rack_name, rack.name)

tests/unit/site_processors/test_site_processor.py

@@ -15,117 +15,178 @@
 import logging
 import os
 from tempfile import mkdtemp
+import textwrap
+import unittest
+from unittest import mock
 
 from jinja2 import UndefinedError
 import pytest
 
+from spyglass.data_extractor import models
 from spyglass.site_processors.site_processor import SiteProcessor
 
 LOG = logging.getLogger(__name__)
 LOG.level = logging.DEBUG
 
-J2_TPL = """---
+
-schema: pegleg/SiteDefinition/v1
+class TestSiteProcessor(unittest.TestCase):
-metadata:
+
+    J2_TPL = textwrap.dedent(
+        """
+    ---
+    schema: pegleg/SiteDefinition/v1
+    metadata:
       schema: metadata/Document/v1
       layeringDefinition:
         abstract: false
         layer: site
-  name: {{ data['region_name'] }}
+      name: {{ data.site_info.region_name }}
       storagePolicy: cleartext
-data:
+    data:
-  site_type:{{ data['site_info']['sitetype'] }}
+      site_type:{{ data.site_info.sitetype }}
-..."""
+    ...""")
 
+    J2_TPL_UNDEFINED = textwrap.dedent(
+        """
+    ---
+    schema: pegleg/SiteDefinition/v1
+    metadata:
+      schema: metadata/Document/v1
+      layeringDefinition:
+        abstract: false
+        layer: site
+      name: {{ data.site_info.region_name }}
+      storagePolicy: cleartext
+    data:
+      site_type:{{ undefined_param }}
+    ...""")
 
-def test_render_template():
+    @mock.patch(
+        'spyglass.data_extractor.models.SiteDocumentData',
+        spec=models.SiteDocumentData)
+    @mock.patch('spyglass.data_extractor.models.SiteInfo')
+    @mock.patch('spyglass.data_extractor.models.ServerList')
+    def test_render_template(self, ServerList, SiteInfo, SiteDocumentData):
         _tpl_parent_dir = mkdtemp()
         _tpl_dir = mkdtemp(dir=_tpl_parent_dir)
         _tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
         with open(_tpl_file, 'w') as f:
-        f.write(J2_TPL)
+            f.write(self.J2_TPL)
             LOG.debug("Writing test template to %s", _tpl_file)
-    _input_yaml = {
+
-        "region_name": "test",
+        site_data = SiteDocumentData()
-        "site_info": {
+        type(SiteDocumentData()).site_info = SiteInfo()
-            "sitetype": "test_type"
+        region_name = 'test'
-        }
+        type(SiteInfo()).region_name = mock.PropertyMock(
-    }
+            return_value=region_name)
+        site_type = 'test_type'
+        type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
+
         _out_dir = mkdtemp()
-    site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=False)
+        site_processor = SiteProcessor(site_data, _out_dir, force_write=False)
         site_processor.render_template(_tpl_parent_dir)
 
-    expected_output = """---
+        expected_output = textwrap.dedent(
-schema: pegleg/SiteDefinition/v1
+            """
-metadata:
+        ---
+        schema: pegleg/SiteDefinition/v1
+        metadata:
           schema: metadata/Document/v1
           layeringDefinition:
             abstract: false
             layer: site
           name: test
           storagePolicy: cleartext
-data:
+        data:
           site_type:test_type
-..."""
+        ...""")
 
         output_file = os.path.join(
-        _out_dir, "pegleg_manifests", "site", _input_yaml["region_name"],
+            _out_dir, "pegleg_manifests", "site", region_name,
             os.path.split(_tpl_dir)[1], "test.yaml")
         LOG.debug(output_file)
-    assert (os.path.exists(output_file))
+        self.assertTrue(os.path.exists(output_file))
         with open(output_file, 'r') as f:
             content = f.read()
-        assert (expected_output == content)
+            self.assertEqual(expected_output, content)
 
-
+    @mock.patch(
-def test_render_template_missing_data():
+        'spyglass.data_extractor.models.SiteDocumentData',
+        spec=models.SiteDocumentData)
+    @mock.patch('spyglass.data_extractor.models.SiteInfo')
+    @mock.patch('spyglass.data_extractor.models.ServerList')
+    def test_render_template_missing_data(
+            self, ServerList, SiteInfo, SiteDocumentData):
         _tpl_parent_dir = mkdtemp()
         _tpl_dir = mkdtemp(dir=_tpl_parent_dir)
         _tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
         with open(_tpl_file, 'w') as f:
-        f.write(J2_TPL)
+            f.write(self.J2_TPL_UNDEFINED)
             LOG.debug("Writing test template to %s", _tpl_file)
-    _input_yaml = {"region_name": "test", "site_info": {}}
+
+        site_data = SiteDocumentData()
+        type(SiteDocumentData()).site_info = SiteInfo()
+        region_name = 'test'
+        type(SiteInfo()).region_name = mock.PropertyMock(
+            return_value=region_name)
+        site_type = 'test_type'
+        type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
+
         _out_dir = mkdtemp()
-    site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=False)
+        site_processor = SiteProcessor(site_data, _out_dir, force_write=False)
         with pytest.raises(UndefinedError):
             site_processor.render_template(_tpl_parent_dir)
 
         output_file = os.path.join(
-        _out_dir, "pegleg_manifests", "site", _input_yaml["region_name"],
+            _out_dir, "pegleg_manifests", "site", region_name,
             os.path.split(_tpl_dir)[1], "test.yaml")
-    assert (not os.path.exists(output_file))
+        self.assertFalse(os.path.exists(output_file))
 
-
+    @mock.patch(
-def test_render_template_missing_data_force():
+        'spyglass.data_extractor.models.SiteDocumentData',
+        spec=models.SiteDocumentData)
+    @mock.patch('spyglass.data_extractor.models.SiteInfo')
+    @mock.patch('spyglass.data_extractor.models.ServerList')
+    def test_render_template_missing_data_force(
+            self, ServerList, SiteInfo, SiteDocumentData):
         _tpl_parent_dir = mkdtemp()
         _tpl_dir = mkdtemp(dir=_tpl_parent_dir)
         _tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
         with open(_tpl_file, 'w') as f:
-        f.write(J2_TPL)
+            f.write(self.J2_TPL_UNDEFINED)
             LOG.debug("Writing test template to %s", _tpl_file)
-    _input_yaml = {"region_name": "test", "site_info": {}}
+
+        site_data = SiteDocumentData()
+        type(SiteDocumentData()).site_info = SiteInfo()
+        region_name = 'test'
+        type(SiteInfo()).region_name = mock.PropertyMock(
+            return_value=region_name)
+        site_type = 'test_type'
+        type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
+
         _out_dir = mkdtemp()
-    site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=True)
+        site_processor = SiteProcessor(site_data, _out_dir, force_write=True)
         site_processor.render_template(_tpl_parent_dir)
 
-    expected_output = """---
+        expected_output = textwrap.dedent(
-schema: pegleg/SiteDefinition/v1
+            """
-metadata:
+        ---
+        schema: pegleg/SiteDefinition/v1
+        metadata:
           schema: metadata/Document/v1
           layeringDefinition:
             abstract: false
             layer: site
           name: test
           storagePolicy: cleartext
-data:
+        data:
           site_type:
-..."""
+        ...""")
 
         output_file = os.path.join(
-        _out_dir, "pegleg_manifests", "site", _input_yaml["region_name"],
+            _out_dir, "pegleg_manifests", "site", region_name,
             os.path.split(_tpl_dir)[1], "test.yaml")
-    assert (os.path.exists(output_file))
+        self.assertTrue(os.path.exists(output_file))
         with open(output_file, 'r') as f:
             content = f.read()
-        assert (expected_output == content)
+            self.assertEqual(expected_output, content)