From e84b80c32de14c51a5a086b4f29fd98bf5fe80a6 Mon Sep 17 00:00:00 2001 From: Ian H Pittwood Date: Wed, 26 Jun 2019 16:41:42 -0500 Subject: [PATCH] Testing for plugin Adds tests for the Excel plugin's parser and extractor. Enables pep8 and fmt checks on the tests directory. Increases plugin test coverage to 94%, sets new minimum to 92%. Fixes DNS and NTP server extraction with regex. Updates file licenses. Change-Id: I35ee97574e6d63b7a82cfa94caf79db5db9755e7 --- spyglass_plugin_xls/check_exceptions.py | 16 +- spyglass_plugin_xls/cli.py | 2 +- spyglass_plugin_xls/examples/excel_spec.yaml | 3 +- spyglass_plugin_xls/examples/site_config.yaml | 13 + spyglass_plugin_xls/excel.py | 25 +- spyglass_plugin_xls/excel_parser.py | 130 +++---- tests/conftest.py | 159 ++++++++ tests/shared/excel_spec.yaml | 5 +- tests/shared/invalid_excel_spec.yaml | 64 ++++ tests/shared/site_config.yaml | 13 + tests/unit/test_cli.py | 4 +- tests/unit/test_excel.py | 355 ++++++++++++++++++ tests/unit/test_excel_parser.py | 168 +++++++++ tox.ini | 4 +- 14 files changed, 859 insertions(+), 102 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/shared/invalid_excel_spec.yaml create mode 100644 tests/unit/test_excel.py create mode 100644 tests/unit/test_excel_parser.py diff --git a/spyglass_plugin_xls/check_exceptions.py b/spyglass_plugin_xls/check_exceptions.py index 49ac3b4..e7dd2d2 100644 --- a/spyglass_plugin_xls/check_exceptions.py +++ b/spyglass_plugin_xls/check_exceptions.py @@ -1,4 +1,4 @@ -# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -36,3 +36,17 @@ class NoSpecMatched(BaseError): print( "No spec matched. Following are the available specs:\n".format( self.specs)) + + +class ExcelFileNotSpecified(BaseError): + + @staticmethod + def display_error(): + print("Engineering excel file not specified") + + +class ExcelSpecNotSpecified(BaseError): + + @staticmethod + def display_error(): + print("Engineering excel spec not specified") diff --git a/spyglass_plugin_xls/cli.py b/spyglass_plugin_xls/cli.py index 9c74fa9..a7849aa 100644 --- a/spyglass_plugin_xls/cli.py +++ b/spyglass_plugin_xls/cli.py @@ -1,4 +1,4 @@ -# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/spyglass_plugin_xls/examples/excel_spec.yaml b/spyglass_plugin_xls/examples/excel_spec.yaml index be15b26..0ea99fb 100644 --- a/spyglass_plugin_xls/examples/excel_spec.yaml +++ b/spyglass_plugin_xls/examples/excel_spec.yaml @@ -1,5 +1,4 @@ -# Copyright 2018 The Openstack-Helm Authors. -# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/spyglass_plugin_xls/examples/site_config.yaml b/spyglass_plugin_xls/examples/site_config.yaml index 13979f6..1096074 100644 --- a/spyglass_plugin_xls/examples/site_config.yaml +++ b/spyglass_plugin_xls/examples/site_config.yaml @@ -1,3 +1,16 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. ############################################## # Site Specific Spyglass XLS Plugin Settings # ############################################## diff --git a/spyglass_plugin_xls/excel.py b/spyglass_plugin_xls/excel.py index 7bcd0f5..41790ce 100644 --- a/spyglass_plugin_xls/excel.py +++ b/spyglass_plugin_xls/excel.py @@ -1,4 +1,4 @@ -# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,9 @@ import pprint import re from spyglass.data_extractor.base import BaseDataSourcePlugin from spyglass.data_extractor import models + +from spyglass_plugin_xls.check_exceptions import ExcelFileNotSpecified +from spyglass_plugin_xls.check_exceptions import ExcelSpecNotSpecified from spyglass_plugin_xls.excel_parser import ExcelParser LOG = logging.getLogger(__name__) @@ -66,13 +69,11 @@ class ExcelPlugin(BaseDataSourcePlugin): and excel specs are not specified. The below code has been written as an additional safeguard. """ - if not kwargs["excel_file"]: - LOG.error("Engineering excel file not specified: Spyglass exited!") - exit() + if 'excel_file' not in kwargs: + raise ExcelFileNotSpecified() excel_file_info = kwargs["excel_file"] - if not kwargs["excel_spec"]: - LOG.error("Engineering spec file not specified: Spyglass exited!") - exit() + if 'excel_spec' not in kwargs: + raise ExcelSpecNotSpecified() excel_spec_info = kwargs["excel_spec"] plugin_conf = { "excel_path": excel_file_info, @@ -161,7 +162,7 @@ class ExcelPlugin(BaseDataSourcePlugin): vlan_list.append(models.VLANNetworkData(**tmp_vlan)) return vlan_list - def get_ips(self, region, host=None): + def get_ips(self, region, host): """Return list of IPs on the host :param string region: Region name @@ -255,7 +256,7 @@ class ExcelPlugin(BaseDataSourcePlugin): data_dict['dns'] = self.get_dns_servers(region) data_dict['ntp'] = self.get_ntp_servers(region) data_dict['ldap'] = self.get_ldap_information(region) - return models.SiteInfo(region, **data_dict) + return models.SiteInfo(region_name=region, **data_dict) def _get_excel_obj(self): """Creation of an ExcelParser object to store site information. @@ -271,7 +272,8 @@ class ExcelPlugin(BaseDataSourcePlugin): """Extracts raw information from excel file based on excel spec""" self.parsed_xl_data = self.excel_obj.get_data() - def _get_network_name_from_vlan_name(self, vlan_name): + @staticmethod + def _get_network_name_from_vlan_name(vlan_name): """Network names are ksn, oam, oob, overlay, storage, pxe @@ -317,7 +319,8 @@ class ExcelPlugin(BaseDataSourcePlugin): "Unable to recognize VLAN name extracted from Plugin data source") return "" - def _get_formatted_server_list(self, server_list): + @staticmethod + def _get_formatted_server_list(server_list): """Format dns and ntp server list as comma separated string""" # dns/ntp server info from excel is of the format diff --git a/spyglass_plugin_xls/excel_parser.py b/spyglass_plugin_xls/excel_parser.py index fc5c368..afdc512 100644 --- a/spyglass_plugin_xls/excel_parser.py +++ b/spyglass_plugin_xls/excel_parser.py @@ -1,4 +1,4 @@ -# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -30,16 +30,20 @@ LOG = logging.getLogger(__name__) class ExcelParser(object): """Parse data from excel into a dict""" - def __init__(self, file_name, excel_specs): + def __init__(self, file_name: str, excel_specs: str): + """Initializes an ExcelParser to extract data from the Excel workbook + + :param file_name: path to the Excel workbook + :param excel_specs: path to the Excel workbook spec + """ self.file_name = file_name with open(excel_specs, "r") as f: spec_raw_data = f.read() self.excel_specs = yaml.safe_load(spec_raw_data) # A combined design spec, returns a workbook object after combining # all the inputs excel specs - combined_design_spec = self.combine_excel_design_specs(file_name) + combined_design_spec = self.load_excel_data(file_name) self.wb_combined = combined_design_spec - self.filenames = file_name self.spec = "xl_spec" @staticmethod @@ -75,19 +79,22 @@ class ExcelParser(object): return spec raise NoSpecMatched(self.excel_specs) - def get_ipmi_data(self): - """Read IPMI data from the sheet""" - - ipmi_data = {} - hosts = [] + def _get_workbook(self): provided_sheetname = self.excel_specs["specs"][ self.spec]["ipmi_sheet_name"] workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( provided_sheetname) if workbook_object is not None: - ws = workbook_object[extracted_sheetname] + return workbook_object[extracted_sheetname] else: - ws = self.wb_combined[provided_sheetname] + return self.wb_combined[provided_sheetname] + + def get_ipmi_data(self): + """Read IPMI data from the sheet""" + + ipmi_data = {} + hosts = [] + ws = self._get_workbook() row = self.excel_specs["specs"][self.spec]["start_row"] end_row = self.excel_specs["specs"][self.spec]["end_row"] hostname_col = self.excel_specs["specs"][self.spec]["hostname_col"] @@ -119,7 +126,7 @@ class ExcelParser(object): self.spec, row, host_profile_col)) except RuntimeError as rerror: LOG.critical(rerror) - sys.exit("Tugboat exited!!") + sys.exit("Spyglass exited") ipmi_data[hostname] = { "ipmi_address": ipmi_address, "ipmi_gateway": ipmi_gateway, @@ -157,14 +164,7 @@ class ExcelParser(object): def get_private_network_data(self): """Read network data from the private ip sheet""" - provided_sheetname = self.excel_specs["specs"][ - self.spec]["private_ip_sheet"] - workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( - provided_sheetname) - if workbook_object is not None: - ws = workbook_object[extracted_sheetname] - else: - ws = self.wb_combined[provided_sheetname] + ws = self._get_workbook() vlan_data = self.get_private_vlan_data(ws) network_data = {} row = self.excel_specs["specs"][self.spec]["net_start_row"] @@ -208,14 +208,7 @@ class ExcelParser(object): """Read public network data from public ip data""" network_data = {} - provided_sheetname = self.excel_specs["specs"][ - self.spec]["public_ip_sheet"] - workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( - provided_sheetname) - if workbook_object is not None: - ws = workbook_object[extracted_sheetname] - else: - ws = self.wb_combined[provided_sheetname] + ws = self._get_workbook() oam_row = self.excel_specs["specs"][self.spec]["oam_ip_row"] oam_col = self.excel_specs["specs"][self.spec]["oam_ip_col"] oam_vlan_col = self.excel_specs["specs"][self.spec]["oam_vlan_col"] @@ -250,13 +243,8 @@ class ExcelParser(object): site_info = {} provided_sheetname = self.excel_specs["specs"][ - self.spec]["dns_ntp_ldap_sheet"] - workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( - provided_sheetname) - if workbook_object is not None: - ws = workbook_object[extracted_sheetname] - else: - ws = self.wb_combined[provided_sheetname] + self.spec]["ipmi_sheet_name"] + ws = self._get_workbook() dns_row = self.excel_specs["specs"][self.spec]["dns_row"] dns_col = self.excel_specs["specs"][self.spec]["dns_col"] ntp_row = self.excel_specs["specs"][self.spec]["ntp_row"] @@ -282,17 +270,8 @@ class ExcelParser(object): except RuntimeError as rerror: LOG.critical(rerror) sys.exit("Tugboat exited!!") - - dns_servers = dns_servers.replace("\n", " ") - ntp_servers = ntp_servers.replace("\n", " ") - if "," in dns_servers: - dns_servers = dns_servers.split(",") - else: - dns_servers = dns_servers.split() - if "," in ntp_servers: - ntp_servers = ntp_servers.split(",") - else: - ntp_servers = ntp_servers.split() + dns_servers = list(filter(None, re.split(" |,|\n", dns_servers))) + ntp_servers = list(filter(None, re.split(" |,|\n", ntp_servers))) site_info = { "location": self.get_location_data(), "dns": dns_servers, @@ -316,14 +295,7 @@ class ExcelParser(object): def get_location_data(self): """Read location data from the site and zone sheet""" - provided_sheetname = self.excel_specs["specs"][ - self.spec]["location_sheet"] - workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( - provided_sheetname) - if workbook_object is not None: - ws = workbook_object[extracted_sheetname] - else: - ws = self.wb_combined[provided_sheetname] + ws = self._get_workbook() corridor_row = self.excel_specs["specs"][self.spec]["corridor_row"] column = self.excel_specs["specs"][self.spec]["column"] site_name_row = self.excel_specs["specs"][self.spec]["site_name_row"] @@ -356,22 +328,18 @@ class ExcelParser(object): sheet_name_list.append(dns_ntp_ldap_sheet_name) location_sheet_name = spec_item["location_sheet"] sheet_name_list.append(location_sheet_name) - try: - for sheetname in sheet_name_list: - workbook_object, extracted_sheetname = ( - self.get_xl_obj_and_sheetname(sheetname)) - if workbook_object is not None: - wb = workbook_object - sheetname = extracted_sheetname - else: - wb = self.wb_combined + for sheetname in sheet_name_list: + workbook_object, extracted_sheetname = ( + self.get_xl_obj_and_sheetname(sheetname)) + if workbook_object is not None: + wb = workbook_object + sheetname = extracted_sheetname + else: + wb = self.wb_combined - if sheetname not in wb.sheetnames: - raise RuntimeError( - "SheetName '{}' not found ".format(sheetname)) - except RuntimeError as rerror: - LOG.critical(rerror) - sys.exit("Tugboat exited!!") + if sheetname not in wb.sheetnames: + raise RuntimeError( + "SheetName '{}' not found ".format(sheetname)) LOG.info("Sheet names in excel spec validated") @@ -398,29 +366,29 @@ class ExcelParser(object): ) return data - def combine_excel_design_specs(self, filenames): + @staticmethod + def load_excel_data(filename): """Combines multiple excel file to a single design spec""" design_spec = Workbook() - for exel_file in filenames: - loaded_workbook = load_workbook(exel_file, data_only=True) - for names in loaded_workbook.sheetnames: - design_spec_worksheet = design_spec.create_sheet(names) - loaded_workbook_ws = loaded_workbook[names] - for row in loaded_workbook_ws: - for cell in row: - design_spec_worksheet[ - cell.coordinate].value = cell.value + loaded_workbook = load_workbook(filename, data_only=True) + for names in loaded_workbook.sheetnames: + design_spec_worksheet = design_spec.create_sheet(names) + loaded_workbook_ws = loaded_workbook[names] + for row in loaded_workbook_ws: + for cell in row: + design_spec_worksheet[cell.coordinate].value = cell.value return design_spec - def get_xl_obj_and_sheetname(self, sheetname): + @staticmethod + def get_xl_obj_and_sheetname(sheetname): """The logic confirms if the sheetname is specified for example as: 'MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs' """ if re.search(".xlsx", sheetname) or re.search(".xls", sheetname): - """ Extract file name """ + # Extract file name source_xl_file = sheetname.split(":")[0] wb = load_workbook(source_xl_file, data_only=True) return [wb, sheetname.split(":")[1]] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..167aa21 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,159 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture(scope='class') +def site_data(request): + request.cls.site_data = { + 'ipmi_data': [ + { + 'cab2r72c12': { + 'ipmi_address': '10.0.220.138', + 'ipmi_gateway': '10.0.220.129', + 'host_profile': 'dp-r720' + }, + 'cab2r72c13': { + 'ipmi_address': '10.0.220.139', + 'ipmi_gateway': '10.0.220.129', + 'host_profile': 'dp-r720' + }, + 'cab2r72c14': { + 'ipmi_address': '10.0.220.140', + 'ipmi_gateway': '10.0.220.129', + 'host_profile': 'dp-r720' + }, + 'cab2r72c15': { + 'ipmi_address': '10.0.220.141', + 'ipmi_gateway': '10.0.220.129', + 'host_profile': 'dp-r720' + }, + 'cab2r72c16': { + 'ipmi_address': '10.0.220.142', + 'ipmi_gateway': '10.0.220.129', + 'host_profile': 'cp-r720' + }, + 'cab2r72c17': { + 'ipmi_address': '10.0.220.143', + 'ipmi_gateway': '10.0.220.129', + 'host_profile': 'cp-r720' + }, + 'cab2r73c12': { + 'ipmi_address': '10.0.220.170', + 'ipmi_gateway': '10.0.220.161', + 'host_profile': 'dp-r720' + }, + 'cab2r73c13': { + 'ipmi_address': '10.0.220.171', + 'ipmi_gateway': '10.0.220.161', + 'host_profile': 'dp-r720' + }, + 'cab2r73c14': { + 'ipmi_address': '10.0.220.172', + 'ipmi_gateway': '10.0.220.161', + 'host_profile': 'dp-r720' + }, + 'cab2r73c15': { + 'ipmi_address': '10.0.220.173', + 'ipmi_gateway': '10.0.220.161', + 'host_profile': 'dp-r720' + }, + 'cab2r73c16': { + 'ipmi_address': '10.0.220.174', + 'ipmi_gateway': '10.0.220.161', + 'host_profile': 'cp-r720' + }, + 'cab2r73c17': { + 'ipmi_address': '10.0.220.175', + 'ipmi_gateway': '10.0.220.161', + 'host_profile': 'cp-r720' + }, + }, + [ + 'cab2r72c12', + 'cab2r72c13', + 'cab2r72c14', + 'cab2r72c15', + 'cab2r72c16', + 'cab2r72c17', + 'cab2r73c12', + 'cab2r73c13', + 'cab2r73c14', + 'cab2r73c15', + 'cab2r73c16', + 'cab2r73c17', + ] + ], + 'network_data': { + 'private': { + 'iSCSI/Storage': { + 'vlan': 'vlan 23', + 'subnet': ['30.31.1.0/25'], + 'is_common': True + }, + 'PXE': { + 'vlan': 'vlan 21', + 'subnet': [ + '30.30.4.0/25', '30.30.4.128/25', '30.30.5.0/25', + '30.30.5.128/25' + ], + 'is_common': True + }, + 'Calico BGP peering addresses': { + 'vlan': 'vlan 22', + 'subnet': ['30.29.1.0/25'], + 'is_common': True + }, + 'Overlay': { + 'vlan': 'vlan 24', + 'subnet': ['30.19.0.0/25'], + 'is_common': True + } + }, + 'public': { + 'oam': { + 'subnet': ['10.0.220.0/26'], + 'vlan': 'VLAN-21' + }, + 'ingress': '10.0.220.72/29', + 'oob': { + 'subnet': [ + '10.0.220.128/27', '10.0.220.160/27', + '10.0.220.192/27', '10.0.220.224/27' + ] + } + } + }, + 'site_info': { + 'location': { + 'corridor': 'Corridor 1', + 'name': 'SampleSiteName', + 'state': 'New Jersey', + 'country': 'SampleCountry', + 'physical_location': 'XXXXXX21' + }, + 'dns': [ + '40.40.40.40', '(ntp1.example.com)', '41.41.41.41', + '(ntp2.example.com)' + ], + 'ntp': ['150.234.210.5', '(ns1.example.com)'], + 'domain': 'dmy00.example.com', + 'ldap': { + 'subdomain': 'testitservices', + 'common_name': 'AA-AAA-dmy00', + 'url': 'url: ldap://ldap.example.com' + } + } + } diff --git a/tests/shared/excel_spec.yaml b/tests/shared/excel_spec.yaml index be15b26..58b4333 100644 --- a/tests/shared/excel_spec.yaml +++ b/tests/shared/excel_spec.yaml @@ -1,5 +1,4 @@ -# Copyright 2018 The Openstack-Helm Authors. -# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +18,8 @@ specs: # Design Spec file name: SiteDesignSpec_v0.1.xlsx xl_spec: + header_row: 3 + ipmi_address_header: "IPMI Address" ipmi_sheet_name: 'Site-Information' start_row: 4 end_row: 15 diff --git a/tests/shared/invalid_excel_spec.yaml b/tests/shared/invalid_excel_spec.yaml new file mode 100644 index 0000000..082d094 --- /dev/null +++ b/tests/shared/invalid_excel_spec.yaml @@ -0,0 +1,64 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Important: Please modify the dictionary with appropriate +# design spec file. +--- +specs: + # Design Spec file name: SiteDesignSpec_v0.1.xlsx + xl_spec: + header_row: 3 + ipmi_address_header: "IPMI Address" + ipmi_sheet_name: 'Sheet-DNE' + start_row: 4 + end_row: 15 + hostname_col: 2 + ipmi_address_col: 2 + host_profile_col: 5 + ipmi_gateway_col: 7 + private_ip_sheet: 'Site-Information' + net_type_col: 1 + vlan_col: 2 + vlan_start_row: 19 + vlan_end_row: 30 + net_start_row: 33 + net_end_row: 40 + net_col: 2 + net_vlan_col: 1 + public_ip_sheet: 'Site-Information' + oam_vlan_col: 1 + oam_ip_row: 43 + oam_ip_col: 2 + oob_net_row: 48 + oob_net_start_col: 2 + oob_net_end_col: 5 + ingress_ip_row: 45 + dns_ntp_ldap_sheet: 'Site-Information' + login_domain_row: 52 + ldap_col: 2 + global_group: 53 + ldap_search_url_row: 54 + ntp_row: 55 + ntp_col: 2 + dns_row: 56 + dns_col: 2 + domain_row: 51 + domain_col: 2 + location_sheet: 'Site-Information' + column: 2 + corridor_row: 59 + site_name_row: 58 + state_name_row: 60 + country_name_row: 61 + clli_name_row: 62 diff --git a/tests/shared/site_config.yaml b/tests/shared/site_config.yaml index 13979f6..1096074 100644 --- a/tests/shared/site_config.yaml +++ b/tests/shared/site_config.yaml @@ -1,3 +1,16 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. ############################################## # Site Specific Spyglass XLS Plugin Settings # ############################################## diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 3dec5e3..5e3ebc7 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -18,8 +18,8 @@ from unittest import mock from click.testing import CliRunner from spyglass.site_processors.site_processor import SiteProcessor -from spyglass_plugin_xls.cli import generate_intermediary, \ - generate_manifests_and_intermediary +from spyglass_plugin_xls.cli import generate_intermediary +from spyglass_plugin_xls.cli import generate_manifests_and_intermediary FIXTURE_DIR = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'shared') diff --git a/tests/unit/test_excel.py b/tests/unit/test_excel.py new file mode 100644 index 0000000..708cfac --- /dev/null +++ b/tests/unit/test_excel.py @@ -0,0 +1,355 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from copy import copy +import os +import unittest +from unittest import mock + +import pytest +from spyglass.data_extractor import models + +from spyglass_plugin_xls.check_exceptions import ExcelFileNotSpecified +from spyglass_plugin_xls.check_exceptions import ExcelSpecNotSpecified +from spyglass_plugin_xls.excel import ExcelPlugin +from spyglass_plugin_xls.excel_parser import ExcelParser + +FIXTURE_DIR = os.path.join( + os.path.dirname(os.path.dirname(__file__)), 'shared') + +EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'excel_spec.yaml') + +INVALID_EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'invalid_excel_spec.yaml') + +EXCEL_FILE_PATH = os.path.join(FIXTURE_DIR, 'SiteDesignSpec_v0.1.xlsx') + +SITE_CONFIG_PATH = os.path.join(FIXTURE_DIR, 'site_config.yaml') + + +@pytest.mark.usefixtures('site_data') +class TestExcelPlugin(unittest.TestCase): + """Tests for ExcelPlugin""" + + def test___init__(self): + region = 'test_region' + result = ExcelPlugin(region) + self.assertEqual(region, result.region) + self.assertEqual('excel', result.source_type) + self.assertEqual('spyglass-plugin-xls', result.source_name) + self.assertEqual(None, result.excel_path) + self.assertEqual(None, result.excel_spec) + self.assertEqual(None, result.parsed_xl_data) + + @mock.patch('spyglass_plugin_xls.excel_parser.ExcelParser', autospec=True) + def test_set_config_opts(self, excel_parser): + region = 'test_region' + result = ExcelPlugin(region) + config = {'excel_spec': EXCEL_SPEC_PATH, 'excel_path': EXCEL_FILE_PATH} + result.excel_spec = EXCEL_SPEC_PATH + result.excel_path = EXCEL_FILE_PATH + result.set_config_opts(config) + self.assertEqual(config['excel_path'], result.excel_path) + self.assertEqual(config['excel_spec'], result.excel_spec) + self.assertIsInstance(result.excel_obj, ExcelParser) + + def test_get_plugin_conf(self): + expected_result = { + 'excel_path': 'ExcelFile.xlsx', + 'excel_spec': 'ExcelSpec.yaml' + } + region = 'test_region' + obj = ExcelPlugin(region) + result = obj.get_plugin_conf( + excel_file='ExcelFile.xlsx', excel_spec='ExcelSpec.yaml') + self.assertDictEqual(expected_result, result) + + def test_get_plugin_conf_no_excel_file(self): + region = 'test_region' + obj = ExcelPlugin(region) + with self.assertRaises(ExcelFileNotSpecified): + obj.get_plugin_conf(excel_spec='ExcelSpec.yaml') + + def test_get_plugin_conf_no_excel_spec(self): + region = 'test_region' + obj = ExcelPlugin(region) + with self.assertRaises(ExcelSpecNotSpecified): + obj.get_plugin_conf(excel_file='ExcelFile.xlsx') + + def test_get_racks(self): + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj.get_racks(region) + self.assertEqual(2, len(result)) + for rack in result: + self.assertIsInstance(rack, models.Rack) + self.assertEqual(6, len(rack.hosts)) + for host in rack.hosts: + self.assertIn(host.name, self.site_data['ipmi_data'][0]) + self.assertEqual( + self.site_data['ipmi_data'][0][host.name]['host_profile'], + host.host_profile) + + def test_get_hosts(self): + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj.get_hosts(region) + self.assertEqual(12, len(result)) + for host in result: + self.assertIn(host.name, self.site_data['ipmi_data'][0]) + self.assertEqual( + self.site_data['ipmi_data'][0][host.name]['host_profile'], + host.host_profile) + + def test_get_hosts_using_rack(self): + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj.get_hosts(region, 'rack73') + self.assertEqual(6, len(result)) + for host in result: + self.assertIn(host.name, self.site_data['ipmi_data'][0]) + self.assertEqual('rack73', host.rack_name) + self.assertNotIn('r72', host.name) + self.assertEqual( + self.site_data['ipmi_data'][0][host.name]['host_profile'], + host.host_profile) + + def test_get_networks(self): + expected_network_types = { + 'oob': { + 'type': 'public', + 'name': 'oob' + }, + 'oam': { + 'type': 'public', + 'name': 'oam' + }, + 'calico': { + 'type': 'private', + 'name': 'Calico BGP peering addresses' + }, + 'ingress': { + 'type': 'public', + 'name': 'ingress' + }, + 'overlay': { + 'type': 'private', + 'name': 'Overlay' + }, + 'pxe': { + 'type': 'private', + 'name': 'PXE' + }, + 'storage': { + 'type': 'private', + 'name': 'iSCSI/Storage' + } + } + network_data = self.site_data['network_data'] + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj.get_networks(region) + self.assertEqual(7, len(result)) + for vlan_data in result: + self.assertIn(vlan_data.name, expected_network_types) + data = expected_network_types[vlan_data.name] + if vlan_data.name != 'ingress': + self.assertEqual( + network_data[data['type']][data['name']]['subnet'], + vlan_data.subnet) + else: + self.assertEqual( + network_data[data['type']][data['name']], + vlan_data.subnet[0]) + + def test_get_ips(self): + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + host_name = 'cab2r72c15' + result = obj.get_ips(region, host_name) + self.assertIsInstance(result, models.IPList) + self.assertEqual( + self.site_data['ipmi_data'][0][host_name]['ipmi_address'], + result.oob) + + def test_get_ldap_information(self): + expected_ldap_data = copy(self.site_data['site_info']['ldap']) + expected_ldap_data['domain'] = 'example' + expected_ldap_data['url'] = expected_ldap_data['url'].split(' ')[1] + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj.get_ldap_information(region) + self.assertDictEqual(expected_ldap_data, result) + + def test_get_ntp_servers(self): + expected_ntp_servers = self.site_data['site_info']['ntp'][:1] + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj.get_ntp_servers(region) + self.assertIsInstance(result, models.ServerList) + self.assertEqual(expected_ntp_servers, result.servers) + + def test_get_dns_servers(self): + expected_dns_servers = [ + self.site_data['site_info']['dns'][0], + self.site_data['site_info']['dns'][2] + ] + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj.get_dns_servers(region) + self.assertIsInstance(result, models.ServerList) + self.assertEqual(expected_dns_servers, result.servers) + + def test_get_domain_name(self): + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj.get_domain_name(region) + self.assertEqual(self.site_data['site_info']['domain'], result) + + def test_get_location_information(self): + expected_location_data = copy(self.site_data['site_info']['location']) + expected_location_data['corridor'] = 'c1' + expected_location_data[ + 'physical_location_id'] = expected_location_data.pop( + 'physical_location') + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj.get_location_information(region) + self.assertDictEqual(expected_location_data, result) + + def test_get_site_info(self): + expected_ntp_servers = self.site_data['site_info']['ntp'][:1] + expected_dns_servers = [ + self.site_data['site_info']['dns'][0], + self.site_data['site_info']['dns'][2] + ] + + expected_location_data = copy(self.site_data['site_info']['location']) + expected_location_data['corridor'] = 'c1' + expected_location_data[ + 'physical_location_id'] = expected_location_data.pop( + 'physical_location') + + expected_ldap_data = copy(self.site_data['site_info']['ldap']) + expected_ldap_data['domain'] = 'example' + expected_ldap_data['url'] = expected_ldap_data['url'].split(' ')[1] + + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj.get_site_info(region) + self.assertIsInstance(result, models.SiteInfo) + self.assertEqual(region, result.region_name) + self.assertEqual(expected_dns_servers, result.dns.servers) + self.assertEqual(expected_ntp_servers, result.ntp.servers) + self.assertDictEqual(expected_ldap_data, result.ldap) + self.assertEqual(expected_location_data['corridor'], result.corridor) + self.assertEqual(expected_location_data['state'], result.state) + self.assertEqual(expected_location_data['country'], result.country) + self.assertEqual( + expected_location_data['physical_location_id'], + result.physical_location_id) + self.assertEqual(expected_location_data['name'], result.name) + + @mock.patch('spyglass_plugin_xls.excel_parser.ExcelParser') + def test__get_excel_obj(self, excel_parser): + region = 'test_region' + obj = ExcelPlugin(region) + obj.excel_spec = EXCEL_SPEC_PATH + obj.excel_path = EXCEL_FILE_PATH + obj._get_excel_obj() + self.assertIsInstance(obj.excel_obj, ExcelParser) + + def test__extract_raw_data_from_excel(self): + region = 'test_region' + obj = ExcelPlugin(region) + obj.excel_obj = mock.MagicMock(spec=ExcelParser) + obj.excel_obj.get_data.return_value = 'success' + obj._extract_raw_data_from_excel() + obj.excel_obj.get_data.assert_called_once() + self.assertEqual('success', obj.parsed_xl_data) + + def test__get_network_name_from_vlan_name(self): + result = ExcelPlugin._get_network_name_from_vlan_name('ksn') + self.assertEqual('calico', result) + result = ExcelPlugin._get_network_name_from_vlan_name('calico') + self.assertEqual('calico', result) + result = ExcelPlugin._get_network_name_from_vlan_name('storage') + self.assertEqual('storage', result) + result = ExcelPlugin._get_network_name_from_vlan_name('oam') + self.assertEqual('oam', result) + result = ExcelPlugin._get_network_name_from_vlan_name('server') + self.assertEqual('oam', result) + result = ExcelPlugin._get_network_name_from_vlan_name('ovs') + self.assertEqual('overlay', result) + result = ExcelPlugin._get_network_name_from_vlan_name('overlay') + self.assertEqual('overlay', result) + result = ExcelPlugin._get_network_name_from_vlan_name('oob') + self.assertEqual('oob', result) + result = ExcelPlugin._get_network_name_from_vlan_name('pxe') + self.assertEqual('pxe', result) + + def test__get_network_name_from_vlan_name_dne(self): + result = ExcelPlugin._get_network_name_from_vlan_name('dne') + self.assertEqual('', result) + + def test__get_formatted_server_list(self): + test_list = [ + '124.1.23.54', '(example.com)', '192.168.1.0', + '(anotherexample.com)' + ] + expected_list = ['124.1.23.54', '192.168.1.0'] + result = ExcelPlugin._get_formatted_server_list(test_list) + self.assertIsInstance(result, models.ServerList) + self.assertEqual(expected_list, result.servers) + + def test__get_rack(self): + region = 'test_region' + obj = ExcelPlugin(region) + result = obj._get_rack('cab2r72c15') + self.assertEqual('r72', result) + + def test__get_rackwise_hosts(self): + expected_data = { + 'rack72': [ + 'cab2r72c12', 'cab2r72c13', 'cab2r72c14', 'cab2r72c15', + 'cab2r72c16', 'cab2r72c17' + ], + 'rack73': [ + 'cab2r73c12', 'cab2r73c13', 'cab2r73c14', 'cab2r73c15', + 'cab2r73c16', 'cab2r73c17' + ] + } + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj._get_rackwise_hosts() + self.assertDictEqual(expected_data, result) + + def test__get_rack_data(self): + expected_data = {'r72': 'rack72', 'r73': 'rack73'} + region = 'test_region' + obj = ExcelPlugin(region) + obj.parsed_xl_data = self.site_data + result = obj._get_rack_data() + self.assertDictEqual(expected_data, result) diff --git a/tests/unit/test_excel_parser.py b/tests/unit/test_excel_parser.py new file mode 100644 index 0000000..0d0a163 --- /dev/null +++ b/tests/unit/test_excel_parser.py @@ -0,0 +1,168 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest + +from openpyxl import Workbook +from openpyxl.worksheet.worksheet import Worksheet +import pytest +from spyglass.data_extractor.custom_exceptions import NoSpecMatched +import yaml + +from spyglass_plugin_xls.excel_parser import ExcelParser + +FIXTURE_DIR = os.path.join( + os.path.dirname(os.path.dirname(__file__)), 'shared') + +EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'excel_spec.yaml') + +INVALID_EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'invalid_excel_spec.yaml') + +EXCEL_FILE_PATH = os.path.join(FIXTURE_DIR, 'SiteDesignSpec_v0.1.xlsx') + +SITE_CONFIG_PATH = os.path.join(FIXTURE_DIR, 'site_config.yaml') + + +@pytest.mark.usefixtures('site_data') +class TestExcelParser(unittest.TestCase): + """Tests for ExcelParser""" + + def test___init__(self): + with open(EXCEL_SPEC_PATH, 'r') as f: + loaded_spec = yaml.safe_load(f) + result = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + self.assertEqual(EXCEL_FILE_PATH, result.file_name) + self.assertDictEqual(loaded_spec, result.excel_specs) + self.assertIsInstance(result.wb_combined, Workbook) + self.assertEqual('xl_spec', result.spec) + + def test_sanitize(self): + test_string = 'Hello THIS is A TeSt' + expected_output = 'hellothisisatest' + result = ExcelParser.sanitize(test_string) + self.assertEqual(expected_output, result) + + def test_compare(self): + test_string1 = 'These strings are equal.' + test_string2 = 'These strIngs are Equal .' + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.compare(test_string1, test_string2) + self.assertTrue(result) + + def test_compare_false(self): + test_string1 = 'These strings are not equal.' + test_string2 = 'These strIngs are Equal.' + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.compare(test_string1, test_string2) + self.assertFalse(result) + + def test_validate_sheet(self): + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.validate_sheet('xl_spec', 'Site-Information') + self.assertTrue(result) + + def test_validate_sheet_invalid(self): + obj = ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH) + result = obj.validate_sheet('xl_spec', 'Site-Information') + self.assertFalse(result) + + def test_find_correct_spec(self): + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.find_correct_spec() + self.assertEqual('xl_spec', result) + + def test_find_correct_spec_no_spec_matched(self): + obj = ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH) + with self.assertRaises(NoSpecMatched): + obj.find_correct_spec() + + def test__get_workbook(self): + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj._get_workbook() + self.assertIsInstance(result, Worksheet) + + def test_get_ipmi_data(self): + expected_hosts = self.site_data['ipmi_data'][1] + expected_ipmi_data = self.site_data['ipmi_data'][0] + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_ipmi_data() + self.assertDictEqual(result[0], expected_ipmi_data) + self.assertEqual(result[1], expected_hosts) + + def test_get_private_vlan_data(self): + expected_vlan_data = { + 'vlan 23': 'iSCSI/Storage', + 'vlan 21': 'PXE', + 'vlan 22': 'Calico BGP peering addresses', + 'vlan 24': 'Overlay', + 'n/a': 'CNI Pod addresses' + } + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_private_vlan_data(obj._get_workbook()) + self.assertDictEqual(expected_vlan_data, result) + + def test_get_private_network_data(self): + expected_network_data = self.site_data['network_data']['private'] + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_private_network_data() + self.assertDictEqual(expected_network_data, result) + + def test_get_public_network_data(self): + expected_network_data = self.site_data['network_data']['public'] + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_public_network_data() + self.assertEqual(expected_network_data, result) + + def test_get_site_info(self): + expected_site_info = self.site_data['site_info'] + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_site_info() + self.assertDictEqual(expected_site_info, result) + + def test_get_location_data(self): + expected_location_data = self.site_data['site_info']['location'] + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_location_data() + self.assertEqual(expected_location_data, result) + + def test_validate_sheet_names_with_spec(self): + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + self.assertIsNone(obj.validate_sheet_names_with_spec()) + + def test_validate_sheet_names_with_spec_invalid(self): + obj = ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH) + with self.assertRaises(RuntimeError): + obj.validate_sheet_names_with_spec() + + def test_get_data(self): + expected_data = self.site_data + obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_data() + self.assertDictEqual(expected_data, result) + + def test_load_excel_data(self): + result = ExcelParser.load_excel_data(EXCEL_FILE_PATH) + self.assertIsInstance(result, Workbook) + + def test_get_xl_obj_and_sheetname(self): + result = ExcelParser.get_xl_obj_and_sheetname('Site-Information') + self.assertEqual([None, 'Site-Information'], result) + + def test_get_xl_obj_and_sheetname_file_specified(self): + sheet = EXCEL_FILE_PATH + ':Site-Information' + result = ExcelParser.get_xl_obj_and_sheetname(sheet) + self.assertIsInstance(result, list) + self.assertIsInstance(result[0], Workbook) + self.assertEqual(result[1], 'Site-Information') diff --git a/tox.ini b/tox.ini index c2b3aee..dea76a7 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ deps = commands = bash -c "{toxinidir}/tools/gate/whitespace-linter.sh" yapf -dr {toxinidir}/spyglass_plugin_xls {toxinidir}/setup.py {toxinidir}/tests - flake8 {toxinidir}/spyglass_plugin_xls + flake8 {toxinidir}/spyglass_plugin_xls {toxinidir}/tests bandit -r spyglass_plugin_xls -n 5 safety check -r requirements.txt --bare whitelist_externals = @@ -72,6 +72,6 @@ deps = commands = bash -c 'PATH=$PATH:~/.local/bin; pytest --cov=spyglass_plugin_xls \ --cov-report html:cover --cov-report xml:cover/coverage.xml \ - --cov-report term --cov-fail-under 20 tests/' + --cov-report term --cov-fail-under 92 tests/' whitelist_externals = bash