commit 6174c80f5d4434b89dbd396b142e7f4aefaa1a2d Author: Ian H. Pittwood Date: Fri Apr 26 16:01:24 2019 -0500 Initial commit Adds code for tugboat plugin from Airship Spyglass. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a741440 --- /dev/null +++ b/.gitignore @@ -0,0 +1,116 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +*.tgz + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.testrepository/* +cover/* +results/* +.stestr/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# pycharm-ide +.idea/ + +# osx +.DS_Store + +# git +Changelog +AUTHORS + +# Ansible +*.retry diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000..922d40b --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- project: + check: + jobs: + - openstack-tox-pep8 + gate: + jobs: + - openstack-tox-pep8 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..8acd160 --- /dev/null +++ b/README.rst @@ -0,0 +1,10 @@ +Spyglass Excel Plugin +--------------------- + +Tugboat is a Spyglass plugin to generate airship-seaworthy site manifest files +from an excel based engineering spec. The plugin is configured with an Excel +sheet and its corresponding excel specification as inputs. Spyglass uses this +plugin to construct an intermediary yaml which is processed further using J2 +templates to generate site manifests. + +Spyglass: https://opendev.org/airship/spyglass diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..b13935d --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,3 @@ +# Documentation +sphinx>=2.0.1 +sphinx_rtd_theme==0.4.3 diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000..fff90ae --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# +# shipyard documentation build configuration file, created by +# sphinx-quickstart on Sat Sep 16 03:40:50 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +import sphinx_rtd_theme + +sys.path.insert(0, os.path.abspath('../../')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.todo', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +# templates_path = [] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'spyglass-plugin-xls' +copyright = u'2019 AT&T Intellectual Property.' +author = u'The Airship Authors' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0.0.1' +# The full version, including alpha/beta/rc tags. +release = u'0.0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = [] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'ucpintdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..feb54b6 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,96 @@ +.. + Copyright 2018 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +============== +Tugboat Plugin +============== + +What is Tugboat Plugin? +----------------------- + +Tugboat is a Spyglass plugin to generate airship-seaworthy site manifest files +from an excel based engineering spec. The plugin is configured with an Excel +sheet and its corresponding excel specification as inputs. Spyglass uses this +plugin to construct an intermediary yaml which is processed further using J2 +templates to generate site manifests. + +Excel specification +------------------- +Excel Spec is like an index to the Excel sheet to look for the data to be +collected by the tool. Excel Spec Sample specifies all the details that +need to be filled by the Deployment Engineer. + +Below is the definition for each key in the Excel spec + +:: + + + ipmi_sheet_name - name of the sheet from where IPMI and host profile information is to be read + start_row - row number from where the IPMI and host profile information starts + end_row - row number from where the IPMI and host profile information ends + hostname_col - column number where the hostnames are to be read from + ipmi_address_col - column number from where the ipmi addresses are to be read + host_profile_col - column number from where the host profiles are to be read + ipmi_gateway_col - column number from where the ipmi gateways are to be read + private_ip_sheet - name of the sheet which has the private IP information + net_type_col - column number from where the network type is to be read + vlan_col - column number from where the network vlan is to be read + vlan_start_row - row number from where the vlan information starts + vlan_end_row - row number from where the vlan information ends + net_start_row - row number from where the network information starts + net_end_row - row number from where the network information ends + net_col - column number where the IP ranges for network is to be read + net_vlan_col - column number where the vlan information is present in the pod wise network section + public_ip_sheet - name of the sheet which has the public IP information + oam_vlan_col - column number from where the OAM vlan information is to be read from + oam_ip_row - row number from where the OAM network information is to be read from + oam_ip_col - column number from where the OAM network information is to be read from + oob_net_row - row number which has the OOB network subnet ranges + oob_net_start_col - column number from where the OOB network ranges start + oob_net_end_col - column number from where the OOB network ranges end + ingress_ip_row - row number from where the Ingress network information is to be read from + dns_ntp_ldap_sheet - name of the sheet which has the DNS, NTP and LDAP information + login_domain_row - row number which has the ldap login domain + ldap_col - column number which has the all ldap related information + global_group - row number which has the ldap group information + ldap_search_url_row - row number which has the ldap url + ntp_row - row number which has the ntp information + ntp_col - column number which has the ntp information + dns_row - row number which has the dns information + dns_col - column number which has the dns information + domain_row - row number which has the domain information + domain_col - column number which has the domain information + location_sheet - name of the sheet which has the location information + column - column number which has all the information + corridor_row - row number which has the corridor information + site_name_row - row number which has the site name + state_name_row - row number which has the state name + country_name_row - row number which has the country name + clli_name_row - row number which has CLLI information + +Example: Tugboat Plugin Usage +----------------------------- + +1. Required Input(Refer to 'spyglass/examples' folder to get these inputs) + + a) Excel File: SiteDesignSpec_v0.1.xlsx + b) Excel Spec: excel_spec_upstream.yaml + c) Site Config: site_config.yaml + +2. Spyglass CLI Command: +:: + + spyglass -mg -t tugboat -x SiteDesignSpec_v0.1.xlsx -e excel_spec_upstream.yaml -d site_config.yaml -s airship-seaworthy --template_dir= 1: + network_data[network]['is_common'] = False + else: + network_data[network]['is_common'] = True + LOG.debug( + "private network data extracted from\ + excel:\n%s", pprint.pformat(network_data)) + """ + return network_data + + def get_public_network_data(self): + """Read public network data from public ip data""" + + network_data = {} + provided_sheetname = self.excel_specs["specs"][ + self.spec]["public_ip_sheet"] + workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( + provided_sheetname) + if workbook_object is not None: + ws = workbook_object[extracted_sheetname] + else: + ws = self.wb_combined[provided_sheetname] + oam_row = self.excel_specs["specs"][self.spec]["oam_ip_row"] + oam_col = self.excel_specs["specs"][self.spec]["oam_ip_col"] + oam_vlan_col = self.excel_specs["specs"][self.spec]["oam_vlan_col"] + ingress_row = self.excel_specs["specs"][self.spec]["ingress_ip_row"] + oob_row = self.excel_specs["specs"][self.spec]["oob_net_row"] + col = self.excel_specs["specs"][self.spec]["oob_net_start_col"] + end_col = self.excel_specs["specs"][self.spec]["oob_net_end_col"] + network_data = { + "oam": { + "subnet": [ws.cell(row=oam_row, column=oam_col).value], + "vlan": ws.cell(row=oam_row, column=oam_vlan_col).value, + }, + "ingress": ws.cell(row=ingress_row, column=oam_col).value, + } + network_data["oob"] = {"subnet": []} + while col <= end_col: + cell_value = ws.cell(row=oob_row, column=col).value + if cell_value: + network_data["oob"]["subnet"].append(self.sanitize(cell_value)) + col += 1 + LOG.debug( + "public network data extracted from\ + excel:\n%s", + pprint.pformat(network_data), + ) + return network_data + + def get_site_info(self): + """Read location, dns, ntp and ldap data""" + + site_info = {} + provided_sheetname = self.excel_specs["specs"][ + self.spec]["dns_ntp_ldap_sheet"] + workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( + provided_sheetname) + if workbook_object is not None: + ws = workbook_object[extracted_sheetname] + else: + ws = self.wb_combined[provided_sheetname] + dns_row = self.excel_specs["specs"][self.spec]["dns_row"] + dns_col = self.excel_specs["specs"][self.spec]["dns_col"] + ntp_row = self.excel_specs["specs"][self.spec]["ntp_row"] + ntp_col = self.excel_specs["specs"][self.spec]["ntp_col"] + domain_row = self.excel_specs["specs"][self.spec]["domain_row"] + domain_col = self.excel_specs["specs"][self.spec]["domain_col"] + login_domain_row = self.excel_specs["specs"][ + self.spec]["login_domain_row"] + ldap_col = self.excel_specs["specs"][self.spec]["ldap_col"] + global_group = self.excel_specs["specs"][self.spec]["global_group"] + ldap_search_url_row = self.excel_specs["specs"][ + self.spec]["ldap_search_url_row"] + dns_servers = ws.cell(row=dns_row, column=dns_col).value + ntp_servers = ws.cell(row=ntp_row, column=ntp_col).value + try: + if dns_servers is None: + raise RuntimeError(( + "No value for dns_server from:{} Sheet:'{}' ", + "Row:{} Col:{}", + ).format(self.file_name, provided_sheetname, dns_row, dns_col)) + except RuntimeError as rerror: + LOG.critical(rerror) + sys.exit("Tugboat exited!!") + + dns_servers = dns_servers.replace("\n", " ") + ntp_servers = ntp_servers.replace("\n", " ") + if "," in dns_servers: + dns_servers = dns_servers.split(",") + else: + dns_servers = dns_servers.split() + if "," in ntp_servers: + ntp_servers = ntp_servers.split(",") + else: + ntp_servers = ntp_servers.split() + site_info = { + "location": self.get_location_data(), + "dns": dns_servers, + "ntp": ntp_servers, + "domain": ws.cell(row=domain_row, column=domain_col).value, + "ldap": { + "subdomain": ws.cell(row=login_domain_row, + column=ldap_col).value, + "common_name": ws.cell(row=global_group, + column=ldap_col).value, + "url": ws.cell(row=ldap_search_url_row, column=ldap_col).value, + }, + } + LOG.debug( + "Site Info extracted from\ + excel:\n%s", + pprint.pformat(site_info), + ) + return site_info + + def get_location_data(self): + """Read location data from the site and zone sheet""" + + provided_sheetname = self.excel_specs["specs"][ + self.spec]["location_sheet"] + workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname( + provided_sheetname) + if workbook_object is not None: + ws = workbook_object[extracted_sheetname] + else: + ws = self.wb_combined[provided_sheetname] + corridor_row = self.excel_specs["specs"][self.spec]["corridor_row"] + column = self.excel_specs["specs"][self.spec]["column"] + site_name_row = self.excel_specs["specs"][self.spec]["site_name_row"] + state_name_row = self.excel_specs["specs"][self.spec]["state_name_row"] + country_name_row = self.excel_specs["specs"][ + self.spec]["country_name_row"] + clli_name_row = self.excel_specs["specs"][self.spec]["clli_name_row"] + return { + "corridor": ws.cell(row=corridor_row, column=column).value, + "name": ws.cell(row=site_name_row, column=column).value, + "state": ws.cell(row=state_name_row, column=column).value, + "country": ws.cell(row=country_name_row, column=column).value, + "physical_location": ws.cell(row=clli_name_row, + column=column).value, + } + + def validate_sheet_names_with_spec(self): + """Checks is sheet name in spec file matches with excel file""" + + spec = list(self.excel_specs["specs"].keys())[0] + spec_item = self.excel_specs["specs"][spec] + sheet_name_list = [] + ipmi_header_sheet_name = spec_item["ipmi_sheet_name"] + sheet_name_list.append(ipmi_header_sheet_name) + private_ip_sheet_name = spec_item["private_ip_sheet"] + sheet_name_list.append(private_ip_sheet_name) + public_ip_sheet_name = spec_item["public_ip_sheet"] + sheet_name_list.append(public_ip_sheet_name) + dns_ntp_ldap_sheet_name = spec_item["dns_ntp_ldap_sheet"] + sheet_name_list.append(dns_ntp_ldap_sheet_name) + location_sheet_name = spec_item["location_sheet"] + sheet_name_list.append(location_sheet_name) + try: + for sheetname in sheet_name_list: + workbook_object, extracted_sheetname = ( + self.get_xl_obj_and_sheetname(sheetname)) + if workbook_object is not None: + wb = workbook_object + sheetname = extracted_sheetname + else: + wb = self.wb_combined + + if sheetname not in wb.sheetnames: + raise RuntimeError( + "SheetName '{}' not found ".format(sheetname)) + except RuntimeError as rerror: + LOG.critical(rerror) + sys.exit("Tugboat exited!!") + + LOG.info("Sheet names in excel spec validated") + + def get_data(self): + """Create a dict with combined data""" + + self.validate_sheet_names_with_spec() + ipmi_data = self.get_ipmi_data() + network_data = self.get_private_network_data() + public_network_data = self.get_public_network_data() + site_info_data = self.get_site_info() + data = { + "ipmi_data": ipmi_data, + "network_data": { + "private": network_data, + "public": public_network_data, + }, + "site_info": site_info_data, + } + LOG.debug( + "Location data extracted from\ + excel:\n%s", + pprint.pformat(data), + ) + return data + + def combine_excel_design_specs(self, filenames): + """Combines multiple excel file to a single design spec""" + + design_spec = Workbook() + for exel_file in filenames: + loaded_workbook = load_workbook(exel_file, data_only=True) + for names in loaded_workbook.sheetnames: + design_spec_worksheet = design_spec.create_sheet(names) + loaded_workbook_ws = loaded_workbook[names] + for row in loaded_workbook_ws: + for cell in row: + design_spec_worksheet[ + cell.coordinate].value = cell.value + return design_spec + + def get_xl_obj_and_sheetname(self, sheetname): + """The logic confirms if the sheetname is specified for example as: + + 'MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs' + """ + + if re.search(".xlsx", sheetname) or re.search(".xls", sheetname): + """ Extract file name """ + source_xl_file = sheetname.split(":")[0] + wb = load_workbook(source_xl_file, data_only=True) + return [wb, sheetname.split(":")[1]] + else: + return [None, sheetname] diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..b4a7d5e --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,8 @@ +# Formatting +yapf==0.27.0 + +# Linting +hacking>=1.1.0,<1.2.0 # Apache-2.0 + +# Security +bandit>=1.5.0 diff --git a/tools/gate/whitespace-linter.sh b/tools/gate/whitespace-linter.sh new file mode 100755 index 0000000..55f062d --- /dev/null +++ b/tools/gate/whitespace-linter.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -x + +RES=$(git grep -E -l " +$") + +if [[ -n $RES ]]; then + exit 1 +fi diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..822df86 --- /dev/null +++ b/tox.ini @@ -0,0 +1,57 @@ +[tox] +envlist = pep8, bandit, docs +minversion = 2.3.1 +skipsdist = True + +[testenv] +usedevelop = True +setenv = + VIRTUAL_ENV={envdir} + LANGUAGE=en_US + LC_ALL=en_US.utf-8 +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +passenv = http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY PBR_VERSION +whitelist_externals = + find +commands = + find . -type f -name "*.pyc" -delete + {toxinidir}/tools/gate/run-unit-tests.sh '{posargs}' + +[testenv:fmt] +basepython = python3 +deps = + -r{toxinidir}/test-requirements.txt +commands = + yapf -ir {toxinidir}/spyglass-plugin-xls + +[testenv:pep8] +basepython = python3 +deps = + -r{toxinidir}/test-requirements.txt +commands = + bash -c "{toxinidir}/tools/gate/whitespace-linter.sh" + yapf -dr {toxinidir}/spyglass-plugin-xls {toxinidir}/setup.py + flake8 {toxinidir}/spyglass-plugin-xls + bandit -r spyglass-plugin-xls -n 5 +whitelist_externals = + bash + +[testenv:bandit] +deps = + bandit +commands = bandit -r spyglass-plugin-xls -n 5 + +[flake8] +ignore = E125,E251,W503 + +[testenv:docs] +basepython = python3 +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/doc/requirements.txt +commands = + rm -rf doc/build + sphinx-build -b html doc/source doc/build -n -W -v +whitelist_externals = rm