From c19b61424708941bb40acf1ee5f71f10bbba5c4b Mon Sep 17 00:00:00 2001 From: "Reed, Joshua" Date: Fri, 15 Dec 2023 12:54:27 -0700 Subject: [PATCH] Update app Zuul Check Jobs. Modify code to conform to flake8 and pylint. Jobs are now flake8, pylint, py39 and metadata. Test Plan PASS - All zuul jobs pass as expected. Story: 2010929 Task: 49279 Change-Id: I7e105edccbff006423ada7ab1f99449822dd09bc Signed-off-by: Reed, Joshua --- .gitignore | 1 + .zuul.yaml | 50 ++++++- portieris-helm/files/metadata.yaml | 0 .../k8sapp_portieris/pylint.rc | 131 ++++++++++++++++-- .../k8sapp_portieris/requirements.txt | 1 + .../k8sapp_portieris/test-requirements.txt | 18 ++- .../k8sapp_portieris/tox.ini | 111 +++++++++++++-- stx-portieris-helm/debian/deb_folder/rules | 6 +- .../stx-portieris-helm/files/metadata.yaml | 8 +- test-requirements.txt | 2 +- tox.ini | 28 +++- 11 files changed, 301 insertions(+), 55 deletions(-) delete mode 100644 portieris-helm/files/metadata.yaml diff --git a/.gitignore b/.gitignore index 172bf57..f51ee66 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .tox +venv/ \ No newline at end of file diff --git a/.zuul.yaml b/.zuul.yaml index d01d721..a99b354 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -5,18 +5,22 @@ - openstack-tox-linters - k8sapp-portieris-tox-py39 - k8sapp-portieris-tox-pylint + - k8sapp-portieris-tox-flake8 + - k8sapp-portieris-tox-metadata gate: jobs: - openstack-tox-linters - k8sapp-portieris-tox-py39 - k8sapp-portieris-tox-pylint + - k8sapp-portieris-tox-flake8 + - k8sapp-portieris-tox-metadata post: jobs: - stx-portieris-armada-app-upload-git-mirror - job: name: k8sapp-portieris-tox-py39 - parent: tox-py39 + parent: openstack-tox-py39 description: | Run py39 test for portieris nodeset: debian-bullseye @@ -25,12 +29,12 @@ - starlingx/fault - starlingx/update - starlingx/utilities + - starlingx/root files: - python3-k8sapp-portieris/* vars: - tox_envlist: py39 - python_version: 3.9 tox_extra_args: -c python3-k8sapp-portieris/k8sapp_portieris/tox.ini + tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/starlingx/root/build-tools/requirements/debian/upper-constraints.txt' - job: name: k8sapp-portieris-tox-pylint @@ -43,11 +47,51 @@ - starlingx/fault - starlingx/update - starlingx/utilities + - starlingx/root files: - python3-k8sapp-portieris/* vars: tox_envlist: pylint tox_extra_args: -c python3-k8sapp-portieris/k8sapp_portieris/tox.ini + tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/starlingx/root/build-tools/requirements/debian/upper-constraints.txt' + +- job: + name: k8sapp-portieris-tox-flake8 + parent: tox + description: | + Run flake8 test for k8sapp_portieris + nodeset: debian-bullseye + required-projects: + - starlingx/config + - starlingx/fault + - starlingx/update + - starlingx/utilities + - starlingx/root + files: + - python3-k8sapp-portieris/* + vars: + tox_envlist: flake8 + tox_extra_args: -c python3-k8sapp-portieris/k8sapp_portieris/tox.ini + tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/starlingx/root/build-tools/requirements/debian/upper-constraints.txt' + +- job: + name: k8sapp-portieris-tox-metadata + parent: tox + description: | + Run metadata test for k8sapp_portieris + nodeset: debian-bullseye + required-projects: + - starlingx/config + - starlingx/fault + - starlingx/update + - starlingx/utilities + - starlingx/root + files: + - python3-k8sapp-portieris/* + vars: + tox_envlist: metadata + tox_extra_args: -c python3-k8sapp-portieris/k8sapp_portieris/tox.ini + tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/starlingx/root/build-tools/requirements/debian/upper-constraints.txt' - job: name: stx-portieris-armada-app-upload-git-mirror diff --git a/portieris-helm/files/metadata.yaml b/portieris-helm/files/metadata.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/python3-k8sapp-portieris/k8sapp_portieris/pylint.rc b/python3-k8sapp-portieris/k8sapp_portieris/pylint.rc index 4cf02c0..d9e84e0 100644 --- a/python3-k8sapp-portieris/k8sapp_portieris/pylint.rc +++ b/python3-k8sapp-portieris/k8sapp_portieris/pylint.rc @@ -31,21 +31,127 @@ extension-pkg-whitelist=lxml.etree,greenlet [MESSAGES CONTROL] -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. - # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). # See "Messages Control" section of # https://pylint.readthedocs.io/en/latest/user_guide -# We are disabling (C)onvention -# We are disabling (R)efactor -# W0212: protected-access -# W1618: no-absolute-import -disable=C, R, W0212, W1618 +disable= + # C codes refer to Convention + C0103, # invalid-name + C0104, # disallowed-nameA + C0112, # empty-docstring + C0114, # missing-module-docstring + C0115, # missing-class-docstring + C0116, # missing-function-docstring + C0123, # unidiomatic-typecheck !!! + C0201, # consider-iterating-dictionary + C0202, # bad-classmethod-argument + C0206, # consider-using-dict-items + C0207, # use-maxsplit-arg + C0209, # consider-using-f-string + C0301, # line-too-long + C0302, # too-many-lines + C0325, # superfluous-parens + C0411, # wrong-import-order + C0412, # ungrouped-imports + C0413, # wrong-import-position + C0414, # useless-import-alias !!! + C0415, # import-outside-toplevel + C1802, # use-implicit-booleaness-not-len !!! + C2801, # unnecessary-dunder-call !!! + C3002, # unnecessary-direct-lambda-call !!! + # R codes refer to refactoring + R0022, # useless-option-value !!! + R0205, # useless-object-inheritance + R0402, # consider-using-from-import + R0901, # too-many-ancestors + R0902, # too-many-instance-attributes + R0903, # too-few-public-methods + R0904, # too-many-public-methods + R0911, # too-many-return-statements + R0912, # too-many-branches + R0913, # too-many-arguments + R0914, # too-many-locals + R0915, # too-many-statements + R0916, # too-many-boolean-expressions + R1702, # too-many-nested-blocks + R1703, # simplifiable-if-statement + R1704, # redefined-argument-from-local !!! + R1705, # no-else-return + R1707, # trailing-comma-tuple !!! + R1708, # stop-iteration-return !!! + R1710, # inconsistent-return-statements + R1711, # useless-return + R1714, # consider-using-in + R1717, # consider-using-dict-comprehension !!! + R1718, # consider-using-set-comprehension + R1719, # simplifiable-if-expression + R1720, # no-else-raise + R1721, # unnecessary-comprehension + R1722, # consider-using-sys-exit !!! + R1723, # no-else-break + R1724, # no-else-continue + R1725, # super-with-arguments + R1726, # simplifiable-condition !!! + R1728, # consider-using-generator + R1729, # use-a-generator + R1730, # consider-using-min-builtin !!! + R1731, # consider-using-max-builtin !!! + R1732, # consider-using-with + R1733, # unnecessary-dict-index-lookup !! + R1734, # use-list-literal + R1735, # use-dict-literal + # W codes are warnings + W0101, # unreachable + W0105, # pointless-string-statement + W0106, # expression-not-assigned + W0107, # unnecessary-pass + W0108, # unnecessary-lambda + W0109, # duplicate-key !!! + W0123, # eval-used + W0125, # using-constant-test !!! + W0133, # pointless-exception-statement !!! + W0143, # comparison-with-callable !!! + W0150, # lost-exception + W0201, # attribute-defined-outside-init + W0211, # bad-staticmethod-argument + W0212, # protected-access + W0221, # arguments-differ + W0223, # abstract-method + W0231, # super-init-not-called + W0235, # useless-super-delegation + W0237, # arguments-renamed !!! + W0311, # bad-indentation + W0402, # deprecated-module + W0404, # reimported + W0511, # fixme + W0602, # global-variable-not-assigned !!! + W0603, # global-statement + W0612, # unused-variable + W0613, # unused-argument + W0621, # redefined-outer-name + W0622, # redefined-builtin + W0631, # undefined-loop-variable + W0703, # broad-except (pylint 2.16 renamed to broad-except-caught) + W0706, # try-except-raise + W0707, # raise-missing-from + W0719, # broad-exception-raised + W1113, # keyword-arg-before-vararg + W1310, # format-string-without-interpolation !!! + W1401, # anomalous-backslash-in-string + W1406, # redundant-u-string-prefix + W1505, # deprecated-method + W1514, # unspecified-encoding + W3101, # missing-timeout + E0601, # used-before-assignment !!! + E0605, # invalid-all-format !!! + E1101, # no-member + E1111, # assignment-from-no-return + E1121, # too-many-function-args !!! + E1123, # unexpected-keyword-arg !!! + E1136, # unsubscriptable-object !!! [REPORTS] # Set the output format. Available formats are text, parseable, colorized, msvs @@ -53,7 +159,7 @@ disable=C, R, W0212, W1618 output-format=text # Tells whether to display a full report or only the messages -reports=yes +reports=no # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which @@ -226,6 +332,5 @@ valid-classmethod-first-arg=cls [EXCEPTIONS] -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=builtins.Exception +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception diff --git a/python3-k8sapp-portieris/k8sapp_portieris/requirements.txt b/python3-k8sapp-portieris/k8sapp_portieris/requirements.txt index fafb620..8f225d8 100644 --- a/python3-k8sapp-portieris/k8sapp_portieris/requirements.txt +++ b/python3-k8sapp-portieris/k8sapp_portieris/requirements.txt @@ -1 +1,2 @@ pbr>=2.0.0 +PyYAML>=3.10.0 diff --git a/python3-k8sapp-portieris/k8sapp_portieris/test-requirements.txt b/python3-k8sapp-portieris/k8sapp_portieris/test-requirements.txt index 05c633d..a140bcc 100644 --- a/python3-k8sapp-portieris/k8sapp_portieris/test-requirements.txt +++ b/python3-k8sapp-portieris/k8sapp_portieris/test-requirements.txt @@ -2,21 +2,19 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. hacking>=1.1.0,<=2.0.0 # Apache-2.0 +astroid +bandit<1.7.2;python_version>="3.0" coverage>=3.6 -discover +fixtures>=3.0.0 # Apache-2.0/BSD mock>=2.0.0 # BSD -passlib>=1.7.0 -psycopg2-binary python-subunit>=0.0.18 requests-mock>=0.6.0 # Apache-2.0 +sphinx +oslosphinx oslotest>=3.2.0 # Apache-2.0 stestr>=1.0.0 # Apache-2.0 +testrepository>=0.0.18 testtools!=1.2.0,>=0.9.36 -ipaddr -pytest -pyudev -migrate -markupsafe -bandit -flake8-bugbear +isort<5;python_version>="3.0" pylint +pycryptodomex diff --git a/python3-k8sapp-portieris/k8sapp_portieris/tox.ini b/python3-k8sapp-portieris/k8sapp_portieris/tox.ini index 289ac73..8acc96d 100644 --- a/python3-k8sapp-portieris/k8sapp_portieris/tox.ini +++ b/python3-k8sapp-portieris/k8sapp_portieris/tox.ini @@ -1,20 +1,24 @@ [tox] -envlist = flake8,py39,pylint,bandit,cover -minversion = 2.3 +envlist = flake8,py39,pylint,metadata +minversion = 1.6 +skipsdist = True # tox does not work if the path to the workdir is too long, so move it to /tmp +# tox 3.1.0 adds TOX_LIMITED_SHEBANG toxworkdir = /tmp/{env:USER}_k8sportieristox stxdir = {toxinidir}/../../.. distshare={toxworkdir}/.tox/distshare [testenv] +basepython = python3.9 +usedevelop = True + +# tox is silly... these need to be separated by a newline.... allowlist_externals = bash find -basepython = python3 -sitepackages = False + echo install_command = pip install -v -v -v \ - -c{toxinidir}/upper-constraints.txt \ -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \ {opts} {packages} @@ -22,6 +26,7 @@ install_command = pip install -v -v -v \ # random hash seed successfully. setenv = VIRTUAL_ENV={envdir} PYTHONHASHSEED=0 + PIP_RESOLVER_DEBUG=1 PYTHONDONTWRITEBYTECODE=1 OS_TEST_PATH=./k8sapp_portieris/tests LANG=en_US.UTF-8 @@ -32,56 +37,126 @@ setenv = VIRTUAL_ENV={envdir} TOX_WORK_DIR={toxworkdir} PYLINTHOME={toxworkdir} -# the path to cgcs-patch needs to be updated to sw-patch deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -e{[tox]stxdir}/config/sysinv/sysinv/sysinv -e{[tox]stxdir}/config/tsconfig/tsconfig -e{[tox]stxdir}/fault/fm-api/source -e{[tox]stxdir}/fault/python-fmclient/fmclient - -e{[tox]stxdir}/utilities/ceph/python-cephclient/python-cephclient -e{[tox]stxdir}/update/sw-patch/cgcs-patch - + -e{[tox]stxdir}/utilities/ceph/python-cephclient/python-cephclient commands = find . -type f -name "*.pyc" -delete [flake8] +# H series are hacking +# H101 is TODO +# H102 is apache license +# H104 file contains only comments (ie: license) +# H105 author tags +# H306 imports not in alphabetical order +# H401 docstring should not start with a space +# H403 multi line docstrings should end on a new line +# H404 multi line docstring should start without a leading new line +# H405 multi line docstring summary not separated with an empty line +# H701 Empty localization string +# H702 Formatting operation should be outside of localization method call +# H703 Multiple positional placeholders + +# B series are bugbear +# B006 Do not use mutable data structures for argument defaults. Needs to be FIXED. +# B007 Loop control variable not used within the loop body. +# B009 Do not call getattr with a constant attribute value +# B010 Do not call setattr with a constant attribute value +# B012 return/continue/break inside finally blocks cause exceptions to be silenced +# B014 Redundant exception types +# B301 Python 3 does not include `.iter*` methods on dictionaries. (this should be suppressed on a per line basis) + +# W series are warnings +# W503 line break before binary operator +# W504 line break after binary operator +# W605 invalid escape sequence + +# E series are pep8 +# E117 over-indented +# E126 continuation line over-indented for hanging indent +# E127 continuation line over-indented for visual indent +# E128 continuation line under-indented for visual indent +# E402 module level import not at top of file +# E741 ambiguous variable name + +ignore = H101,H102,H104,H105,H306,H401,H403,H404,H405,H701,H702,H703, + B006,B007,B009,B010,B012,B014,B301 + W503,W504,W605, + E117,E126,E127,E128,E402,E741 exclude = build,dist,tools,.eggs max-line-length=120 [testenv:flake8] deps = -r{toxinidir}/test-requirements.txt commands = - flake8 {posargs} . + flake8 {posargs} ./k8sapp_portieris [testenv:py39] -basepython = python3.9 commands = - {[testenv]commands} stestr run {posargs} stestr slowest [testenv:pep8] -deps = {[testenv:flake8]deps} +# testenv:flake8 clone +deps = -r{toxinidir}/test-requirements.txt commands = {[testenv:flake8]commands} [testenv:venv] commands = {posargs} [bandit] -# Add bandit configuration here +# The following bandit tests are being skipped: +# B101: Test for use of assert +# B103: Test for setting permissive file permissions +# B104: Test for binding to all interfaces +# B105: Test for use of hard-coded password strings +# B108: Test for insecure usage of tmp file/directory +# B110: Try, Except, Pass detected. +# B303: Use of insecure MD2, MD4, MD5, or SHA1 hash function. +# B307: Blacklisted call to eval. +# B310: Audit url open for permitted schemes +# B311: Standard pseudo-random generators are not suitable for security/cryptographic purposes +# B314: Blacklisted calls to xml.etree.ElementTree +# B318: Blacklisted calls to xml.dom.minidom +# B320: Blacklisted calls to lxml.etree +# B404: Import of subprocess module +# B405: import xml.etree +# B408: import xml.minidom +# B410: import lxml +# B506: Test for use of yaml load +# B602: Test for use of popen with shell equals true +# B603: Test for use of subprocess without shell equals true +# B604: Test for any function with shell equals true +# B605: Test for starting a process with a shell +# B607: Test for starting a process with a partial path +# B608: Possible SQL injection vector through string-based query +# +# Note: 'skips' entry cannot be split across multiple lines +# +skips = B101,B103,B104,B105,B108,B110,B303,B307,B310,B311,B314,B318,B320,B404,B405,B408,B410,B506,B602,B603,B604,B605,B607,B608 +exclude = tests [testenv:bandit] deps = -r{toxinidir}/test-requirements.txt commands = bandit --ini tox.ini -n 5 -r k8sapp_portieris [testenv:pylint] -deps = {[testenv]deps} +install_command = pip install -v -v -v \ + -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \ + {opts} {packages} commands = pylint {posargs} k8sapp_portieris --rcfile=./pylint.rc [testenv:cover] +# not sure is passenv is still needed +passenv = CURL_CA_BUNDLE deps = {[testenv]deps} setenv = {[testenv]setenv} PYTHON=coverage run --parallel-mode @@ -103,3 +178,11 @@ commands = deps = pip_missing_reqs -rrequirements.txt commands=pip-missing-reqs -d --ignore-file=/k8sapp_portieris/tests k8sapp_portieris + +[testenv:metadata] +install_command = pip install -v -v -v \ + -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \ + {opts} {packages} +# Pass top level app folder to 'sysinv-app tox' command. +commands = + bash -c "echo $(dirname $(dirname $(pwd))) | xargs -n 1 sysinv-app tox" \ No newline at end of file diff --git a/stx-portieris-helm/debian/deb_folder/rules b/stx-portieris-helm/debian/deb_folder/rules index c1ac3b1..5786d80 100755 --- a/stx-portieris-helm/debian/deb_folder/rules +++ b/stx-portieris-helm/debian/deb_folder/rules @@ -40,9 +40,9 @@ override_dh_auto_build: cp $(HELM_FOLDER)/portieris*.tgz $(STAGING)/charts # Populate metadata. - sed -i 's/@APP_NAME@/$(APP_NAME)/g' $(STAGING)/metadata.yaml - sed -i 's/@APP_VERSION@/$(APP_VERSION)/g' $(STAGING)/metadata.yaml - sed -i 's/@HELM_REPO@/$(HELM_REPO)/g' $(STAGING)/metadata.yaml + sed -i 's/APP_REPLACE_NAME/$(APP_NAME)/g' $(STAGING)/metadata.yaml + sed -i 's/APP_REPLACE_VERSION/$(APP_VERSION)/g' $(STAGING)/metadata.yaml + sed -i 's/HELM_REPLACE_REPO/$(HELM_REPO)/g' $(STAGING)/metadata.yaml # Copy the plugins: installed in the buildroot mkdir -p $(STAGING)/plugins diff --git a/stx-portieris-helm/stx-portieris-helm/files/metadata.yaml b/stx-portieris-helm/stx-portieris-helm/files/metadata.yaml index 9412318..ae3ddb7 100644 --- a/stx-portieris-helm/stx-portieris-helm/files/metadata.yaml +++ b/stx-portieris-helm/stx-portieris-helm/files/metadata.yaml @@ -1,7 +1,7 @@ +app_name: APP_REPLACE_NAME +app_version: APP_REPLACE_VERSION +helm_repo: HELM_REPLACE_REPO + maintain_user_overrides: true upgrades: auto_update: true - -app_name: @APP_NAME@ -app_version: @APP_VERSION@ -helm_repo: @HELM_REPO@ diff --git a/test-requirements.txt b/test-requirements.txt index 8ae3e22..fa7c694 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,3 @@ # hacking pulls in flake8 -hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 +hacking>=1.1.0,<=2.0.0 # Apache-2.0 bashate >= 0.2 diff --git a/tox.ini b/tox.ini index 731c409..feb7435 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,14 @@ [tox] envlist = linters -minversion = 2.3 +minversion = 2.9 skipsdist = True sitepackages=False [testenv] -install_command = pip install -U {opts} {packages} +basepython = python3 +install_command = pip install -U \ + {opts} {packages} \ + -c{env:TOX_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} setenv = VIRTUAL_ENV={envdir} OS_STDOUT_CAPTURE=1 @@ -18,6 +21,22 @@ deps = allowlist_externals = bash +[testenv:flake8] +basepython = python3 +description = Dummy environment to allow flake8 to be run in subdir tox + +[testenv:pylint] +basepython = python3 +description = Dummy environment to allow pylint to be run in subdir tox + +[testenv:metadata] +basepython = python3 +description = Dummy environment to allow sysinv-app to be run in subdir tox + +[testenv:bandit] +basepython = python3 +description = Dummy environment to allow bandit to be run in subdir tox + [testenv:bashate] # Treat all E* codes as Errors rather than warnings using: -e 'E*' commands = @@ -33,8 +52,3 @@ commands = [testenv:linters] commands = {[testenv:bashate]commands} - -[testenv:pylint] -basepython = python3 -description = Dummy environment to allow pylint to be run in subdir tox -