From 80c8b9f3fcd232cb4bc5064758a85ca4157d5c82 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Tue, 29 Oct 2024 14:53:48 -0700 Subject: [PATCH] Migrate lint to pre-commit Migrate all existing linters to pre-commit. This consolodates our bandit and codespell job into the general pep8 job. Change-Id: I6b40a3338d98fab500e22918b6bd5b8bff2106fd --- .pre-commit-config.yaml | 58 +++++++++++++++++++++- tools/bandit.yml | 1 + tox.ini | 106 +++++++++++++++++----------------------- zuul.d/ironic-jobs.yaml | 29 ----------- zuul.d/project.yaml | 4 -- 5 files changed, 101 insertions(+), 97 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d2c8f8b875..f94ae95d0d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,9 +7,23 @@ repos: rev: v4.5.0 hooks: - id: trailing-whitespace + # NOTE(JayF): We shouldn't modify release notes after their + # associated release. Instead, ignore these minor lint issues. + exclude: | + (?x)( + ^releasenotes/notes/redfish-raid-get-drives-fix-18d46f3e7275b0ef.yaml$| + ^releasenotes/notes/provide_mountpoint-58cfd25b6dd4cfde.yaml$| + ^releasenotes/notes/ipmi-retries-min-command-interval-070cd7eff5eb74dd.yaml$| + ^releasenotes/notes/deprecate-ibmc-9106cc3a81171738.yaml$| + ^releasenotes/notes/fix-cve-2016-4985-b62abae577025365.yaml$ + ) - id: mixed-line-ending args: ['--fix', 'lf'] - exclude: '.*\.(svg)$' + exclude: | + (?x)( + .*.svg$| + ^releasenotes/notes/ibmc-driver-45fcf9f50ebf0193.yaml$| + ) - id: fix-byte-order-marker - id: check-merge-conflict - id: debug-statements @@ -17,6 +31,7 @@ repos: files: .*\.json$ - id: check-yaml files: .*\.(yaml|yml)$ + exclude: releasenotes/.*$ - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.5.4 hooks: @@ -37,5 +52,44 @@ repos: hooks: - id: sphinx-lint args: [--enable=default-role] - files: ^doc/|releasenotes|api-guide + files: ^doc/|releasenotes|api-ref types: [rst] + - repo: https://opendev.org/openstack/bashate + rev: 2.1.0 + hooks: + - id: bashate + args: ["-iE006,E044", "-eE005,E042"] + name: bashate + description: This hook runs bashate for linting shell scripts + entry: bashate + language: python + types: [shell] + - repo: https://github.com/PyCQA/bandit + rev: 1.7.10 + hooks: + - id: bandit + args: ["-x", "tests/", "-n5", "-ll", "-c", "tools/bandit.yml"] + name: bandit + description: 'Bandit is a tool for finding common security issues in Python code' + entry: bandit + language: python + language_version: python3 + types: [ python ] + require_serial: true + - repo: https://github.com/PyCQA/doc8 + rev: v1.1.1 + hooks: + - id: doc8 + args: ["--ignore", "D001"] + name: doc8 + description: This hook runs doc8 for linting docs + entry: python -m doc8 + language: python + files: \.rst$ + require_serial: true + - repo: local + hooks: + - id: check-releasenotes + name: check-releasenotes + language: python + entry: python tools/check-releasenotes.py \ No newline at end of file diff --git a/tools/bandit.yml b/tools/bandit.yml index 028d1a214d..b72863b585 100644 --- a/tools/bandit.yml +++ b/tools/bandit.yml @@ -90,6 +90,7 @@ tests: skips: - B104 - B604 + - B701 ### (optional) plugin settings - some test plugins require configuration data ### that may be given here, per-plugin. All bandit test plugins have a built in diff --git a/tox.ini b/tox.ini index 517c4cd7c4..8563517129 100644 --- a/tox.ini +++ b/tox.ini @@ -39,26 +39,19 @@ deps = {[testenv]deps} commands = {toxinidir}/tools/states_to_dot.py -f {toxinidir}/doc/source/images/states.svg --format svg [testenv:pep8] -usedevelop = False -deps= - hacking~=6.1.0 # Apache-2.0 - doc8~=1.1.0 # Apache-2.0 - pycodestyle>=2.0.0,<3.0.0 # MIT - flake8-import-order~=0.18.0 # LGPLv3 - bashate~=2.1.0 # Apache-2.0 - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -allowlist_externals = bash - {toxinidir}/tools/run_bashate.sh - {toxinidir}/tools/check-releasenotes.py -commands = - bash tools/flake8wrap.sh {posargs} - # Run bashate during pep8 runs to ensure violations are caught by - # the check and gate queues. - {toxinidir}/tools/run_bashate.sh {toxinidir} - # Check the *.rst files - doc8 README.rst CONTRIBUTING.rst doc/source api-ref/source --ignore D001 - # Check to make sure reno releasenotes created with 'reno new' - {toxinidir}/tools/check-releasenotes.py +deps = pre-commit +allowlist_externals = pre-commit +commands = pre-commit run --all-files --show-diff-on-failure {posargs} + +[testenv:bandit] +deps = pre-commit +commands = pre-commit run --all-files --show-diff-on-failure bandit + +[testenv:codespell] +description = + Run codespell to check spelling +deps = pre-commit +commands = pre-commit run --all-files --show-diff-on-failure codespell [testenv:cover] setenv = {[testenv]setenv} @@ -103,7 +96,6 @@ commands = sphinx-build -W -b latex doc/source doc/build/pdf make -C doc/build/pdf - [testenv:api-ref] # NOTE(Mahnoor): documentation building process requires importing ironic API modules usedevelop = False @@ -126,12 +118,41 @@ commands = [testenv:venv] setenv = PYTHONHASHSEED=0 +# NOTE(JayF) The generic venv target exists to allow people to run arbitrary +# things, e.g. `reno`. For this reason, allow all externals in this +# environment. +allowlist_externals = * deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/test-requirements.txt -r{toxinidir}/doc/requirements.txt commands = {posargs} +[testenv:local-ironic-dev] +constrain_package_deps = true +setenv = VIRTUAL_ENV={envdir} + PYTHONDONTWRITEBYTECODE=1 + LANGUAGE=en_US + LC_ALL=en_US.UTF-8 + PYTHONUNBUFFERED=1 + SQLALCHEMY_WARN_20=true +deps = + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/requirements.txt + -r{toxinidir}/driver-requirements.txt + python-ironicclient + python-openstackclient +allowlist_externals = bash +commands = + bash -c 'ironic-dbsync --config-file={toxinidir}/tools/ironic.conf.localdev create_schema || echo "===== DB already exists; continuing ====="' + ironic --config-file={toxinidir}/tools/ironic.conf.localdev {posargs} +passenv = http_proxy + HTTP_PROXY + https_proxy + HTTPS_PROXY + no_proxy + NO_PROXY + [flake8] # [E129] visually indented line with same indent as next logical line # [E741] ambiguous variable name @@ -154,50 +175,11 @@ per-file-ignores = ironic/cmd/__init__.py:E402 ironic/tests/base.py:E402 -[hacking] -import_exceptions = testtools.matchers, ironic.common.i18n - [flake8:local-plugins] # [N323] Found use of _() without explicit import of _! extension = N323 = checks:check_explicit_underscore_import paths = ./ironic/hacking/ -[testenv:bandit] -usedevelop = False -deps = -r{toxinidir}/test-requirements.txt -commands = bandit -r ironic -x tests -n5 -ll -c tools/bandit.yml - -[testenv:codespell] -description = - Run codespell to check spelling -deps = codespell -# note(JayF): {posargs} lets us run `tox -ecodespell -- -w` to get codespell -# to correct spelling issues in our code it's aware of. -commands = - codespell {posargs} - -[testenv:local-ironic-dev] -constrain_package_deps = true -usedevelop = True -setenv = VIRTUAL_ENV={envdir} - PYTHONDONTWRITEBYTECODE=1 - LANGUAGE=en_US - LC_ALL=en_US.UTF-8 - PYTHONUNBUFFERED=1 -deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} - -r{toxinidir}/requirements.txt - -r{toxinidir}/driver-requirements.txt - python-ironicclient - python-openstackclient -allowlist_externals = bash -commands = - bash -c 'ironic-dbsync --config-file={toxinidir}/tools/ironic.conf.localdev create_schema || echo "===== DB already exists; continuing ====="' - ironic --config-file={toxinidir}/tools/ironic.conf.localdev {posargs} -passenv = http_proxy - HTTP_PROXY - https_proxy - HTTPS_PROXY - no_proxy - NO_PROXY +[hacking] +import_exceptions = testtools.matchers, ironic.common.i18n diff --git a/zuul.d/ironic-jobs.yaml b/zuul.d/ironic-jobs.yaml index 7acc74c4d7..8c47ff2f3e 100644 --- a/zuul.d/ironic-jobs.yaml +++ b/zuul.d/ironic-jobs.yaml @@ -881,28 +881,6 @@ IRONIC_RAMDISK_TYPE: tinyipa IRONIC_AUTOMATED_CLEAN_ENABLED: False -- job: - # Security testing for known issues - name: ironic-tox-bandit - parent: openstack-tox - timeout: 2400 - vars: - tox_envlist: bandit - required-projects: - - opendev.org/openstack/ironic - irrelevant-files: - - ^.*\.rst$ - - ^api-ref/.*$ - - ^doc/.*$ - - ^driver-requirements.txt$ - - ^install-guide/.*$ - - ^ironic/locale/.*$ - - ^ironic/tests/.*$ - - ^redfish-interop-profiles/.*$ - - ^releasenotes/.*$ - - ^setup.cfg$ - - ^tox.ini$ - - job: name: ironic-tempest-ipa-wholedisk-bios-ipmi-direct-dib parent: ironic-base @@ -1319,10 +1297,3 @@ grenade_localrc: # NOTE(iurygregory): This is required for skip upgrades. NOVA_ENABLE_UPGRADE_WORKAROUND: True - -- job: - name: ironic-tox-codespell - parent: openstack-tox - timeout: 7200 - vars: - tox_envlist: codespell diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 6ccb012e16..78d0444678 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -10,8 +10,6 @@ - release-notes-jobs-python3 check: jobs: - - ironic-tox-bandit - - ironic-tox-codespell - ironic-tox-unit-mysql-migrations - ironic-tox-unit-with-driver-libs - ironic-cross-sushy: @@ -70,8 +68,6 @@ voting: false gate: jobs: - - ironic-tox-bandit - - ironic-tox-codespell - ironic-tox-unit-mysql-migrations - ironic-tox-unit-with-driver-libs - ironic-tempest-functional-python3