Merge "Add unified linters check"

scripts/gate-check-commit.sh

@@ -40,7 +40,7 @@ export ANSIBLE_LOG_DIR="/openstack/log/ansible-logging"
 export SCENARIO=${1:-"aio_lxc"}
 
 # Set the action base on the second CLI parameter
-# Actions available: [ 'deploy', 'upgrade' ]
+# Actions available: [ 'deploy', 'upgrade', 'varstest', 'linters' ]
 export ACTION=${2:-"deploy"}
 
 # Set the installation method for the OpenStack services
@@ -103,6 +103,49 @@ if [[ "${ACTION}" == "varstest" ]]; then
   pushd "${OSA_CLONE_DIR}/tests"
       openstack-ansible test-vars-overrides.yml
   popd
+elif [[ "${ACTION}" == "linters" ]]; then
+  pushd "${OSA_CLONE_DIR}/playbooks"
+    # Install linter tools
+    ${PIP_COMMAND} install --isolated ${PIP_OPTS} -r ${OSA_CLONE_DIR}/test-requirements.txt
+    # Disable Ansible color output
+    export ANSIBLE_NOCOLOR=1
+    # Create ansible logging directory
+    mkdir -p ${ANSIBLE_LOG_DIR}
+
+    # Prepare the hosts
+    export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-syntax-check.log"
+
+    # defining working directories
+    VENV_BIN_DIR=$(dirname ${PIP_COMMAND})
+    ROLE_DIR="/etc/ansible/roles/${SCENARIO}"
+
+    ANSIBLE_LINT_EXCLUDES="204,metadata"
+    # Check if we have test playbook and running checks
+    if [[ -f "${ROLE_DIR}/examples/playbook.yml" ]]; then
+      ${VENV_BIN_DIR}/ansible-lint ${ROLE_DIR}/examples/playbook.yml -x ${ANSIBLE_LINT_EXCLUDES}
+      ${VENV_BIN_DIR}/ansible-playbook --syntax-check --list-tasks ${ROLE_DIR}/examples/playbook.yml
+    else
+      ${VENV_BIN_DIR}/ansible-lint ${ROLE_DIR} -x ${ANSIBLE_LINT_EXCLUDES}
+      ${VENV_BIN_DIR}/ansible-playbook --syntax-check --list-tasks setup-everything.yml
+    fi
+
+    # Run bashate
+    grep --recursive --binary-files=without-match \
+      --files-with-match '^.!.*\(ba\)\?sh$' \
+      --exclude-dir .tox \
+      --exclude-dir .git \
+      "${ROLE_DIR}" | xargs -r -n1 ${VENV_BIN_DIR}/bashate --error . --verbose --ignore=E003,E006,E040
+
+    # Run pep8 check
+    grep --recursive --binary-files=without-match \
+      --files-with-match '^.!.*python$' \
+      --exclude-dir .eggs \
+      --exclude-dir .git \
+      --exclude-dir .tox \
+      --exclude-dir *.egg-info \
+      --exclude-dir doc \
+      "${ROLE_DIR}" | xargs -r ${VENV_BIN_DIR}/flake8 --verbose
+  popd
 else
   pushd "${OSA_CLONE_DIR}/playbooks"
     # Disable Ansible color output

test-requirements.txt

@@ -8,3 +8,9 @@ bashate>=0.5.1 # Apache-2.0
 coverage!=4.4,>=4.0 # Apache-2.0
 mock>=2.0.0 # BSD
 virtualenv>=14.0.6 # MIT
+# TODO(noonedeadpunk) Upgrade linters versions and drop pycodestyle
+# once we stop doing linters check for the integrated repo with
+# openstack-ansible-tests/test-ansible-lint.sh
+flake8<2.6.0,>=2.5.4 # MIT
+ansible-lint==3.4.21 # MIT
+pycodestyle>=2.5.0,<2.6.0

zuul.d/jobs.yaml

@@ -98,6 +98,12 @@
     parent: openstack-ansible-deploy-aio
     nodeset: ubuntu-bionic
 
+# Running linters check
+- job:
+    name: openstack-ansible-linters-ubuntu-bionic
+    parent: openstack-ansible-deploy-aio
+    nodeset: ubuntu-bionic
+
 # Experimental job for checking our AIO can deploy with
 # a different ansible version
 - job:

zuul.d/playbooks/pre-gate-scenario.yml

@@ -36,18 +36,29 @@
                   regex_replace('^openstack-ansible-deploy-([^-]+)-.*$', '\\1')).split('_')
                )
           %}
-          {% endif %}
+          {#   If testing a role, add the role service name to the scenario list #}
-          {# If testing a role, add the role service name to the scenario list #}
+          {%   if zuul.project.short_name is match('^openstack-ansible-os_(.*)$') %}
-          {% if zuul.project.short_name is match('^openstack-ansible-os_(.*)$') %}
+          {%     set role_service_name = zuul.project.short_name |
-          {%   set role_service_name = zuul.project.short_name |
                                        regex_replace('^openstack-ansible-os_(.*)$', '\\1')
           %}
-          {%   set _ = _scenario.append(role_service_name) %}
+          {%     set _ = _scenario.append(role_service_name) %}
-          {#   Add special scenarios based on the project name #}
+          {#     Add special scenarios based on the project name #}
-          {%   if role_service_name in scenario_map %}
+          {%     if role_service_name in scenario_map %}
-          {%     set _ = _scenario.append(scenario_map[role_service_name]) %}
+          {%       set _ = _scenario.append(scenario_map[role_service_name]) %}
+          {%     endif %}
           {%   endif %}
           {% endif %}
+          {# If running linters we need to provide name of the role as a scenario w/o splitting #}
+          {% if zuul.job is match('^openstack-ansible-linters-.*$') %}
+          {%   for prefix in ['openstack-ansible', 'ansible-role'] %}
+          {%     if zuul.project.short_name is match('^' ~ prefix ~ '-([^-]+)$') %}
+          {%        set _ = _scenario.append(
+                      zuul.project.short_name | regex_replace('^' ~ prefix ~'-([^-]+)$', '\\1')
+                    )
+          %}
+          {%     endif%}
+          {%   endfor %}
+          {% endif %}
           {{ _scenario | join('_') }}
       when:
         - scenario is not defined
@@ -78,6 +89,7 @@
           Scenario: {{ scenario }}
           Action: {{ action }}
           Install Method: {{ install_method }}
+          Zuul project name: {{ zuul.project.short_name }}
 
     # Other playbooks can consume this vars file as shown in run.yml
     # This file is stored in work/osa-gate-scenario.yml on the Zuul executor

zuul.d/project-templates.yaml

@@ -152,3 +152,12 @@
     experimental:
       jobs:
         - openstack-ansible-deploy-aio_distro_ceph-opensuse-150
+
+- project-template:
+    name: openstack-ansible-linters-jobs
+    check:
+      jobs:
+        - openstack-ansible-linters-ubuntu-bionic
+    gate:
+      jobs:
+        - openstack-ansible-linters-ubuntu-bionic