Scan templated python files with bandit

This change adds a zuul check job to export any templated python
contained in the helm charts and scan it with bandit for any
potential security flaws.

This also adds two nosec comments on the instances of subprocess
used as they currently do not appear to be malicious, as well
as changing the endpoint_update python code to prevent sql
injection, which satisfies bandit code B608.

Change-Id: I2212d26514c3510353d16a4592893dd2e85cb369

keystone/templates/bin/_endpoint-update.py.tpl

@@ -73,9 +73,8 @@ except:
 # Set Internal Endpoint
 try:
     endpoint_url = os.environ['OS_BOOTSTRAP_INTERNAL_URL']
-    user_engine.execute(
+    cmd = "update endpoint set url = %s where interface ='internal' and service_id = (select id from service where service.type = 'identity')"
-        "update endpoint set url = '{0}' where interface ='internal' and service_id = (select id from service where service.type = 'identity')".
+    user_engine.execute(cmd, (endpoint_url,))
-        format(endpoint_url))
 except:
     logger.critical("Could not update internal endpoint")
     raise
@@ -83,9 +82,8 @@ except:
 # Set Admin Endpoint
 try:
     endpoint_url = os.environ['OS_BOOTSTRAP_ADMIN_URL']
-    user_engine.execute(
+    cmd = "update endpoint set url = %s where interface ='admin' and service_id = (select id from service where service.type = 'identity')"
-        "update endpoint set url = '{0}' where interface ='admin' and service_id = (select id from service where service.type = 'identity')".
+    user_engine.execute(cmd, (endpoint_url,))
-        format(endpoint_url))
 except:
     logger.critical("Could not update admin endpoint")
     raise
@@ -93,9 +91,8 @@ except:
 # Set Public Endpoint
 try:
     endpoint_url = os.environ['OS_BOOTSTRAP_PUBLIC_URL']
-    user_engine.execute(
+    cmd = "update endpoint set url = %s where interface ='public' and service_id = (select id from service where service.type = 'identity')"
-        "update endpoint set url = '{0}' where interface ='public' and service_id = (select id from service where service.type = 'identity')".
+    user_engine.execute(cmd, (endpoint_url,))
-        format(endpoint_url))
 except:
     logger.critical("Could not update public endpoint")
     raise

keystone/templates/bin/_fernet-manage.py.tpl

@@ -25,7 +25,7 @@ import os
 import pwd
 import re
 import six
-import subprocess
+import subprocess  #nosec
 import sys
 import time
 
@@ -127,7 +127,7 @@ def execute_command(cmd):
     LOG.info("Executing 'keystone-manage %s --keystone-user=%s "
              "--keystone-group=%s' command.",
              cmd, KEYSTONE_USER, KEYSTONE_GROUP)
-    subprocess.call(['keystone-manage', cmd,
+    subprocess.call(['keystone-manage', cmd,  #nosec
                      '--keystone-user=%s' % KEYSTONE_USER,
                      '--keystone-group=%s' % KEYSTONE_GROUP])
 

tools/gate/files/template-python.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+EXCLUDES="helm-toolkit doc tests tools logs tmp roles playbooks releasenotes zuul.d python-files"
+DIRS=`ls -d */ | cut -f1 -d'/'`
+
+for EX in $EXCLUDES; do
+  DIRS=`echo $DIRS | sed "s/\b$EX\b//g"`
+done
+
+for DIR in $DIRS; do
+  PYFILES=$(helm template $DIR | yq 'select(.data != null) | .data | to_entries | map(select(.key | test(".*\\.py"))) | select(length > 0) | values[] | {(.key) : (.value)}' | jq -s add)
+  PYKEYS=$(echo "$PYFILES" | jq -r 'select(. != null) | keys[]')
+  for KEY in $PYKEYS; do
+    echo "$PYFILES" | jq -r --arg KEY "$KEY" '.[$KEY]' > ./python-files/"$DIR-$KEY"
+  done
+done

tools/gate/playbooks/osh-bandit.yaml

@@ -0,0 +1,28 @@
+- hosts: all
+  name: openstack-helm-bandit
+  tasks:
+
+    - name: Install Required Packages and Setup Host
+      shell: |
+        set -xe;
+        ./tools/deployment/common/install-packages.sh
+        ./tools/deployment/common/deploy-k8s.sh
+        sudo -H pip install yq bandit
+      environment:
+        zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - name: Template out python files
+      shell: |
+        set -xe;
+        make all
+        mkdir -p python-files
+        ./tools/gate/files/template-python.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - name: Run bandit against python files
+      shell: bandit -r ./python-files
+      args:
+        chdir: "{{ zuul.project.src_dir }}"

zuul.d/jobs-openstack-helm.yaml

@@ -24,6 +24,20 @@
       - ^doc/.*$
       - ^releasenotes/.*$
 
+- job:
+    name: openstack-helm-bandit
+    timeout: 3600
+    run: tools/gate/playbooks/osh-bandit.yaml
+    required-projects:
+      - openstack/openstack-helm-infra
+    # NOTE(gagehugo): Look into only running this for py.tpl file changes
+    # files:
+    #  - ^.*\.py\.tpl$
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+
 - job:
     name: openstack-helm-chart-deploy
     parent: openstack-helm-functional-temp

zuul.d/project.yaml

@@ -24,6 +24,7 @@
     check:
       jobs:
         - openstack-helm-lint
+        - openstack-helm-bandit
         - openstack-helm-keystone
         - openstack-helm-keystone-ldap
         - openstack-helm-glance