Rework bindep role to be more ansible and less shell

There's still a chunk of shell, because it's honestly better than doing
the equiv with package manager modules would be.

Also, make zuul_work_dir a parameter we pass in from the playbook. The
bindep role itself is otherwise totally non-zuul specific.

Change-Id: I95e78dd25a1a54b3ce5f2ddb18228a183d3c06ad

playbooks/unittests/pre.yaml

@@ -1,11 +1,16 @@
 - hosts: all
   # We're gathering facts here so that we can emit things in the debug-ansible
-  # role. Don't copy-pasta this. Zuul runs ansible with gathering = explicit
+  # role. We also need facts in bindep.
+  # Don't copy-pasta this. Zuul runs ansible with gathering = explicit
   gather_facts: true
   gather_subset:
     - "!all"
   roles:
     - debug-ansible
-    - bindep
+    - role: bindep
+      bindep_profile: test
+      bindep_dir: "{{ zuul_work_dir }}"
+      bindep_command: /usr/bindep-env/bin/bindep
+      bindep_fallback: /usr/local/jenkins/common_data/bindep-fallback.txt
     - test-setup
     - revoke-sudo

roles/bindep/README.rst

@@ -4,3 +4,24 @@ Looks for a ``bindep.txt`` in a project's source directory, or failing
 that a ``other-requirements.txt``. If one exists, run ``bindep`` on the
 file to produce a list of required distro packages that do not exist and
 then install the missing packages.
+
+Role Variables
+--------------
+
+bindep_dir
+  The directory to look for bindep files in. Defaults to current directory.
+
+bindep_profile
+  A specific bindep profile to request. Defaults to empty.
+
+bindep_file
+  Path to a specific bindep file to read from.
+
+bindep_command
+  Path to the bindep command. Defaults to unset which will look for a system
+  installed bindep. If bindep_command is not found, bindep will be installed
+  into a temporary virtualenv.
+
+bindep_fallback
+  Path to a bindep fallback file to be used if no bindep file can be found in
+  `bindep_dir`.

roles/bindep/defaults/main.yaml

@@ -1,4 +1,3 @@
 ---
-zuul_work_dir: "src/{{ zuul.project.canonical_name }}"
+bindep_dir: .
-bindep_command: /usr/bindep-env/bin/bindep
+bindep_profile: ""
-bindep_file: ""

roles/bindep/tasks/find.yaml

@@ -0,0 +1,30 @@
+- name: Look for bindep.txt
+  stat:
+    path: "{{ bindep_dir }}/bindep.txt"
+  register: bindep_file_stat
+
+- set_fact:
+    bindep_file: "{{ bindep_file_stat.stat.path }}"
+  when: bindep_file_stat.stat.exists
+
+- name: Look for other-requirements.txt
+  stat:
+    path: "{{ bindep_dir }}/other-requirements.txt"
+  register: bindep_other_file_stat
+  when: not bindep_file_stat.stat.exists
+
+- set_fact:
+    bindep_file: "{{ bindep_other_file_stat.stat.path }}"
+  when: not bindep_other_file_stat|skipped and  bindep_other_file_stat.stat.exists
+
+- name: Look for bindep fallback file
+  stat:
+    path: "{{ bindep_fallback }}"
+  register: bindep_fallback_file_stat
+  when: bindep_fallback is defined
+        and not bindep_other_file_stat|skipped
+        and not bindep_other_file_stat.stat.exists
+
+- set_fact:
+    bindep_file: "{{ bindep_fallback_file_stat.stat.path }}"
+  when: not bindep_fallback_file_stat|skipped and  bindep_fallback_file_stat.stat.exists

roles/bindep/tasks/main.yaml

@@ -1,80 +1,77 @@
 ---
-- stat:
+- name: Look for bindep command
+  stat:
     path: "{{ bindep_command }}"
+  when: bindep_command is defined
   register: bindep_command_stat
-  failed_when: false
+
+- name: Check for system bindep
+  args:
+    executable: /bin/bash
+  shell: type -p bindep
+  ignore_errors: yes
+  register: bindep_command_type
+  when: bindep_command is not defined or not bindep_command_stat.stat.exists
 
 - set_fact:
-    bindep_found_command: "{{ bindep_command }}"
+    bindep_command: "{{ bindep_command_type.stdout }}"
-  when: bindep_command_stat is defined
+  when: bindep_command_type|succeeded and not bindep_command_type|skipped
 
 - include: install.yaml
-  when: bindep_command_stat is not defined
+  when: bindep_command is not defined
 
+- include: find.yaml
+  when: bindep_file is not defined
+
+- set_fact:
+    bindep_run: "{{ bindep_command }} -b -f {{ bindep_file }} {{ bindep_profile }}"
+  when: bindep_file is defined
+
+# Retry to check that all requested packages are obtained. Sometimes there are
+# transient repo issues. This is left as a shell section on purpose rather
+# than using ansible package modules because bindep produces a list of packages
+# that we feed directly to the package managers.
 - name: Install distro packages from bindep
   args:
-    chdir: "{{ zuul_work_dir }}"
     executable: /bin/bash
+  become: yes
+  when: bindep_file is defined
+  register: result
+  until: result|succeeded
+  retries: 3
+  delay: 5
   environment:
-    BINDEP: "{{ bindep_command }}"
+    DEBIAN_FRONTEND: noninteractive
-    PACKAGES: "{{ bindep_file }}"
+    PATH: /usr/sbin:/sbin:{{ ansible_env.PATH }}
   shell: |
-    function is_fedora {
-        [ -f /usr/bin/yum ] && cat /etc/*release | grep -q -e "Fedora"
-    }
-
-    YUM=yum
-    if is_fedora; then
-        YUM=dnf
-    fi
-
-    # figure out which bindep list to use
-    if [ -n "$PACKAGES" ] ; then
-        # already set in the calling environment
-        :
-    elif [ -e bindep.txt ] ; then
-        # project has its own bindep list
-        export PACKAGES=bindep.txt
-    elif [ -e other-requirements.txt ] ; then
-        # project has its own bindep list
-        export PACKAGES=other-requirements.txt
-    else
-        # use the bindep fallback list preinstalled on the worker
-        export PACKAGES=/usr/local/jenkins/common_data/bindep-fallback.txt
-    fi
-
-    # an install loop, retrying to check that all requested packages are
-    # obtained
-    try=0
     # Install test profile using bindep
-    until $BINDEP -b -f $PACKAGES test; do
+    case {{ ansible_pkg_mgr }} in
-        if [ $try -gt 2 ] ; then
+      apt)
-            set +x
+        apt-get -qq update
-            echo -e "\nERROR: These requested packages were not installed:\n" \
+        apt-get -q --option "Dpkg::Options::=--force-confold" --assume-yes \
-                "\n`$BINDEP -b -f $PACKAGES test`\n" 1>&2
+            install $({{ bindep_run }})
-            set -x
+        ;;
-            exit 1
+      portage)
-        fi
+        emerge -uDNq --jobs=4 @world
+        emerge -q --jobs=4 $({{ bindep_run }})
+        ;;
+      zypper)
+        zypper --non-interactive install $({{ bindep_run }})
+        ;;
+      dnf|yum)
+        {{ ansible_pkg_mgr }} install -y $({{ bindep_run }})
+        ;;
+      *)
+        echo "Unsupported package manager"
+        exit 1
+        ;;
+    esac
+    if {{ bindep_run }}; then
+        exit 0
+    else
+        exit 1
+    fi
 
-        # do not abort inside the loop, we check for the desired outcome
+- fail:
-        set +e
+    msg: "The packages from the {{ bindep_file }} were not installed"
-        if apt-get -v >/dev/null 2>&1 ; then
+  when: bindep_file is defined and result|failed
-            sudo apt-get -qq update
-            sudo PATH=/usr/sbin:/sbin:$PATH DEBIAN_FRONTEND=noninteractive \
-                apt-get -q --option "Dpkg::Options::=--force-confold" \
-                --assume-yes install `$BINDEP -b -f $PACKAGES test`
-        elif emerge --version >/dev/null 2>&1 ; then
-            sudo emerge -uDNq --jobs=4 @world
-            sudo PATH=/usr/sbin:/sbin:$PATH emerge -q --jobs=4 \
-                `$BINDEP -b -f $PACKAGES test`
-        elif zypper --version >/dev/null 2>&1 ; then
-            sudo PATH=/usr/sbin:/sbin:$PATH zypper --non-interactive install \
-                `$BINDEP -b -f $PACKAGES test`
-        else
-            sudo PATH=/usr/sbin:/sbin:$PATH $YUM install -y \
-                `$BINDEP -b -f $PACKAGES test`
-        fi
-        set -e
-
-        try=$(( $try+1 ))
-    done