Merge "Add support for installing python with pyenv"

roles/ensure-python/README.rst

@@ -8,3 +8,9 @@ Ensure specified python interpreter and development files are installed
 .. zuul:rolevar:: python_version
 
   Optional version of python interpreter to install, such as ``3.7``.
+
+.. zuul:rolevar:: python_use_pyenv
+   :default: false
+
+  Whether to optionally use pyenv to install python instead of distro
+  packages.

roles/ensure-python/defaults/main.yaml

@@ -0,0 +1 @@
+python_use_pyenv: false

roles/ensure-python/tasks/main.yaml

@@ -1,8 +1,23 @@
+- name: Validate python_version value
+  assert:
+    that:
+      - (python_version|string).split(".") | length == 2
+      - (python_version|string).split(".")[0]
+      - (python_version|string).split(".")[1]
+  when: python_version is defined
+
 - name: Install specified version of python interpreter and development files
   when:
     - python_version is defined
     - ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
+    - not python_use_pyenv
   package:
     name: python{{ python_version }}-dev
     state: present
   become: yes
+
+- name: Install python using pyenv
+  when:
+    - python_version is defined
+    - python_use_pyenv
+  include_tasks: pyenv.yaml

roles/ensure-python/tasks/pyenv.yaml

@@ -0,0 +1,38 @@
+- name: Include OS-specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+    - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
+    - "{{ ansible_distribution }}.{{ ansible_architecture }}.yaml"
+    - "{{ ansible_distribution }}.yaml"
+    - "{{ ansible_os_family }}.yaml"
+
+- name: Install python build depends
+  become: true
+  package:
+    name: "{{ python_build_depends }}"
+
+- name: Clone pyenv repo
+  git:
+    repo: https://github.com/pyenv/pyenv.git
+    dest: "{{ ansible_user_dir }}/.pyenv"
+    version: master
+
+# NOTE(mnaser): pyenv does not allow us to let it install Python from a specific
+#               series so we have to do some magic to find out what's the latest
+#               release from a series
+- name: Determine Python version
+  shell: |
+    set -o pipefail
+    {{ ansible_user_dir }}/.pyenv/plugins/python-build/bin/python-build --definitions | grep ^{{ python_version }} | tail -1
+  args:
+    executable: /bin/bash
+  register: _python_version
+
+# NOTE(mnaser): We install Python globally in the system, as that's somewhat
+#               similar to behaviour of installing a Python package.  It also
+#               avoids us having to mess around $PATH.
+- name: Install python
+  become: true
+  command: "{{ ansible_user_dir }}/.pyenv/plugins/python-build/bin/python-build {{ _python_version.stdout }} /usr/local"
+  environment:
+    CFLAGS: -O2

roles/ensure-python/vars/Debian.yaml

@@ -0,0 +1,9 @@
+python_build_depends:
+  - aria2
+  - gcc
+  - libbz2-dev
+  - libreadline-dev
+  - libssl-dev
+  - libsqlite3-dev
+  - make
+  - zlib1g-dev

roles/ensure-python/vars/Gentoo.yaml

@@ -0,0 +1 @@
+python_build_depends: []

roles/ensure-python/vars/RedHat.yaml

@@ -0,0 +1,8 @@
+python_build_depends:
+  - bzip2-devel
+  - gcc
+  - make
+  - openssl-devel
+  - readline-devel
+  - sqlite-devel
+  - zlib-devel

roles/ensure-python/vars/Suse.yaml

@@ -0,0 +1,10 @@
+python_build_depends:
+  - aria2
+  - gcc
+  - libbz2-devel
+  - make
+  - openssl-devel
+  - readline-devel
+  - sqlite3-devel
+  - tar
+  - zlib-devel

test-playbooks/ensure-python-pyenv.yaml

@@ -0,0 +1,13 @@
+- hosts: all
+  tasks:
+    - name: Include ensure-python role
+      include_role:
+        name: ensure-python
+      vars:
+        python_use_pyenv: true
+        python_version: 3.8
+
+    - name: Check installed version of Python
+      command: /usr/local/bin/python3 --version
+      register: _check_version
+      failed_when: "'Python 3.8' not in _check_version.stdout"

zuul-tests.d/python-roles-jobs.yaml

@@ -0,0 +1,121 @@
+- job:
+    name: zuul-jobs-test-ensure-python-pyenv
+    description: Test the ensure-python role with pyenv
+    files:
+      - zuul-tests.d/python-roles-jobs.yaml
+    run: test-playbooks/ensure-python-pyenv.yaml
+    tags: all-platforms
+
+- job:
+    name: zuul-jobs-test-ensure-python-pyenv-centos-7
+    description: Test the ensure-python role with pyenv on centos-7
+    parent: zuul-jobs-test-ensure-python-pyenv
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-7
+          label: centos-7
+
+- job:
+    name: zuul-jobs-test-ensure-python-pyenv-centos-8
+    description: Test the ensure-python role with pyenv on centos-8
+    parent: zuul-jobs-test-ensure-python-pyenv
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-8
+          label: centos-8
+
+- job:
+    name: zuul-jobs-test-ensure-python-pyenv-debian-stretch
+    description: Test the ensure-python role with pyenv on debian-stretch
+    parent: zuul-jobs-test-ensure-python-pyenv
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: debian-stretch
+          label: debian-stretch
+
+- job:
+    name: zuul-jobs-test-ensure-python-pyenv-fedora-30
+    description: Test the ensure-python role with pyenv on fedora-30
+    parent: zuul-jobs-test-ensure-python-pyenv
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: fedora-30
+          label: fedora-30
+
+- job:
+    name: zuul-jobs-test-ensure-python-pyenv-gentoo-17-0-systemd
+    description: Test the ensure-python role with pyenv on gentoo-17-0-systemd
+    parent: zuul-jobs-test-ensure-python-pyenv
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: gentoo-17-0-systemd
+          label: gentoo-17-0-systemd
+
+- job:
+    name: zuul-jobs-test-ensure-python-pyenv-opensuse-15
+    description: Test the ensure-python role with pyenv on opensuse-15
+    parent: zuul-jobs-test-ensure-python-pyenv
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: opensuse-15
+          label: opensuse-15
+
+- job:
+    name: zuul-jobs-test-ensure-python-pyenv-opensuse-tumbleweed-nv
+    voting: false
+    description: Test the ensure-python role with pyenv on opensuse-tumbleweed
+    parent: zuul-jobs-test-ensure-python-pyenv
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: opensuse-tumbleweed
+          label: opensuse-tumbleweed
+
+- job:
+    name: zuul-jobs-test-ensure-python-pyenv-ubuntu-bionic
+    description: Test the ensure-python role with pyenv on ubuntu-bionic
+    parent: zuul-jobs-test-ensure-python-pyenv
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-bionic
+          label: ubuntu-bionic
+
+- job:
+    name: zuul-jobs-test-ensure-python-pyenv-ubuntu-xenial
+    description: Test the ensure-python role with pyenv on ubuntu-xenial
+    parent: zuul-jobs-test-ensure-python-pyenv
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-xenial
+          label: ubuntu-xenial
+
+- project:
+    check:
+      jobs:
+        - zuul-jobs-test-ensure-python-pyenv-centos-7
+        - zuul-jobs-test-ensure-python-pyenv-centos-8
+        - zuul-jobs-test-ensure-python-pyenv-debian-stretch
+        - zuul-jobs-test-ensure-python-pyenv-fedora-30
+        - zuul-jobs-test-ensure-python-pyenv-gentoo-17-0-systemd
+        - zuul-jobs-test-ensure-python-pyenv-opensuse-15
+        - zuul-jobs-test-ensure-python-pyenv-opensuse-tumbleweed-nv
+        - zuul-jobs-test-ensure-python-pyenv-ubuntu-bionic
+        - zuul-jobs-test-ensure-python-pyenv-ubuntu-xenial
+    gate:
+      jobs:
+        - zuul-jobs-test-ensure-python-pyenv-centos-7
+        - zuul-jobs-test-ensure-python-pyenv-centos-8
+        - zuul-jobs-test-ensure-python-pyenv-debian-stretch
+        - zuul-jobs-test-ensure-python-pyenv-fedora-30
+        - zuul-jobs-test-ensure-python-pyenv-gentoo-17-0-systemd
+        - zuul-jobs-test-ensure-python-pyenv-opensuse-15
+        - zuul-jobs-test-ensure-python-pyenv-ubuntu-bionic
+        - zuul-jobs-test-ensure-python-pyenv-ubuntu-xenial