From e71d0d2607bea73af21072df94e7f23e3eab96fc Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 27 Jan 2020 09:13:26 +0700 Subject: [PATCH] Add support for installing python with pyenv In order to install particular python versions on distros that don't otherwise have them, add an option for installing via pyenv. The packages to be installed to allow for Python to be built were tested on the official Docker containers so they should provide a fair amount of coverage from a minimal environment. Co-Authored-By: Mohammed Naser Change-Id: Ic3312458b499a4b743895fa5829bb25155f77654 --- roles/ensure-python/README.rst | 6 ++ roles/ensure-python/defaults/main.yaml | 1 + roles/ensure-python/tasks/main.yaml | 15 +++ roles/ensure-python/tasks/pyenv.yaml | 38 ++++++++ roles/ensure-python/vars/Debian.yaml | 9 ++ roles/ensure-python/vars/Gentoo.yaml | 1 + roles/ensure-python/vars/RedHat.yaml | 8 ++ roles/ensure-python/vars/Suse.yaml | 10 ++ test-playbooks/ensure-python-pyenv.yaml | 13 +++ zuul-tests.d/python-roles-jobs.yaml | 121 ++++++++++++++++++++++++ 10 files changed, 222 insertions(+) create mode 100644 roles/ensure-python/defaults/main.yaml create mode 100644 roles/ensure-python/tasks/pyenv.yaml create mode 100644 roles/ensure-python/vars/Debian.yaml create mode 100644 roles/ensure-python/vars/Gentoo.yaml create mode 100644 roles/ensure-python/vars/RedHat.yaml create mode 100644 roles/ensure-python/vars/Suse.yaml create mode 100644 test-playbooks/ensure-python-pyenv.yaml create mode 100644 zuul-tests.d/python-roles-jobs.yaml diff --git a/roles/ensure-python/README.rst b/roles/ensure-python/README.rst index 029398029..bac3c4a2e 100644 --- a/roles/ensure-python/README.rst +++ b/roles/ensure-python/README.rst @@ -8,3 +8,9 @@ Ensure specified python interpreter and development files are installed .. zuul:rolevar:: python_version Optional version of python interpreter to install, such as ``3.7``. + +.. zuul:rolevar:: python_use_pyenv + :default: false + + Whether to optionally use pyenv to install python instead of distro + packages. diff --git a/roles/ensure-python/defaults/main.yaml b/roles/ensure-python/defaults/main.yaml new file mode 100644 index 000000000..a34eef7e1 --- /dev/null +++ b/roles/ensure-python/defaults/main.yaml @@ -0,0 +1 @@ +python_use_pyenv: false diff --git a/roles/ensure-python/tasks/main.yaml b/roles/ensure-python/tasks/main.yaml index 01a8cc0a1..9d70ffbbf 100644 --- a/roles/ensure-python/tasks/main.yaml +++ b/roles/ensure-python/tasks/main.yaml @@ -1,8 +1,23 @@ +- name: Validate python_version value + assert: + that: + - (python_version|string).split(".") | length == 2 + - (python_version|string).split(".")[0] + - (python_version|string).split(".")[1] + when: python_version is defined + - name: Install specified version of python interpreter and development files when: - python_version is defined - ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + - not python_use_pyenv package: name: python{{ python_version }}-dev state: present become: yes + +- name: Install python using pyenv + when: + - python_version is defined + - python_use_pyenv + include_tasks: pyenv.yaml diff --git a/roles/ensure-python/tasks/pyenv.yaml b/roles/ensure-python/tasks/pyenv.yaml new file mode 100644 index 000000000..775271a21 --- /dev/null +++ b/roles/ensure-python/tasks/pyenv.yaml @@ -0,0 +1,38 @@ +- name: Include OS-specific variables + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml" + - "{{ ansible_distribution }}.{{ ansible_architecture }}.yaml" + - "{{ ansible_distribution }}.yaml" + - "{{ ansible_os_family }}.yaml" + +- name: Install python build depends + become: true + package: + name: "{{ python_build_depends }}" + +- name: Clone pyenv repo + git: + repo: https://github.com/pyenv/pyenv.git + dest: "{{ ansible_user_dir }}/.pyenv" + version: master + +# NOTE(mnaser): pyenv does not allow us to let it install Python from a specific +# series so we have to do some magic to find out what's the latest +# release from a series +- name: Determine Python version + shell: | + set -o pipefail + {{ ansible_user_dir }}/.pyenv/plugins/python-build/bin/python-build --definitions | grep ^{{ python_version }} | tail -1 + args: + executable: /bin/bash + register: _python_version + +# NOTE(mnaser): We install Python globally in the system, as that's somewhat +# similar to behaviour of installing a Python package. It also +# avoids us having to mess around $PATH. +- name: Install python + become: true + command: "{{ ansible_user_dir }}/.pyenv/plugins/python-build/bin/python-build {{ _python_version.stdout }} /usr/local" + environment: + CFLAGS: -O2 \ No newline at end of file diff --git a/roles/ensure-python/vars/Debian.yaml b/roles/ensure-python/vars/Debian.yaml new file mode 100644 index 000000000..513f372ed --- /dev/null +++ b/roles/ensure-python/vars/Debian.yaml @@ -0,0 +1,9 @@ +python_build_depends: + - aria2 + - gcc + - libbz2-dev + - libreadline-dev + - libssl-dev + - libsqlite3-dev + - make + - zlib1g-dev \ No newline at end of file diff --git a/roles/ensure-python/vars/Gentoo.yaml b/roles/ensure-python/vars/Gentoo.yaml new file mode 100644 index 000000000..419edf6e9 --- /dev/null +++ b/roles/ensure-python/vars/Gentoo.yaml @@ -0,0 +1 @@ +python_build_depends: [] \ No newline at end of file diff --git a/roles/ensure-python/vars/RedHat.yaml b/roles/ensure-python/vars/RedHat.yaml new file mode 100644 index 000000000..9efd469d1 --- /dev/null +++ b/roles/ensure-python/vars/RedHat.yaml @@ -0,0 +1,8 @@ +python_build_depends: + - bzip2-devel + - gcc + - make + - openssl-devel + - readline-devel + - sqlite-devel + - zlib-devel \ No newline at end of file diff --git a/roles/ensure-python/vars/Suse.yaml b/roles/ensure-python/vars/Suse.yaml new file mode 100644 index 000000000..c32ea4283 --- /dev/null +++ b/roles/ensure-python/vars/Suse.yaml @@ -0,0 +1,10 @@ +python_build_depends: + - aria2 + - gcc + - libbz2-devel + - make + - openssl-devel + - readline-devel + - sqlite3-devel + - tar + - zlib-devel \ No newline at end of file diff --git a/test-playbooks/ensure-python-pyenv.yaml b/test-playbooks/ensure-python-pyenv.yaml new file mode 100644 index 000000000..f0c1ac300 --- /dev/null +++ b/test-playbooks/ensure-python-pyenv.yaml @@ -0,0 +1,13 @@ +- hosts: all + tasks: + - name: Include ensure-python role + include_role: + name: ensure-python + vars: + python_use_pyenv: true + python_version: 3.8 + + - name: Check installed version of Python + command: /usr/local/bin/python3 --version + register: _check_version + failed_when: "'Python 3.8' not in _check_version.stdout" \ No newline at end of file diff --git a/zuul-tests.d/python-roles-jobs.yaml b/zuul-tests.d/python-roles-jobs.yaml new file mode 100644 index 000000000..13e43c450 --- /dev/null +++ b/zuul-tests.d/python-roles-jobs.yaml @@ -0,0 +1,121 @@ +- job: + name: zuul-jobs-test-ensure-python-pyenv + description: Test the ensure-python role with pyenv + files: + - zuul-tests.d/python-roles-jobs.yaml + run: test-playbooks/ensure-python-pyenv.yaml + tags: all-platforms + +- job: + name: zuul-jobs-test-ensure-python-pyenv-centos-7 + description: Test the ensure-python role with pyenv on centos-7 + parent: zuul-jobs-test-ensure-python-pyenv + tags: auto-generated + nodeset: + nodes: + - name: centos-7 + label: centos-7 + +- job: + name: zuul-jobs-test-ensure-python-pyenv-centos-8 + description: Test the ensure-python role with pyenv on centos-8 + parent: zuul-jobs-test-ensure-python-pyenv + tags: auto-generated + nodeset: + nodes: + - name: centos-8 + label: centos-8 + +- job: + name: zuul-jobs-test-ensure-python-pyenv-debian-stretch + description: Test the ensure-python role with pyenv on debian-stretch + parent: zuul-jobs-test-ensure-python-pyenv + tags: auto-generated + nodeset: + nodes: + - name: debian-stretch + label: debian-stretch + +- job: + name: zuul-jobs-test-ensure-python-pyenv-fedora-30 + description: Test the ensure-python role with pyenv on fedora-30 + parent: zuul-jobs-test-ensure-python-pyenv + tags: auto-generated + nodeset: + nodes: + - name: fedora-30 + label: fedora-30 + +- job: + name: zuul-jobs-test-ensure-python-pyenv-gentoo-17-0-systemd + description: Test the ensure-python role with pyenv on gentoo-17-0-systemd + parent: zuul-jobs-test-ensure-python-pyenv + tags: auto-generated + nodeset: + nodes: + - name: gentoo-17-0-systemd + label: gentoo-17-0-systemd + +- job: + name: zuul-jobs-test-ensure-python-pyenv-opensuse-15 + description: Test the ensure-python role with pyenv on opensuse-15 + parent: zuul-jobs-test-ensure-python-pyenv + tags: auto-generated + nodeset: + nodes: + - name: opensuse-15 + label: opensuse-15 + +- job: + name: zuul-jobs-test-ensure-python-pyenv-opensuse-tumbleweed-nv + voting: false + description: Test the ensure-python role with pyenv on opensuse-tumbleweed + parent: zuul-jobs-test-ensure-python-pyenv + tags: auto-generated + nodeset: + nodes: + - name: opensuse-tumbleweed + label: opensuse-tumbleweed + +- job: + name: zuul-jobs-test-ensure-python-pyenv-ubuntu-bionic + description: Test the ensure-python role with pyenv on ubuntu-bionic + parent: zuul-jobs-test-ensure-python-pyenv + tags: auto-generated + nodeset: + nodes: + - name: ubuntu-bionic + label: ubuntu-bionic + +- job: + name: zuul-jobs-test-ensure-python-pyenv-ubuntu-xenial + description: Test the ensure-python role with pyenv on ubuntu-xenial + parent: zuul-jobs-test-ensure-python-pyenv + tags: auto-generated + nodeset: + nodes: + - name: ubuntu-xenial + label: ubuntu-xenial + +- project: + check: + jobs: + - zuul-jobs-test-ensure-python-pyenv-centos-7 + - zuul-jobs-test-ensure-python-pyenv-centos-8 + - zuul-jobs-test-ensure-python-pyenv-debian-stretch + - zuul-jobs-test-ensure-python-pyenv-fedora-30 + - zuul-jobs-test-ensure-python-pyenv-gentoo-17-0-systemd + - zuul-jobs-test-ensure-python-pyenv-opensuse-15 + - zuul-jobs-test-ensure-python-pyenv-opensuse-tumbleweed-nv + - zuul-jobs-test-ensure-python-pyenv-ubuntu-bionic + - zuul-jobs-test-ensure-python-pyenv-ubuntu-xenial + gate: + jobs: + - zuul-jobs-test-ensure-python-pyenv-centos-7 + - zuul-jobs-test-ensure-python-pyenv-centos-8 + - zuul-jobs-test-ensure-python-pyenv-debian-stretch + - zuul-jobs-test-ensure-python-pyenv-fedora-30 + - zuul-jobs-test-ensure-python-pyenv-gentoo-17-0-systemd + - zuul-jobs-test-ensure-python-pyenv-opensuse-15 + - zuul-jobs-test-ensure-python-pyenv-ubuntu-bionic + - zuul-jobs-test-ensure-python-pyenv-ubuntu-xenial