diff --git a/doc/source/python-roles.rst b/doc/source/python-roles.rst index f8fb8450c..b7d871e26 100644 --- a/doc/source/python-roles.rst +++ b/doc/source/python-roles.rst @@ -5,6 +5,7 @@ Python Roles .. zuul:autorole:: build-releasenotes .. zuul:autorole:: ensure-babel .. zuul:autorole:: ensure-if-python +.. zuul:autorole:: ensure-pip .. zuul:autorole:: ensure-python .. zuul:autorole:: ensure-sphinx .. zuul:autorole:: ensure-tox diff --git a/roles/ensure-pip/README.rst b/roles/ensure-pip/README.rst new file mode 100644 index 000000000..7797a6f33 --- /dev/null +++ b/roles/ensure-pip/README.rst @@ -0,0 +1,51 @@ +Ensure pip is available + +This role is intended install the requirements for the `pip module +`__ +on hosts. + +Jobs that also wish to call ``pip`` via shell commands directly can +also use this to ensure ``pip`` is available. However, it should be +noted that calling ``pip`` is ambiguous when supporting many +platforms. On some platforms it may install the package under the +Python 2 interpreter and in others Python 3. You should use a +qualified name (``pip2`` or ``pip3``) to avoid confusion. + +**Role Variables** + +.. zuul:rolevar:: ensure_pip_from_packages + :default: True + + Ensure the system packages for pip with the running + ``ansible_python_interpreter`` are installed. + +.. zuul:rolevar:: ensure_pip_from_packages_with_python2 + :default: False + + Also ensure Python 2 pip is available. This is for backwards + compatability with platforms that have + ``ansible_python_interpreter`` as Python 3 but may run some jobs + that still require Python 2 libraries. Note that this may bring in + the Python 2 interpreter environment, which may not be desirable or + even available on many platforms. + +.. zuul:rolevar:: ensure_pip_from_upstream + :default: False + + Install pip from latest upstream sources locally. Note this is + probably not what you want and should be used with extreme caution. + The installed pip does not coordinate with the system packaged + versions, and can lead to wide variety of problems if CI jobs + re-install ``pip`` packages, for example. + +.. zuul:rolevar:: ensure_pip_from_upstream_interpreters + :default: [ ansible_python_interpreter ] + + A list of interpreters to install pip from upstream with. Note + that by default the *last* entry in the list will likely own the + ``/usr/local/bin/pip`` command; this can create confusion for + legacy jobs if they assume ``pip`` installs Python 2 libraries but + it is actually installing into the Python 3 environment. This role + does not install the Python 2 interpreter, which may not be + available on the system, so caution should be used when specifying + ``python2`` in this list. diff --git a/roles/ensure-pip/defaults/main.yaml b/roles/ensure-pip/defaults/main.yaml new file mode 100644 index 000000000..2d92a5e38 --- /dev/null +++ b/roles/ensure-pip/defaults/main.yaml @@ -0,0 +1,6 @@ +ensure_pip_from_packages: True +ensure_pip_from_packages_with_python2: False +ensure_pip_from_upstream_url: 'https://bootstrap.pypa.io/get-pip.py' +ensure_pip_from_upstream: False +ensure_pip_from_upstream_interpreters: + - '{{ ansible_python.executable }}' diff --git a/roles/ensure-pip/tasks/Debian.yaml b/roles/ensure-pip/tasks/Debian.yaml new file mode 100644 index 000000000..ad7c4163d --- /dev/null +++ b/roles/ensure-pip/tasks/Debian.yaml @@ -0,0 +1,15 @@ +- name: Install Python 3 pip + package: + name: + - python3-pip + - python3-setuptools + become: yes + +- name: Install Python 2 pip + package: + name: + - python-setuptools + - python-pip + become: yes + when: (ensure_pip_from_packages_with_python2) or + (ansible_python.version.major == 2) diff --git a/roles/ensure-pip/tasks/Gentoo.yaml b/roles/ensure-pip/tasks/Gentoo.yaml new file mode 100644 index 000000000..67d129940 --- /dev/null +++ b/roles/ensure-pip/tasks/Gentoo.yaml @@ -0,0 +1,11 @@ +- name: Install Python 3 pip + package: + name: dev-python/pip + become: yes + +- name: Install Python 2 pip + package: + name: dev-python/pip + become: yes + when: + - ensure_pip_from_packages_with_python2 diff --git a/roles/ensure-pip/tasks/RedHat.yaml b/roles/ensure-pip/tasks/RedHat.yaml new file mode 100644 index 000000000..33fb01f77 --- /dev/null +++ b/roles/ensure-pip/tasks/RedHat.yaml @@ -0,0 +1,40 @@ +# NOTE(ianw) : Skip attempting installation if the package is +# excluded. This is a temporary workaround for nodes that use +# pip-and-virtualenv and pre-install python3-pip but place it in the +# excludes list. In this case, the "package: " call will fail on +# dnf/Fedora because it notices it is excluded. Doesn't seem to +# happen with yum or even centos8 era dnf. Can go away when we stop +# this on the base nodes. +- name: Read dnf.conf + shell: | + if [[ -e /etc/dnf/dnf.conf ]]; then + if grep -e 'exclude=.*python3-pip' /etc/dnf/dnf.conf; then + exit 1 + else + exit 0 + fi + fi + exit 0 + failed_when: false + register: _pkg_excluded + +- name: Install Python 3 pip + package: + name: + - python3-pip + - python3-setuptools + state: present + when: + - _pkg_excluded.rc == 0 + - ansible_python.version.major == 3 + become: yes + +- name: Install Python 2 pip + package: + name: + - python-pip + - python-setuptools + state: present + become: yes + when: (ensure_pip_from_packages_with_python2) or + (ansible_python.version.major == 2) diff --git a/roles/ensure-pip/tasks/Suse.yaml b/roles/ensure-pip/tasks/Suse.yaml new file mode 100644 index 000000000..7dacfe319 --- /dev/null +++ b/roles/ensure-pip/tasks/Suse.yaml @@ -0,0 +1,11 @@ +- name: Install Python 3 pip + package: + name: python3-pip + become: yes + +- name: Install Python 2 pip + package: + name: python2-pip + become: yes + when: + - ensure_pip_from_packages_with_python2 diff --git a/roles/ensure-pip/tasks/default.yaml b/roles/ensure-pip/tasks/default.yaml new file mode 100644 index 000000000..fd6bb8fbb --- /dev/null +++ b/roles/ensure-pip/tasks/default.yaml @@ -0,0 +1,3 @@ +- name: Unsupported platform + fail: + msg: 'This platform is currently unsupported' diff --git a/roles/ensure-pip/tasks/main.yaml b/roles/ensure-pip/tasks/main.yaml new file mode 100644 index 000000000..908c14185 --- /dev/null +++ b/roles/ensure-pip/tasks/main.yaml @@ -0,0 +1,13 @@ +- name: Install pip from packages + include: "{{ item }}" + with_first_found: + - "{{ ansible_distribution }}.yaml" + - "{{ ansible_os_family }}.yaml" + - "default.yaml" + when: + - ensure_pip_from_packages + +- name: Install pip from source + include: source.yaml + when: + - ensure_pip_from_upstream diff --git a/roles/ensure-pip/tasks/source.yaml b/roles/ensure-pip/tasks/source.yaml new file mode 100644 index 000000000..c35c706d8 --- /dev/null +++ b/roles/ensure-pip/tasks/source.yaml @@ -0,0 +1,20 @@ +- name: Temporary install directory + tempfile: + state: directory + suffix: ensure-pip + register: _install_dir + +- name: Download pip + get_url: + url: '{{ ensure_pip_from_upstream_url }}' + dest: '{{ _install_dir.path }}/get-pip.py' + +- name: 'Run get-pip.py for {{ item }}' + command: '{{ item }} {{ _install_dir.path }}/get-pip.py' + become: yes + loop: '{{ ensure_pip_from_upstream_interpreters }}' + +- name: Remove temporary install dir + file: + state: absent + path: '{{ _install_dir.path }}' diff --git a/test-playbooks/ensure-pip.yaml b/test-playbooks/ensure-pip.yaml new file mode 100644 index 000000000..6b83580e6 --- /dev/null +++ b/test-playbooks/ensure-pip.yaml @@ -0,0 +1,14 @@ +- hosts: all + roles: + - ensure-pip + +# NOTE(ianw) : this does not play nicely with pip-and-virtualenv which +# has already installed from source. We might be able to test this +# once it's gone... + +#- hosts: all +# roles: +# - role: ensure-pip +# vars: +# ensure_pip_from_upstream: True + diff --git a/zuul-tests.d/python-jobs.yaml b/zuul-tests.d/python-jobs.yaml index 2c8f21b83..5cf7ee7ae 100644 --- a/zuul-tests.d/python-jobs.yaml +++ b/zuul-tests.d/python-jobs.yaml @@ -1,3 +1,113 @@ +- job: + name: zuul-jobs-test-ensure-pip + description: Test the ensure-pip role + files: + - roles/ensure-pip/.* + run: test-playbooks/ensure-pip.yaml + tags: all-platforms + +- job: + name: zuul-jobs-test-ensure-pip-centos-7 + description: Test the ensure-pip role on centos-7 + parent: zuul-jobs-test-ensure-pip + tags: auto-generated + nodeset: + nodes: + - name: centos-7 + label: centos-7 + +- job: + name: zuul-jobs-test-ensure-pip-centos-8 + description: Test the ensure-pip role on centos-8 + parent: zuul-jobs-test-ensure-pip + tags: auto-generated + nodeset: + nodes: + - name: centos-8 + label: centos-8 + +- job: + name: zuul-jobs-test-ensure-pip-debian-stretch + description: Test the ensure-pip role on debian-stretch + parent: zuul-jobs-test-ensure-pip + tags: auto-generated + nodeset: + nodes: + - name: debian-stretch + label: debian-stretch + +- job: + name: zuul-jobs-test-ensure-pip-fedora-30 + description: Test the ensure-pip role on fedora-30 + parent: zuul-jobs-test-ensure-pip + tags: auto-generated + nodeset: + nodes: + - name: fedora-30 + label: fedora-30 + +- job: + name: zuul-jobs-test-ensure-pip-gentoo-17-0-systemd + description: Test the ensure-pip role on gentoo-17-0-systemd + parent: zuul-jobs-test-ensure-pip + tags: auto-generated + nodeset: + nodes: + - name: gentoo-17-0-systemd + label: gentoo-17-0-systemd + +- job: + name: zuul-jobs-test-ensure-pip-opensuse-15 + description: Test the ensure-pip role on opensuse-15 + parent: zuul-jobs-test-ensure-pip + tags: auto-generated + nodeset: + nodes: + - name: opensuse-15 + label: opensuse-15 + +- job: + name: zuul-jobs-test-ensure-pip-opensuse-tumbleweed-nv + voting: false + description: Test the ensure-pip role on opensuse-tumbleweed + parent: zuul-jobs-test-ensure-pip + tags: auto-generated + nodeset: + nodes: + - name: opensuse-tumbleweed + label: opensuse-tumbleweed + +- job: + name: zuul-jobs-test-ensure-pip-ubuntu-bionic + description: Test the ensure-pip role on ubuntu-bionic + parent: zuul-jobs-test-ensure-pip + tags: auto-generated + nodeset: + nodes: + - name: ubuntu-bionic + label: ubuntu-bionic + +- job: + name: zuul-jobs-test-ensure-pip-ubuntu-bionic-plain-nv + voting: false + description: Test the ensure-pip role on ubuntu-bionic-plain + parent: zuul-jobs-test-ensure-pip + tags: auto-generated + nodeset: + nodes: + - name: ubuntu-bionic-plain + label: ubuntu-bionic-plain + +- job: + name: zuul-jobs-test-ensure-pip-ubuntu-xenial + description: Test the ensure-pip role on ubuntu-xenial + parent: zuul-jobs-test-ensure-pip + tags: auto-generated + nodeset: + nodes: + - name: ubuntu-xenial + label: ubuntu-xenial + - job: name: zuul-jobs-test-ensure-sphinx description: Test the ensure-sphinx role @@ -176,6 +286,16 @@ - project: check: jobs: + - zuul-jobs-test-ensure-pip-centos-7 + - zuul-jobs-test-ensure-pip-centos-8 + - zuul-jobs-test-ensure-pip-debian-stretch + - zuul-jobs-test-ensure-pip-fedora-30 + - zuul-jobs-test-ensure-pip-gentoo-17-0-systemd + - zuul-jobs-test-ensure-pip-opensuse-15 + - zuul-jobs-test-ensure-pip-opensuse-tumbleweed-nv + - zuul-jobs-test-ensure-pip-ubuntu-bionic + - zuul-jobs-test-ensure-pip-ubuntu-bionic-plain-nv + - zuul-jobs-test-ensure-pip-ubuntu-xenial - zuul-jobs-test-ensure-sphinx - zuul-jobs-test-ensure-tox-centos-7 - zuul-jobs-test-ensure-tox-centos-8 @@ -194,6 +314,14 @@ - zuul-jobs-test-fetch-subunit-output-synchronize gate: jobs: + - zuul-jobs-test-ensure-pip-centos-7 + - zuul-jobs-test-ensure-pip-centos-8 + - zuul-jobs-test-ensure-pip-debian-stretch + - zuul-jobs-test-ensure-pip-fedora-30 + - zuul-jobs-test-ensure-pip-gentoo-17-0-systemd + - zuul-jobs-test-ensure-pip-opensuse-15 + - zuul-jobs-test-ensure-pip-ubuntu-bionic + - zuul-jobs-test-ensure-pip-ubuntu-xenial - zuul-jobs-test-ensure-sphinx - zuul-jobs-test-ensure-tox-centos-7 - zuul-jobs-test-ensure-tox-centos-8