diff --git a/playbooks/inventory/group_vars/hosts.yml b/playbooks/inventory/group_vars/hosts.yml index d21ebcd6ed..3772e0414b 100644 --- a/playbooks/inventory/group_vars/hosts.yml +++ b/playbooks/inventory/group_vars/hosts.yml @@ -71,6 +71,12 @@ pip_get_pip_options: > pip_links: - { name: "openstack_release", link: "{{ openstack_repo_url }}/os-releases/{{ openstack_release }}/" } +# These are pinned to ensure exactly the same behaviour forever! +# These pins are updated through the sources-branch-updater script +pip_packages: + - pip==8.0.3 + - setuptools==20.1.1 + - wheel==0.29.0 ## Memcached options memcached_listen: "{{ ansible_ssh_host }}" diff --git a/releasenotes/notes/pip-setuptools-wheel-pins-25494191c4739d52.yaml b/releasenotes/notes/pip-setuptools-wheel-pins-25494191c4739d52.yaml new file mode 100644 index 0000000000..71467e00a9 --- /dev/null +++ b/releasenotes/notes/pip-setuptools-wheel-pins-25494191c4739d52.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - The python packages `pip`, `setuptools` and `wheel` are now all pinned on + a per-tag basis. The pins are updated along with every OpenStack Service + update. This is done to ensure a consistent build experience with the + latest available packages at the time the tag is released. A deployer may + override the pins by adding a list of required pins using the + `pip_packages` variable in `user_variables.yml`. diff --git a/requirements.txt b/requirements.txt index ec57d2c7b9..0219c4d97c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,13 @@ Jinja2>=2.6 # ansible netaddr>=0.7.12 # playbooks/inventory/dynamic_inventory.py paramiko>=1.13.0 # ansible -pip>7,<8 # ansible PrettyTable>=0.7,<0.8 # scripts/inventory-manage.py pycrypto>=2.6 # ansible PyYAML>=3.1.0 # ansible +### +### These are pinned to ensure exactly the same behaviour forever! ### +### These pins are updated through the sources-branch-updater script ### +### +pip==8.0.3 +setuptools==20.1.1 +wheel==0.29.0 diff --git a/scripts/get-pypi-pkg-version.py b/scripts/get-pypi-pkg-version.py new file mode 100755 index 0000000000..3c341d73c2 --- /dev/null +++ b/scripts/get-pypi-pkg-version.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2016, Jesse Pretorius +# + + +"""Returns the versions of a list of pypi packages you specify.""" + + +from __future__ import print_function + +import argparse +import xmlrpclib + + +def get_package_version(pypiConn, pkg_name): + """Get the current package version from PyPI.""" + pkg_result = pypiConn.package_releases(pkg_name) + if pkg_result: + pkg_version = pkg_result[0] + else: + pkg_version = 'Not available.' + + return pkg_version + + +def main(): + """Run the main application.""" + + # Setup argument parsing + parser = argparse.ArgumentParser( + description='PyPI Current Package Version Checker', + epilog='Licensed "Apache 2.0"') + + parser.add_argument( + '-f', + '--format', + choices=['requirements', 'bare'], + default='requirements', + help=' Output format', + required=False + ) + + parser.add_argument( + '-l', + '--layout', + choices=['vertical', 'horizontal'], + default='vertical', + help=' Output layout', + required=False + ) + + parser.add_argument( + '-p', + '--packages', + nargs='+', + help=' Space-delimited list of packages', + required=True + ) + + # Parse arguments + args = parser.parse_args() + + # Setup pypi object + pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') + + # Setup the newline if the results layout should be vertical + # Also add a space delimiter appropriately + if args.layout == 'vertical': + delimiter = '' + endline = '\n' + else: + delimiter = ' ' + endline = '' + + # Print the results to stdout + for pkg_name in args.packages: + pkg_version = get_package_version(pypi, pkg_name) + if args.format == 'requirements': + print(pkg_name + '==' + pkg_version + delimiter, end=endline) + else: + print(pkg_version + delimiter, end=endline) + + +if __name__ == "__main__": + main() diff --git a/scripts/scripts-library.sh b/scripts/scripts-library.sh index d86add6258..7c9cae09fc 100755 --- a/scripts/scripts-library.sh +++ b/scripts/scripts-library.sh @@ -21,7 +21,7 @@ MAX_RETRIES=${MAX_RETRIES:-5} REPORT_DATA=${REPORT_DATA:-""} ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:-""} STARTTIME="${STARTTIME:-$(date +%s)}" -PIP_MAJOR_VERSION=${PIP_MAJOR_VERSION:-"7"} +PIP_INSTALL_OPTIONS=${PIP_INSTALL_OPTIONS:-'pip==8.0.3 setuptools==20.1.1 wheel==0.29.0 '} # The default SSHD configuration has MaxSessions = 10. If a deployer changes # their SSHD config, then the FORKS may be set to a higher number. We set the @@ -216,10 +216,8 @@ function get_pip { # check if pip is already installed if [ "$(which pip)" ]; then - # if the version installed is the wrong version, fix it - if [ "$(pip --version | awk '{print $2}' | cut -d. -f1)" != ${PIP_MAJOR_VERSION} ]; then - pip install -I "pip>=${PIP_MAJOR_VERSION},<$((PIP_MAJOR_VERSION+1))" - fi + # make sure that the right pip base packages are installed + pip install --upgrade ${PIP_INSTALL_OPTIONS} # when pip is not installed, install it else @@ -228,7 +226,7 @@ function get_pip { if [ -n "${GET_PIP_URL:-}" ]; then curl --silent ${GET_PIP_URL} > /opt/get-pip.py if head -n 1 /opt/get-pip.py | grep python; then - python /opt/get-pip.py "pip>=${PIP_MAJOR_VERSION},<$((PIP_MAJOR_VERSION+1))" + python /opt/get-pip.py "${PIP_INSTALL_OPTIONS}" return fi fi @@ -236,14 +234,14 @@ function get_pip { # Try getting pip from bootstrap.pypa.io as a primary source curl --silent https://bootstrap.pypa.io/get-pip.py > /opt/get-pip.py if head -n 1 /opt/get-pip.py | grep python; then - python /opt/get-pip.py "pip>=${PIP_MAJOR_VERSION},<$((PIP_MAJOR_VERSION+1))" + python /opt/get-pip.py "${PIP_INSTALL_OPTIONS}" return fi # Try the get-pip.py from the github repository as a primary source curl --silent https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py > /opt/get-pip.py if head -n 1 /opt/get-pip.py | grep python; then - python /opt/get-pip.py "pip>=${PIP_MAJOR_VERSION},<$((PIP_MAJOR_VERSION+1))" + python /opt/get-pip.py "${PIP_INSTALL_OPTIONS}" return fi diff --git a/scripts/sources-branch-updater.sh b/scripts/sources-branch-updater.sh index 9d4ff5533f..ee1835aec7 100755 --- a/scripts/sources-branch-updater.sh +++ b/scripts/sources-branch-updater.sh @@ -134,3 +134,14 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do echo -e "Processed $repo_name @ $branch_entry\n" done + +# Finally, update the PIP_INSTALL_OPTIONS with the current versions of pip, wheel and setuptools +PIP_CURRENT_OPTIONS=$(./scripts/get-pypi-pkg-version.py -p pip setuptools wheel -l horizontal) +sed -i.bak "s|^PIP_INSTALL_OPTIONS=.*|PIP_INSTALL_OPTIONS=\$\{PIP_INSTALL_OPTIONS:-'${PIP_CURRENT_OPTIONS}'\}|" scripts/scripts-library.sh + +for pin in ${PIP_CURRENT_OPTIONS}; do + sed -i.bak "s|^$(echo ${pin} | cut -f1 -d=).*|${pin}|" *requirements.txt + sed -i.bak "s|^ - $(echo ${pin} | cut -f1 -d=).*| - ${pin}|" playbooks/inventory/group_vars/hosts.yml +done + +echo "Updated pip install options/pins"