From 792c753354381a030f11b949cd159c329c5cfdfe Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Mon, 23 Apr 2018 10:51:44 +0100 Subject: [PATCH] Ensure that repo checks use the right values When executing the check for the appropriate files on the repo host and then overriding the group_vars to cater for when the repo is not present, we need to make sure that we're doing the checks against the right URL. In this patch we backup the default URL so that it can be used when re-executing the check later in the same playbook. Also, given that the set-pip-upstream-url.yml and set-upper-constraints.yml task sets are always used together, we combine them into one task set. Depends-On: https://review.openstack.org/563578 Change-Id: I040c4f5b6e96c85c68beb975f7b80d1edf42276d --- .../common-tasks/set-pip-upstream-url.yml | 40 ------- playbooks/common-tasks/set-pip-vars.yml | 113 ++++++++++++++++++ .../common-tasks/set-upper-constraints.yml | 79 ------------ playbooks/containers-lxc-create.yml | 3 +- playbooks/containers-lxc-host.yml | 3 +- playbooks/containers-nspawn-create.yml | 3 +- playbooks/openstack-hosts-setup.yml | 3 +- playbooks/repo-build.yml | 3 +- playbooks/repo-server.yml | 3 +- 9 files changed, 119 insertions(+), 131 deletions(-) delete mode 100644 playbooks/common-tasks/set-pip-upstream-url.yml create mode 100644 playbooks/common-tasks/set-pip-vars.yml delete mode 100644 playbooks/common-tasks/set-upper-constraints.yml diff --git a/playbooks/common-tasks/set-pip-upstream-url.yml b/playbooks/common-tasks/set-pip-upstream-url.yml deleted file mode 100644 index dcb68d02e2..0000000000 --- a/playbooks/common-tasks/set-pip-upstream-url.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# Copyright 2016, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This set of tasks checks whether the repo is up yet. If it is it -# will use the get-pip.py script available there. If it is not yet -# up, then it will use the standard upstream get-pip.py script URL. - -- name: Test internal repo URL for the current get-pip.py script - uri: - url: "{{ pip_upstream_url }}" - method: "HEAD" - timeout: 3 - register: _repo_data_check - failed_when: false - when: pip_upstream_url != "https://bootstrap.pypa.io/get-pip.py" - tags: - - common-pip - -# If we can't reach the internal repo, we need to ensure -# pip isn't locked to it. -- name: Set pip upstream URL - set_fact: - pip_upstream_url: "https://bootstrap.pypa.io/get-pip.py" - when: - - pip_upstream_url != "https://bootstrap.pypa.io/get-pip.py" - - (_repo_data_check.status | default(503)) != 200 - tags: - - common-pip diff --git a/playbooks/common-tasks/set-pip-vars.yml b/playbooks/common-tasks/set-pip-vars.yml new file mode 100644 index 0000000000..ff0b3dcf1e --- /dev/null +++ b/playbooks/common-tasks/set-pip-vars.yml @@ -0,0 +1,113 @@ +--- +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This set of tasks checks whether the repo is up yet. If it is it +# will use it as the source for constraints and as the default index. +# If it is not yet up, then it will use the upstream upper constraints +# for the SHA currently pinned and will set the default index to pypi. + +- name: Backup the default pip_upstream_url + set_fact: + __pip_upstream_url: "{{ pip_upstream_url }}" + when: + - __pip_upstream_url is not defined + +- name: Backup the default pip_install_upper_constraints + set_fact: + __pip_install_upper_constraints: "{{ pip_install_upper_constraints }}" + when: + - __pip_install_upper_constraints is not defined + +- name: Backup the default pip_default_index + set_fact: + __pip_default_index: "{{ pip_default_index }}" + when: + - __pip_default_index is not defined + +- name: Test internal repo URL for the current get-pip.py script + uri: + url: "{{ __pip_upstream_url }}" + method: "HEAD" + timeout: 3 + register: _repo_data_check + failed_when: false + tags: + - common-pip + +- name: Test internal repo URL for the current upper constraints file + uri: + url: "{{ __pip_install_upper_constraints }}" + method: "HEAD" + timeout: 3 + register: upper_constraints_check + failed_when: false + tags: + - common-constraints + +- name: Test internal repo URL for the availability of the pypi index + uri: + url: "{{ __pip_default_index }}" + method: "HEAD" + timeout: 3 + register: pip_default_index_check + failed_when: false + tags: + - common-constraints + +- name: Remove global requirement pins file from host + file: + path: "/opt/global-requirement-pins.txt" + state: absent + when: (upper_constraints_check.status | default(503)) == 200 + tags: + - common-constraints + +- name: Copy global requirement pins file to host + copy: + src: "../global-requirement-pins.txt" + dest: "/opt/global-requirement-pins.txt" + when: (upper_constraints_check.status | default(503)) != 200 + tags: + - common-constraints + +# If we can't reach the internal repo, we need to ensure +# pip isn't locked to it. +- name: Set pip upstream URL + set_fact: + pip_upstream_url: >- + {{ ((_repo_data_check.status | default(503)) != 200) | ternary('https://bootstrap.pypa.io/get-pip.py', __pip_upstream_url) }} + tags: + - common-pip + +- name: Set pip install upper constraints + set_fact: + pip_install_upper_constraints: >- + {{ ((upper_constraints_check.status | default(503)) != 200) | ternary(__upstream_constraints, __pip_install_upper_constraints) }} + vars: + # Use https when Python with native SNI support is available + __pip_install_upper_constraints_proto: "{{ ansible_python_version | version_compare('2.7.9', '>=') | ternary('https','http') }}" + __upstream_constraints: >- + /opt/global-requirement-pins.txt + --constraint {{ __pip_install_upper_constraints_proto }}://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?id={{ requirements_git_install_branch | regex_replace(' #.*$','') }} + + tags: + - common-constraints + +- name: Fallback to PyPI + set_fact: + pip_default_index: >- + {{ ((pip_default_index_check.status | default(503)) != 200) | ternary('https://pypi.python.org/simple', __pip_default_index) }} + tags: + - common-constraints diff --git a/playbooks/common-tasks/set-upper-constraints.yml b/playbooks/common-tasks/set-upper-constraints.yml deleted file mode 100644 index d97e167efc..0000000000 --- a/playbooks/common-tasks/set-upper-constraints.yml +++ /dev/null @@ -1,79 +0,0 @@ ---- -# Copyright 2016, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This set of tasks checks whether the repo is up yet. If it is it -# will use the compiled upper constraints available there. If it -# is not yet up, then it will use the upstream upper constraints -# for the SHA currently pinned. - -- name: Test internal repo URL for the current upper constraints file - uri: - url: "{{ pip_install_upper_constraints }}" - method: "HEAD" - timeout: 3 - register: upper_constraints_check - failed_when: false - tags: - - common-constraints - -- name: Test internal repo URL for the availability of the pypi index - uri: - url: "{{ pip_default_index }}" - method: "HEAD" - timeout: 3 - register: pip_default_index_check - failed_when: false - tags: - - common-constraints - -- name: Remove global requirement pins file from host - file: - path: "/opt/global-requirement-pins.txt" - state: absent - when: (upper_constraints_check.status | default(503)) == 200 - tags: - - common-constraints - -- name: Copy global requirement pins file to host - copy: - src: "../global-requirement-pins.txt" - dest: "/opt/global-requirement-pins.txt" - when: (upper_constraints_check.status | default(503)) != 200 - tags: - - common-constraints - -# Use https when Python with native SNI support is available -- name: Set pip install upper constraints protocol - set_fact: - pip_install_upper_constraints_proto: "{{ ansible_python_version | version_compare('2.7.9', '>=') | ternary('https','http') }}" - when: (upper_constraints_check.status | default(503)) != 200 - tags: - - common-constraints - -- name: Set pip install upper constraints - set_fact: - pip_install_upper_constraints: >- - /opt/global-requirement-pins.txt - --constraint {{ pip_install_upper_constraints_proto }}://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?id={{ requirements_git_install_branch | regex_replace(' #.*$','') }} - when: (upper_constraints_check.status | default(503)) != 200 - tags: - - common-constraints - -- name: Fallback to PyPI - set_fact: - pip_default_index: "https://pypi.python.org/simple" - when: (pip_default_index_check.status | default(503)) != 200 - tags: - - common-constraints diff --git a/playbooks/containers-lxc-create.yml b/playbooks/containers-lxc-create.yml index 28c000068d..c9b0b5ca4a 100644 --- a/playbooks/containers-lxc-create.yml +++ b/playbooks/containers-lxc-create.yml @@ -54,8 +54,7 @@ user: root pre_tasks: - include: common-tasks/package-cache-proxy.yml - - include: common-tasks/set-upper-constraints.yml - - include: common-tasks/set-pip-upstream-url.yml + - include: common-tasks/set-pip-vars.yml roles: - role: "openstack_hosts" is_container: true diff --git a/playbooks/containers-lxc-host.yml b/playbooks/containers-lxc-host.yml index 06ebeda726..fae6cda22a 100644 --- a/playbooks/containers-lxc-host.yml +++ b/playbooks/containers-lxc-host.yml @@ -18,8 +18,7 @@ gather_facts: "{{ osa_gather_facts | default(True) }}" user: root pre_tasks: - - include: common-tasks/set-upper-constraints.yml - - include: common-tasks/set-pip-upstream-url.yml + - include: common-tasks/set-pip-vars.yml - name: Check the state of the default LXC service log directory stat: path: "/var/log/lxc" diff --git a/playbooks/containers-nspawn-create.yml b/playbooks/containers-nspawn-create.yml index 33af98e0d1..0e263dc231 100644 --- a/playbooks/containers-nspawn-create.yml +++ b/playbooks/containers-nspawn-create.yml @@ -47,8 +47,7 @@ user: root pre_tasks: - include: common-tasks/package-cache-proxy.yml - - include: common-tasks/set-upper-constraints.yml - - include: common-tasks/set-pip-upstream-url.yml + - include: common-tasks/set-pip-vars.yml roles: - role: "openstack_hosts" is_container: true diff --git a/playbooks/openstack-hosts-setup.yml b/playbooks/openstack-hosts-setup.yml index 4979b6352f..035ab0ada1 100644 --- a/playbooks/openstack-hosts-setup.yml +++ b/playbooks/openstack-hosts-setup.yml @@ -50,8 +50,7 @@ msg: "The only supported platforms for this release are Ubuntu 16.04 LTS (Xenial), CentOS 7 (WIP) and openSUSE Leap 42.X (WIP)" - include: common-tasks/package-cache-proxy.yml - - include: common-tasks/set-upper-constraints.yml - - include: common-tasks/set-pip-upstream-url.yml + - include: common-tasks/set-pip-vars.yml roles: - role: "openstack_hosts" vars_files: diff --git a/playbooks/repo-build.yml b/playbooks/repo-build.yml index c1020baeba..483721c239 100644 --- a/playbooks/repo-build.yml +++ b/playbooks/repo-build.yml @@ -50,8 +50,7 @@ serial: 1 pre_tasks: - - include: common-tasks/set-upper-constraints.yml - - include: common-tasks/set-pip-upstream-url.yml + - include: common-tasks/set-pip-vars.yml - name: Load local packages debug: diff --git a/playbooks/repo-server.yml b/playbooks/repo-server.yml index ddb70b310f..cbfb770cfe 100644 --- a/playbooks/repo-server.yml +++ b/playbooks/repo-server.yml @@ -19,8 +19,7 @@ user: root pre_tasks: - - include: common-tasks/set-upper-constraints.yml - - include: common-tasks/set-pip-upstream-url.yml + - include: common-tasks/set-pip-vars.yml - name: Check if the git cache exists on deployment host local_action: