From 6832d5dd41b74b3658e0be3c47538b1504ec794c Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 8 Apr 2015 13:32:36 -0500 Subject: [PATCH] Updated the repo scripts The playbook `playbooks/repo-clone-mirror.yml` was not cloning the repo from the upstream mirror correctly it was specifically not respecting symlinks and in some situations, if the user was using xattrs, hard links, or acls the clone operation would cause other issues. The `openstack-branch-grabber.py` was removed as the entire process of looping through all of the branches and tags and rebuilding all of the wheels is no longer relevant. As such this process was removed in favor of forcing the build process to specify a release. The `openstack-wheel-builder.py` script was updated to enforce the updating/cloning of the git sources. Closes-Bug: 1441812 Change-Id: Ibdac88607ffea57ab380f539f3f52346f15792ca --- playbooks/repo-build.yml | 2 +- playbooks/repo-clone-mirror.yml | 12 +- .../files/openstack-branch-grabber.py | 139 ------------------ .../files/openstack-wheel-builder.py | 14 +- .../files/openstack-wheel-builder.sh | 18 +-- .../repo_server/tasks/repo_pre_install.yml | 1 - 6 files changed, 24 insertions(+), 162 deletions(-) delete mode 100644 playbooks/roles/repo_server/files/openstack-branch-grabber.py diff --git a/playbooks/repo-build.yml b/playbooks/repo-build.yml index 6bfb41f5d1..6587aff682 100644 --- a/playbooks/repo-build.yml +++ b/playbooks/repo-build.yml @@ -79,7 +79,7 @@ yaprt --quiet \ create-html-indexes \ --repo-dir "{{ repo_service_home_folder }}/repo" \ - --dir-exclude "{{ repo_service_home_folder }}/repo/openstackgit" "{{ repo_service_home_folder }}/repo/rpcgit" + --dir-exclude "{{ repo_service_home_folder }}/repo/openstackgit" sudo: yes sudo_user: "{{ repo_service_user_name }}" tags: diff --git a/playbooks/repo-clone-mirror.yml b/playbooks/repo-clone-mirror.yml index ae88960896..d55182d878 100644 --- a/playbooks/repo-clone-mirror.yml +++ b/playbooks/repo-clone-mirror.yml @@ -25,14 +25,14 @@ tasks: - name: Sync the upstream repo(s) shell: | - rsync -rz \ - --exclude="repos/* mirror/* rpcgit/* openstackgit/*" \ - --links \ + rsync -avzlHAX \ + --exclude="{{ mirror_excludes }}" \ {{ mirror_source_host }}::{{ mirror_name }} {{ mirror_path }} sudo: yes sudo_user: "{{ repo_service_user_name }}" vars: - mirror_path: /var/www/repo/ - mirror_name: openstack_mirror - mirror_source_host: "rpc-repo.rackspace.com" + mirror_excludes: "{{ repo_mirror_excludes|default('repos/* mirror/* rpcgit/* openstackgit/*') }}" + mirror_path: "{{ repo_service_home_folder }}/repo" + mirror_name: "{{ repo_mirror_name|default('openstack_mirror') }}" + mirror_source_host: "{{ repo_mirror_source_host|default('rpc-repo.rackspace.com') }}" is_metal: "{{ properties.is_metal|default(false) }}" diff --git a/playbooks/roles/repo_server/files/openstack-branch-grabber.py b/playbooks/roles/repo_server/files/openstack-branch-grabber.py deleted file mode 100644 index 86d0777a70..0000000000 --- a/playbooks/roles/repo_server/files/openstack-branch-grabber.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Simple input script to return a list of branches in a github repo. - -This script will return a space seperated list of all of the branches available -from within a git repo as found in the github api. When running the script you -can provide a list of branches that you want to exclude from the returned list. -This exclusion list a matched based list and will exclude anything that matches -the list of strings. - -Example Usage: -~$ # Endpoint -~$ ENDPOINT="https://api.github.com/repos/stackforge/os-ansible-deployment" -~$ # Exclusions -~$ EXCLUDE_RELEASES="v9.0.0 gh-pages revert" -~$ # Run script -~$ /opt/openstack-branch-grabber.py "${ENDPOINT}" "${EXCLUDE_RELEASES}" - -Example Library Usage: ->>> endpoint_url = ( -... "https://api.github.com/repos/stackforge/os-ansible-deployment" -... ) ->>> exclude_list = ["v9.0.0", "gh-pages", "revert"] ->>> print(main(endpoint_url, exclude_list)) -9.0.0 9.0.1 9.0.2 9.0.3 stable/icehouse proposed/juno master -""" - - -import functools -import requests -import sys -import time - - -def retry(exception_check, tries=3, delay=1, backoff=1): - """Retry calling the decorated function using an exponential backoff. - - original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry - - :param exception_check: ``Exception || Tuple`` the exception to check. - may be a tuple of exceptions to check - :param tries: ``int`` number of times to try (not retry) before giving up - :param delay: ``int`` initial delay between retries in seconds - :param backoff: ``int`` backoff multiplier e.g. value of 2 will double the - delay each retry - """ - def deco_retry(f): - @functools.wraps(f) - def f_retry(*args, **kwargs): - mtries, mdelay = tries, delay - while mtries > 1: - try: - return f(*args, **kwargs) - except exception_check: - time.sleep(mdelay) - mtries -= 1 - mdelay *= backoff - return f(*args, **kwargs) - return f_retry # true decorator - return deco_retry - - -@retry(exception_check=Exception) -def get_url(url): - return requests.get(url) - - -@retry(exception_check=(AttributeError, ValueError)) -def return_releases(url, exclude_list=None): - """Return a list of releases found in the github api. - - :param url: ``str`` URL to hit public github api - :param exclude_list: ``str`` Branches to exclude - """ - _releases = get_url(url) - loaded_releases = _releases.json() - releases = list() - - if exclude_list is None: - exclude_list = list() - - for i in loaded_releases: - for k, v in i.iteritems(): - if k == 'name': - # if the name is not excluded append it - if not any([v.startswith(i) for i in exclude_list]): - releases.append(v) - else: - # Return a unique list. - return list(set(releases)) - - -def main(endpoint_url, exclude_list): - """Run the main application.""" - - # Create an array of all releases and branches. - all_releases = list() - all_releases.extend( - return_releases( - url="%s/tags" % endpoint_url, - exclude_list=exclude_list - ) - ) - all_releases.extend( - return_releases( - url="%s/branches" % endpoint_url, - exclude_list=exclude_list - ) - ) - - # Print all of the releases that were found within the github api. - print(' '.join(all_releases)) - - -if __name__ == '__main__': - # git api endpoint to use for searching for releases and branches - endpoint = sys.argv[1] - - # Create an array of excluded items - if len(sys.argv) >= 3: - exclude = sys.argv[2].split() - else: - exclude = list() - - main(endpoint_url=endpoint, exclude_list=exclude) diff --git a/playbooks/roles/repo_server/files/openstack-wheel-builder.py b/playbooks/roles/repo_server/files/openstack-wheel-builder.py index 2a73421685..11461977ab 100755 --- a/playbooks/roles/repo_server/files/openstack-wheel-builder.py +++ b/playbooks/roles/repo_server/files/openstack-wheel-builder.py @@ -24,9 +24,6 @@ from cloudlib import arguments from cloudlib import shell -VERSION_DESCRIPTORS = ['>=', '<=', '==', '!=', '<', '>'] - - REQUIREMENTS_FILE_TYPES = [ 'requirements.txt', 'global-requirements.txt', @@ -396,5 +393,16 @@ def main(): ] _run_command(index_command) + # Store the git repositories + index_command = [ + 'yaprt', + 'store-repos', + '--report-file', + _abs_path(user_vars['report_file']), + '--git-repo-path', + '/var/www/repo/openstackgit' + ] + _run_command(index_command) + if __name__ == '__main__': main() diff --git a/playbooks/roles/repo_server/files/openstack-wheel-builder.sh b/playbooks/roles/repo_server/files/openstack-wheel-builder.sh index 34f489abfa..41b6dd5c7b 100644 --- a/playbooks/roles/repo_server/files/openstack-wheel-builder.sh +++ b/playbooks/roles/repo_server/files/openstack-wheel-builder.sh @@ -110,6 +110,12 @@ function cleanup() { rm -rf "${WORK_DIR}" } +# Check for releases +if [ -z "${RELEASES}" ];then + echo "No releases specified. Provide a space separated list branches to build for." + exit 1 +fi + # Check for system lock file. if [ ! -f "${LOCKFILE}" ]; then echo $$ | tee "${LOCKFILE}" @@ -126,18 +132,6 @@ else fi fi -# Grab releases -if [[ ! "${RELEASES}" ]];then - # From the GITHUB API pull a list of all branches/tags - if [ -f "/opt/openstack-branch-grabber.py" ];then - RELEASES=$(/opt/openstack-branch-grabber.py "${GITHUB_API_ENDPOINT}" "${EXCLUDE_RELEASES}") - else - echo "No releases specified and the openstack-branch-grabber.py script was not found." - exit 1 - fi -fi - - # Iterate through the list of releases and build everything that's needed logger "Building Python Wheels for ${RELEASES}" for release in ${RELEASES}; do diff --git a/playbooks/roles/repo_server/tasks/repo_pre_install.yml b/playbooks/roles/repo_server/tasks/repo_pre_install.yml index f807d4f317..e31a803ae9 100644 --- a/playbooks/roles/repo_server/tasks/repo_pre_install.yml +++ b/playbooks/roles/repo_server/tasks/repo_pre_install.yml @@ -45,7 +45,6 @@ with_items: - { src: "openstack-wheel-builder.sh", dest: "/opt/openstack-wheel-builder.sh" } - { src: "openstack-wheel-builder.py", dest: "/opt/openstack-wheel-builder.py" } - - { src: "openstack-branch-grabber.py", dest: "/opt/openstack-branch-grabber.py" } tags: - repo-wheel-builder