Updated the repo scripts

The playbook `playbooks/repo-clone-mirror.yml` was not cloning the repo
from the upstream mirror correctly it was specifically not respecting
symlinks and in some situations, if the user was using xattrs, hard
links, or acls the clone operation would cause other issues.

The `openstack-branch-grabber.py` was removed as the entire process of
looping through all of the branches and tags and rebuilding all of the
wheels is no longer relevant. As such this process was removed in favor
of forcing the build process to specify a release.

The `openstack-wheel-builder.py` script was updated to enforce the
updating/cloning of the git sources.

Closes-Bug: 1441812

Change-Id: Ibdac88607ffea57ab380f539f3f52346f15792ca

playbooks/repo-build.yml

@@ -79,7 +79,7 @@
         yaprt --quiet \
               create-html-indexes \
               --repo-dir "{{ repo_service_home_folder }}/repo" \
-              --dir-exclude "{{ repo_service_home_folder }}/repo/openstackgit" "{{ repo_service_home_folder }}/repo/rpcgit"
+              --dir-exclude "{{ repo_service_home_folder }}/repo/openstackgit"
       sudo: yes
       sudo_user: "{{ repo_service_user_name }}"
       tags:

playbooks/repo-clone-mirror.yml

@@ -25,14 +25,14 @@
   tasks:
     - name: Sync the upstream repo(s)
       shell: |
-        rsync -rz \
+        rsync -avzlHAX \
-              --exclude="repos/* mirror/* rpcgit/* openstackgit/*" \
+              --exclude="{{ mirror_excludes }}" \
-              --links \
               {{ mirror_source_host }}::{{ mirror_name }} {{ mirror_path }}
       sudo: yes
       sudo_user: "{{ repo_service_user_name }}"
   vars:
-    mirror_path: /var/www/repo/
+    mirror_excludes: "{{ repo_mirror_excludes|default('repos/* mirror/* rpcgit/* openstackgit/*') }}"
-    mirror_name: openstack_mirror
+    mirror_path: "{{ repo_service_home_folder }}/repo"
-    mirror_source_host: "rpc-repo.rackspace.com"
+    mirror_name: "{{ repo_mirror_name|default('openstack_mirror') }}"
+    mirror_source_host: "{{ repo_mirror_source_host|default('rpc-repo.rackspace.com') }}"
     is_metal: "{{ properties.is_metal|default(false) }}"

playbooks/roles/repo_server/files/openstack-branch-grabber.py

@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2014, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-"""Simple input script to return a list of branches in a github repo.
-
-This script will return a space seperated list of all of the branches available
-from within a git repo as found in the github api. When running the script you
-can provide a list of branches that you want to exclude from the returned list.
-This exclusion list a matched based list and will exclude anything that matches
-the list of strings.
-
-Example Usage:
-~$ # Endpoint
-~$ ENDPOINT="https://api.github.com/repos/stackforge/os-ansible-deployment"
-~$ # Exclusions
-~$ EXCLUDE_RELEASES="v9.0.0 gh-pages revert"
-~$ # Run script
-~$ /opt/openstack-branch-grabber.py "${ENDPOINT}" "${EXCLUDE_RELEASES}"
-
-Example Library Usage:
->>> endpoint_url = (
-...     "https://api.github.com/repos/stackforge/os-ansible-deployment"
-... )
->>> exclude_list = ["v9.0.0", "gh-pages", "revert"]
->>> print(main(endpoint_url, exclude_list))
-9.0.0 9.0.1 9.0.2 9.0.3 stable/icehouse proposed/juno master
-"""
-
-
-import functools
-import requests
-import sys
-import time
-
-
-def retry(exception_check, tries=3, delay=1, backoff=1):
-    """Retry calling the decorated function using an exponential backoff.
-
-    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
-
-    :param exception_check: ``Exception || Tuple`` the exception to check.
-                             may be a tuple of exceptions to check
-    :param tries: ``int`` number of times to try (not retry) before giving up
-    :param delay: ``int`` initial delay between retries in seconds
-    :param backoff: ``int`` backoff multiplier e.g. value of 2 will double the
-                    delay each retry
-    """
-    def deco_retry(f):
-        @functools.wraps(f)
-        def f_retry(*args, **kwargs):
-            mtries, mdelay = tries, delay
-            while mtries > 1:
-                try:
-                    return f(*args, **kwargs)
-                except exception_check:
-                    time.sleep(mdelay)
-                    mtries -= 1
-                    mdelay *= backoff
-            return f(*args, **kwargs)
-        return f_retry  # true decorator
-    return deco_retry
-
-
-@retry(exception_check=Exception)
-def get_url(url):
-    return requests.get(url)
-
-
-@retry(exception_check=(AttributeError, ValueError))
-def return_releases(url, exclude_list=None):
-    """Return a list of releases found in the github api.
-
-    :param url: ``str`` URL to hit public github api
-    :param exclude_list: ``str`` Branches to exclude
-    """
-    _releases = get_url(url)
-    loaded_releases = _releases.json()
-    releases = list()
-
-    if exclude_list is None:
-        exclude_list = list()
-
-    for i in loaded_releases:
-        for k, v in i.iteritems():
-            if k == 'name':
-                # if the name is not excluded append it
-                if not any([v.startswith(i) for i in exclude_list]):
-                    releases.append(v)
-    else:
-        # Return a unique list.
-        return list(set(releases))
-
-
-def main(endpoint_url, exclude_list):
-    """Run the main application."""
-
-    # Create an array of all releases and branches.
-    all_releases = list()
-    all_releases.extend(
-        return_releases(
-            url="%s/tags" % endpoint_url,
-            exclude_list=exclude_list
-        )
-    )
-    all_releases.extend(
-        return_releases(
-            url="%s/branches" % endpoint_url,
-            exclude_list=exclude_list
-        )
-    )
-
-    # Print all of the releases that were found within the github api.
-    print(' '.join(all_releases))
-
-
-if __name__ == '__main__':
-    # git api endpoint to use for searching for releases and branches
-    endpoint = sys.argv[1]
-
-    # Create an array of excluded items
-    if len(sys.argv) >= 3:
-        exclude = sys.argv[2].split()
-    else:
-        exclude = list()
-
-    main(endpoint_url=endpoint, exclude_list=exclude)

playbooks/roles/repo_server/files/openstack-wheel-builder.py

@@ -24,9 +24,6 @@ from cloudlib import arguments
 from cloudlib import shell
 
 
-VERSION_DESCRIPTORS = ['>=', '<=', '==', '!=', '<', '>']
-
-
 REQUIREMENTS_FILE_TYPES = [
     'requirements.txt',
     'global-requirements.txt',
@@ -396,5 +393,16 @@ def main():
             ]
             _run_command(index_command)
 
+            # Store the git repositories
+            index_command = [
+                'yaprt',
+                'store-repos',
+                '--report-file',
+                _abs_path(user_vars['report_file']),
+                '--git-repo-path',
+                '/var/www/repo/openstackgit'
+            ]
+            _run_command(index_command)
+
 if __name__ == '__main__':
     main()

playbooks/roles/repo_server/files/openstack-wheel-builder.sh

@@ -110,6 +110,12 @@ function cleanup() {
     rm -rf "${WORK_DIR}"
 }
 
+# Check for releases
+if [ -z "${RELEASES}" ];then
+    echo "No releases specified. Provide a space separated list branches to build for."
+    exit 1
+fi
+
 # Check for system lock file.
 if [ ! -f "${LOCKFILE}" ]; then
     echo $$ | tee "${LOCKFILE}"
@@ -126,18 +132,6 @@ else
     fi
 fi
 
-# Grab releases
-if [[ ! "${RELEASES}" ]];then
-    # From the GITHUB API pull a list of all branches/tags
-    if [ -f "/opt/openstack-branch-grabber.py" ];then
-        RELEASES=$(/opt/openstack-branch-grabber.py "${GITHUB_API_ENDPOINT}" "${EXCLUDE_RELEASES}")
-    else
-        echo "No releases specified and the openstack-branch-grabber.py script was not found."
-        exit 1
-    fi
-fi
-
-
 # Iterate through the list of releases and build everything that's needed
 logger "Building Python Wheels for ${RELEASES}"
 for release in ${RELEASES}; do

playbooks/roles/repo_server/tasks/repo_pre_install.yml

@@ -45,7 +45,6 @@
   with_items:
     - { src: "openstack-wheel-builder.sh", dest: "/opt/openstack-wheel-builder.sh" }
     - { src: "openstack-wheel-builder.py", dest: "/opt/openstack-wheel-builder.py" }
-    - { src: "openstack-branch-grabber.py", dest: "/opt/openstack-branch-grabber.py" }
   tags:
     - repo-wheel-builder