Update source-branch-updater to work with IRR's

This updated the sources-branch-updater convenience script which
is used to update SHA's and ensure that all carried files are
appropriately included in the update.

The patch includes the following changes:

 - Automatically submit updates of applicable files to all the
   independent repositories.

 - Automatically update the ansible-role-requirements.yml file
   with the latest SHA from the independent repositories or the
   latest tag from any repositories that are not part of OSA.
   This functionality is disabled by design for the master branch.

 - Adds ansible-role-requirements-editer.py which provides the
   ability to edit the ansible-role-requirements.yml file
   appropriately and output a consistent result.

Change-Id: I67fadeb43cc65a557d0380a12eb3e94335097d0d
This commit is contained in:
Jesse Pretorius 2016-04-13 17:23:04 +01:00
parent 1d66d1963c
commit fac5030571
3 changed files with 256 additions and 96 deletions

View File

@ -1,118 +1,118 @@
- name: apt_package_pinning
src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning
version: master
- name: pip_install
src: https://git.openstack.org/openstack/openstack-ansible-pip_install
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-pip_install
version: master
- name: pip_lock_down
src: https://git.openstack.org/openstack/openstack-ansible-pip_lock_down
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-pip_lock_down
version: master
- name: galera_client
src: https://git.openstack.org/openstack/openstack-ansible-galera_client
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-galera_client
version: master
- name: galera_server
src: https://git.openstack.org/openstack/openstack-ansible-galera_server
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-galera_server
version: master
- name: keepalived
src: https://github.com/evrardjp/ansible-keepalived
version: master
- name: lxc_container_create
src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create
version: master
- name: lxc_hosts
src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts
version: master
- name: memcached_server
src: https://git.openstack.org/openstack/openstack-ansible-memcached_server
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-memcached_server
version: master
- name: openstack-ansible-security
src: https://git.openstack.org/openstack/openstack-ansible-security
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-security
version: master
- name: openstack_hosts
src: https://git.openstack.org/openstack/openstack-ansible-openstack_hosts
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-openstack_hosts
version: master
- name: os_keystone
src: https://git.openstack.org/openstack/openstack-ansible-os_keystone
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_keystone
version: master
- name: openstack_openrc
src: https://git.openstack.org/openstack/openstack-ansible-openstack_openrc
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-openstack_openrc
version: master
- name: os_aodh
src: https://git.openstack.org/openstack/openstack-ansible-os_aodh
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_aodh
version: master
- name: os_ceilometer
src: https://git.openstack.org/openstack/openstack-ansible-os_ceilometer
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_ceilometer
version: master
- name: os_cinder
src: https://git.openstack.org/openstack/openstack-ansible-os_cinder
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_cinder
version: master
- name: os_glance
src: https://git.openstack.org/openstack/openstack-ansible-os_glance
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_glance
version: master
- name: os_heat
src: https://git.openstack.org/openstack/openstack-ansible-os_heat
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_heat
version: master
- name: os_horizon
src: https://git.openstack.org/openstack/openstack-ansible-os_horizon
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_horizon
version: master
- name: os_neutron
src: https://git.openstack.org/openstack/openstack-ansible-os_neutron
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_neutron
version: master
- name: os_nova
src: https://git.openstack.org/openstack/openstack-ansible-os_nova
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_nova
version: master
- name: os_swift
src: https://git.openstack.org/openstack/openstack-ansible-os_swift
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_swift
version: master
- name: os_tempest
src: https://git.openstack.org/openstack/openstack-ansible-os_tempest
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_tempest
version: master
- name: plugins
src: https://git.openstack.org/openstack/openstack-ansible-plugins
path: /etc/ansible
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-plugins
version: master
- name: rabbitmq_server
src: https://git.openstack.org/openstack/openstack-ansible-rabbitmq_server
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-rabbitmq_server
version: master
- name: repo_build
src: https://git.openstack.org/openstack/openstack-ansible-repo_build
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-repo_build
version: master
- name: repo_server
src: https://git.openstack.org/openstack/openstack-ansible-repo_server
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-repo_server
version: master
- name: rsyslog_client
src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_client
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_client
version: master
- name: rsyslog_server
src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_server
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_server
version: master
- name: sshd
src: https://github.com/willshersystems/ansible-sshd

View File

@ -0,0 +1,104 @@
#!/usr/bin/env python
#
# Copyright 2016, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# (c) 2016, Jesse Pretorius <jesse.pretorius@rackspace.co.uk>
#
"""Read/write ansible-role-requirements.yml content from the CLI."""
from __future__ import print_function
import argparse
import yaml
# To ensure that the dicts are always output in the same order
# we setup a representation for dict objects and register it
# with the yaml class.
def represent_dict(self, data):
def key_function((key, value)):
# Prioritizes certain keys when sorting.
prio = {"model": 0, "pk": 1, "fields": 2}.get(key, 99)
return (prio, key)
items = data.items()
items.sort(key=key_function)
return self.represent_mapping(u'tag:yaml.org,2002:map', items)
yaml.add_representer(dict, represent_dict)
def main():
"""Run the main application."""
# Setup argument parsing
parser = argparse.ArgumentParser(
description='ansible-role-requirements.yml CLI editer',
epilog='Licensed "Apache 2.0"')
parser.add_argument(
'-f',
'--file',
help='<Required> ansible-role-requirements.yml file location',
required=True
)
parser.add_argument(
'-n',
'--name',
help='<Required> The name of the Ansible role to edit',
required=True
)
parser.add_argument(
'-v',
'--version',
help='<Required> The version to set for the Ansible role',
required=True
)
parser.add_argument(
'-s',
'--src',
help='<Optional> The source URL to set for the Ansible role',
required=False
)
# Parse arguments
args = parser.parse_args()
# Read the ansible-role-requirements.yml file into memory
with open(args.file, "r+") as role_req_file:
reqs = yaml.safe_load(role_req_file)
# Loop through the list to find the applicable role
for role_data in reqs:
if role_data['name'] == args.name:
# Change the specified role data
role_data['version'] = args.version
if args.src:
role_data['src'] = args.src
# Write out the resulting file
with open(args.file, "r+") as role_req_file:
try:
yaml.dump(reqs, role_req_file, default_flow_style=False)
except yaml.YAMLError as exc:
print(exc)
if __name__ == "__main__":
main()

View File

@ -18,22 +18,19 @@
# the head of given branch via the SHA. This makes it possible to update
# all of the services that we support in an "automated" fashion.
ONLINE_BRANCH=${ONLINE_BRANCH:-"master"}
OS_BRANCH=${OS_BRANCH:-"master"}
OSA_BRANCH=${OSA_BRANCH:-"$OS_BRANCH"}
SERVICE_FILE=${SERVICE_FILE:-"playbooks/defaults/repo_packages/openstack_services.yml"}
OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-"aodh ceilometer cinder glance heat keystone neutron nova"}
OSA_ROLE_PATH="${OSA_ROLE_PATH:-playbooks/roles}"
OSA_ROLE_PREFIX="${OSA_ROLE_PREFIX:-os}"
OSA_ROLE_LOCAL="${OSA_ROLE_PATH}/${OSA_ROLE_PREFIX}"
IFS=$'\n'
if echo "$@" | grep -e '-h' -e '--help';then
echo "
Options:
-b|--branch (name of branch, eg: stable/liberty)
-s|--service-file (path to service yaml file to parse)
-r|--osa-role-path (path to stored roles to update, eg: 'playbooks/roles')
-p|--osa-role-prefix (the prefix in front of OpenStack service roles, eg: 'os')
-b|--openstack-branch (name of OpenStack branch, eg: stable/mitaka)
-o|--osa-branch (name of the OSA branch, eg: stable/mitaka)
-s|--service-file (path to service file to parse)
"
exit 0
fi
@ -42,22 +39,18 @@ fi
while [[ $# > 1 ]]; do
key="$1"
case $key in
-b|--branch)
ONLINE_BRANCH="$2"
-b|--openstack-branch)
OS_BRANCH="$2"
shift
;;
-o|--osa-branch)
OSA_BRANCH="$2"
shift
;;
-s|--service-file)
SERVICE_FILE="$2"
shift
;;
-r|--osa-role-path)
OSA_ROLE_PATH="$2"
shift
;;
-p|--osa-role-prefix)
OSA_ROLE_PREFIX="$2"
shift
;;
*)
;;
esac
@ -76,7 +69,7 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
repo_address=$(echo ${repo} | awk '{print $2}')
# Get the branch data
branch_data=$(git ls-remote ${repo_address} | grep "${ONLINE_BRANCH}$")
branch_data=$(git ls-remote ${repo_address} | grep "${OS_BRANCH}$")
# If there is branch data continue
if [ ! -z "${branch_data}" ];then
@ -85,64 +78,100 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
branch_sha=$(echo "${branch_data}" | awk '{print $1}')
# Set the branch entry
branch_entry="${branch_sha} # HEAD of \"$ONLINE_BRANCH\" as of $(date +%d.%m.%Y)"
branch_entry="${branch_sha} # HEAD of \"$OS_BRANCH\" as of $(date +%d.%m.%Y)"
# Write the branch entry into the repo_packages file
sed -i.bak "s|${repo_name}_git_install_branch:.*|${repo_name}_git_install_branch: $branch_entry|" ${SERVICE_FILE}
# If the repo is in the specified list, then action the additional updates
if [[ "${OPENSTACK_SERVICE_LIST}" =~ "${repo_name}" ]]; then
if [[ -d "${OSA_ROLE_LOCAL}_${repo_name}" ]]; then
repo_tmp_path="/tmp/${repo_name}"
os_repo_tmp_path="/tmp/os_${repo_name}"
osa_repo_tmp_path="/tmp/osa_${repo_name}"
# Ensure that the temp path doesn't exist
[[ -d "repo_tmp_path" ]] && rm -rf "${repo_tmp_path}"
rm -rf ${os_repo_tmp_path} ${osa_repo_tmp_path}
# Do a shallow clone of the repo to work with
git clone --quiet --depth=5 --branch ${ONLINE_BRANCH} --no-checkout --single-branch ${repo_address} "${repo_tmp_path}"
pushd ${repo_tmp_path} > /dev/null
git checkout --quiet "${branch_sha}"
# Do a shallow clone of the OpenStack repo to work with
git clone --quiet --depth=10 --branch ${OS_BRANCH} --no-checkout --single-branch ${repo_address} ${os_repo_tmp_path}
pushd ${os_repo_tmp_path} > /dev/null
git checkout --quiet ${branch_sha}
popd > /dev/null
# Set the OSA address
if [[ "${repo_name}" == "ironic" ]]; then
osa_repo_address="https://git.openstack.org/openstack/openstack-ansible-ironic"
else
osa_repo_address="https://git.openstack.org/openstack/openstack-ansible-os_${repo_name}"
fi
# Do a shallow clone of the OSA repo to work with
git clone --quiet --depth=10 --branch ${OSA_BRANCH} --single-branch ${osa_repo_address} ${osa_repo_tmp_path}
pushd ${osa_repo_tmp_path} > /dev/null
git checkout --quiet origin/${OSA_BRANCH}
popd > /dev/null
# Update the policy files
find ${repo_tmp_path}/etc -name "policy.json" -exec \
cp {} "${OSA_ROLE_LOCAL}_${repo_name}/templates/policy.json.j2" \;
find ${os_repo_tmp_path}/etc -name "policy.json" -exec \
cp {} "${osa_repo_tmp_path}/templates/policy.json.j2" \;
# Tweak the paste files
find ${repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \;
# Update the paste files
find ${repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(basename \${name}).j2\"" \;
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \;
# Tweak the rootwrap conf files
find ${repo_tmp_path}/etc -name "rootwrap.conf" -exec \
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \;
# Update the rootwrap conf files
find ${repo_tmp_path}/etc -name "rootwrap.conf" -exec \
cp {} "${OSA_ROLE_LOCAL}_${repo_name}/templates/rootwrap.conf.j2" \;
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
cp {} "${osa_repo_tmp_path}/templates/rootwrap.conf.j2" \;
# Update the rootwrap filters
find ${repo_tmp_path}/etc -name "*.filters" -exec \
bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/files/rootwrap.d/\$(basename \${name})\"" \;
find ${os_repo_tmp_path}/etc -name "*.filters" -exec \
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/files/rootwrap.d/\$(basename \${name})\"" \;
# Update the yaml files for Ceilometer
if [ "${repo_name}" = "ceilometer" ]; then
find ${repo_tmp_path}/etc -name "*.yaml" -exec \
bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(basename \${name}).j2\"" \;
find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \;
fi
# Update the yaml files for Heat
if [ "${repo_name}" = "heat" ]; then
find ${repo_tmp_path}/etc -name "*.yaml" -exec \
bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
fi
# Clean up the temporary files
[[ -d "repo_tmp_path" ]] && rm -rf "${repo_tmp_path}"
# Switch into the OSA git directory to work with it
pushd ${osa_repo_tmp_path} > /dev/null
# Check for changed files
git_changed=$(git status --porcelain | wc -l)
# Check for untracked files
git_untracked=$(git ls-files --other --exclude-standard --directory | wc -l)
if [ ${git_untracked} -gt 0 ]; then
# If there are untracked files, ensure that the commit message includes
# a WIP prefix so that the patch is revised in more detail.
git_msg_prefix="[New files - needs update] "
else
git_msg_prefix=""
fi
# If any files have changed, submit a patch including the changes
if [ ${git_changed} -gt 0 ]; then
git review -s > /dev/null
git add --all
git commit -a -m "${git_msg_prefix}Update paste, policy and rootwrap configurations $(date +%Y-%m-%d)" --quiet
git review > /dev/null
fi
popd > /dev/null
# Clean up the temporary files
rm -rf ${os_repo_tmp_path} ${osa_repo_tmp_path}
fi
fi
@ -152,7 +181,7 @@ done
unset IFS
# Finally, update the PIP_INSTALL_OPTIONS with the current versions of pip, wheel and setuptools
# Update the PIP_INSTALL_OPTIONS with the current versions of pip, wheel and setuptools
PIP_CURRENT_OPTIONS=$(./scripts/get-pypi-pkg-version.py -p pip setuptools wheel -l horizontal)
sed -i.bak "s|^PIP_INSTALL_OPTIONS=.*|PIP_INSTALL_OPTIONS=\$\{PIP_INSTALL_OPTIONS:-'${PIP_CURRENT_OPTIONS}'\}|" scripts/scripts-library.sh
@ -162,3 +191,30 @@ for pin in ${PIP_CURRENT_OPTIONS}; do
done
echo "Updated pip install options/pins"
# Update the ansible-role-requirements.yml file
# We don't want to be doing this for the master branch
if [[ "${OSA_BRANCH}" != "master" ]]; then
echo "Updating ansible-role-requirements.yml"
# Loop through each of the role git sources
for role_src in $(awk '/src: / {print $2}' ansible-role-requirements.yml); do
# Determine the role's name
role_name=$(sed 's/^[ \t-]*//' ansible-role-requirements.yml | awk '/src: / || /name: / {print $2}' | grep -B1 "${role_src}" | head -n 1)
# Grab the latest SHA that matches the specified branch
role_version=$(git ls-remote ${role_src} | grep "${OSA_BRANCH}$" | awk '{print $1}')
# If that branch doesn't exist, then it's probably not an OpenStack-Ansible role, so grab the latest tag instead
if [[ -z "${role_version}" ]]; then
role_version=$(git ls-remote --tags ${role_src} | awk '{print $2}' | grep -v '{}' | cut -d/ -f 3 | sort -n | tail -n 1)
fi
# Now use the information we have to update the ansible-role-requirements file
$(dirname ${0})/ansible-role-requirements-editer.py -f ansible-role-requirements.yml -n "${role_name}" -v "${role_version}"
done
echo "Completed updating ansible-role-requirements.yml"
else
echo "Skipping the ansible-role-requirements.yml update as we're working on the master branch"
fi