Merge "Update source-branch-updater to work with IRR's"

This commit is contained in:
Jenkins 2016-04-14 16:55:35 +00:00 committed by Gerrit Code Review
commit 8bbe826759
3 changed files with 256 additions and 96 deletions

View File

@ -1,118 +1,118 @@
- name: apt_package_pinning
src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning
version: master
- name: pip_install
src: https://git.openstack.org/openstack/openstack-ansible-pip_install
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-pip_install
version: master
- name: pip_lock_down
src: https://git.openstack.org/openstack/openstack-ansible-pip_lock_down
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-pip_lock_down
version: master
- name: galera_client
src: https://git.openstack.org/openstack/openstack-ansible-galera_client
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-galera_client
version: master
- name: galera_server
src: https://git.openstack.org/openstack/openstack-ansible-galera_server
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-galera_server
version: master
- name: keepalived
src: https://github.com/evrardjp/ansible-keepalived
version: master
- name: lxc_container_create
src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create
version: master
- name: lxc_hosts
src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts
version: master
- name: memcached_server
src: https://git.openstack.org/openstack/openstack-ansible-memcached_server
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-memcached_server
version: master
- name: openstack-ansible-security
src: https://git.openstack.org/openstack/openstack-ansible-security
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-security
version: master
- name: openstack_hosts
src: https://git.openstack.org/openstack/openstack-ansible-openstack_hosts
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-openstack_hosts
version: master
- name: os_keystone
src: https://git.openstack.org/openstack/openstack-ansible-os_keystone
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_keystone
version: master
- name: openstack_openrc
src: https://git.openstack.org/openstack/openstack-ansible-openstack_openrc
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-openstack_openrc
version: master
- name: os_aodh
src: https://git.openstack.org/openstack/openstack-ansible-os_aodh
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_aodh
version: master
- name: os_ceilometer
src: https://git.openstack.org/openstack/openstack-ansible-os_ceilometer
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_ceilometer
version: master
- name: os_cinder
src: https://git.openstack.org/openstack/openstack-ansible-os_cinder
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_cinder
version: master
- name: os_glance
src: https://git.openstack.org/openstack/openstack-ansible-os_glance
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_glance
version: master
- name: os_heat
src: https://git.openstack.org/openstack/openstack-ansible-os_heat
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_heat
version: master
- name: os_horizon
src: https://git.openstack.org/openstack/openstack-ansible-os_horizon
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_horizon
version: master
- name: os_neutron
src: https://git.openstack.org/openstack/openstack-ansible-os_neutron
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_neutron
version: master
- name: os_nova
src: https://git.openstack.org/openstack/openstack-ansible-os_nova
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_nova
version: master
- name: os_swift
src: https://git.openstack.org/openstack/openstack-ansible-os_swift
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_swift
version: master
- name: os_tempest
src: https://git.openstack.org/openstack/openstack-ansible-os_tempest
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-os_tempest
version: master
- name: plugins
src: https://git.openstack.org/openstack/openstack-ansible-plugins
path: /etc/ansible
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-plugins
version: master
- name: rabbitmq_server
src: https://git.openstack.org/openstack/openstack-ansible-rabbitmq_server
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-rabbitmq_server
version: master
- name: repo_build
src: https://git.openstack.org/openstack/openstack-ansible-repo_build
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-repo_build
version: master
- name: repo_server
src: https://git.openstack.org/openstack/openstack-ansible-repo_server
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-repo_server
version: master
- name: rsyslog_client
src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_client
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_client
version: master
- name: rsyslog_server
src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_server
scm: git
src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_server
version: master
- name: sshd
src: https://github.com/willshersystems/ansible-sshd

View File

@ -0,0 +1,104 @@
#!/usr/bin/env python
#
# Copyright 2016, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# (c) 2016, Jesse Pretorius <jesse.pretorius@rackspace.co.uk>
#
"""Read/write ansible-role-requirements.yml content from the CLI."""
from __future__ import print_function
import argparse
import yaml
# To ensure that the dicts are always output in the same order
# we setup a representation for dict objects and register it
# with the yaml class.
def represent_dict(self, data):
def key_function((key, value)):
# Prioritizes certain keys when sorting.
prio = {"model": 0, "pk": 1, "fields": 2}.get(key, 99)
return (prio, key)
items = data.items()
items.sort(key=key_function)
return self.represent_mapping(u'tag:yaml.org,2002:map', items)
yaml.add_representer(dict, represent_dict)
def main():
"""Run the main application."""
# Setup argument parsing
parser = argparse.ArgumentParser(
description='ansible-role-requirements.yml CLI editer',
epilog='Licensed "Apache 2.0"')
parser.add_argument(
'-f',
'--file',
help='<Required> ansible-role-requirements.yml file location',
required=True
)
parser.add_argument(
'-n',
'--name',
help='<Required> The name of the Ansible role to edit',
required=True
)
parser.add_argument(
'-v',
'--version',
help='<Required> The version to set for the Ansible role',
required=True
)
parser.add_argument(
'-s',
'--src',
help='<Optional> The source URL to set for the Ansible role',
required=False
)
# Parse arguments
args = parser.parse_args()
# Read the ansible-role-requirements.yml file into memory
with open(args.file, "r+") as role_req_file:
reqs = yaml.safe_load(role_req_file)
# Loop through the list to find the applicable role
for role_data in reqs:
if role_data['name'] == args.name:
# Change the specified role data
role_data['version'] = args.version
if args.src:
role_data['src'] = args.src
# Write out the resulting file
with open(args.file, "r+") as role_req_file:
try:
yaml.dump(reqs, role_req_file, default_flow_style=False)
except yaml.YAMLError as exc:
print(exc)
if __name__ == "__main__":
main()

View File

@ -18,22 +18,19 @@
# the head of given branch via the SHA. This makes it possible to update
# all of the services that we support in an "automated" fashion.
ONLINE_BRANCH=${ONLINE_BRANCH:-"master"}
OS_BRANCH=${OS_BRANCH:-"master"}
OSA_BRANCH=${OSA_BRANCH:-"$OS_BRANCH"}
SERVICE_FILE=${SERVICE_FILE:-"playbooks/defaults/repo_packages/openstack_services.yml"}
OPENSTACK_SERVICE_LIST=${OPENSTACK_SERVICE_LIST:-"aodh ceilometer cinder glance heat keystone neutron nova"}
OSA_ROLE_PATH="${OSA_ROLE_PATH:-playbooks/roles}"
OSA_ROLE_PREFIX="${OSA_ROLE_PREFIX:-os}"
OSA_ROLE_LOCAL="${OSA_ROLE_PATH}/${OSA_ROLE_PREFIX}"
IFS=$'\n'
if echo "$@" | grep -e '-h' -e '--help';then
echo "
Options:
-b|--branch (name of branch, eg: stable/liberty)
-s|--service-file (path to service yaml file to parse)
-r|--osa-role-path (path to stored roles to update, eg: 'playbooks/roles')
-p|--osa-role-prefix (the prefix in front of OpenStack service roles, eg: 'os')
-b|--openstack-branch (name of OpenStack branch, eg: stable/mitaka)
-o|--osa-branch (name of the OSA branch, eg: stable/mitaka)
-s|--service-file (path to service file to parse)
"
exit 0
fi
@ -42,22 +39,18 @@ fi
while [[ $# > 1 ]]; do
key="$1"
case $key in
-b|--branch)
ONLINE_BRANCH="$2"
-b|--openstack-branch)
OS_BRANCH="$2"
shift
;;
-o|--osa-branch)
OSA_BRANCH="$2"
shift
;;
-s|--service-file)
SERVICE_FILE="$2"
shift
;;
-r|--osa-role-path)
OSA_ROLE_PATH="$2"
shift
;;
-p|--osa-role-prefix)
OSA_ROLE_PREFIX="$2"
shift
;;
*)
;;
esac
@ -76,7 +69,7 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
repo_address=$(echo ${repo} | awk '{print $2}')
# Get the branch data
branch_data=$(git ls-remote ${repo_address} | grep "${ONLINE_BRANCH}$")
branch_data=$(git ls-remote ${repo_address} | grep "${OS_BRANCH}$")
# If there is branch data continue
if [ ! -z "${branch_data}" ];then
@ -85,64 +78,100 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
branch_sha=$(echo "${branch_data}" | awk '{print $1}')
# Set the branch entry
branch_entry="${branch_sha} # HEAD of \"$ONLINE_BRANCH\" as of $(date +%d.%m.%Y)"
branch_entry="${branch_sha} # HEAD of \"$OS_BRANCH\" as of $(date +%d.%m.%Y)"
# Write the branch entry into the repo_packages file
sed -i.bak "s|${repo_name}_git_install_branch:.*|${repo_name}_git_install_branch: $branch_entry|" ${SERVICE_FILE}
# If the repo is in the specified list, then action the additional updates
if [[ "${OPENSTACK_SERVICE_LIST}" =~ "${repo_name}" ]]; then
if [[ -d "${OSA_ROLE_LOCAL}_${repo_name}" ]]; then
repo_tmp_path="/tmp/${repo_name}"
os_repo_tmp_path="/tmp/os_${repo_name}"
osa_repo_tmp_path="/tmp/osa_${repo_name}"
# Ensure that the temp path doesn't exist
[[ -d "repo_tmp_path" ]] && rm -rf "${repo_tmp_path}"
# Ensure that the temp path doesn't exist
rm -rf ${os_repo_tmp_path} ${osa_repo_tmp_path}
# Do a shallow clone of the repo to work with
git clone --quiet --depth=5 --branch ${ONLINE_BRANCH} --no-checkout --single-branch ${repo_address} "${repo_tmp_path}"
pushd ${repo_tmp_path} > /dev/null
git checkout --quiet "${branch_sha}"
popd > /dev/null
# Do a shallow clone of the OpenStack repo to work with
git clone --quiet --depth=10 --branch ${OS_BRANCH} --no-checkout --single-branch ${repo_address} ${os_repo_tmp_path}
pushd ${os_repo_tmp_path} > /dev/null
git checkout --quiet ${branch_sha}
popd > /dev/null
# Update the policy files
find ${repo_tmp_path}/etc -name "policy.json" -exec \
cp {} "${OSA_ROLE_LOCAL}_${repo_name}/templates/policy.json.j2" \;
# Tweak the paste files
find ${repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \;
# Update the paste files
find ${repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(basename \${name}).j2\"" \;
# Tweak the rootwrap conf files
find ${repo_tmp_path}/etc -name "rootwrap.conf" -exec \
sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \;
# Update the rootwrap conf files
find ${repo_tmp_path}/etc -name "rootwrap.conf" -exec \
cp {} "${OSA_ROLE_LOCAL}_${repo_name}/templates/rootwrap.conf.j2" \;
# Update the rootwrap filters
find ${repo_tmp_path}/etc -name "*.filters" -exec \
bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/files/rootwrap.d/\$(basename \${name})\"" \;
# Update the yaml files for Ceilometer
if [ "${repo_name}" = "ceilometer" ]; then
find ${repo_tmp_path}/etc -name "*.yaml" -exec \
bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(basename \${name}).j2\"" \;
fi
# Update the yaml files for Heat
if [ "${repo_name}" = "heat" ]; then
find ${repo_tmp_path}/etc -name "*.yaml" -exec \
bash -c "name=\"{}\"; cp \${name} \"${OSA_ROLE_LOCAL}_${repo_name}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
fi
# Clean up the temporary files
[[ -d "repo_tmp_path" ]] && rm -rf "${repo_tmp_path}"
# Set the OSA address
if [[ "${repo_name}" == "ironic" ]]; then
osa_repo_address="https://git.openstack.org/openstack/openstack-ansible-ironic"
else
osa_repo_address="https://git.openstack.org/openstack/openstack-ansible-os_${repo_name}"
fi
# Do a shallow clone of the OSA repo to work with
git clone --quiet --depth=10 --branch ${OSA_BRANCH} --single-branch ${osa_repo_address} ${osa_repo_tmp_path}
pushd ${osa_repo_tmp_path} > /dev/null
git checkout --quiet origin/${OSA_BRANCH}
popd > /dev/null
# Update the policy files
find ${os_repo_tmp_path}/etc -name "policy.json" -exec \
cp {} "${osa_repo_tmp_path}/templates/policy.json.j2" \;
# Tweak the paste files
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
sed -i.bak "s|hmac_keys = SECRET_KEY|hmac_keys = {{ ${repo_name}_profiler_hmac_key }}|" {} \;
# Update the paste files
find ${os_repo_tmp_path}/etc -name "*[_-]paste.ini" -exec \
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \;
# Tweak the rootwrap conf files
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
sed -i.bak "s|exec_dirs=|exec_dirs={{ ${repo_name}_bin }},|" {} \;
# Update the rootwrap conf files
find ${os_repo_tmp_path}/etc -name "rootwrap.conf" -exec \
cp {} "${osa_repo_tmp_path}/templates/rootwrap.conf.j2" \;
# Update the rootwrap filters
find ${os_repo_tmp_path}/etc -name "*.filters" -exec \
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/files/rootwrap.d/\$(basename \${name})\"" \;
# Update the yaml files for Ceilometer
if [ "${repo_name}" = "ceilometer" ]; then
find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(basename \${name}).j2\"" \;
fi
# Update the yaml files for Heat
if [ "${repo_name}" = "heat" ]; then
find ${os_repo_tmp_path}/etc -name "*.yaml" -exec \
bash -c "name=\"{}\"; cp \${name} \"${osa_repo_tmp_path}/templates/\$(echo \${name} | rev | cut -sd / -f -2 | rev).j2\"" \;
fi
# Switch into the OSA git directory to work with it
pushd ${osa_repo_tmp_path} > /dev/null
# Check for changed files
git_changed=$(git status --porcelain | wc -l)
# Check for untracked files
git_untracked=$(git ls-files --other --exclude-standard --directory | wc -l)
if [ ${git_untracked} -gt 0 ]; then
# If there are untracked files, ensure that the commit message includes
# a WIP prefix so that the patch is revised in more detail.
git_msg_prefix="[New files - needs update] "
else
git_msg_prefix=""
fi
# If any files have changed, submit a patch including the changes
if [ ${git_changed} -gt 0 ]; then
git review -s > /dev/null
git add --all
git commit -a -m "${git_msg_prefix}Update paste, policy and rootwrap configurations $(date +%Y-%m-%d)" --quiet
git review > /dev/null
fi
popd > /dev/null
# Clean up the temporary files
rm -rf ${os_repo_tmp_path} ${osa_repo_tmp_path}
fi
fi
@ -152,7 +181,7 @@ done
unset IFS
# Finally, update the PIP_INSTALL_OPTIONS with the current versions of pip, wheel and setuptools
# Update the PIP_INSTALL_OPTIONS with the current versions of pip, wheel and setuptools
PIP_CURRENT_OPTIONS=$(./scripts/get-pypi-pkg-version.py -p pip setuptools wheel -l horizontal)
sed -i.bak "s|^PIP_INSTALL_OPTIONS=.*|PIP_INSTALL_OPTIONS=\$\{PIP_INSTALL_OPTIONS:-'${PIP_CURRENT_OPTIONS}'\}|" scripts/scripts-library.sh
@ -162,3 +191,30 @@ for pin in ${PIP_CURRENT_OPTIONS}; do
done
echo "Updated pip install options/pins"
# Update the ansible-role-requirements.yml file
# We don't want to be doing this for the master branch
if [[ "${OSA_BRANCH}" != "master" ]]; then
echo "Updating ansible-role-requirements.yml"
# Loop through each of the role git sources
for role_src in $(awk '/src: / {print $2}' ansible-role-requirements.yml); do
# Determine the role's name
role_name=$(sed 's/^[ \t-]*//' ansible-role-requirements.yml | awk '/src: / || /name: / {print $2}' | grep -B1 "${role_src}" | head -n 1)
# Grab the latest SHA that matches the specified branch
role_version=$(git ls-remote ${role_src} | grep "${OSA_BRANCH}$" | awk '{print $1}')
# If that branch doesn't exist, then it's probably not an OpenStack-Ansible role, so grab the latest tag instead
if [[ -z "${role_version}" ]]; then
role_version=$(git ls-remote --tags ${role_src} | awk '{print $2}' | grep -v '{}' | cut -d/ -f 3 | sort -n | tail -n 1)
fi
# Now use the information we have to update the ansible-role-requirements file
$(dirname ${0})/ansible-role-requirements-editer.py -f ansible-role-requirements.yml -n "${role_name}" -v "${role_version}"
done
echo "Completed updating ansible-role-requirements.yml"
else
echo "Skipping the ansible-role-requirements.yml update as we're working on the master branch"
fi