362bf84586
This patch adds filtered report support to the osa-differ script and refactors some code that was repeated more than once. A user can provide ``--projects-only`` on the command line to see only the commits that were made to OpenStack projects that OSA builds. The user can also use ``--roles-only`` to limit the report only to the changes in OSA roles. All commits are rendered through the same jinja template, which should make future cosmetic updates much easier. Each section for a project or role now includes links back to the repository itself and links to each commit. The script now also shows the commits on OSA itself. That seems fairly important and it should have been there earlier. Change-Id: I139f0f1bab3d4afbc5004dc52756800e426aa605
264 lines
8.8 KiB
Python
Executable File
264 lines
8.8 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright 2016, Rackspace US, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Analyzes the differences between two OpenStack-Ansible commits."""
|
|
|
|
import argparse
|
|
import jinja2
|
|
import pygithub3
|
|
import os
|
|
import requests
|
|
import sys
|
|
from urlparse import urlparse
|
|
import yaml
|
|
|
|
|
|
def get_arguments():
|
|
"""Setup argument Parsing."""
|
|
parser = argparse.ArgumentParser(
|
|
usage='%(prog)s',
|
|
description='OpenStack-Ansible Release Diff Generator',
|
|
epilog='Licensed "Apache 2.0"')
|
|
parser.add_argument(
|
|
'old_commit',
|
|
action='store',
|
|
nargs=1,
|
|
help="Git SHA of the older commit",
|
|
)
|
|
parser.add_argument(
|
|
'new_commit',
|
|
action='store',
|
|
nargs=1,
|
|
help="Git SHA of the newer commit",
|
|
)
|
|
display_opts = parser.add_mutually_exclusive_group()
|
|
display_opts.add_argument(
|
|
"--projects-only",
|
|
action="store_true",
|
|
help="Only display commits for OpenStack projects"
|
|
)
|
|
display_opts.add_argument(
|
|
"--roles-only",
|
|
action="store_true",
|
|
help="Only display commits for OpenStack-Ansible roles"
|
|
)
|
|
|
|
return vars(parser.parse_args())
|
|
|
|
|
|
def get_commit_data(commit_sha):
|
|
"""Get information about a commit from Github."""
|
|
commit_data = gh.repos.commits.get(user='openstack',
|
|
repo='openstack-ansible',
|
|
sha=commit_sha)
|
|
return commit_data
|
|
|
|
|
|
def get_commit_message(commit_sha):
|
|
"""Get the first line of the commit message for a particular commit."""
|
|
commit_data = get_commit_data(commit_sha)
|
|
commit_message = commit_data.commit.message.splitlines()[0]
|
|
return commit_message
|
|
|
|
|
|
def get_project_names(project_dict):
|
|
"""Get project names from the list of project variables."""
|
|
return [x[:-9] for x in project_dict if x.endswith('git_repo')]
|
|
|
|
|
|
def get_projects(base_url, commit):
|
|
"""Get all projects from multiple YAML files."""
|
|
# Assemble the full URLs to our YAML files that contain our OpenStack
|
|
# projects' details.
|
|
repo_files = [
|
|
'playbooks/defaults/repo_packages/openstack_services.yml',
|
|
'playbooks/defaults/repo_packages/openstack_other.yml'
|
|
]
|
|
yaml_urls = [base_url.format(commit, x) for x in repo_files]
|
|
|
|
# Loop through both YAML files and merge the data into one dictionary.
|
|
yaml_parsed = []
|
|
for yaml_url in yaml_urls:
|
|
r = requests.get(yaml_url)
|
|
yaml_parsed.append(yaml.load(r.text))
|
|
merged_dicts = {k: v for d in yaml_parsed for k, v in d.items()}
|
|
|
|
return merged_dicts
|
|
|
|
|
|
def render_commit_template(user, repo, old_commit, new_commit, extra_vars={},
|
|
template_file='repo_details.j2'):
|
|
"""Render a template to generate RST content for commits."""
|
|
global gh
|
|
global jinja_env
|
|
|
|
# Compare the two commits in the project's repository to see what
|
|
# the differences are between them.
|
|
comparison = gh.repos.commits.compare(
|
|
user=user,
|
|
repo=repo,
|
|
base=old_commit,
|
|
head=new_commit
|
|
)
|
|
|
|
# Render the jinja2 template
|
|
rendered_template = jinja_env.get_template(template_file).render(
|
|
repo=repo,
|
|
commits=comparison.commits,
|
|
latest_sha=short_commit(new_commit),
|
|
older_sha=short_commit(old_commit),
|
|
extra_vars=extra_vars
|
|
)
|
|
|
|
return rendered_template
|
|
|
|
|
|
def short_commit(commit_sha):
|
|
"""Return a short commit hash string."""
|
|
return commit_sha[0:8]
|
|
|
|
if __name__ == "__main__":
|
|
|
|
# Set up some initial variables
|
|
gh = pygithub3.Github()
|
|
args = get_arguments()
|
|
|
|
# Load our Jinja templates
|
|
TEMPLATE_DIR = "{0}/templates".format(
|
|
os.path.dirname(os.path.abspath(__file__))
|
|
)
|
|
jinja_env = jinja2.Environment(
|
|
loader=jinja2.FileSystemLoader(TEMPLATE_DIR),
|
|
trim_blocks=True
|
|
)
|
|
|
|
# Store our arguments into variables to make things easier.
|
|
old_commit = args['old_commit'][0]
|
|
new_commit = args['new_commit'][0]
|
|
|
|
# Get the first line of the commit message in the older commit
|
|
try:
|
|
old_commit_message = get_commit_message(old_commit)
|
|
except pygithub3.exceptions.NotFound:
|
|
print("The old commit SHA was not found: {0}".format(old_commit))
|
|
sys.exit(1)
|
|
|
|
# Get the first line of the commit message in the newer commit
|
|
try:
|
|
new_commit_message = get_commit_message(new_commit)
|
|
except pygithub3.exceptions.NotFound:
|
|
print("The new commit SHA was not found: {0}".format(new_commit))
|
|
sys.exit(1)
|
|
|
|
# Generate header and initial report for OpenStack-Ansible itself
|
|
report = render_commit_template(
|
|
user='openstack',
|
|
repo='openstack-ansible',
|
|
old_commit=old_commit,
|
|
new_commit=new_commit,
|
|
template_file='header.j2',
|
|
extra_vars={
|
|
'roles_only': args['roles_only'],
|
|
'projects_only': args['projects_only']
|
|
}
|
|
)
|
|
|
|
# Add a horizontal line to report after the OpenStack-Ansible commits.
|
|
report += "----\n"
|
|
|
|
# Set up the base url that allows us to retrieve data from
|
|
# OpenStack-Ansible at a particular commit.
|
|
base_url = 'https://raw.githubusercontent.com/openstack/' \
|
|
'openstack-ansible/{0}/{1}'
|
|
|
|
if args['roles_only']:
|
|
# Short circuit here and don't get any projects since the user only
|
|
# wants to see role commits.
|
|
old_commit_projects = []
|
|
new_commit_projects = []
|
|
else:
|
|
report += "\nOpenStack Projects\n------------------\n"
|
|
|
|
# Get all of the OpenStack projects that OpenStack-Ansible builds
|
|
old_commit_projects = get_projects(base_url, old_commit)
|
|
new_commit_projects = get_projects(base_url, new_commit)
|
|
|
|
# Get the bare project names from the YAML data we retrieved
|
|
old_commit_project_names = get_project_names(old_commit_projects)
|
|
new_commit_project_names = get_project_names(new_commit_projects)
|
|
|
|
# Loop through each OpenStack project found in the latest commit
|
|
for project in sorted(new_commit_project_names):
|
|
|
|
# Find the git repo URL from the new commit
|
|
git_repo = new_commit_projects["{0}_git_repo".format(project)]
|
|
_, user, project_repo_name = urlparse(git_repo).path.split('/')
|
|
|
|
# Determine the latest sha for this project
|
|
project_sha = "{0}_git_install_branch".format(project)
|
|
latest_sha = new_commit_projects[project_sha]
|
|
|
|
# If this module didn't exist in the old OpenStack-Ansible commit,
|
|
# just skip it for now.
|
|
try:
|
|
project_sha = "{0}_git_install_branch".format(project)
|
|
older_sha = old_commit_projects[project_sha]
|
|
except:
|
|
continue
|
|
|
|
# Render a template showing the commits in this project's repository.
|
|
report += render_commit_template(
|
|
user=user,
|
|
repo=project_repo_name,
|
|
old_commit=older_sha,
|
|
new_commit=latest_sha
|
|
)
|
|
|
|
# Set up the URLs for the old and new ansible-role-requirements.yml
|
|
old_role_url = base_url.format(old_commit, 'ansible-role-requirements.yml')
|
|
new_role_url = base_url.format(new_commit, 'ansible-role-requirements.yml')
|
|
|
|
if args['projects_only']:
|
|
# Short circuit here and don't get any roles since the user only wants
|
|
# to see OpenStack project commits.
|
|
old_role_yaml = {}
|
|
new_role_yaml = {}
|
|
else:
|
|
report += "\nOpenStack-Ansible Roles\n-----------------------\n"
|
|
|
|
# Retrieve the roles YAML
|
|
old_role_yaml = yaml.load(requests.get(old_role_url).text)
|
|
new_role_yaml = yaml.load(requests.get(new_role_url).text)
|
|
|
|
# Loop through each OpenStack-Ansible role found in the latest commit
|
|
for role in new_role_yaml:
|
|
|
|
# Get the user and repo name that we will need to use with GitHub
|
|
_, user, role_repo_name = urlparse(role['src']).path.split('/')
|
|
|
|
# Determine the older and newer SHA for this role
|
|
latest_sha = role['version']
|
|
older_sha = next(x['version'] for x in old_role_yaml
|
|
if x['name'] == role['name'])
|
|
|
|
# Render a template showing the commits in this role's repository.
|
|
report += render_commit_template(
|
|
user=user,
|
|
repo=role_repo_name,
|
|
old_commit=older_sha,
|
|
new_commit=latest_sha
|
|
)
|
|
|
|
print(report)
|