Add tool to analyze check and gate success rates

This tool scans gerrit changes for comments from zuul over the last 30
days to build out success rates for check and gate pipelines. This only
looks at changes that have merged to avoid those that never can merge
because they only fail or are expected to fail.

This tool emits information like:

  Changes: 4475
  Check Failures: 5317.0
  Check Successes: 9173.0
  Check Rate of failure: 0.3669427191166322
  Gate Failures: 687.0
  Gate Successes: 4450.0
  Gate Rate of failure: 0.13373564337161767
  Total Failures: 6004.0
  Total Successes: 13623.0
  Total Rate of failure: 0.3059051306873185

Change-Id: I759ba670c6b81f4425ce618c412db9cbd0e51401
This commit is contained in:
Clark Boylan 2019-07-19 09:06:30 -07:00
parent dd63186f66
commit 00348a4d0d

View File

@ -0,0 +1,75 @@
import datetime
import json
import requests
today = datetime.date.today()
thirty_days = datetime.timedelta(days=30)
last_month = today - thirty_days
str_today = today.strftime('%Y-%m-%d')
str_last_month = last_month.strftime('%Y-%m-%d')
more_changes = True
change_count = 0
check_successes = 0.0
check_failures = 0.0
gate_successes = 0.0
gate_failures = 0.0
while(more_changes):
more_changes = False
# We look only at merged changes to ignore Do Not Merge changes that are
# expected to fail intentionally.
r = requests.get('https://review.opendev.org/changes/?q=status:merged+'
'after:{after}+before:{before}&n=100&S={offset}'.format(
after=str_last_month,
before=str_today,
offset=(change_count + 1)))
content = r.content[4:]
changes = json.loads(content)
for change in changes:
r = requests.get('https://review.opendev.org/changes/%s/detail' % change['id'])
content = r.content[4:]
details = json.loads(content)
change_count += 1
for message in details['messages']:
if 'author' not in message or 'message' not in message:
continue
if message['author']['_account_id'] != 22348:
# Don't care about non zuul comments
continue
if '(check pipeline)' in message['message']:
if 'Build succeeded' in message['message']:
check_successes += 1
elif 'Build failed' in message['message']:
# explicitly check for failed builds here to avoid merge
# conflict messages
check_failures += 1
if '(gate pipeline)' in message['message']:
if 'Build succeeded' in message['message']:
gate_successes += 1
elif 'Build failed' in message['message']:
# explicitly check for failed builds here to avoid merge
# conflict messages
gate_failures += 1
if '_more_changes' in change:
more_changes = True
print('Changes: %s' % change_count)
print('Check Failures: %s' % check_failures)
print('Check Successes: %s' % check_successes)
check_total = check_failures + check_successes
print('Check Rate of failure: %s' % (check_failures/check_total))
print('Gate Failures: %s' % gate_failures)
print('Gate Successes: %s' % gate_successes)
gate_total = gate_failures + gate_successes
print('Gate Rate of failure: %s' % (gate_failures/gate_total))
print('Total Failures: %s' % (check_failures + gate_failures))
print('Total Successes: %s' % (check_successes + gate_successes))
total = check_total + gate_total
print('Total Rate of failure: %s' % ((check_failures + gate_failures)/total))