versions.yaml updater tool: verify that images & tags do exist
This patch allows the tool to verify that container images with specific tags equal to the HEAD git commit id's we reference, do really exist on quay.io repository and are available for download. Change-Id: I2205f4fa63a085d94ef3c0e9d2ff511665df43e6
This commit is contained in:
Roman Gorshunov
parent
46a1513a9b
commit
3efa98a31b
108
tools/updater.py
108
tools/updater.py
@ -23,12 +23,18 @@
|
||||
# 2) tags of container images listed in dict `image_repo_git_url` in the
|
||||
# code below
|
||||
#
|
||||
# In addition to that, the tool verifies that container images with
|
||||
# specific tags equal to the HEAD git commit id's we reference, do really
|
||||
# exist on quay.io repository and are available for download.
|
||||
#
|
||||
|
||||
from functools import reduce
|
||||
import argparse
|
||||
import datetime
|
||||
import logging
|
||||
import operator
|
||||
import os
|
||||
import requests
|
||||
import sys
|
||||
|
||||
try:
|
||||
@ -54,7 +60,8 @@ image_repo_git_url = {
|
||||
'quay.io/airshipit/airflow': 'https://git.openstack.org/openstack/airship-shipyard',
|
||||
'quay.io/airshipit/armada': 'https://git.openstack.org/openstack/airship-armada',
|
||||
'quay.io/airshipit/deckhand': 'https://git.openstack.org/openstack/airship-deckhand',
|
||||
'quay.io/airshipit/divingbell': 'https://git.openstack.org/openstack/airship-divingbell',
|
||||
# yes, divingbell image is just Ubuntu 16.04 image, and we don't check it's tag
|
||||
#'docker.io/ubuntu': 'https://git.openstack.org/openstack/airship-divingbell',
|
||||
'quay.io/airshipit/drydock': 'https://git.openstack.org/openstack/airship-drydock',
|
||||
# maas-{rack,region}-controller images are built from airship-maas repository
|
||||
'quay.io/airshipit/maas-rack-controller': 'https://git.openstack.org/openstack/airship-maas',
|
||||
@ -64,18 +71,33 @@ image_repo_git_url = {
|
||||
'quay.io/airshipit/shipyard': 'https://git.openstack.org/openstack/airship-shipyard',
|
||||
# sstream-cache image is built from airship-maas repository
|
||||
'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas',
|
||||
'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios'
|
||||
# Disabled by Kaspars: https://review.openstack.org/#/c/596909/21/tools/updater.py@53
|
||||
#'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter'
|
||||
'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios',
|
||||
'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter'
|
||||
}
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
# Temporary dict of git url's and cached commit id's: {'git_url': 'commit_id'}
|
||||
global git_url_commit_ids
|
||||
git_url_commit_ids = {}
|
||||
# Temporary dict of image repo's and status of image on quay.io
|
||||
global image_repo_status
|
||||
image_repo_status = {}
|
||||
dict_path = None
|
||||
|
||||
|
||||
def inverse_dict(dic):
|
||||
"""Accepts dictionary, returns dictionary where keys become values,
|
||||
and values become keys"""
|
||||
new_dict = {}
|
||||
for k, v in dic.items():
|
||||
new_dict[v] = k
|
||||
return new_dict
|
||||
|
||||
|
||||
git_url_image_repo = inverse_dict(image_repo_git_url)
|
||||
|
||||
|
||||
# https://stackoverflow.com/a/35585837
|
||||
def lsremote(url, remote_ref):
|
||||
"""Accepts git url and remote reference, returns git commit id."""
|
||||
@ -92,14 +114,92 @@ def get_commit_id(url):
|
||||
# If we don't have this git url in our url's dictionary,
|
||||
# fetch latest commit ID and add new dictionary entry
|
||||
logging.debug('git_url_commit_ids: %s', git_url_commit_ids)
|
||||
logging.debug('image_repo_status: %s', image_repo_status)
|
||||
if url not in git_url_commit_ids:
|
||||
logging.debug('git url: ' + url +
|
||||
' is not in git_url_commit_ids dict;' +
|
||||
' adding it with HEAD commit id')
|
||||
git_url_commit_ids[url] = lsremote(url, 'HEAD')
|
||||
if url in git_url_image_repo:
|
||||
if git_url_image_repo[url] not in image_repo_status:
|
||||
image_repo_status[git_url_image_repo[url]] = \
|
||||
verify_image_tag(git_url_image_repo[url], git_url_commit_ids[url])
|
||||
else:
|
||||
logging.debug('We checked image ' + git_url_image_repo[url] +
|
||||
' on quay.io already, skipping')
|
||||
return git_url_commit_ids[url]
|
||||
|
||||
|
||||
def verify_image_tag(image, git_commit_id):
|
||||
"""Verify if image with certain tag exists on quay.io,
|
||||
returns 0 (image not hosted on quay.io), True, or False
|
||||
"""
|
||||
if not image.startswith('quay.io/'):
|
||||
logging.info('Unable to verify if image ' + image + ':' +
|
||||
git_commit_id + ' with this specific tag exists' +
|
||||
' in containers repository: only quay.io is' +
|
||||
' supported at the moment')
|
||||
return 0
|
||||
|
||||
logging.info('Verifying if image ' + image + ':' + git_commit_id +
|
||||
' with this specific tag exists on quay.io...')
|
||||
|
||||
retries = 0
|
||||
max_retries = 5
|
||||
|
||||
payload = {'specificTag': git_commit_id}
|
||||
hash_image = image.split('/')
|
||||
url = 'https://quay.io/api/v1/repository/' + \
|
||||
hash_image[1] + '/' + hash_image[2] + '/tag/'
|
||||
|
||||
while retries <= max_retries:
|
||||
retries = retries +1
|
||||
try:
|
||||
res = requests.get(url, params = payload, timeout = 5)
|
||||
except requests.exceptions.Timeout:
|
||||
logging.warning("Failed to fetch url %s" % res.url)
|
||||
sleep(1)
|
||||
if retries == max_retries:
|
||||
logging.error("Failed to connect to quay.io")
|
||||
return 0
|
||||
|
||||
if res.status_code != 200:
|
||||
logging.error('Image %s is not available on quay.io or it' +
|
||||
' requires authentication', image)
|
||||
return 0
|
||||
|
||||
try:
|
||||
res = res.json()
|
||||
except json.decoder.JSONDecodeError:
|
||||
logging.error('Unable to parse response from quay.io (%s)' % res.url)
|
||||
return 0
|
||||
|
||||
try:
|
||||
for tag in res['tags']:
|
||||
# Normally there should be only one tag description
|
||||
if tag['name'] == git_commit_id:
|
||||
if 'end_ts' not in tag:
|
||||
# Active image tag
|
||||
logging.info('Image ' + image + ':' + git_commit_id +
|
||||
' with this specific tag corresponding to the' +
|
||||
' commit id of the git HEAD exists on quay.io,' +
|
||||
' last modified on ' +
|
||||
datetime.datetime.fromtimestamp(tag['start_ts']).isoformat() +
|
||||
' UTC')
|
||||
return True
|
||||
else:
|
||||
# Tag used to exist
|
||||
logging.error('Image ' + image + ':' + git_commit_id +
|
||||
' with this specific tag no longer exists on quay.io')
|
||||
return False
|
||||
except KeyError:
|
||||
logging.error('Unable to parse response from quay.io (%s)' % res.url)
|
||||
return 0
|
||||
logging.error('There is no image ' + image + ':' + git_commit_id +
|
||||
' with this specific tag on quay.io or it requires authentication')
|
||||
return False
|
||||
|
||||
|
||||
# https://stackoverflow.com/a/14692747
|
||||
def get_by_path(root, items):
|
||||
"""Access a nested object in root by item sequence."""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user