42c2419353
Change-Id: Ie6e77a4d45210122746b544a1aa8da563833e395 Partially-Implements: blueprint enable-flake8
256 lines
8.7 KiB
Python
Executable File
256 lines
8.7 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# TODO(SamYaple): Allow image pushing
|
|
# TODO(SamYaple): Single image building w/ optional parent building
|
|
# TODO(SamYaple): Build only missing images
|
|
# TODO(SamYaple): Execute the source install script that will pull
|
|
# down and create tarball
|
|
# TODO(SamYaple): Improve logging instead of printing to stdout
|
|
|
|
from __future__ import print_function
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import os
|
|
import Queue
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
from threading import Thread
|
|
import time
|
|
import traceback
|
|
|
|
import docker
|
|
|
|
|
|
class WorkerThread(Thread):
|
|
|
|
def __init__(self, queue, cache, rm):
|
|
self.queue = queue
|
|
self.nocache = not cache
|
|
self.forcerm = rm
|
|
self.dc = docker.Client()
|
|
Thread.__init__(self)
|
|
|
|
def run(self):
|
|
""" Executes tasks until the queue is empty """
|
|
while True:
|
|
try:
|
|
data = self.queue.get(block=False)
|
|
self.builder(data)
|
|
self.queue.task_done()
|
|
except Queue.Empty:
|
|
break
|
|
except:
|
|
traceback.print_exc()
|
|
self.queue.task_done()
|
|
|
|
def builder(self, image):
|
|
print('Processing:', image['name'])
|
|
image['status'] = "building"
|
|
|
|
if image['parent'] != None and image['parent']['status'] == "error":
|
|
image['status'] = "parent_error"
|
|
return
|
|
|
|
# Pull the latest image for the base distro only
|
|
pull = True if image['parent'] is None else False
|
|
|
|
image['logs'] = str()
|
|
for response in self.dc.build(path=image['path'],
|
|
tag=image['fullname'],
|
|
nocache=self.nocache,
|
|
rm=True,
|
|
pull=pull,
|
|
forcerm=self.forcerm):
|
|
stream = json.loads(response)
|
|
|
|
if 'stream' in stream:
|
|
image['logs'] = image['logs'] + stream['stream']
|
|
elif 'errorDetail' in stream:
|
|
image['status'] = "error"
|
|
raise Exception(stream['errorDetail']['message'])
|
|
|
|
image['status'] = "built"
|
|
print(image['logs'], '\nProcessed:', image['name'])
|
|
|
|
|
|
def argParser():
|
|
parser = argparse.ArgumentParser(description='Kolla build script')
|
|
parser.add_argument('-n', '--namespace',
|
|
help='Set the Docker namespace name',
|
|
type=str,
|
|
default='kollaglue')
|
|
parser.add_argument('--tag',
|
|
help='Set the Docker tag',
|
|
type=str,
|
|
default='latest')
|
|
parser.add_argument('-b', '--base',
|
|
help='The base distro to use when building',
|
|
type=str,
|
|
default='centos')
|
|
parser.add_argument('-t', '--type',
|
|
help='The method of the Openstack install',
|
|
type=str,
|
|
default='binary')
|
|
parser.add_argument('-c', '--cache',
|
|
help='Use Docker cache when building',
|
|
type=bool,
|
|
default=True)
|
|
parser.add_argument('-r', '--rm',
|
|
help='Remove intermediate containers while building',
|
|
type=bool,
|
|
default=True)
|
|
parser.add_argument('-T', '--threads',
|
|
help='The number of threads to use while building',
|
|
type=int,
|
|
default=8)
|
|
return vars(parser.parse_args())
|
|
|
|
|
|
class KollaWorker():
|
|
|
|
def __init__(self, args):
|
|
self.kolla_dir = os.path.join(sys.path[0], '..')
|
|
self.images_dir = os.path.join(self.kolla_dir, 'docker')
|
|
|
|
self.namespace = args['namespace']
|
|
self.base = args['base']
|
|
self.type_ = args['type']
|
|
self.tag = args['tag']
|
|
self.prefix = self.base + '-' + self.type_ + '-'
|
|
|
|
def setupWorkingDir(self):
|
|
""" Creates a working directory for use while building """
|
|
ts = time.time()
|
|
ts = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S_')
|
|
self.temp_dir = tempfile.mkdtemp(prefix='kolla-' + ts)
|
|
self.working_dir = os.path.join(self.temp_dir, 'docker')
|
|
shutil.copytree(self.images_dir, self.working_dir)
|
|
|
|
def findDockerfiles(self):
|
|
""" Recursive search for Dockerfiles in the working directory """
|
|
self.docker_build_paths = list()
|
|
path = os.path.join(self.working_dir, self.base, self.type_)
|
|
|
|
for root, dirs, names in os.walk(path):
|
|
if 'Dockerfile' in names:
|
|
self.docker_build_paths.append(root)
|
|
|
|
def cleanup(self):
|
|
""" Remove temp files """
|
|
shutil.rmtree(self.temp_dir)
|
|
|
|
def sortImages(self):
|
|
""" Build images dependency tiers """
|
|
images_to_process = list(self.images)
|
|
|
|
self.tiers = list()
|
|
while images_to_process:
|
|
self.tiers.append(list())
|
|
processed_images = list()
|
|
|
|
for image in images_to_process:
|
|
if image['parent'] is None:
|
|
self.tiers[-1].append(image)
|
|
processed_images.append(image)
|
|
if len(self.tiers) > 1:
|
|
for parent in self.tiers[-2]:
|
|
if image['parent'] == parent['fullname']:
|
|
image['parent'] = parent
|
|
self.tiers[-1].append(image)
|
|
processed_images.append(image)
|
|
|
|
# TODO(SamYaple): Improve error handling in this section
|
|
if not processed_images:
|
|
print('Could not find parent image from some images. Aborting',
|
|
file=sys.stderr)
|
|
for image in images_to_process:
|
|
print(image['name'], image['parent'], file=sys.stderr)
|
|
sys.exit()
|
|
|
|
# You cannot modify a list while using the list in a for loop as it
|
|
# will produce unexpected results by messing up the index so we
|
|
# build a seperate list and remove them here instead
|
|
for image in processed_images:
|
|
images_to_process.remove(image)
|
|
|
|
def summary(self):
|
|
""" Walk the list of images and check for errors """
|
|
print("Successfully built images")
|
|
print("=========================")
|
|
for image in self.images:
|
|
if image['status'] == "built":
|
|
print(image['name'])
|
|
|
|
print("\nImages that failed to build")
|
|
print("===========================")
|
|
for image in self.images:
|
|
if image['status'] != "built":
|
|
print(image['name'], "\r\t\t\t Failed with status:",
|
|
image['status'])
|
|
|
|
def buildImageList(self):
|
|
self.images = list()
|
|
|
|
# Walk all of the Dockerfiles and replace the %%KOLLA%% variables
|
|
for path in self.docker_build_paths:
|
|
with open(os.path.join(path, 'Dockerfile')) as f:
|
|
content = f.read().replace('%%KOLLA_NAMESPACE%%',
|
|
self.namespace)
|
|
content = content.replace('%%KOLLA_PREFIX%%', self.prefix)
|
|
content = content.replace('%%KOLLA_TAG%%', self.tag)
|
|
with open(os.path.join(path, 'Dockerfile'), 'w') as f:
|
|
f.write(content)
|
|
|
|
image = dict()
|
|
image['status'] = "unprocessed"
|
|
image['name'] = os.path.basename(path)
|
|
image['fullname'] = self.namespace + '/' + self.prefix + \
|
|
image['name'] + ':' + self.tag
|
|
image['path'] = path
|
|
image['parent'] = content.split(' ')[1].split('\n')[0]
|
|
if self.namespace not in image['parent']:
|
|
image['parent'] = None
|
|
|
|
self.images.append(image)
|
|
|
|
def buildQueues(self):
|
|
"""
|
|
Return a list of Queues that have been organized into a hierarchy
|
|
based on dependencies
|
|
"""
|
|
self.buildImageList()
|
|
self.sortImages()
|
|
|
|
pools = list()
|
|
for tier in self.tiers:
|
|
pool = Queue.Queue()
|
|
for image in tier:
|
|
pool.put(image)
|
|
|
|
pools.append(pool)
|
|
|
|
return pools
|
|
|
|
|
|
def main():
|
|
args = argParser()
|
|
|
|
kolla = KollaWorker(args)
|
|
kolla.setupWorkingDir()
|
|
kolla.findDockerfiles()
|
|
|
|
# Returns a list of Queues for us to loop through
|
|
for pool in kolla.buildQueues():
|
|
for x in xrange(args['threads']):
|
|
WorkerThread(pool, args['cache'], args['rm']).start()
|
|
# block until queue is empty
|
|
pool.join()
|
|
|
|
kolla.summary()
|
|
kolla.cleanup()
|
|
|
|
if __name__ == '__main__':
|
|
main()
|