openstack/kolla-ansible
Code Issues Proposed changes

Remove tiered building in build.py

Browse Source 
Many tiers have been shed over this issue and frankly enough is enough
on this tiering thing, it's tiering us apart! Now we use threading a
bit better as well.
No more tiers! Puns very much intended.

Additionally, this refactors the function names to make inc0 happier
and be consistent.

We also modify the test_build.py to catch the new statuses introduced
by the regex patch. Then we remove the Ubuntu Source from the Docker
folder test as that will never ever be functional.

Also fixed regex to properly match true regex expressions.

Co-Authored-By: Michal Jastrzebski (inc0) <michal.jastrzebski@intel.com>
Change-Id: I650fd6af76eddb809756762222e66aefd6fc1dca
Partially-Implements: blueprint build-script
This commit is contained in:
Sam Yaple 2015-08-25 15:26:47 +00:00
parent eecf6581a4
commit aada29b4db
3 changed files with 131 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

252
kolla/cmd/build.py
View File

@ -56,28 +56,36 @@ class WorkerThread(Thread):
self.retries = args['retries'] self.retries = args['retries']
self.threads = args['threads'] self.threads = args['threads']
self.dc = docker.Client(**docker.utils.kwargs_from_env()) self.dc = docker.Client(**docker.utils.kwargs_from_env())
Thread.__init__(self) super(WorkerThread, self).__init__()
def end_task(self, image):
"""Properly inform the queue we are finished"""
# No matter whether the parent failed or not, we still process
# the children. We have the code in place to catch a parent in
# an 'error' status
for child in image['children']:
self.queue.put(child)
LOG.debug('Added image {} to queue'.format(child['name']))
self.queue.task_done()
LOG.debug('Processed: {}'.format(image['name']))
def run(self): def run(self):
"""Executes tasks until the queue is empty""" """Executes tasks until the queue is empty"""
while True: while True:
try: try:
data = self.queue.get(block=False) image = self.queue.get()
for _ in range(self.retries): for _ in range(self.retries):
self.builder(data) self.builder(image)
if data['status'] in ['built', 'parent_error']: if image['status'] in ['built', 'unmatched',
'parent_error']:
break break
self.queue.task_done()
except Queue.Empty:
break
except ConnectionError as e: except ConnectionError as e:
LOG.error(e) LOG.error(e)
LOG.error('Make sure Docker is running and that you have ' LOG.error('Make sure Docker is running and that you have '
'the correct privileges to run Docker (root)') 'the correct privileges to run Docker (root)')
data['status'] = "connection_error" image['status'] = "connection_error"
self.queue.task_done()
break break
self.end_task(image)
def process_source(self, image): def process_source(self, image):
source = image['source'] source = image['source']
@ -126,17 +134,21 @@ class WorkerThread(Thread):
os.utime(os.path.join(dest_dir, source['dest']), (0, 0)) os.utime(os.path.join(dest_dir, source['dest']), (0, 0))
def builder(self, image): def builder(self, image):
LOG.info('Processing: {}'.format(image['name'])) LOG.debug('Processing: {}'.format(image['name']))
image['status'] = "building" if image['status'] == 'unmatched':
return
if image['parent'] is not None and \ if (image['parent'] is not None and
image['parent']['status'] in ['error', 'parent_error', image['parent']['status'] in ['error', 'parent_error',
'connection_error']: 'connection_error']):
LOG.error('Parent image error\'d with message "%s"', LOG.error('Parent image error\'d with message "{}"'.format(
image['parent']['status']) image['parent']['status']))
image['status'] = "parent_error" image['status'] = "parent_error"
return return
image['status'] = "building"
LOG.info('Building {}'.format(image['name']))
if 'source' in image and 'source' in image['source']: if 'source' in image and 'source' in image['source']:
self.process_source(image) self.process_source(image)
if image['status'] == "error": if image['status'] == "error":
@ -168,12 +180,12 @@ class WorkerThread(Thread):
image['status'] = "built" image['status'] = "built"
if self.threads == 1: if self.threads == 1:
LOG.info('Processed: {}'.format(image['name'])) LOG.info('Built: {}'.format(image['name']))
else: else:
LOG.info('{}Processed: {}'.format(image['logs'], image['name'])) LOG.info('{}Built: {}'.format(image['logs'], image['name']))
def argParser(): def arg_parser():
parser = argparse.ArgumentParser(description='Kolla build script') parser = argparse.ArgumentParser(description='Kolla build script')
parser.add_argument('regex', parser.add_argument('regex',
help=('Build only images matching ' help=('Build only images matching '
@ -266,11 +278,11 @@ class KollaWorker(object):
self.include_header = args['include_header'] self.include_header = args['include_header']
self.regex = args['regex'] self.regex = args['regex']
self.image_statuses_bad = {} self.image_statuses_bad = dict()
self.image_statuses_good = {} self.image_statuses_good = dict()
self.image_statuses_unproc = {} self.image_statuses_unmatched = dict()
def setupWorkingDir(self): def setup_working_dir(self):
"""Creates a working directory for use while building""" """Creates a working directory for use while building"""
ts = time.time() ts = time.time()
ts = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S_') ts = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S_')
@ -290,7 +302,7 @@ class KollaWorker(object):
os.utime(os.path.join(root, dir_), (0, 0)) os.utime(os.path.join(root, dir_), (0, 0))
LOG.debug('Set atime and mtime to 0 for all content in working dir') LOG.debug('Set atime and mtime to 0 for all content in working dir')
def createDockerfiles(self): def create_dockerfiles(self):
for path in self.docker_build_paths: for path in self.docker_build_paths:
template_name = "Dockerfile.j2" template_name = "Dockerfile.j2"
env = jinja2.Environment(loader=jinja2.FileSystemLoader(path)) env = jinja2.Environment(loader=jinja2.FileSystemLoader(path))
@ -307,7 +319,7 @@ class KollaWorker(object):
with open(os.path.join(path, 'Dockerfile'), 'w') as f: with open(os.path.join(path, 'Dockerfile'), 'w') as f:
f.write(content) f.write(content)
def findDockerfiles(self): def find_dockerfiles(self):
"""Recursive search for Dockerfiles in the working directory""" """Recursive search for Dockerfiles in the working directory"""
self.docker_build_paths = list() self.docker_build_paths = list()
@ -329,91 +341,68 @@ class KollaWorker(object):
"""Remove temp files""" """Remove temp files"""
shutil.rmtree(self.temp_dir) shutil.rmtree(self.temp_dir)
def sortImages(self): def filter_images(self):
"""Build images dependency tiers""" """Filter which images to build"""
if self.regex: if self.regex:
patterns = re.compile(r'({})'.format("|".join(self.regex))) patterns = re.compile(r"|".join(self.regex).join('()'))
images_to_process = list()
for image in self.images: for image in self.images:
if re.search(patterns, image['fullname']): if image['status'] == 'matched':
images_to_process.append(image) continue
added = True if re.search(patterns, image['name']):
while added: image['status'] = 'matched'
added = False while (image['parent'] is not None and
parents = [p['parent'] for p in images_to_process] image['parent']['status'] != 'matched'):
for image in self.images: image = image['parent']
if (image['fullname'] in parents and image['status'] = 'matched'
image not in images_to_process): LOG.debug('Matched image {}'.format(image['name']))
images_to_process.append(image) else:
added = True image['status'] = 'unmatched'
else: else:
images_to_process = list(self.images) for image in self.images:
image['status'] = 'matched'
self.tiers = list()
while images_to_process:
self.tiers.append(list())
processed_images = list()
for image in images_to_process:
if image['parent'] is None:
self.tiers[-1].append(image)
processed_images.append(image)
LOG.debug('Sorted parentless image: {}'.format(
image['name']))
if len(self.tiers) > 1:
for parent in self.tiers[-2]:
if image['parent'] == parent['fullname']:
image['parent'] = parent
self.tiers[-1].append(image)
processed_images.append(image)
LOG.debug('Sorted image {} with parent {}'.format(
image['name'], parent['fullname']))
LOG.debug('===')
# TODO(SamYaple): Improve error handling in this section
if not processed_images:
LOG.warning('Could not find parent image from some images.'
' Aborting')
for image in images_to_process:
LOG.warning('{} {}'.format(image['name'], image['parent']))
sys.exit()
# You cannot modify a list while using the list in a for loop as it
# will produce unexpected results by messing up the index so we
# build a seperate list and remove them here instead
for image in processed_images:
images_to_process.remove(image)
def summary(self): def summary(self):
"""Walk the dictionary of images statuses and print results""" """Walk the dictionary of images statuses and print results"""
self.get_image_statuses() self.get_image_statuses()
LOG.info("Successfully built images")
LOG.info("=========================")
for name in self.image_statuses_good.keys():
LOG.info(name)
LOG.info("Images that failed to build") if self.image_statuses_good:
LOG.info("===========================") LOG.info("Successfully built images")
for name, status in self.image_statuses_bad.iteritems(): LOG.info("=========================")
LOG.error('{}\r\t\t\t Failed with status: {}'.format( for name in self.image_statuses_good.keys():
name, status)) LOG.info(name)
LOG.debug("Not processed images") if self.image_statuses_bad:
LOG.debug("=========================") LOG.info("Images that failed to build")
for name in self.image_statuses_unproc.keys(): LOG.info("===========================")
LOG.debug(name) for name, status in self.image_statuses_bad.iteritems():
LOG.error('{}\r\t\t\t Failed with status: {}'.format(
name, status))
if self.image_statuses_unmatched:
LOG.debug("Images not matched for build by regex")
LOG.debug("=====================================")
for name in self.image_statuses_unmatched.keys():
LOG.debug(name)
def get_image_statuses(self): def get_image_statuses(self):
if len(self.image_statuses_bad) or len(self.image_statuses_good): if any(self.image_statuses_bad,
return (self.image_statuses_bad, self.image_statuses_good) self.image_statuses_good,
self.image_statuses_unmatched):
return (self.image_statuses_bad,
self.image_statuses_good,
self.image_statuses_unmatched)
for image in self.images: for image in self.images:
if image['status'] == "built": if image['status'] == "built":
self.image_statuses_good[image['name']] = image['status'] self.image_statuses_good[image['name']] = image['status']
elif image['status'] == "unprocessed": elif image['status'] == "unmatched":
self.image_statuses_unproc[image['name']] = image['status'] self.image_statuses_unmatched[image['name']] = image['status']
else: else:
self.image_statuses_bad[image['name']] = image['status'] self.image_statuses_bad[image['name']] = image['status']
return (self.image_statuses_bad, self.image_statuses_good) return (self.image_statuses_bad,
self.image_statuses_good,
self.image_statuses_unmatched)
def buildImageList(self): def build_image_list(self):
self.images = list() self.images = list()
# Walk all of the Dockerfiles and replace the %%KOLLA%% variables # Walk all of the Dockerfiles and replace the %%KOLLA%% variables
@ -432,10 +421,10 @@ class KollaWorker(object):
image['fullname'] = self.namespace + '/' + self.prefix + \ image['fullname'] = self.namespace + '/' + self.prefix + \
image['name'] + ':' + self.tag image['name'] + ':' + self.tag
image['path'] = path image['path'] = path
image['parent'] = content.split(' ')[1].split('\n')[0] image['parent_name'] = content.split(' ')[1].split('\n')[0]
if self.namespace not in image['parent_name']:
if self.namespace not in image['parent']:
image['parent'] = None image['parent'] = None
image['children'] = list()
if self.type_ == 'source': if self.type_ == 'source':
image['source'] = dict() image['source'] = dict()
@ -456,31 +445,43 @@ class KollaWorker(object):
self.images.append(image) self.images.append(image)
def buildQueues(self): def find_parents(self):
"""Associate all images with parents and children"""
sort_images = dict()
for image in self.images:
sort_images[image['fullname']] = image
for parent_name, parent in sort_images.iteritems():
for image in sort_images.values():
if image['parent_name'] == parent_name:
parent['children'].append(image)
image['parent'] = parent
def build_queue(self):
"""Organizes Queue list """Organizes Queue list
Return a list of Queues that have been organized into a hierarchy Return a list of Queues that have been organized into a hierarchy
based on dependencies based on dependencies
""" """
self.buildImageList() self.build_image_list()
self.sortImages() self.find_parents()
self.filter_images()
pools = list() queue = Queue.Queue()
for count, tier in enumerate(self.tiers):
pool = Queue.Queue()
for image in tier:
pool.put(image)
LOG.debug('Tier {}: add image {}'.format(count, image['name']))
pools.append(pool) for image in self.images:
if image['parent'] is None:
queue.put(image)
LOG.debug('Added image {} to queue'.format(image['name']))
return pools return queue
def push_image(image): def push_image(image):
dc = docker.Client(**docker.utils.kwargs_from_env()) dc = docker.Client(**docker.utils.kwargs_from_env())
image['push_logs'] = str() image['push_logs'] = str()
for response in dc.push(image['fullname'], for response in dc.push(image['fullname'],
stream=True, stream=True,
insecure_registry=True): insecure_registry=True):
@ -488,43 +489,42 @@ def push_image(image):
if 'stream' in stream: if 'stream' in stream:
image['push_logs'] = image['logs'] + stream['stream'] image['push_logs'] = image['logs'] + stream['stream']
# This is only single threaded for right now so we can show logs LOG.info('{}'.format(stream['stream']))
print(stream['stream'])
elif 'errorDetail' in stream: elif 'errorDetail' in stream:
image['status'] = "error" image['status'] = "error"
LOG.error(stream['errorDetail']['message']) LOG.error(stream['errorDetail']['message'])
def main(): def main():
args = argParser() args = arg_parser()
if args['debug']: if args['debug']:
LOG.setLevel(logging.DEBUG) LOG.setLevel(logging.DEBUG)
kolla = KollaWorker(args) kolla = KollaWorker(args)
kolla.setupWorkingDir() kolla.setup_working_dir()
kolla.findDockerfiles() kolla.find_dockerfiles()
if args['template']: if args['template']:
kolla.createDockerfiles() kolla.create_dockerfiles()
# We set the atime and mtime to 0 epoch to preserve allow the Docker cache # We set the atime and mtime to 0 epoch to preserve allow the Docker cache
# to work like we want. A different size or hash will still force a rebuild # to work like we want. A different size or hash will still force a rebuild
kolla.set_time() kolla.set_time()
pools = kolla.buildQueues() queue = kolla.build_queue()
# Returns a list of Queues for us to loop through for x in xrange(args['threads']):
for pool in pools: worker = WorkerThread(queue, args)
for x in xrange(args['threads']): worker.setDaemon(True)
WorkerThread(pool, args).start() worker.start()
# block until queue is empty
pool.join() # block until queue is empty
queue.join()
if args['push']: if args['push']:
for tier in kolla.tiers: for image in kolla.images:
for image in tier: if image['status'] == "built":
if image['status'] == "built": push_image(image)
push_image(image)
kolla.summary() kolla.summary()
kolla.cleanup() kolla.cleanup()

13
tests/test_build.py
View File

@ -35,7 +35,7 @@ class BuildTest(base.BaseTestCase):
def runTest(self): def runTest(self):
with patch.object(sys, 'argv', self.build_args): with patch.object(sys, 'argv', self.build_args):
LOG.info("Running with args %s" % self.build_args) LOG.info("Running with args %s" % self.build_args)
bad_results, good_results = build.main() bad_results, good_results, unmatched_results = build.main()
# these are images that are known to not build properly # these are images that are known to not build properly
excluded_images = ["gnocchi-api", excluded_images = ["gnocchi-api",
@ -56,6 +56,10 @@ class BuildTest(base.BaseTestCase):
failures = failures + 1 failures = failures + 1
LOG.critical(">>> Expected image '%s' to succeed!" % image) LOG.critical(">>> Expected image '%s' to succeed!" % image)
for image in unmatched_results.keys():
failures = failures + 1
LOG.critical(">>> Expected image '%s' to be matched!" % image)
self.assertEqual(failures, 0, "%d failure(s) occurred" % failures) self.assertEqual(failures, 0, "%d failure(s) occurred" % failures)
@ -73,13 +77,6 @@ class BuildTestCentosSourceDocker(BuildTest):
"--type", "source"]) "--type", "source"])
class BuildTestUbuntuSourceDocker(BuildTest):
def setUp(self):
super(BuildTestUbuntuSourceDocker, self).setUp()
self.build_args.extend(["--base", "ubuntu",
"--type", "source"])
class BuildTestCentosBinaryTemplate(BuildTest): class BuildTestCentosBinaryTemplate(BuildTest):
def setUp(self): def setUp(self):
super(BuildTestCentosBinaryTemplate, self).setUp() super(BuildTestCentosBinaryTemplate, self).setUp()

9
tox.ini
View File

@ -80,15 +80,6 @@ commands =
bash -c "if [ ! -d .testrepository ]; then testr init; fi" bash -c "if [ ! -d .testrepository ]; then testr init; fi"
sudo -g docker testr run test_build.BuildTestCentosSourceDocker sudo -g docker testr run test_build.BuildTestCentosSourceDocker
[testenv:images-ubuntu-source-docker]
whitelist_externals = find
bash
sudo
commands =
find . -type f -name "*.pyc" -delete
bash -c "if [ ! -d .testrepository ]; then testr init; fi"
sudo -g docker testr run test_build.BuildTestUbuntuSourceDocker
[testenv:images-centos-binary-template] [testenv:images-centos-binary-template]
whitelist_externals = find whitelist_externals = find
bash bash