zuul/nodepool
Code Issues Proposed changes

Merge "Load diskimage configs before building them"

Browse Source
This commit is contained in:
Zuul 2020-09-21 16:09:04 +00:00 committed by Gerrit Code Review
commit e0e4d08a87
2 changed files with 37 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
nodepool/builder.py
View File

@ -147,6 +147,12 @@ class BaseWorker(threading.Thread):
self.log.debug("Detected ZooKeeper server changes")
self._zk.resetHosts(list(new_config.zookeeper_servers.values()))
def _readConfig(self):
new_config = nodepool_config.loadConfig(self._config_path)
if self._secure_path:
nodepool_config.loadSecureConfig(new_config, self._secure_path)
return new_config
@property
def running(self):
return self._running
@ -565,9 +571,7 @@ class CleanupWorker(BaseWorker):
'''
Body of run method for exception handling purposes.
'''
new_config = nodepool_config.loadConfig(self._config_path)
if self._secure_path:
nodepool_config.loadSecureConfig(new_config, self._secure_path)
new_config = self._readConfig()
if not self._config:
self._config = new_config
@ -623,6 +627,11 @@ class BuildWorker(BaseWorker):
if not self._running or self._zk.suspended or self._zk.lost:
return
try:
new_config = self._readConfig()
if new_config != self._config:
# If our config isn't up to date then return and start
# over with a new config load.
return
self._checkImageForScheduledImageUpdates(diskimage)
except Exception:
self.log.exception("Exception checking for scheduled "
@ -730,6 +739,11 @@ class BuildWorker(BaseWorker):
if not self._running or self._zk.suspended or self._zk.lost:
return
try:
new_config = self._readConfig()
if new_config != self._config:
# If our config isn't up to date then return and start
# over with a new config load.
return
self._checkImageForManualBuildRequest(diskimage)
except Exception:
self.log.exception("Exception checking for manual "
@ -998,9 +1012,7 @@ class BuildWorker(BaseWorker):
Body of run method for exception handling purposes.
'''
# NOTE: For the first iteration, we expect self._config to be None
new_config = nodepool_config.loadConfig(self._config_path)
if self._secure_path:
nodepool_config.loadSecureConfig(new_config, self._secure_path)
new_config = self._readConfig()
if not self._config:
self._config = new_config
@ -1022,9 +1034,7 @@ class UploadWorker(BaseWorker):
'''
Reload the nodepool configuration file.
'''
new_config = nodepool_config.loadConfig(self._config_path)
if self._secure_path:
nodepool_config.loadSecureConfig(new_config, self._secure_path)
new_config = self._readConfig()
if not self._config:
self._config = new_config

21
nodepool/tests/unit/test_builder.py
View File

@ -437,12 +437,27 @@ class TestNodePoolBuilder(tests.DBTestCase):
def test_cleanup_failed_image_build(self):
configfile = self.setup_config('node_diskimage_fail.yaml')
self.useBuilder(configfile)
# NOTE(pabelanger): We are racing here, but don't really care. We just
# need our first image build to fail.
# Wait for the build to fail before we replace our config. Otherwise
# we may replace the config before we build the image.
found = False
while not found:
builds = self.zk.getBuilds('fake-image')
for build in builds:
# Lexicographical order
if build and build.id > '0000000001':
# We know we've built more than one image and we know
# they have all failed. We can't check if they have
# failed directly because they may be cleaned up.
found = build.id
break
time.sleep(0.1)
# Now replace the config with a valid config and check that the image
# builds successfully. Finally check that the failed image is gone.
self.replace_config(configfile, 'node.yaml')
self.waitForImage('fake-provider', 'fake-image')
# Make sure our cleanup worker properly removes the first build.
self.waitForBuildDeletion('fake-image', '0000000001')
self.waitForBuildDeletion('fake-image', found)
self.assertReportedStat('nodepool.dib_image_build.'
'fake-image.status.rc',
'127', 'g')