Merge "Use iterate_timeout in test waits"
This commit is contained in:
commit
9036dd7159
@ -29,13 +29,13 @@ log = logging.getLogger("nodepool.utils")
|
|||||||
ITERATE_INTERVAL = 2
|
ITERATE_INTERVAL = 2
|
||||||
|
|
||||||
|
|
||||||
def iterate_timeout(max_seconds, exc, purpose):
|
def iterate_timeout(max_seconds, exc, purpose, interval=ITERATE_INTERVAL):
|
||||||
start = time.time()
|
start = time.time()
|
||||||
count = 0
|
count = 0
|
||||||
while (time.time() < start + max_seconds):
|
while (time.time() < start + max_seconds):
|
||||||
count += 1
|
count += 1
|
||||||
yield count
|
yield count
|
||||||
time.sleep(ITERATE_INTERVAL)
|
time.sleep(interval)
|
||||||
raise exc("Timeout waiting for %s" % purpose)
|
raise exc("Timeout waiting for %s" % purpose)
|
||||||
|
|
||||||
|
|
||||||
|
@ -38,8 +38,11 @@ from nodepool import launcher
|
|||||||
from nodepool import webapp
|
from nodepool import webapp
|
||||||
from nodepool import zk
|
from nodepool import zk
|
||||||
from nodepool.cmd.config_validator import ConfigValidator
|
from nodepool.cmd.config_validator import ConfigValidator
|
||||||
|
from nodepool.nodeutils import iterate_timeout
|
||||||
|
|
||||||
TRUE_VALUES = ('true', '1', 'yes')
|
TRUE_VALUES = ('true', '1', 'yes')
|
||||||
|
SECOND = 1
|
||||||
|
ONE_MINUTE = 60 * SECOND
|
||||||
|
|
||||||
|
|
||||||
class LoggingPopen(subprocess.Popen):
|
class LoggingPopen(subprocess.Popen):
|
||||||
@ -224,7 +227,9 @@ class BaseTestCase(testtools.TestCase):
|
|||||||
'pydevd.Writer',
|
'pydevd.Writer',
|
||||||
]
|
]
|
||||||
|
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"Transient threads to finish",
|
||||||
|
interval=0.1):
|
||||||
done = True
|
done = True
|
||||||
for t in threading.enumerate():
|
for t in threading.enumerate():
|
||||||
if t.name.startswith("Thread-"):
|
if t.name.startswith("Thread-"):
|
||||||
@ -245,7 +250,6 @@ class BaseTestCase(testtools.TestCase):
|
|||||||
done = False
|
done = False
|
||||||
if done:
|
if done:
|
||||||
return
|
return
|
||||||
time.sleep(0.1)
|
|
||||||
|
|
||||||
def assertReportedStat(self, key, value=None, kind=None):
|
def assertReportedStat(self, key, value=None, kind=None):
|
||||||
"""Check statsd output
|
"""Check statsd output
|
||||||
@ -268,8 +272,9 @@ class BaseTestCase(testtools.TestCase):
|
|||||||
if value:
|
if value:
|
||||||
self.assertNotEqual(kind, None)
|
self.assertNotEqual(kind, None)
|
||||||
|
|
||||||
start = time.time()
|
for _ in iterate_timeout(5 * SECOND, Exception,
|
||||||
while time.time() < (start + 5):
|
"Find statsd event",
|
||||||
|
interval=0.1):
|
||||||
# Note our fake statsd just queues up results in a queue.
|
# Note our fake statsd just queues up results in a queue.
|
||||||
# We just keep going through them until we find one that
|
# We just keep going through them until we find one that
|
||||||
# matches, or fail out. If statsd pipelines are used,
|
# matches, or fail out. If statsd pipelines are used,
|
||||||
@ -305,7 +310,6 @@ class BaseTestCase(testtools.TestCase):
|
|||||||
|
|
||||||
# this key matches
|
# this key matches
|
||||||
return True
|
return True
|
||||||
time.sleep(0.1)
|
|
||||||
|
|
||||||
raise Exception("Key %s not found in reported stats" % key)
|
raise Exception("Key %s not found in reported stats" % key)
|
||||||
|
|
||||||
@ -407,7 +411,9 @@ class DBTestCase(BaseTestCase):
|
|||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
|
|
||||||
def waitForImage(self, provider_name, image_name, ignore_list=None):
|
def waitForImage(self, provider_name, image_name, ignore_list=None):
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"Image upload to be ready",
|
||||||
|
interval=1):
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
image = self.zk.getMostRecentImageUpload(image_name, provider_name)
|
image = self.zk.getMostRecentImageUpload(image_name, provider_name)
|
||||||
if image:
|
if image:
|
||||||
@ -415,27 +421,28 @@ class DBTestCase(BaseTestCase):
|
|||||||
break
|
break
|
||||||
elif not ignore_list:
|
elif not ignore_list:
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
return image
|
return image
|
||||||
|
|
||||||
def waitForUploadRecordDeletion(self, provider_name, image_name,
|
def waitForUploadRecordDeletion(self, provider_name, image_name,
|
||||||
build_id, upload_id):
|
build_id, upload_id):
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"Image upload record deletion",
|
||||||
|
interval=1):
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
uploads = self.zk.getUploads(image_name, build_id, provider_name)
|
uploads = self.zk.getUploads(image_name, build_id, provider_name)
|
||||||
if not uploads or upload_id not in [u.id for u in uploads]:
|
if not uploads or upload_id not in [u.id for u in uploads]:
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
|
|
||||||
def waitForImageDeletion(self, provider_name, image_name, match=None):
|
def waitForImageDeletion(self, provider_name, image_name, match=None):
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"Image upload deletion",
|
||||||
|
interval=1):
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
image = self.zk.getMostRecentImageUpload(image_name, provider_name)
|
image = self.zk.getMostRecentImageUpload(image_name, provider_name)
|
||||||
if not image or (match and image != match):
|
if not image or (match and image != match):
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
|
|
||||||
def waitForBuild(self, image_name, build_id, states=None):
|
def waitForBuild(self, image_name, build_id, states=None):
|
||||||
@ -444,12 +451,13 @@ class DBTestCase(BaseTestCase):
|
|||||||
|
|
||||||
base = "-".join([image_name, build_id])
|
base = "-".join([image_name, build_id])
|
||||||
|
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"Image build record to reach state",
|
||||||
|
interval=1):
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
build = self.zk.getBuild(image_name, build_id)
|
build = self.zk.getBuild(image_name, build_id)
|
||||||
if build and build.state in states:
|
if build and build.state in states:
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
# We should only expect a dib manifest with a successful build.
|
# We should only expect a dib manifest with a successful build.
|
||||||
while build.state == zk.READY:
|
while build.state == zk.READY:
|
||||||
@ -465,34 +473,39 @@ class DBTestCase(BaseTestCase):
|
|||||||
|
|
||||||
def waitForBuildDeletion(self, image_name, build_id):
|
def waitForBuildDeletion(self, image_name, build_id):
|
||||||
base = "-".join([image_name, build_id])
|
base = "-".join([image_name, build_id])
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"DIB build files deletion",
|
||||||
|
interval=1):
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
files = builder.DibImageFile.from_image_id(
|
files = builder.DibImageFile.from_image_id(
|
||||||
self._config_images_dir.path, base)
|
self._config_images_dir.path, base)
|
||||||
if not files:
|
if not files:
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"DIB build file deletion leaks",
|
||||||
|
interval=1):
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
# Now, check the disk to ensure we didn't leak any files.
|
# Now, check the disk to ensure we didn't leak any files.
|
||||||
matches = glob.glob('%s/%s.*' % (self._config_images_dir.path,
|
matches = glob.glob('%s/%s.*' % (self._config_images_dir.path,
|
||||||
base))
|
base))
|
||||||
if not matches:
|
if not matches:
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"Image build record deletion",
|
||||||
|
interval=1):
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
build = self.zk.getBuild(image_name, build_id)
|
build = self.zk.getBuild(image_name, build_id)
|
||||||
if not build:
|
if not build:
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
|
|
||||||
def waitForNodeDeletion(self, node):
|
def waitForNodeDeletion(self, node):
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"Node record deletion",
|
||||||
|
interval=1):
|
||||||
exists = False
|
exists = False
|
||||||
for n in self.zk.nodeIterator():
|
for n in self.zk.nodeIterator():
|
||||||
if node.id == n.id:
|
if node.id == n.id:
|
||||||
@ -500,17 +513,19 @@ class DBTestCase(BaseTestCase):
|
|||||||
break
|
break
|
||||||
if not exists:
|
if not exists:
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
def waitForInstanceDeletion(self, manager, instance_id):
|
def waitForInstanceDeletion(self, manager, instance_id):
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"Cloud instance deletion",
|
||||||
|
interval=1):
|
||||||
servers = manager.listNodes()
|
servers = manager.listNodes()
|
||||||
if not (instance_id in [s.id for s in servers]):
|
if not (instance_id in [s.id for s in servers]):
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
def waitForNodeRequestLockDeletion(self, request_id):
|
def waitForNodeRequestLockDeletion(self, request_id):
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"Node request lock deletion",
|
||||||
|
interval=1):
|
||||||
exists = False
|
exists = False
|
||||||
for lock_id in self.zk.getNodeRequestLockIDs():
|
for lock_id in self.zk.getNodeRequestLockIDs():
|
||||||
if request_id == lock_id:
|
if request_id == lock_id:
|
||||||
@ -518,15 +533,15 @@ class DBTestCase(BaseTestCase):
|
|||||||
break
|
break
|
||||||
if not exists:
|
if not exists:
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
def waitForNodes(self, label, count=1):
|
def waitForNodes(self, label, count=1):
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"Ready nodes",
|
||||||
|
interval=1):
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
ready_nodes = self.zk.getReadyNodesOfTypes([label])
|
ready_nodes = self.zk.getReadyNodesOfTypes([label])
|
||||||
if label in ready_nodes and len(ready_nodes[label]) == count:
|
if label in ready_nodes and len(ready_nodes[label]) == count:
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
self.wait_for_threads()
|
self.wait_for_threads()
|
||||||
return ready_nodes[label]
|
return ready_nodes[label]
|
||||||
|
|
||||||
@ -536,11 +551,12 @@ class DBTestCase(BaseTestCase):
|
|||||||
'''
|
'''
|
||||||
if states is None:
|
if states is None:
|
||||||
states = (zk.FULFILLED, zk.FAILED)
|
states = (zk.FULFILLED, zk.FAILED)
|
||||||
while True:
|
for _ in iterate_timeout(ONE_MINUTE, Exception,
|
||||||
|
"Node request state transition",
|
||||||
|
interval=1):
|
||||||
req = self.zk.getNodeRequest(req.id)
|
req = self.zk.getNodeRequest(req.id)
|
||||||
if req.state in states:
|
if req.state in states:
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
return req
|
return req
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user