diff --git a/doc/source/operation.rst b/doc/source/operation.rst index db21fc02a..bb4bfecb2 100644 --- a/doc/source/operation.rst +++ b/doc/source/operation.rst @@ -149,6 +149,11 @@ dib-image-list .. program-output:: nodepool dib-image-list --help :nostderr: +dib-request-list +^^^^^^^^^^^^^^^^ +.. program-output:: nodepool dib-request-list --help + :nostderr: + image-list ^^^^^^^^^^ .. program-output:: nodepool image-list --help @@ -391,6 +396,15 @@ launchers, all will provide the same information. :resheader Content-Type: ``application/json`` or ``text/plain`` depending on the :http:header:`Accept` header +.. http:get:: /dib-request-list + + The status of manual build requests + + :query fields: comma-separated list of fields to display + :reqheader Accept: ``application/json`` or ``text/*`` + :resheader Content-Type: ``application/json`` or ``text/plain`` + depending on the :http:header:`Accept` header + .. http:get:: /node-list The status of currently active nodes diff --git a/nodepool/cmd/nodepoolcmd.py b/nodepool/cmd/nodepoolcmd.py index 063d0d33d..b585d8fa6 100644 --- a/nodepool/cmd/nodepoolcmd.py +++ b/nodepool/cmd/nodepoolcmd.py @@ -61,6 +61,11 @@ class NodePoolCmd(NodepoolApp): help='list images built with diskimage-builder') cmd_dib_image_list.set_defaults(func=self.dib_image_list) + cmd_dib_request_list = subparsers.add_parser( + 'dib-request-list', + help='list image build requests') + cmd_dib_request_list.set_defaults(func=self.dib_request_list) + cmd_image_build = subparsers.add_parser( 'image-build', help='build image using diskimage-builder') @@ -203,6 +208,10 @@ class NodePoolCmd(NodepoolApp): results = status.dib_image_list(self.zk) print(status.output(results, 'pretty')) + def dib_request_list(self): + results = status.dib_request_list(self.zk) + print(status.output(results, 'pretty')) + def image_list(self): results = status.image_list(self.zk) print(status.output(results, 'pretty')) @@ -422,6 +431,7 @@ class NodePoolCmd(NodepoolApp): # commands needing ZooKeeper if self.args.command in ('image-build', 'dib-image-list', + 'dib-request-list', 'image-list', 'dib-image-delete', 'image-delete', 'alien-image-list', 'list', 'delete', diff --git a/nodepool/status.py b/nodepool/status.py index fd4b0420e..ba4712396 100644 --- a/nodepool/status.py +++ b/nodepool/status.py @@ -202,6 +202,25 @@ def dib_image_list(zk): return (objs, headers_table) +def dib_request_list(zk): + headers_table = OrderedDict([ + ("image", "Image"), + ("state", "State"), + ("age", "Age") + ]) + objs = [] + for image_name in zk.getImageNames(): + request = zk.getBuildRequest(image_name) + if request is None: + continue + objs.append({ + "image": request.image_name, + "state": "pending" if request.pending else "building", + "age": int(request.state_time) + }) + return (objs, headers_table) + + def image_list(zk): headers_table = OrderedDict([ ("id", "Build ID"), diff --git a/nodepool/tests/unit/test_commands.py b/nodepool/tests/unit/test_commands.py index a1bd8df56..d45505cea 100644 --- a/nodepool/tests/unit/test_commands.py +++ b/nodepool/tests/unit/test_commands.py @@ -193,6 +193,18 @@ class TestNodepoolCMD(tests.DBTestCase): nodepoolcmd.main() self.assert_listed(configfile, ['dib-image-list'], 4, zk.READY, 1) + def test_dib_request_list(self): + configfile = self.setup_config('node.yaml') + builder = self.useBuilder(configfile) + # Make sure we have enough time to test for the build request + # before it's processed by the build worker. + for worker in builder._build_workers: + worker._interval = 60 + self.waitForImage('fake-provider', 'fake-image') + self.zk.submitBuildRequest("fake-image") + self.assert_listed(configfile, ['dib-request-list'], + 0, 'fake-image', 1) + def test_dib_image_build_pause(self): configfile = self.setup_config('node_diskimage_pause.yaml') self.useBuilder(configfile) diff --git a/nodepool/tests/unit/test_webapp.py b/nodepool/tests/unit/test_webapp.py index 65dd2248c..dc41aa539 100644 --- a/nodepool/tests/unit/test_webapp.py +++ b/nodepool/tests/unit/test_webapp.py @@ -138,6 +138,47 @@ class TestWebApp(tests.DBTestCase): 'formats': ['qcow2'], 'state': 'ready'}, objs[0]) + def test_dib_request_list_json(self): + configfile = self.setup_config("node.yaml") + pool = self.useNodepool(configfile, watermark_sleep=1) + builder = self.useBuilder(configfile) + # Make sure we have enough time to test for the build request + # before it's processed by the build worker. + for worker in builder._build_workers: + worker._interval = 60 + + pool.start() + webapp = self.useWebApp(pool, port=0) + webapp.start() + port = webapp.server.socket.getsockname()[1] + + self.waitForImage("fake-provider", "fake-image") + self.waitForNodes('fake-label') + + self.zk.submitBuildRequest("fake-image") + + req = request.Request( + "http://localhost:{}/dib-request-list".format(port)) + req.add_header("Accept", "application/json") + + f = request.urlopen(req) + self.assertEqual(f.info().get("Content-Type"), + "application/json") + + data = f.read() + objs = json.loads(data.decode("utf8")) + self.assertDictContainsSubset({"image": "fake-image", + "state": "pending"}, objs[0]) + + webapp.cache.cache.clear() + with self.zk.imageBuildLock('fake-image', blocking=True, timeout=1): + f = request.urlopen(req) + data = f.read() + + objs = json.loads(data.decode("utf8")) + self.assertDictContainsSubset({"image": "fake-image", + "state": "building"}, objs[0]) + def test_node_list_json(self): configfile = self.setup_config('node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) diff --git a/nodepool/tests/unit/test_zk.py b/nodepool/tests/unit/test_zk.py index 69ede0eca..07e43877a 100644 --- a/nodepool/tests/unit/test_zk.py +++ b/nodepool/tests/unit/test_zk.py @@ -348,12 +348,17 @@ class TestZooKeeper(tests.DBTestCase): [upload_id]) def test_build_request(self): - '''Test the build request API methods (has/submit/remove)''' + '''Test the build request API methods (has/get/submit/remove)''' image = "ubuntu-trusty" self.zk.submitBuildRequest(image) self.assertTrue(self.zk.hasBuildRequest(image)) + build_request = self.zk.getBuildRequest(image) + self.assertEqual(build_request.image_name, image) + self.assertTrue(build_request.pending) self.zk.removeBuildRequest(image) self.assertFalse(self.zk.hasBuildRequest(image)) + build_request = self.zk.getBuildRequest(image) + self.assertIsNone(build_request) def test_buildLock_orphan(self): image = "ubuntu-trusty" diff --git a/nodepool/webapp.py b/nodepool/webapp.py index 49c7a88e7..0a505632e 100644 --- a/nodepool/webapp.py +++ b/nodepool/webapp.py @@ -101,6 +101,8 @@ class WebApp(threading.Thread): results = status.image_list(zk) elif path == '/dib-image-list': results = status.dib_image_list(zk) + elif path == '/dib-request-list': + results = status.dib_request_list(zk) elif path == '/node-list': results = status.node_list(zk, node_id=params.get('node_id')) diff --git a/nodepool/zk/zookeeper.py b/nodepool/zk/zookeeper.py index e8d791123..38d8cf08c 100644 --- a/nodepool/zk/zookeeper.py +++ b/nodepool/zk/zookeeper.py @@ -251,6 +251,21 @@ class ImageBuild(BaseModel): return o +class ImageBuildRequest(object): + """Class representing a manual build request. + + This doesn't need to derive from BaseModel since this class exists only + to aggregate information about a build request. + """ + def __init__(self, image_name, pending, state_time): + self.image_name = image_name + self.state_time = state_time + self.pending = pending + + def __repr__(self): + return "".format(self.image_name) + + class ImageUpload(BaseModel): ''' Class representing a provider image upload within the ZooKeeper cluster. @@ -736,12 +751,14 @@ class ZooKeeper(object): def _imagePausePath(self, image): return "%s/pause" % self._imagePath(image) + def _imageBuildNumberPath(self, image, build_number): + return "%s/%s" % (self._imageBuildsPath(image), build_number) + def _imageBuildLockPath(self, image): return "%s/lock" % self._imageBuildsPath(image) def _imageBuildNumberLockPath(self, image, build_number): - return "%s/%s/lock" % (self._imageBuildsPath(image), - build_number) + return "%s/lock" % self._imageBuildNumberPath(image, build_number) def _imageProviderPath(self, image, build_number): return "%s/%s/providers" % (self._imageBuildsPath(image), @@ -1474,6 +1491,42 @@ class ZooKeeper(object): return True return False + def _latestImageBuildStat(self, image): + builds = self.getBuildNumbers(image) + if not builds: + return + + latest_build, *_ = builds + builds_path = self._imageBuildNumberPath(image, latest_build) + return self.client.exists(builds_path) + + def getBuildRequest(self, image): + """Get a build request for the given image. + + :param str image: The image name to check. + + :returns: An ImagebuildRequest object, or None if not found + """ + path = self._imageBuildRequestPath(image) + try: + _, stat = self.client.get(path) + except kze.NoNodeError: + return + + pending = True + lock_path = self._imageBuildLockPath(image) + lock_stat = self.client.exists(lock_path) + if lock_stat and lock_stat.children_count: + build_stat = self._latestImageBuildStat(image) + # If there is a lock, but no build we assume that the build + # will was not yet created. + pending = ( + build_stat is None or + build_stat.created < lock_stat.created + ) + + return ImageBuildRequest(image, pending, stat.created) + def submitBuildRequest(self, image): ''' Submit a request for a new image build.