Merge "Expose image build requests in web UI and cli"

doc/source/operation.rst

@@ -149,6 +149,11 @@ dib-image-list
 .. program-output:: nodepool dib-image-list --help
    :nostderr:
 
+dib-request-list
+^^^^^^^^^^^^^^^^
+.. program-output:: nodepool dib-request-list --help
+   :nostderr:
+
 image-list
 ^^^^^^^^^^
 .. program-output:: nodepool image-list --help
@@ -391,6 +396,15 @@ launchers, all will provide the same information.
    :resheader Content-Type: ``application/json`` or ``text/plain``
                             depending on the :http:header:`Accept` header
 
+.. http:get:: /dib-request-list
+
+   The status of manual build requests
+
+   :query fields: comma-separated list of fields to display
+   :reqheader Accept: ``application/json`` or ``text/*``
+   :resheader Content-Type: ``application/json`` or ``text/plain``
+                            depending on the :http:header:`Accept` header
+
 .. http:get:: /node-list
 
    The status of currently active nodes

nodepool/cmd/nodepoolcmd.py

@@ -61,6 +61,11 @@ class NodePoolCmd(NodepoolApp):
             help='list images built with diskimage-builder')
         cmd_dib_image_list.set_defaults(func=self.dib_image_list)
 
+        cmd_dib_request_list = subparsers.add_parser(
+            'dib-request-list',
+            help='list image build requests')
+        cmd_dib_request_list.set_defaults(func=self.dib_request_list)
+
         cmd_image_build = subparsers.add_parser(
             'image-build',
             help='build image using diskimage-builder')
@@ -203,6 +208,10 @@ class NodePoolCmd(NodepoolApp):
         results = status.dib_image_list(self.zk)
         print(status.output(results, 'pretty'))
 
+    def dib_request_list(self):
+        results = status.dib_request_list(self.zk)
+        print(status.output(results, 'pretty'))
+
     def image_list(self):
         results = status.image_list(self.zk)
         print(status.output(results, 'pretty'))
@@ -422,6 +431,7 @@ class NodePoolCmd(NodepoolApp):
 
         # commands needing ZooKeeper
         if self.args.command in ('image-build', 'dib-image-list',
+                                 'dib-request-list',
                                  'image-list', 'dib-image-delete',
                                  'image-delete', 'alien-image-list',
                                  'list', 'delete',

nodepool/status.py

@@ -202,6 +202,25 @@ def dib_image_list(zk):
     return (objs, headers_table)
 
 
+def dib_request_list(zk):
+    headers_table = OrderedDict([
+        ("image", "Image"),
+        ("state", "State"),
+        ("age", "Age")
+    ])
+    objs = []
+    for image_name in zk.getImageNames():
+        request = zk.getBuildRequest(image_name)
+        if request is None:
+            continue
+        objs.append({
+            "image": request.image_name,
+            "state": "pending" if request.pending else "building",
+            "age": int(request.state_time)
+        })
+    return (objs, headers_table)
+
+
 def image_list(zk):
     headers_table = OrderedDict([
         ("id", "Build ID"),

nodepool/tests/unit/test_commands.py

@@ -193,6 +193,18 @@ class TestNodepoolCMD(tests.DBTestCase):
         nodepoolcmd.main()
         self.assert_listed(configfile, ['dib-image-list'], 4, zk.READY, 1)
 
+    def test_dib_request_list(self):
+        configfile = self.setup_config('node.yaml')
+        builder = self.useBuilder(configfile)
+        # Make sure we have enough time to test for the build request
+        # before it's processed by the build worker.
+        for worker in builder._build_workers:
+            worker._interval = 60
+        self.waitForImage('fake-provider', 'fake-image')
+        self.zk.submitBuildRequest("fake-image")
+        self.assert_listed(configfile, ['dib-request-list'],
+                           0, 'fake-image', 1)
+
     def test_dib_image_build_pause(self):
         configfile = self.setup_config('node_diskimage_pause.yaml')
         self.useBuilder(configfile)

nodepool/tests/unit/test_webapp.py

@@ -138,6 +138,47 @@ class TestWebApp(tests.DBTestCase):
                                        'formats': ['qcow2'],
                                        'state': 'ready'}, objs[0])
 
+    def test_dib_request_list_json(self):
+        configfile = self.setup_config("node.yaml")
+        pool = self.useNodepool(configfile, watermark_sleep=1)
+        builder = self.useBuilder(configfile)
+        # Make sure we have enough time to test for the build request
+        # before it's processed by the build worker.
+        for worker in builder._build_workers:
+            worker._interval = 60
+
+        pool.start()
+        webapp = self.useWebApp(pool, port=0)
+        webapp.start()
+        port = webapp.server.socket.getsockname()[1]
+
+        self.waitForImage("fake-provider", "fake-image")
+        self.waitForNodes('fake-label')
+
+        self.zk.submitBuildRequest("fake-image")
+
+        req = request.Request(
+            "http://localhost:{}/dib-request-list".format(port))
+        req.add_header("Accept", "application/json")
+
+        f = request.urlopen(req)
+        self.assertEqual(f.info().get("Content-Type"),
+                         "application/json")
+
+        data = f.read()
+        objs = json.loads(data.decode("utf8"))
+        self.assertDictContainsSubset({"image": "fake-image",
+                                       "state": "pending"}, objs[0])
+
+        webapp.cache.cache.clear()
+        with self.zk.imageBuildLock('fake-image', blocking=True, timeout=1):
+            f = request.urlopen(req)
+            data = f.read()
+
+        objs = json.loads(data.decode("utf8"))
+        self.assertDictContainsSubset({"image": "fake-image",
+                                       "state": "building"}, objs[0])
+
     def test_node_list_json(self):
         configfile = self.setup_config('node.yaml')
         pool = self.useNodepool(configfile, watermark_sleep=1)

nodepool/tests/unit/test_zk.py

@@ -348,12 +348,17 @@ class TestZooKeeper(tests.DBTestCase):
                          [upload_id])
 
     def test_build_request(self):
-        '''Test the build request API methods (has/submit/remove)'''
+        '''Test the build request API methods (has/get/submit/remove)'''
         image = "ubuntu-trusty"
         self.zk.submitBuildRequest(image)
         self.assertTrue(self.zk.hasBuildRequest(image))
+        build_request = self.zk.getBuildRequest(image)
+        self.assertEqual(build_request.image_name, image)
+        self.assertTrue(build_request.pending)
         self.zk.removeBuildRequest(image)
         self.assertFalse(self.zk.hasBuildRequest(image))
+        build_request = self.zk.getBuildRequest(image)
+        self.assertIsNone(build_request)
 
     def test_buildLock_orphan(self):
         image = "ubuntu-trusty"

nodepool/webapp.py

@@ -101,6 +101,8 @@ class WebApp(threading.Thread):
             results = status.image_list(zk)
         elif path == '/dib-image-list':
             results = status.dib_image_list(zk)
+        elif path == '/dib-request-list':
+            results = status.dib_request_list(zk)
         elif path == '/node-list':
             results = status.node_list(zk,
                                        node_id=params.get('node_id'))

nodepool/zk/zookeeper.py

@@ -251,6 +251,21 @@ class ImageBuild(BaseModel):
         return o
 
 
+class ImageBuildRequest(object):
+    """Class representing a manual build request.
+
+    This doesn't need to derive from BaseModel since this class exists only
+    to aggregate information about a build request.
+    """
+    def __init__(self, image_name, pending, state_time):
+        self.image_name = image_name
+        self.state_time = state_time
+        self.pending = pending
+
+    def __repr__(self):
+        return "<ImageBuildRequest {}>".format(self.image_name)
+
+
 class ImageUpload(BaseModel):
     '''
     Class representing a provider image upload within the ZooKeeper cluster.
@@ -736,12 +751,14 @@ class ZooKeeper(object):
     def _imagePausePath(self, image):
         return "%s/pause" % self._imagePath(image)
 
+    def _imageBuildNumberPath(self, image, build_number):
+        return "%s/%s" % (self._imageBuildsPath(image), build_number)
+
     def _imageBuildLockPath(self, image):
         return "%s/lock" % self._imageBuildsPath(image)
 
     def _imageBuildNumberLockPath(self, image, build_number):
-        return "%s/%s/lock" % (self._imageBuildsPath(image),
-                               build_number)
+        return "%s/lock" % self._imageBuildNumberPath(image, build_number)
 
     def _imageProviderPath(self, image, build_number):
         return "%s/%s/providers" % (self._imageBuildsPath(image),
@@ -1474,6 +1491,42 @@ class ZooKeeper(object):
             return True
         return False
 
+    def _latestImageBuildStat(self, image):
+        builds = self.getBuildNumbers(image)
+        if not builds:
+            return
+
+        latest_build, *_ = builds
+        builds_path = self._imageBuildNumberPath(image, latest_build)
+        return self.client.exists(builds_path)
+
+    def getBuildRequest(self, image):
+        """Get a build request for the given image.
+
+        :param str image: The image name to check.
+
+        :returns: An ImagebuildRequest object, or None if not found
+        """
+        path = self._imageBuildRequestPath(image)
+        try:
+            _, stat = self.client.get(path)
+        except kze.NoNodeError:
+            return
+
+        pending = True
+        lock_path = self._imageBuildLockPath(image)
+        lock_stat = self.client.exists(lock_path)
+        if lock_stat and lock_stat.children_count:
+            build_stat = self._latestImageBuildStat(image)
+            # If there is a lock, but no build we assume that the build
+            # will was not yet created.
+            pending = (
+                build_stat is None or
+                build_stat.created < lock_stat.created
+            )
+
+        return ImageBuildRequest(image, pending, stat.created)
+
     def submitBuildRequest(self, image):
         '''
         Submit a request for a new image build.