Add nodepool.image_build_requests metric

This reports the number of outstanding manual image build requests.

Change-Id: I365516bdb1fa20a3129099a81825e8506b3af4df

doc/source/operation.rst

@@ -482,6 +482,13 @@ The following metrics are produced by a ``nodepool-builder`` process:
    Number of image uploads to a specific provider in the cloud plus the time in
    seconds spent to upload the image.
 
+.. zuul:stat:: nodepool.image_build_requests
+   :type: gauge
+
+   Number of manual build requests outstanding (does not include
+   currently running builds).
+
+
 Nodepool launcher
 ~~~~~~~~~~~~~~~~~
 

nodepool/builder.py

@@ -377,6 +377,24 @@ class CleanupWorker(BaseWorker):
             except Exception:
                 self.log.exception("Exception cleaning up image %s:", image)
 
+    def _emitBuildRequestStats(self):
+        '''Emit general build request stats
+
+        This runs in the cleanup worker because it's the least likely
+        thread to be stopped for hours on end.
+        '''
+        if not self._statsd:
+            return
+        count = 0
+        for image_name in self._zk.getImageNames():
+            request = self._zk.getBuildRequest(image_name)
+            if request and request.pending:
+                count += 1
+        pipeline = self._statsd.pipeline()
+        key = 'nodepool.image_build_requests'
+        pipeline.gauge(key, count)
+        pipeline.send()
+
     def _filterLocalBuilds(self, image, builds):
         '''Return the subset of builds that are local'''
         ret = []
@@ -569,6 +587,7 @@ class CleanupWorker(BaseWorker):
         self._config = new_config
 
         self._cleanup()
+        self._emitBuildRequestStats()
 
 
 class BuildWorker(BaseWorker):

nodepool/tests/unit/test_webapp.py

@@ -179,6 +179,9 @@ class TestWebApp(tests.DBTestCase):
                                        "paused": False,
                                        "build_request": "pending"}, objs[0])
 
+        builder._janitor._emitBuildRequestStats()
+        self.assertReportedStat('nodepool.image_build_requests', '1', 'g')
+
         webapp.cache.cache.clear()
         with self.zk.imageBuildLock('fake-image', blocking=True, timeout=1):
             f = request.urlopen(req)

releasenotes/notes/build-requests-stats-789188316aa9831a.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    A new statsd metric, ``nodepool.image_build_requests`` is
+    available.  It reports the number of outstanding manual image
+    build requests.