Line-by-line dib logs

Log each line of output from DIB as it is read, and log it to the
image log file instead of the standard one.

Also, update the fake-dib system so that it exercises more code.
Fix the filename check for fake-dib as the fake filename key
actually ends up in the middle of the filename string.

Add a fake-dib config file.

Change-Id: I1e43c26ac0f9db4cf4e52f77b2cf5ca59f19989c

nodepool/nodepool.py

@@ -719,9 +719,6 @@ class DiskImageBuilder(threading.Thread):
             self.queue.task_done()
 
     def _buildImage(self, image, image_name, filename):
-        if filename.startswith('./fake-dib-image'):
-            return True
-
         try:
             env = os.environ.copy()
 
@@ -744,6 +741,13 @@ class DiskImageBuilder(threading.Thread):
 
             cmd = ('disk-image-create -x --no-tmpfs %s -o %s %s' %
                    (extra_options, out_file_path, img_elements))
+
+            if 'fake-dib-image' in filename:
+                cmd = 'echo ' + cmd
+
+            log = logging.getLogger("nodepool.image.build.%s" %
+                                    (image_name,))
+
             self.log.info('Running %s' % cmd)
 
             p = subprocess.Popen(
@@ -751,13 +755,14 @@ class DiskImageBuilder(threading.Thread):
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 env=env)
-            (stdout, stderr) = p.communicate()
+
-            if self.log:
+            while True:
-                self.log.info(stdout)
+                ln = p.stdout.readline()
-            if stderr:
+                log.info(ln.strip())
-                for line in stderr:
+                if not ln:
-                    if self.log:
+                    break
-                        self.log.error(line.rstrip())
+
+            p.wait()
             ret = p.returncode
             if ret:
                 raise Exception("Unable to create %s" % filename)
@@ -1694,8 +1699,7 @@ class NodePool(threading.Thread):
                         # to rebuild and delete this buggy image
                         if (dib_image.state == nodedb.READY and
                                 not os.path.exists(dib_image.filename) and
-                                not dib_image.filename.startswith(
+                                not 'fake-dib-image' in dib_image.filename):
-                                    './fake-dib-image')):
                             self.log.warning("Image filename %s does not "
                                              "exist. Removing image" %
                                              dib_image.filename)

tools/fake-dib.yaml

@@ -0,0 +1,54 @@
+script-dir: .
+elements-dir: .
+images-dir: /tmp/nodepool_dib
+dburi: 'mysql://nodepool@localhost/nodepool'
+
+cron:
+  check: '*/15 * * * *'
+  cleanup: '*/1 * * * *'
+  image-update: '14 2 * * *'
+
+zmq-publishers:
+  - tcp://localhost:8881
+
+gearman-servers:
+  - host: localhost
+
+diskimages:
+  - name: fake-dib-image
+    elements:
+      - ubuntu
+      - vm
+      - openstack-repos
+      - puppet
+      - nodepool-base
+      - node-devstack
+    release: trusty
+
+labels:
+  - name: fake-dib-image
+    image: fake-dib-image
+    min-ready: 2
+    providers:
+      - name: fake-provider
+
+providers:
+  - name: fake-provider
+    keypair: 'if-present-use-this-keypair'
+    region-name: 'fake-region'
+    username: 'fake'
+    password: 'fake'
+    auth-url: 'fake'
+    project-id: 'fake'
+    max-servers: 96
+    images:
+      - name: fake-dib-image
+        min-ram: 8192
+        diskimage: fake-dib-image
+
+targets:
+  - name: fake-jenkins
+    jenkins:
+      url: https://jenkins.example.org/
+      user: fake
+      apikey: fake