From c46a53141ea3615812785e1488b581b4c1695968 Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jeblair@hp.com>
Date: Tue, 23 Sep 2014 11:13:13 -0700
Subject: [PATCH] Line-by-line dib logs

Log each line of output from DIB as it is read, and log it to the
image log file instead of the standard one.

Also, update the fake-dib system so that it exercises more code.
Fix the filename check for fake-dib as the fake filename key
actually ends up in the middle of the filename string.

Add a fake-dib config file.

Change-Id: I1e43c26ac0f9db4cf4e52f77b2cf5ca59f19989c
---
 nodepool/nodepool.py | 28 +++++++++++++----------
 tools/fake-dib.yaml  | 54 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+), 12 deletions(-)
 create mode 100644 tools/fake-dib.yaml

diff --git a/nodepool/nodepool.py b/nodepool/nodepool.py
index a083f0ca4..574dc6f4a 100644
--- a/nodepool/nodepool.py
+++ b/nodepool/nodepool.py
@@ -719,9 +719,6 @@ class DiskImageBuilder(threading.Thread):
             self.queue.task_done()
 
     def _buildImage(self, image, image_name, filename):
-        if filename.startswith('./fake-dib-image'):
-            return True
-
         try:
             env = os.environ.copy()
 
@@ -744,6 +741,13 @@ class DiskImageBuilder(threading.Thread):
 
             cmd = ('disk-image-create -x --no-tmpfs %s -o %s %s' %
                    (extra_options, out_file_path, img_elements))
+
+            if 'fake-dib-image' in filename:
+                cmd = 'echo ' + cmd
+
+            log = logging.getLogger("nodepool.image.build.%s" %
+                                    (image_name,))
+
             self.log.info('Running %s' % cmd)
 
             p = subprocess.Popen(
@@ -751,13 +755,14 @@ class DiskImageBuilder(threading.Thread):
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 env=env)
-            (stdout, stderr) = p.communicate()
-            if self.log:
-                self.log.info(stdout)
-            if stderr:
-                for line in stderr:
-                    if self.log:
-                        self.log.error(line.rstrip())
+
+            while True:
+                ln = p.stdout.readline()
+                log.info(ln.strip())
+                if not ln:
+                    break
+
+            p.wait()
             ret = p.returncode
             if ret:
                 raise Exception("Unable to create %s" % filename)
@@ -1694,8 +1699,7 @@ class NodePool(threading.Thread):
                         # to rebuild and delete this buggy image
                         if (dib_image.state == nodedb.READY and
                                 not os.path.exists(dib_image.filename) and
-                                not dib_image.filename.startswith(
-                                    './fake-dib-image')):
+                                not 'fake-dib-image' in dib_image.filename):
                             self.log.warning("Image filename %s does not "
                                              "exist. Removing image" %
                                              dib_image.filename)
diff --git a/tools/fake-dib.yaml b/tools/fake-dib.yaml
new file mode 100644
index 000000000..6c7059d82
--- /dev/null
+++ b/tools/fake-dib.yaml
@@ -0,0 +1,54 @@
+script-dir: .
+elements-dir: .
+images-dir: /tmp/nodepool_dib
+dburi: 'mysql://nodepool@localhost/nodepool'
+
+cron:
+  check: '*/15 * * * *'
+  cleanup: '*/1 * * * *'
+  image-update: '14 2 * * *'
+
+zmq-publishers:
+  - tcp://localhost:8881
+
+gearman-servers:
+  - host: localhost
+
+diskimages:
+  - name: fake-dib-image
+    elements:
+      - ubuntu
+      - vm
+      - openstack-repos
+      - puppet
+      - nodepool-base
+      - node-devstack
+    release: trusty
+
+labels:
+  - name: fake-dib-image
+    image: fake-dib-image
+    min-ready: 2
+    providers:
+      - name: fake-provider
+
+providers:
+  - name: fake-provider
+    keypair: 'if-present-use-this-keypair'
+    region-name: 'fake-region'
+    username: 'fake'
+    password: 'fake'
+    auth-url: 'fake'
+    project-id: 'fake'
+    max-servers: 96
+    images:
+      - name: fake-dib-image
+        min-ram: 8192
+        diskimage: fake-dib-image
+
+targets:
+  - name: fake-jenkins
+    jenkins:
+      url: https://jenkins.example.org/
+      user: fake
+      apikey: fake