Gracefully handle upload failures

When an upload fails we do not attempt to upload other images and or
providers.

Change-Id: I328a94563a846317d9b8597bb5f742cc12e83249

nodepool/fakeprovider.py

@@ -109,11 +109,15 @@ class FakeClient(object):
 
 
 class FakeGlanceClient(object):
-    def __init__(self):
+    def __init__(self, **kwargs):
         self.id = 'fake-glance-id'
+        self.should_fail = kwargs.get('SHOULD_FAIL', '').lower() == 'true'
 
     def update(self, **kwargs):
-        return True
+        if self.should_fail:
+            raise RuntimeError('This image has SHOULD_FAIL set to True.')
+        else:
+            return True
 
 
 class FakeFile(StringIO.StringIO):

nodepool/nodepool.py

@@ -1852,19 +1852,28 @@ class NodePool(threading.Thread):
                 self.uploadImage(session, provider.name,
                                  image.name)
 
+    def checkForMissingImage(self, session, provider, image):
+        if image.name in self.config.images_in_use:
+            if not image.diskimage:
+                self.checkForMissingSnapshotImage(session, provider, image)
+            else:
+                self.checkForMissingDiskImage(session, provider, image)
+
     def checkForMissingImages(self, session):
         # If we are missing an image, run the image update function
         # outside of its schedule.
         self.log.debug("Checking missing images.")
 
-        for provider in self.config.providers.values():
+        # this is sorted so we can have a deterministic pass on providers
+        # (very useful for testing/debugging)
+        providers = sorted(self.config.providers.values(),
+                           key=lambda x: x.name)
+        for provider in providers:
             for image in provider.images.values():
-                if image.name not in self.config.images_in_use:
-                    continue
-                if not image.diskimage:
-                    self.checkForMissingSnapshotImage(session, provider, image)
-                else:
-                    self.checkForMissingDiskImage(session, provider, image)
+                try:
+                    self.checkForMissingImage(session, provider, image)
+                except Exception:
+                    self.log.exception("Exception in missing image check:")
 
     def _doUpdateImages(self):
         try:

nodepool/provider_manager.py

@@ -513,7 +513,7 @@ class ProviderManager(TaskManager):
     def uploadImage(self, image_name, filename, disk_format, container_format,
                     meta):
         if image_name.startswith('fake-'):
-            image = fakeprovider.FakeGlanceClient()
+            image = fakeprovider.FakeGlanceClient(**meta)
             image.update(data='fake')
         else:
             # configure glance and upload image.  Note the meta flags

nodepool/tests/fixtures/node_dib_and_snap_upload_fail.yaml

@@ -0,0 +1,80 @@
+script-dir: .
+elements-dir: .
+images-dir: .
+
+dburi: '{dburi}'
+
+cron:
+  check: '*/15 * * * *'
+  cleanup: '*/1 * * * *'
+  image-update: '14 2 * * *'
+
+zmq-publishers:
+  - tcp://localhost:8881
+
+#gearman-servers:
+#  - host: localhost
+
+labels:
+  - name: fake-label
+    image: fake-dib-image
+    min-ready: 2
+    providers:
+      - name: fake-provider1
+      - name: fake-provider2
+
+providers:
+  - name: fake-provider1
+    keypair: 'if-present-use-this-keypair'
+    username: 'fake'
+    password: 'fake'
+    auth-url: 'fake'
+    project-id: 'fake'
+    max-servers: 1
+    pool: 'fake'
+    networks:
+      - net-id: 'some-uuid'
+    rate: 0.0001
+    images:
+      - name: fake-dib-image
+        base-image: 'Fake Precise'
+        min-ram: 8192
+        diskimage: fake-dib-image
+        meta:
+          SHOULD_FAIL: 'true'
+  - name: fake-provider2
+    keypair: 'if-present-use-this-keypair'
+    username: 'fake'
+    password: 'fake'
+    auth-url: 'fake'
+    project-id: 'fake'
+    max-servers: 2
+    pool: 'fake'
+    networks:
+      - net-id: 'some-uuid'
+    rate: 0.0001
+    images:
+      - name: fake-dib-image
+        base-image: 'Fake Precise'
+        min-ram: 8192
+        diskimage: fake-dib-image
+
+targets:
+  - name: fake-target
+    jenkins:
+      url: https://jenkins.example.org/
+      user: fake
+      apikey: fake
+
+diskimages:
+  - name: fake-dib-image
+    elements:
+      - ubuntu
+      - vm
+    release: 21
+    env-vars:
+      TMPDIR: /opt/dib_tmp
+      DIB_IMAGE_CACHE: /opt/dib_cache
+      DIB_CLOUD_IMAGES: http://download.fedoraproject.org/pub/fedora/linux/releases/test/21-Beta/Cloud/Images/x86_64/
+      BASE_IMAGE_FILE: Fedora-Cloud-Base-20141029-21_Beta.x86_64.qcow2
+

nodepool/tests/test_nodepool.py

@@ -176,6 +176,29 @@ class TestNodepool(tests.DBTestCase):
         self.assertEqual(self.subprocesses[0].returncode, 127)
         self.assertEqual(self.subprocesses[-1].returncode, 127)
 
+    def test_dib_upload_fail(self):
+        """Test that a dib and snap image upload failure is contained."""
+        configfile = self.setup_config('node_dib_and_snap_upload_fail.yaml')
+        pool = nodepool.nodepool.NodePool(configfile, watermark_sleep=1)
+        pool.start()
+        self.addCleanup(pool.stop)
+        self.waitForImage(pool, 'fake-provider2', 'fake-dib-image')
+        self.waitForNodes(pool)
+
+        with pool.getDB().getSession() as session:
+            # fake-provider1 uses dib.
+            nodes = session.getNodes(provider_name='fake-provider1',
+                                     label_name='fake-label',
+                                     target_name='fake-target',
+                                     state=nodedb.READY)
+            self.assertEqual(len(nodes), 0)
+            # fake-provider2 uses snapshots.
+            nodes = session.getNodes(provider_name='fake-provider2',
+                                     label_name='fake-label',
+                                     target_name='fake-target',
+                                     state=nodedb.READY)
+            self.assertEqual(len(nodes), 2)
+
     def test_subnodes(self):
         """Test that an image and node are created"""
         configfile = self.setup_config('subnodes.yaml')