From 455d9ad6202061d0413aff62f8ee27ccb38cb4e0 Mon Sep 17 00:00:00 2001
From: Benoit Bayszczak <benoit.bayszczak@adevinta.com>
Date: Wed, 19 Aug 2020 16:02:03 +0200
Subject: [PATCH] [provider][aws] fix exception when instance not found during
 cleanup

When an instance is not found by AWS API during cleanup, this leads
to an infinite error loop because the execption raised is not catched
correctly
```
"2020-08-19 15:33:05,893 ERROR nodepool.NodeDeleter: Exception deleting instance i-xxxxxxxxxxxxxxxxx from XXXXXXXXXXXXXX:"
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/nodepool/launcher.py", line 78, in delete
    manager.cleanupNode(node.external_id)
  File "/usr/local/lib/python3.7/dist-packages/nodepool/driver/aws/provider.py", line 157, in cleanupNode
    instance.terminate()
  File "/usr/local/lib/python3.7/dist-packages/boto3/resources/factory.py", line 520, in do_action
    response = action(self, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/boto3/resources/action.py", line 83, in __call__
    response = getattr(parent.meta.client, operation_name)(**params)
  File "/usr/local/lib/python3.7/dist-packages/botocore/client.py", line 357, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/local/lib/python3.7/dist-packages/botocore/client.py", line 661, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (InvalidInstanceID.NotFound) when calling the TerminateInstances operation: The instance ID 'i-xxxxxxxxxxxxxxxxx' does not exist

```

Change-Id: I776f2d32d7b07b665f90cb71e9dbc018775e1062
---
 nodepool/driver/aws/provider.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/nodepool/driver/aws/provider.py b/nodepool/driver/aws/provider.py
index 4a19222b1..5fcd3c648 100644
--- a/nodepool/driver/aws/provider.py
+++ b/nodepool/driver/aws/provider.py
@@ -14,6 +14,8 @@
 
 import logging
 import boto3
+import botocore.exceptions
+import nodepool.exceptions
 
 from nodepool.driver import Provider
 from nodepool.driver.aws.handler import AwsNodeRequestHandler
@@ -154,7 +156,13 @@ class AwsProvider(Provider):
         if self.ec2 is None:
             return False
         instance = self.ec2.Instance(server_id)
-        instance.terminate()
+        try:
+            instance.terminate()
+        except botocore.exceptions.ClientError as e:
+            error_code = e.response.get('Error', {}).get('Code', 'Unknown')
+            if error_code == "InvalidInstanceID.NotFound":
+                raise nodepool.exceptions.NotFound()
+            raise e
 
     def waitForNodeCleanup(self, server_id):
         # TODO: track instance deletion