From 3b434098c6354b607457115c3404888df25fd449 Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jim@acmegating.com>
Date: Wed, 6 Sep 2023 08:03:32 -0700
Subject: [PATCH] Add an image upload timeout to the openstack driver

Some uploads in opendev are taking hours.

We used to wait 6 hours for this, but we ended up using the SDK
default of 1 hour in recent versions.  Since we're seeing so much
disparity in time, make it user configurable.

Remove the unused 6 hour constant.

Change-Id: I9ca5fdbf7c66f176eb4f650fd287514708f46c16
---
 doc/source/openstack.rst                                 | 9 +++++++++
 nodepool/builder.py                                      | 3 ---
 nodepool/driver/openstack/adapter.py                     | 1 +
 nodepool/driver/openstack/config.py                      | 4 ++++
 .../notes/openstack-upload-timeout-dc91ce56210dd86f.yaml | 6 ++++++
 5 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100644 releasenotes/notes/openstack-upload-timeout-dc91ce56210dd86f.yaml

diff --git a/doc/source/openstack.rst b/doc/source/openstack.rst
index 2561d573d..1b8069fb5 100644
--- a/doc/source/openstack.rst
+++ b/doc/source/openstack.rst
@@ -205,6 +205,15 @@ Selecting the OpenStack driver adds the following options to the
      the cleanup interval has elapsed. This value can be reduced if the
      instance spawn time on the provider is reliably quicker.
 
+  .. attr:: image-upload-timeout
+     :type: int seconds
+     :default: 3600
+
+     How long to wait for an image upload.  Note that in the case of a
+     timeout, it may be possible to leak an image upload in a manner
+     that Nodepool will be unable to detect and will require manual
+     identification and cleanup.
+
   .. attr:: diskimages
      :type: list
 
diff --git a/nodepool/builder.py b/nodepool/builder.py
index cbbe5adea..d659f53ce 100644
--- a/nodepool/builder.py
+++ b/nodepool/builder.py
@@ -39,9 +39,6 @@ from nodepool.version import get_version_string
 
 MINS = 60
 HOURS = 60 * MINS
-# How long to wait for an image save
-IMAGE_TIMEOUT = 6 * HOURS
-
 # How long to wait between checks for ZooKeeper connectivity if it disappears.
 SUSPEND_WAIT_TIME = 30
 
diff --git a/nodepool/driver/openstack/adapter.py b/nodepool/driver/openstack/adapter.py
index d8cb99097..0311f2e43 100644
--- a/nodepool/driver/openstack/adapter.py
+++ b/nodepool/driver/openstack/adapter.py
@@ -566,6 +566,7 @@ class OpenStackAdapter(statemachine.Adapter):
                 wait=True,
                 md5=md5,
                 sha256=sha256,
+                timeout=self.provider.image_upload_timeout,
                 **metadata)
         return image.id
 
diff --git a/nodepool/driver/openstack/config.py b/nodepool/driver/openstack/config.py
index 0f0e71e7d..25985f962 100644
--- a/nodepool/driver/openstack/config.py
+++ b/nodepool/driver/openstack/config.py
@@ -171,6 +171,7 @@ class OpenStackProviderConfig(ProviderConfig):
         self.rate = None
         self.boot_timeout = None
         self.launch_timeout = None
+        self.image_upload_timeout = None
         self.clean_floating_ips = None
         self.port_cleanup_interval = None
         self.diskimages = {}
@@ -206,6 +207,8 @@ class OpenStackProviderConfig(ProviderConfig):
         self.boot_timeout = self.provider.get('boot-timeout', 60)
         self.launch_timeout = self.provider.get('launch-timeout', 3600)
         self.launch_retries = self.provider.get('launch-retries', 3)
+        self.image_upload_timeout = self.provider.get(
+            'image-upload-timeout', 3600)
         self.clean_floating_ips = self.provider.get('clean-floating-ips')
         self.port_cleanup_interval = self.provider.get(
             'port-cleanup-interval',
@@ -340,6 +343,7 @@ class OpenStackProviderConfig(ProviderConfig):
             'region-name': str,
             v.Required('cloud'): str,
             'boot-timeout': int,
+            'image-upload-timeout': int,
             'launch-timeout': int,
             'launch-retries': int,
             'nodepool-id': str,
diff --git a/releasenotes/notes/openstack-upload-timeout-dc91ce56210dd86f.yaml b/releasenotes/notes/openstack-upload-timeout-dc91ce56210dd86f.yaml
new file mode 100644
index 000000000..2b16e12bd
--- /dev/null
+++ b/releasenotes/notes/openstack-upload-timeout-dc91ce56210dd86f.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    An upload timeout can be configured on OpenStack providers for use
+    in the case that image uploads to a provider take longer than the
+    default of one hour.