Fix TypeError if backend response doesn't have expected headers

There was some debug logging mixed in with some error handling on PUTs
that was relying on a very specific edge would only encounter a set of
backend responses that included the expected set of headers to diagnoise
the failure.

But the backend responses may not always have the expected headers.

The proxy debug logging should be more robust to missing headers.

It's a little hard to follow, but if you look `_connect_put_node` in
swift.proxy.controller.obj - you'll see that only a few connections can
make their way out of the initial put connection handling with a "resp"
attribute that is not None.  In the happy path (e.g. 100-Continue) it's
explictly set to None, and in most errors (Timeout, 503, 413, etc) a new
connection will be established to the next node in the node iter.

Some status code will however allow a conn to be returned for validation
in `_check_failure_put_connections`, i.e.

  * 2XX (e.g. 0-byte PUT would not send Expect 100-Continue)
  * 409 - Conflict with another timestamp
  * 412 - If-None-Match that encounters another object

... so I added tests for those - fixing a TypeError along the way.

Change-Id: Ibdad5a90fa14ce62d081e6aaf40aacfca31b94d2

swift/proxy/controllers/obj.py

@@ -657,13 +657,17 @@ class BaseObjectController(Controller):
 
         if any(conn for conn in conns if conn.resp and
                conn.resp.status == HTTP_CONFLICT):
-            timestamps = [HeaderKeyDict(conn.resp.getheaders()).get(
-                'X-Backend-Timestamp') for conn in conns if conn.resp]
+            status_times = ['%(status)s (%(timestamp)s)' % {
+                'status': conn.resp.status,
+                'timestamp': HeaderKeyDict(
+                    conn.resp.getheaders()).get(
+                        'X-Backend-Timestamp', 'unknown')
+            } for conn in conns if conn.resp]
             self.app.logger.debug(
                 _('Object PUT returning 202 for 409: '
                   '%(req_timestamp)s <= %(timestamps)r'),
                 {'req_timestamp': req.timestamp.internal,
-                 'timestamps': ', '.join(timestamps)})
+                 'timestamps': ', '.join(status_times)})
             raise HTTPAccepted(request=req)
 
         self._check_min_conn(req, conns, min_conns)

test/unit/__init__.py

@@ -860,7 +860,9 @@ def fake_http_connect(*code_iter, **kwargs):
             headers = dict(self.expect_headers)
             if expect_status == 409:
                 headers['X-Backend-Timestamp'] = self.timestamp
-            response = FakeConn(expect_status, headers=headers)
+            response = FakeConn(expect_status,
+                                timestamp=self.timestamp,
+                                headers=headers)
             response.status = expect_status
             return response
 

test/unit/proxy/controllers/test_obj.py

@@ -771,6 +771,43 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
             resp = req.get_response(self.app)
         self.assertEqual(resp.status_int, 202)
 
+    def test_put_x_timestamp_conflict_with_missing_backend_timestamp(self):
+        ts = (utils.Timestamp(t) for t in itertools.count(int(time.time())))
+        req = swob.Request.blank(
+            '/v1/a/c/o', method='PUT', headers={
+                'Content-Length': 0,
+                'X-Timestamp': ts.next().internal})
+        ts_iter = iter([None, None, None])
+        codes = [409] * self.obj_ring.replicas
+        with set_http_connect(*codes, timestamps=ts_iter):
+            resp = req.get_response(self.app)
+        self.assertEqual(resp.status_int, 202)
+
+    def test_put_x_timestamp_conflict_with_other_weird_success_response(self):
+        ts = (utils.Timestamp(t) for t in itertools.count(int(time.time())))
+        req = swob.Request.blank(
+            '/v1/a/c/o', method='PUT', headers={
+                'Content-Length': 0,
+                'X-Timestamp': ts.next().internal})
+        ts_iter = iter([ts.next().internal, None, None])
+        codes = [409] + [(201, 'notused')] * (self.obj_ring.replicas - 1)
+        with set_http_connect(*codes, timestamps=ts_iter):
+            resp = req.get_response(self.app)
+        self.assertEqual(resp.status_int, 202)
+
+    def test_put_x_timestamp_conflict_with_if_none_match(self):
+        ts = (utils.Timestamp(t) for t in itertools.count(int(time.time())))
+        req = swob.Request.blank(
+            '/v1/a/c/o', method='PUT', headers={
+                'Content-Length': 0,
+                'If-None-Match': '*',
+                'X-Timestamp': ts.next().internal})
+        ts_iter = iter([ts.next().internal, None, None])
+        codes = [409] + [(412, 'notused')] * (self.obj_ring.replicas - 1)
+        with set_http_connect(*codes, timestamps=ts_iter):
+            resp = req.get_response(self.app)
+        self.assertEqual(resp.status_int, 412)
+
     def test_container_sync_put_x_timestamp_race(self):
         ts = (utils.Timestamp(t) for t in itertools.count(int(time.time())))
         test_indexes = [None] + [int(p) for p in POLICIES]