Merge "Remove swiftclient dependency"

bin/swift-dispersion-populate

@@ -24,7 +24,11 @@ from time import time
 from eventlet import GreenPool, patcher, sleep
 from eventlet.pools import Pool
 
-from swiftclient import Connection, get_auth
+try:
+    from swiftclient import get_auth
+except ImportError:
+    from swift.common.internal_client import get_auth
+from swift.common.internal_client import SimpleClient
 from swift.common.ring import Ring
 from swift.common.utils import compute_eta, get_time_units, config_true_value
 
@@ -133,12 +137,8 @@ Usage: %%prog [options] [conf_file]
                           insecure=insecure)
     account = url.rsplit('/', 1)[1]
     connpool = Pool(max_size=concurrency)
-    connpool.create = lambda: Connection(conf['auth_url'],
+    connpool.create = lambda: SimpleClient(
-                                         conf['auth_user'], conf['auth_key'],
+        url=url, token=token, retries=retries)
-                                         retries=retries,
-                                         preauthurl=url, preauthtoken=token,
-                                         os_options=os_options,
-                                         insecure=insecure)
 
     if container_populate:
         container_ring = Ring(swift_dir, ring_name='container')

bin/swift-dispersion-report

@@ -28,7 +28,11 @@ from eventlet import GreenPool, hubs, patcher, Timeout
 from eventlet.pools import Pool
 
 from swift.common import direct_client
-from swiftclient import Connection, get_auth
+try:
+    from swiftclient import get_auth
+except ImportError:
+    from swift.common.internal_client import get_auth
+from swift.common.internal_client import SimpleClient
 from swift.common.ring import Ring
 from swift.common.exceptions import ClientException
 from swift.common.utils import compute_eta, get_time_units, config_true_value
@@ -356,10 +360,8 @@ Usage: %%prog [options] [conf_file]
                           insecure=insecure)
     account = url.rsplit('/', 1)[1]
     connpool = Pool(max_size=concurrency)
-    connpool.create = lambda: Connection(
+    connpool.create = lambda: SimpleClient(
-        conf['auth_url'], conf['auth_user'], conf['auth_key'], retries=retries,
+        url=url, token=token, retries=retries)
-        preauthurl=url, preauthtoken=token, os_options=os_options,
-        insecure=insecure)
 
     container_ring = Ring(swift_dir, ring_name='container')
     object_ring = Ring(swift_dir, ring_name='object')

requirements.txt

@@ -5,4 +5,3 @@ netifaces>=0.5
 pastedeploy>=1.3.3
 simplejson>=2.0.9
 xattr>=0.4
-python-swiftclient

swift/common/internal_client.py

@@ -14,12 +14,14 @@
 # limitations under the License.
 
 from eventlet import sleep, Timeout
+from eventlet.green import httplib, socket, urllib2
 import json
 from paste.deploy import loadapp
 import struct
 from sys import exc_info
 import zlib
 from swift import gettext_ as _
+import urlparse
 from zlib import compressobj
 
 from swift.common.utils import quote
@@ -675,3 +677,108 @@ class InternalClient(object):
         headers['Transfer-Encoding'] = 'chunked'
         path = self.make_path(account, container, obj)
         self.make_request('PUT', path, headers, (2,), fobj)
+
+
+def get_auth(url, user, key, auth_version='1.0', **kwargs):
+    if auth_version != '1.0':
+        exit('ERROR: swiftclient missing, only auth v1.0 supported')
+    req = urllib2.Request(url)
+    req.add_header('X-Auth-User', user)
+    req.add_header('X-Auth-Key', key)
+    conn = urllib2.urlopen(req)
+    headers = conn.info()
+    return (
+        headers.getheader('X-Storage-Url'),
+        headers.getheader('X-Auth-Token'))
+
+
+class SimpleClient(object):
+    """
+    Simple client that is used in bin/swift-dispersion-* and container sync
+    """
+    def __init__(self, url=None, token=None, starting_backoff=1,
+                 max_backoff=5, retries=5):
+        self.url = url
+        self.token = token
+        self.attempts = 0
+        self.starting_backoff = starting_backoff
+        self.max_backoff = max_backoff
+        self.retries = retries
+
+    def base_request(self, method, container=None, name=None, prefix=None,
+                     headers={}, proxy=None, contents=None, full_listing=None):
+        # Common request method
+        url = self.url
+
+        if self.token:
+            headers['X-Auth-Token'] = self.token
+
+        if container:
+            url = '%s/%s' % (url.rstrip('/'), quote(container))
+
+        if name:
+            url = '%s/%s' % (url.rstrip('/'), quote(name))
+
+        url += '?format=json'
+
+        if prefix:
+            url += '&prefix=%s' % prefix
+
+        if proxy:
+            proxy = urlparse.urlparse(proxy)
+            proxy = urllib2.ProxyHandler({proxy.scheme: proxy.netloc})
+            opener = urllib2.build_opener(proxy)
+            urllib2.install_opener(opener)
+
+        req = urllib2.Request(url, headers=headers, data=contents)
+        req.get_method = lambda: method
+        urllib2.urlopen(req)
+        conn = urllib2.urlopen(req)
+        body = conn.read()
+        try:
+            body_data = json.loads(body)
+        except ValueError:
+            body_data = None
+        return [None, body_data]
+
+    def retry_request(self, method, **kwargs):
+        self.attempts = 0
+        backoff = self.starting_backoff
+        while self.attempts <= self.retries:
+            self.attempts += 1
+            try:
+                return self.base_request(method, **kwargs)
+            except (socket.error, httplib.HTTPException, urllib2.URLError):
+                if self.attempts > self.retries:
+                    raise
+            sleep(backoff)
+            backoff = min(backoff * 2, self.max_backoff)
+
+    def get_account(self, *args, **kwargs):
+        # Used in swift-dispertion-populate
+        return self.retry_request('GET', **kwargs)
+
+    def put_container(self, container, **kwargs):
+        # Used in swift-dispertion-populate
+        return self.retry_request('PUT', container=container, **kwargs)
+
+    def get_container(self, container, **kwargs):
+        # Used in swift-dispertion-populate
+        return self.retry_request('GET', container=container, **kwargs)
+
+    def put_object(self, container, name, contents, **kwargs):
+        # Used in swift-dispertion-populate
+        return self.retry_request('PUT', container=container, name=name,
+                                  contents=contents.read(), **kwargs)
+
+
+def put_object(url, **kwargs):
+    """For usage with container sync """
+    client = SimpleClient(url=url)
+    client.retry_request('PUT', **kwargs)
+
+
+def delete_object(url, **kwargs):
+    """For usage with container sync """
+    client = SimpleClient(url=url)
+    client.retry_request('DELETE', **kwargs)

swift/container/sync.py

@@ -24,15 +24,15 @@ from eventlet import sleep, Timeout
 
 import swift.common.db
 from swift.container import server as container_server
-from swiftclient import delete_object, put_object, quote
 from swift.container.backend import ContainerBroker
 from swift.common.container_sync_realms import ContainerSyncRealms
 from swift.common.direct_client import direct_get_object
+from swift.common.internal_client import delete_object, put_object
 from swift.common.exceptions import ClientException
 from swift.common.ring import Ring
 from swift.common.utils import audit_location_generator, get_logger, \
     hash_path, config_true_value, validate_sync_to, whataremyips, \
-    FileLikeIter, urlparse
+    FileLikeIter, urlparse, quote
 from swift.common.daemon import Daemon
 from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND
 

test-requirements.txt

@@ -7,3 +7,4 @@ openstack.nose_plugin
 nosehtmloutput
 sphinx>=1.1.2,<1.2
 mock>=0.8.0
+python-swiftclient

test/unit/common/test_internal_client.py

@@ -14,11 +14,13 @@
 # limitations under the License.
 
 import json
+import mock
 from StringIO import StringIO
 import unittest
 from urllib import quote
 import zlib
 
+from eventlet.green import urllib2
 from swift.common import internal_client
 
 
@@ -919,5 +921,110 @@ class TestInternalClient(unittest.TestCase):
         client.upload_object(fobj, account, container, obj, headers)
         self.assertEquals(1, client.make_request_called)
 
+
+class TestGetAuth(unittest.TestCase):
+    @mock.patch('eventlet.green.urllib2.urlopen')
+    @mock.patch('eventlet.green.urllib2.Request')
+    def test_ok(self, request, urlopen):
+        def getheader(name):
+            d = {'X-Storage-Url': 'url', 'X-Auth-Token': 'token'}
+            return d.get(name)
+        urlopen.return_value.info.return_value.getheader = getheader
+
+        url, token = internal_client.get_auth(
+            'http://127.0.0.1', 'user', 'key')
+
+        self.assertEqual(url, "url")
+        self.assertEqual(token, "token")
+        request.assert_called_with('http://127.0.0.1')
+        request.return_value.add_header.assert_any_call('X-Auth-User', 'user')
+        request.return_value.add_header.assert_any_call('X-Auth-Key', 'key')
+
+    def test_invalid_version(self):
+        self.assertRaises(SystemExit, internal_client.get_auth,
+                          'http://127.0.0.1', 'user', 'key', auth_version=2.0)
+
+
+class TestSimpleClient(unittest.TestCase):
+
+    @mock.patch('eventlet.green.urllib2.urlopen')
+    @mock.patch('eventlet.green.urllib2.Request')
+    def test_get(self, request, urlopen):
+        # basic GET request, only url as kwarg
+        request.return_value.get_type.return_value = "http"
+        urlopen.return_value.read.return_value = ''
+        sc = internal_client.SimpleClient(url='http://127.0.0.1')
+        retval = sc.retry_request('GET')
+        request.assert_called_with('http://127.0.0.1?format=json',
+                                   headers={},
+                                   data=None)
+        self.assertEqual([None, None], retval)
+        self.assertEqual('GET', request.return_value.get_method())
+
+        # Check if JSON is decoded
+        urlopen.return_value.read.return_value = '{}'
+        retval = sc.retry_request('GET')
+        self.assertEqual([None, {}], retval)
+
+        # same as above, now with token
+        sc = internal_client.SimpleClient(url='http://127.0.0.1',
+                                          token='token')
+        retval = sc.retry_request('GET')
+        request.assert_called_with('http://127.0.0.1?format=json',
+                                   headers={'X-Auth-Token': 'token'},
+                                   data=None)
+        self.assertEqual([None, {}], retval)
+
+        # same as above, now with prefix
+        sc = internal_client.SimpleClient(url='http://127.0.0.1',
+                                          token='token')
+        retval = sc.retry_request('GET', prefix="pre_")
+        request.assert_called_with('http://127.0.0.1?format=json&prefix=pre_',
+                                   headers={'X-Auth-Token': 'token'},
+                                   data=None)
+        self.assertEqual([None, {}], retval)
+
+        # same as above, now with container name
+        retval = sc.retry_request('GET', container='cont')
+        request.assert_called_with('http://127.0.0.1/cont?format=json',
+                                   headers={'X-Auth-Token': 'token'},
+                                   data=None)
+        self.assertEqual([None, {}], retval)
+
+        # same as above, now with object name
+        retval = sc.retry_request('GET', container='cont', name='obj')
+        request.assert_called_with('http://127.0.0.1/cont/obj?format=json',
+                                   headers={'X-Auth-Token': 'token'},
+                                   data=None)
+        self.assertEqual([None, {}], retval)
+
+    @mock.patch('eventlet.green.urllib2.urlopen')
+    @mock.patch('eventlet.green.urllib2.Request')
+    def test_get_with_retries_all_failed(self, request, urlopen):
+        # Simulate a failing request, ensure retries done
+        request.return_value.get_type.return_value = "http"
+        request.side_effect = urllib2.URLError('')
+        urlopen.return_value.read.return_value = ''
+        sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1)
+        self.assertRaises(urllib2.URLError, sc.retry_request, 'GET')
+        self.assertEqual(request.call_count, 2)
+
+    @mock.patch('eventlet.green.urllib2.urlopen')
+    @mock.patch('eventlet.green.urllib2.Request')
+    def test_get_with_retries(self, request, urlopen):
+        # First request fails, retry successful
+        request.return_value.get_type.return_value = "http"
+        urlopen.return_value.read.return_value = ''
+        req = urllib2.Request('http://127.0.0.1', method='GET')
+        request.side_effect = [urllib2.URLError(''), req]
+        sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1)
+
+        retval = sc.retry_request('GET')
+        self.assertEqual(request.call_count, 3)
+        request.assert_called_with('http://127.0.0.1?format=json', data=None,
+                                   headers={'X-Auth-Token': 'token'})
+        self.assertEqual([None, None], retval)
+
+
 if __name__ == '__main__':
     unittest.main()