openstack/swift
Code Issues Proposed changes

Merge "Remove swiftclient dependency"

Browse Source
This commit is contained in:
Jenkins 2014-02-11 02:18:06 +00:00 committed by Gerrit Code Review
commit 8902aef591
7 changed files with 231 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bin/swift-dispersion-populate
View File

@ -24,7 +24,11 @@ from time import time
from eventlet import GreenPool, patcher, sleep
from eventlet.pools import Pool
from swiftclient import Connection, get_auth
try:
from swiftclient import get_auth
except ImportError:
from swift.common.internal_client import get_auth
from swift.common.internal_client import SimpleClient
from swift.common.ring import Ring
from swift.common.utils import compute_eta, get_time_units, config_true_value
@ -133,12 +137,8 @@ Usage: %%prog [options] [conf_file]
insecure=insecure)
account = url.rsplit('/', 1)[1]
connpool = Pool(max_size=concurrency)
connpool.create = lambda: Connection(conf['auth_url'],
conf['auth_user'], conf['auth_key'],
retries=retries,
preauthurl=url, preauthtoken=token,
os_options=os_options,
insecure=insecure)
connpool.create = lambda: SimpleClient(
url=url, token=token, retries=retries)
if container_populate:
container_ring = Ring(swift_dir, ring_name='container')

12
bin/swift-dispersion-report
View File

@ -28,7 +28,11 @@ from eventlet import GreenPool, hubs, patcher, Timeout
from eventlet.pools import Pool
from swift.common import direct_client
from swiftclient import Connection, get_auth
try:
from swiftclient import get_auth
except ImportError:
from swift.common.internal_client import get_auth
from swift.common.internal_client import SimpleClient
from swift.common.ring import Ring
from swift.common.exceptions import ClientException
from swift.common.utils import compute_eta, get_time_units, config_true_value
@ -356,10 +360,8 @@ Usage: %%prog [options] [conf_file]
insecure=insecure)
account = url.rsplit('/', 1)[1]
connpool = Pool(max_size=concurrency)
connpool.create = lambda: Connection(
conf['auth_url'], conf['auth_user'], conf['auth_key'], retries=retries,
preauthurl=url, preauthtoken=token, os_options=os_options,
insecure=insecure)
connpool.create = lambda: SimpleClient(
url=url, token=token, retries=retries)
container_ring = Ring(swift_dir, ring_name='container')
object_ring = Ring(swift_dir, ring_name='object')

1
requirements.txt
View File

@ -5,4 +5,3 @@ netifaces>=0.5
pastedeploy>=1.3.3
simplejson>=2.0.9
xattr>=0.4
python-swiftclient

107
swift/common/internal_client.py
View File

@ -14,12 +14,14 @@
# limitations under the License.
from eventlet import sleep, Timeout
from eventlet.green import httplib, socket, urllib2
import json
from paste.deploy import loadapp
import struct
from sys import exc_info
import zlib
from swift import gettext_ as _
import urlparse
from zlib import compressobj
from swift.common.utils import quote
@ -675,3 +677,108 @@ class InternalClient(object):
headers['Transfer-Encoding'] = 'chunked'
path = self.make_path(account, container, obj)
self.make_request('PUT', path, headers, (2,), fobj)
def get_auth(url, user, key, auth_version='1.0', **kwargs):
if auth_version != '1.0':
exit('ERROR: swiftclient missing, only auth v1.0 supported')
req = urllib2.Request(url)
req.add_header('X-Auth-User', user)
req.add_header('X-Auth-Key', key)
conn = urllib2.urlopen(req)
headers = conn.info()
return (
headers.getheader('X-Storage-Url'),
headers.getheader('X-Auth-Token'))
class SimpleClient(object):
"""
Simple client that is used in bin/swift-dispersion-* and container sync
"""
def __init__(self, url=None, token=None, starting_backoff=1,
max_backoff=5, retries=5):
self.url = url
self.token = token
self.attempts = 0
self.starting_backoff = starting_backoff
self.max_backoff = max_backoff
self.retries = retries
def base_request(self, method, container=None, name=None, prefix=None,
headers={}, proxy=None, contents=None, full_listing=None):
# Common request method
url = self.url
if self.token:
headers['X-Auth-Token'] = self.token
if container:
url = '%s/%s' % (url.rstrip('/'), quote(container))
if name:
url = '%s/%s' % (url.rstrip('/'), quote(name))
url += '?format=json'
if prefix:
url += '&prefix=%s' % prefix
if proxy:
proxy = urlparse.urlparse(proxy)
proxy = urllib2.ProxyHandler({proxy.scheme: proxy.netloc})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
req = urllib2.Request(url, headers=headers, data=contents)
req.get_method = lambda: method
urllib2.urlopen(req)
conn = urllib2.urlopen(req)
body = conn.read()
try:
body_data = json.loads(body)
except ValueError:
body_data = None
return [None, body_data]
def retry_request(self, method, **kwargs):
self.attempts = 0
backoff = self.starting_backoff
while self.attempts <= self.retries:
self.attempts += 1
try:
return self.base_request(method, **kwargs)
except (socket.error, httplib.HTTPException, urllib2.URLError):
if self.attempts > self.retries:
raise
sleep(backoff)
backoff = min(backoff * 2, self.max_backoff)
def get_account(self, *args, **kwargs):
# Used in swift-dispertion-populate
return self.retry_request('GET', **kwargs)
def put_container(self, container, **kwargs):
# Used in swift-dispertion-populate
return self.retry_request('PUT', container=container, **kwargs)
def get_container(self, container, **kwargs):
# Used in swift-dispertion-populate
return self.retry_request('GET', container=container, **kwargs)
def put_object(self, container, name, contents, **kwargs):
# Used in swift-dispertion-populate
return self.retry_request('PUT', container=container, name=name,
contents=contents.read(), **kwargs)
def put_object(url, **kwargs):
"""For usage with container sync """
client = SimpleClient(url=url)
client.retry_request('PUT', **kwargs)
def delete_object(url, **kwargs):
"""For usage with container sync """
client = SimpleClient(url=url)
client.retry_request('DELETE', **kwargs)

4
swift/container/sync.py
View File

@ -24,15 +24,15 @@ from eventlet import sleep, Timeout
import swift.common.db
from swift.container import server as container_server
from swiftclient import delete_object, put_object, quote
from swift.container.backend import ContainerBroker
from swift.common.container_sync_realms import ContainerSyncRealms
from swift.common.direct_client import direct_get_object
from swift.common.internal_client import delete_object, put_object
from swift.common.exceptions import ClientException
from swift.common.ring import Ring
from swift.common.utils import audit_location_generator, get_logger, \
hash_path, config_true_value, validate_sync_to, whataremyips, \
FileLikeIter, urlparse
FileLikeIter, urlparse, quote
from swift.common.daemon import Daemon
from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND

1
test-requirements.txt
View File

@ -7,3 +7,4 @@ openstack.nose_plugin
nosehtmloutput
sphinx>=1.1.2,<1.2
mock>=0.8.0
python-swiftclient

107
test/unit/common/test_internal_client.py
View File

@ -14,11 +14,13 @@
# limitations under the License.
import json
import mock
from StringIO import StringIO
import unittest
from urllib import quote
import zlib
from eventlet.green import urllib2
from swift.common import internal_client
@ -919,5 +921,110 @@ class TestInternalClient(unittest.TestCase):
client.upload_object(fobj, account, container, obj, headers)
self.assertEquals(1, client.make_request_called)
class TestGetAuth(unittest.TestCase):
@mock.patch('eventlet.green.urllib2.urlopen')
@mock.patch('eventlet.green.urllib2.Request')
def test_ok(self, request, urlopen):
def getheader(name):
d = {'X-Storage-Url': 'url', 'X-Auth-Token': 'token'}
return d.get(name)
urlopen.return_value.info.return_value.getheader = getheader
url, token = internal_client.get_auth(
'http://127.0.0.1', 'user', 'key')
self.assertEqual(url, "url")
self.assertEqual(token, "token")
request.assert_called_with('http://127.0.0.1')
request.return_value.add_header.assert_any_call('X-Auth-User', 'user')
request.return_value.add_header.assert_any_call('X-Auth-Key', 'key')
def test_invalid_version(self):
self.assertRaises(SystemExit, internal_client.get_auth,
'http://127.0.0.1', 'user', 'key', auth_version=2.0)
class TestSimpleClient(unittest.TestCase):
@mock.patch('eventlet.green.urllib2.urlopen')
@mock.patch('eventlet.green.urllib2.Request')
def test_get(self, request, urlopen):
# basic GET request, only url as kwarg
request.return_value.get_type.return_value = "http"
urlopen.return_value.read.return_value = ''
sc = internal_client.SimpleClient(url='http://127.0.0.1')
retval = sc.retry_request('GET')
request.assert_called_with('http://127.0.0.1?format=json',
headers={},
data=None)
self.assertEqual([None, None], retval)
self.assertEqual('GET', request.return_value.get_method())
# Check if JSON is decoded
urlopen.return_value.read.return_value = '{}'
retval = sc.retry_request('GET')
self.assertEqual([None, {}], retval)
# same as above, now with token
sc = internal_client.SimpleClient(url='http://127.0.0.1',
token='token')
retval = sc.retry_request('GET')
request.assert_called_with('http://127.0.0.1?format=json',
headers={'X-Auth-Token': 'token'},
data=None)
self.assertEqual([None, {}], retval)
# same as above, now with prefix
sc = internal_client.SimpleClient(url='http://127.0.0.1',
token='token')
retval = sc.retry_request('GET', prefix="pre_")
request.assert_called_with('http://127.0.0.1?format=json&prefix=pre_',
headers={'X-Auth-Token': 'token'},
data=None)
self.assertEqual([None, {}], retval)
# same as above, now with container name
retval = sc.retry_request('GET', container='cont')
request.assert_called_with('http://127.0.0.1/cont?format=json',
headers={'X-Auth-Token': 'token'},
data=None)
self.assertEqual([None, {}], retval)
# same as above, now with object name
retval = sc.retry_request('GET', container='cont', name='obj')
request.assert_called_with('http://127.0.0.1/cont/obj?format=json',
headers={'X-Auth-Token': 'token'},
data=None)
self.assertEqual([None, {}], retval)
@mock.patch('eventlet.green.urllib2.urlopen')
@mock.patch('eventlet.green.urllib2.Request')
def test_get_with_retries_all_failed(self, request, urlopen):
# Simulate a failing request, ensure retries done
request.return_value.get_type.return_value = "http"
request.side_effect = urllib2.URLError('')
urlopen.return_value.read.return_value = ''
sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1)
self.assertRaises(urllib2.URLError, sc.retry_request, 'GET')
self.assertEqual(request.call_count, 2)
@mock.patch('eventlet.green.urllib2.urlopen')
@mock.patch('eventlet.green.urllib2.Request')
def test_get_with_retries(self, request, urlopen):
# First request fails, retry successful
request.return_value.get_type.return_value = "http"
urlopen.return_value.read.return_value = ''
req = urllib2.Request('http://127.0.0.1', method='GET')
request.side_effect = [urllib2.URLError(''), req]
sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1)
retval = sc.retry_request('GET')
self.assertEqual(request.call_count, 3)
request.assert_called_with('http://127.0.0.1?format=json', data=None,
headers={'X-Auth-Token': 'token'})
self.assertEqual([None, None], retval)
if __name__ == '__main__':
unittest.main()