openstack/swift
Code Issues Proposed changes

Remove swiftclient dep on direct_client

Browse Source 
Partial Implements: blueprint remove-swiftclient-dependency
Change-Id: I9af7150e5d21d50e5f880e57796314b8f05822d2
This commit is contained in:
Chmouel Boudjnah 2013-12-23 17:57:56 +01:00
parent dfb5dead3a
commit 150f338fc2
13 changed files with 102 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bin/swift-dispersion-report
View File

@ -28,8 +28,9 @@ from eventlet import GreenPool, hubs, patcher, Timeout
from eventlet.pools import Pool
from swift.common import direct_client
from swiftclient import ClientException, Connection, get_auth
from swiftclient import Connection, get_auth
from swift.common.ring import Ring
from swift.common.exceptions import ClientException
from swift.common.utils import compute_eta, get_time_units, config_true_value

5
swift/account/reaper.py
View File

@ -25,8 +25,9 @@ from eventlet import GreenPool, sleep, Timeout
import swift.common.db
from swift.account.server import DATADIR
from swift.account.backend import AccountBroker
from swift.common.direct_client import ClientException, \
direct_delete_container, direct_delete_object, direct_get_container
from swift.common.direct_client import direct_delete_container, \
direct_delete_object, direct_get_container
from swift.common.exceptions import ClientException
from swift.common.ring import Ring
from swift.common.utils import get_logger, whataremyips, ismount, \
config_true_value

9
swift/common/direct_client.py
View File

@ -26,13 +26,18 @@ from time import time
from eventlet import sleep, Timeout
from swift.common.bufferedhttp import http_connect
from swiftclient import ClientException, json_loads
from swift.common.exceptions import ClientException
from swift.common.utils import normalize_timestamp, FileLikeIter
from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \
is_success, is_server_error
from swift.common.swob import HeaderKeyDict
from swift.common.utils import quote
try:
import simplejson as json
except ImportError:
import json
def _get_direct_account_container(path, stype, node, part,
account, marker=None, limit=None,
@ -74,7 +79,7 @@ def _get_direct_account_container(path, stype, node, part,
if resp.status == HTTP_NO_CONTENT:
resp.read()
return resp_headers, []
return resp_headers, json_loads(resp.read())
return resp_headers, json.loads(resp.read())
def gen_headers(hdrs_in=None, add_ts=False):

54
swift/common/exceptions.py
View File

@ -133,3 +133,57 @@ class ReplicationException(Exception):
class ReplicationLockTimeout(LockTimeout):
pass
class ClientException(Exception):
def __init__(self, msg, http_scheme='', http_host='', http_port='',
http_path='', http_query='', http_status=0, http_reason='',
http_device='', http_response_content=''):
Exception.__init__(self, msg)
self.msg = msg
self.http_scheme = http_scheme
self.http_host = http_host
self.http_port = http_port
self.http_path = http_path
self.http_query = http_query
self.http_status = http_status
self.http_reason = http_reason
self.http_device = http_device
self.http_response_content = http_response_content
def __str__(self):
a = self.msg
b = ''
if self.http_scheme:
b += '%s://' % self.http_scheme
if self.http_host:
b += self.http_host
if self.http_port:
b += ':%s' % self.http_port
if self.http_path:
b += self.http_path
if self.http_query:
b += '?%s' % self.http_query
if self.http_status:
if b:
b = '%s %s' % (b, self.http_status)
else:
b = str(self.http_status)
if self.http_reason:
if b:
b = '%s %s' % (b, self.http_reason)
else:
b = '- %s' % self.http_reason
if self.http_device:
if b:
b = '%s: device %s' % (b, self.http_device)
else:
b = 'device %s' % self.http_device
if self.http_response_content:
if len(self.http_response_content) <= 60:
b += ' %s' % self.http_response_content
else:
b += ' [first 60 chars of response] %s' \
% self.http_response_content[:60]
return b and '%s: %s' % (a, b) or a

4
swift/container/sync.py
View File

@ -22,10 +22,10 @@ from eventlet import sleep, Timeout
import swift.common.db
from swift.container import server as container_server
from swiftclient import ClientException, delete_object, put_object, \
quote
from swiftclient import delete_object, put_object, quote
from swift.container.backend import ContainerBroker
from swift.common.direct_client import direct_get_object
from swift.common.exceptions import ClientException
from swift.common.ring import Ring
from swift.common.utils import audit_location_generator, get_logger, \
hash_path, config_true_value, validate_sync_to, whataremyips, FileLikeIter

3
test/probe/test_container_failures.py
View File

@ -25,6 +25,7 @@ from sqlite3 import connect
from swiftclient import client
from swift.common import direct_client
from swift.common.exceptions import ClientException
from swift.common.utils import hash_path, readconf
from test.probe.common import get_to_final_state, kill_nonprimary_server, \
kill_server, kill_servers, reset_environment, start_server
@ -107,7 +108,7 @@ class TestContainerFailures(TestCase):
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except client.ClientException as err:
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
headers, containers = client.get_account(self.url, self.token)

5
test/probe/test_empty_device_handoff.py Normal file → Executable file
View File

@ -25,6 +25,7 @@ from uuid import uuid4
from swiftclient import client
from swift.common import direct_client
from swift.common.exceptions import ClientException
from test.probe.common import kill_server, kill_servers, reset_environment,\
start_server
from swift.common.utils import readconf
@ -129,7 +130,7 @@ class TestEmptyDevice(TestCase):
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except direct_client.ClientException as err:
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
self.assertFalse(os.path.exists(obj_dir))
@ -158,7 +159,7 @@ class TestEmptyDevice(TestCase):
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except direct_client.ClientException as err:
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)

11
test/probe/test_object_failures.py
View File

@ -23,6 +23,7 @@ from uuid import uuid4
from swiftclient import client
from swift.common import direct_client
from swift.common.exceptions import ClientException
from swift.common.utils import hash_path, readconf
from swift.obj.diskfile import write_metadata, read_metadata
from test.probe.common import kill_servers, reset_environment
@ -93,7 +94,7 @@ class TestObjectFailures(TestCase):
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
raise Exception("Did not quarantine object")
except client.ClientException as err:
except ClientException as err:
self.assertEquals(err.http_status, 404)
def run_quarantine_range_etag(self):
@ -116,7 +117,7 @@ class TestObjectFailures(TestCase):
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
raise Exception("Did not quarantine object")
except client.ClientException as err:
except ClientException as err:
self.assertEquals(err.http_status, 404)
def run_quarantine_zero_byte_get(self):
@ -133,7 +134,7 @@ class TestObjectFailures(TestCase):
container, obj, conn_timeout=1,
response_timeout=1)
raise Exception("Did not quarantine object")
except client.ClientException as err:
except ClientException as err:
self.assertEquals(err.http_status, 404)
def run_quarantine_zero_byte_head(self):
@ -150,7 +151,7 @@ class TestObjectFailures(TestCase):
container, obj, conn_timeout=1,
response_timeout=1)
raise Exception("Did not quarantine object")
except client.ClientException as err:
except ClientException as err:
self.assertEquals(err.http_status, 404)
def run_quarantine_zero_byte_post(self):
@ -170,7 +171,7 @@ class TestObjectFailures(TestCase):
conn_timeout=1,
response_timeout=1)
raise Exception("Did not quarantine object")
except client.ClientException as err:
except ClientException as err:
self.assertEquals(err.http_status, 404)
def test_runner(self):

9
test/probe/test_object_handoff.py
View File

@ -21,6 +21,7 @@ from uuid import uuid4
from swiftclient import client
from swift.common import direct_client
from swift.common.exceptions import ClientException
from test.probe.common import kill_server, kill_servers, reset_environment, \
start_server
@ -110,7 +111,7 @@ class TestObjectHandoff(TestCase):
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except direct_client.ClientException as err:
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
# Run the extra server last so it'll remove its extra partition
@ -142,7 +143,7 @@ class TestObjectHandoff(TestCase):
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except direct_client.ClientException as err:
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
@ -151,7 +152,7 @@ class TestObjectHandoff(TestCase):
exc = None
try:
client.head_object(self.url, self.token, container, obj)
except direct_client.ClientException as err:
except client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
objs = [o['name'] for o in
@ -189,7 +190,7 @@ class TestObjectHandoff(TestCase):
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except direct_client.ClientException as err:
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)

2
test/unit/account/test_reaper.py
View File

@ -25,8 +25,8 @@ from contextlib import nested
from swift.account import reaper
from swift.account.server import DATADIR
from swift.common.exceptions import ClientException
from swift.common.utils import normalize_timestamp
from swift.common.direct_client import ClientException
class FakeLogger(object):

6
test/unit/common/test_direct_client.py
View File

@ -20,6 +20,7 @@ import StringIO
from hashlib import md5
from swift.common import direct_client
from swift.common.exceptions import ClientException
from swiftclient import json_loads
@ -292,9 +293,8 @@ class TestDirectClient(unittest.TestCase):
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(500)
self.assertRaises(direct_client.ClientException,
direct_client.direct_put_object, node, part, account,
container, name, contents)
self.assertRaises(ClientException, direct_client.direct_put_object,
node, part, account, container, name, contents)
direct_client.http_connect = was_http_connector

13
test/unit/common/test_exceptions.py
View File

@ -32,6 +32,19 @@ class TestExceptions(unittest.TestCase):
finally:
exc.cancel()
def test_client_exception(self):
strerror = 'test: HTTP://random:888/randompath?foo=1 666 reason: ' \
'device /sdb1 content'
exc = exceptions.ClientException('test', http_scheme='HTTP',
http_host='random',
http_port=888,
http_path='/randompath',
http_query='foo=1',
http_status=666,
http_reason='reason',
http_device='/sdb1',
http_response_content='content')
self.assertEqual(str(exc), strerror)
if __name__ == '__main__':
unittest.main()

2
test/unit/container/test_sync.py
View File

@ -22,7 +22,7 @@ import mock
from test.unit import FakeLogger
from swift.container import sync
from swift.common import utils
from swiftclient import ClientException
from swift.common.exceptions import ClientException
utils.HASH_PATH_SUFFIX = 'endcap'