openstack/swift
Code Issues Proposed changes

Remove swift.common.client from here.

Browse Source 
- It has been to its own gerrit project.
- direct_client should follow next.
- Implements blueprint clientbindings.

Change-Id: I3bb50c95eba81302bfec71cb7ce5288b85a41dc0
This commit is contained in:
Chmouel Boudjnah 2012-05-22 10:57:10 +02:00
parent 3ba5ea354b
commit 22572b506a
20 changed files with 69 additions and 3535 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1999
bin/swift
View File

File diff suppressed because it is too large Load Diff

2
bin/swift-dispersion-populate
View File

@ -25,7 +25,7 @@ from uuid import uuid4
from eventlet import GreenPool, patcher, sleep
from eventlet.pools import Pool
from swift.common.client import Connection, get_auth
from swiftclient import Connection, get_auth
from swift.common.ring import Ring
from swift.common.utils import compute_eta, get_time_units

2
bin/swift-dispersion-report
View File

@ -28,7 +28,7 @@ from eventlet import GreenPool, hubs, patcher, Timeout
from eventlet.pools import Pool
from swift.common import direct_client
from swift.common.client import ClientException, Connection, get_auth
from swiftclient import ClientException, Connection, get_auth
from swift.common.ring import Ring
from swift.common.utils import compute_eta, get_time_units, TRUE_VALUES

4
doc/source/associated_projects.rst
View File

@ -7,6 +7,10 @@ Associated Projects
Application Bindings
--------------------
* OpenStack supported binding:
* `Python-SwiftClient <http://pypi.python.org/pypi/python-swiftclient>`_
* Made for Cloud Files, but mostly work with Swift too:
* `CSharp-CloudFiles <https://github.com/rackspace/csharp-cloudfiles>`_

3
doc/source/development_saio.rst
View File

@ -266,6 +266,9 @@ Do these commands as you on guest.
#. Check out the swift repo with `git clone https://github.com/openstack/swift.git`
#. Build a development installation of swift, for example:
`cd ~/swift; sudo python setup.py develop`
#. Check out the python-swiftclient repo with `git clone https://github.com/openstack/python-swiftclient.git`
#. Build a development installation of python-swiftclient, for example:
`cd ~/python-swiftclient; sudo python setup.py develop`
#. Edit `~/.bashrc` and add to the end::
export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf

79
doc/source/howto_installmultinode.rst
View File

@ -27,11 +27,11 @@ This document shows a cluster using the following types of nodes:
- five Storage nodes
- Runs the swift-account-server, swift-container-server, and
- Runs the swift-account-server, swift-container-server, and
swift-object-server processes which control storage of the account
databases, the container databases, as well as the actual stored
objects.
.. note::
Fewer Storage nodes can be used initially, but a minimum of 5 is
recommended for a production cluster.
@ -62,7 +62,7 @@ General OS configuration and partitioning for each node
apt-get install python-software-properties
add-apt-repository ppa:swift-core/release
apt-get update
apt-get install swift openssh-server
apt-get install swift python-swiftclient openssh-server
#. Create and populate configuration directories::
@ -79,7 +79,7 @@ General OS configuration and partitioning for each node
#. On the second and subsequent nodes: Copy that file over. It must be the same on every node in the cluster!::
scp firstnode.example.com:/etc/swift/swift.conf /etc/swift/
scp firstnode.example.com:/etc/swift/swift.conf /etc/swift/
#. Publish the local network IP address for use by scripts found later in this documentation::
@ -87,8 +87,8 @@ General OS configuration and partitioning for each node
export PROXY_LOCAL_NET_IP=10.1.2.4
.. note::
The random string of text in /etc/swift/swift.conf is
used as a salt when hashing to determine mappings in the ring.
The random string of text in /etc/swift/swift.conf is
used as a salt when hashing to determine mappings in the ring.
.. _config-proxy:
@ -130,22 +130,22 @@ Configure the Proxy node
bind_port = 8080
workers = 8
user = swift
[pipeline:main]
pipeline = healthcheck cache tempauth proxy-server
[app:proxy-server]
use = egg:swift#proxy
allow_account_management = true
account_autocreate = true
[filter:tempauth]
use = egg:swift#tempauth
user_system_root = testpass .admin https://$PROXY_LOCAL_NET_IP:8080/v1/AUTH_system
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:cache]
use = egg:swift#memcache
memcache_servers = $PROXY_LOCAL_NET_IP:11211
@ -153,7 +153,7 @@ Configure the Proxy node
.. note::
If you run multiple memcache servers, put the multiple IP:port listings
If you run multiple memcache servers, put the multiple IP:port listings
in the [filter:cache] section of the proxy-server.conf file like:
`10.1.2.3:11211,10.1.2.4:11211`. Only the proxy server uses memcache.
@ -165,11 +165,11 @@ Configure the Proxy node
swift-ring-builder account.builder create 18 3 1
swift-ring-builder container.builder create 18 3 1
swift-ring-builder object.builder create 18 3 1
.. note::
For more information on building rings, see :doc:`overview_ring`.
#. For every storage device in /srv/node on each node add entries to each ring::
export ZONE= # set the zone number for that storage device
@ -189,7 +189,7 @@ Configure the Proxy node
swift-ring-builder account.builder
swift-ring-builder container.builder
swift-ring-builder object.builder
#. Rebalance the rings::
swift-ring-builder account.builder rebalance
@ -219,7 +219,7 @@ Configure the Storage nodes
Extended Attributes (XATTRS). We currently recommend XFS as it
demonstrated the best overall performance for the swift use case after
considerable testing and benchmarking at Rackspace. It is also the
only filesystem that has been thoroughly tested. These instructions
only filesystem that has been thoroughly tested. These instructions
assume that you are going to devote /dev/sdb1 to an XFS filesystem.
#. Install Storage node packages::
@ -244,19 +244,19 @@ Configure the Storage nodes
log file = /var/log/rsyncd.log
pid file = /var/run/rsyncd.pid
address = $STORAGE_LOCAL_NET_IP
[account]
max connections = 2
path = /srv/node/
read only = false
lock file = /var/lock/account.lock
[container]
max connections = 2
path = /srv/node/
read only = false
lock file = /var/lock/container.lock
[object]
max connections = 2
path = /srv/node/
@ -282,17 +282,17 @@ Configure the Storage nodes
[DEFAULT]
bind_ip = $STORAGE_LOCAL_NET_IP
workers = 2
[pipeline:main]
pipeline = account-server
[app:account-server]
use = egg:swift#account
[account-replicator]
[account-auditor]
[account-reaper]
EOF
@ -302,17 +302,17 @@ Configure the Storage nodes
[DEFAULT]
bind_ip = $STORAGE_LOCAL_NET_IP
workers = 2
[pipeline:main]
pipeline = container-server
[app:container-server]
use = egg:swift#container
[container-replicator]
[container-updater]
[container-auditor]
EOF
@ -322,17 +322,17 @@ Configure the Storage nodes
[DEFAULT]
bind_ip = $STORAGE_LOCAL_NET_IP
workers = 2
[pipeline:main]
pipeline = object-server
[app:object-server]
use = egg:swift#object
[object-replicator]
[object-updater]
[object-auditor]
EOF
@ -412,11 +412,11 @@ You run these commands from the Proxy node.
Adding a Proxy Server
---------------------
For reliability's sake you may want to have more than one proxy server. You can set up the additional proxy node in the same manner that you set up the first proxy node but with additional configuration steps.
For reliability's sake you may want to have more than one proxy server. You can set up the additional proxy node in the same manner that you set up the first proxy node but with additional configuration steps.
Once you have more than two proxies, you also want to load balance between the two, which means your storage endpoint also changes. You can select from different strategies for load balancing. For example, you could use round robin dns, or an actual load balancer (like pound) in front of the two proxies, and point your storage url to the load balancer.
See :ref:`config-proxy` for the initial setup, and then follow these additional steps.
See :ref:`config-proxy` for the initial setup, and then follow these additional steps.
#. Update the list of memcache servers in /etc/swift/proxy-server.conf for all the added proxy servers. If you run multiple memcache servers, use this pattern for the multiple IP:port listings: `10.1.2.3:11211,10.1.2.4:11211` in each proxy server's conf file.::
@ -430,15 +430,14 @@ See :ref:`config-proxy` for the initial setup, and then follow these additional
use = egg:swift#tempauth
user_system_root = testpass .admin http[s]://<LOAD_BALANCER_HOSTNAME>:<PORT>/v1/AUTH_system
#. Next, copy all the ring information to all the nodes, including your new proxy nodes, and ensure the ring info gets to all the storage nodes as well.
#. Next, copy all the ring information to all the nodes, including your new proxy nodes, and ensure the ring info gets to all the storage nodes as well.
#. After you sync all the nodes, make sure the admin has the keys in /etc/swift and the ownership for the ring file is correct.
#. After you sync all the nodes, make sure the admin has the keys in /etc/swift and the ownership for the ring file is correct.
Troubleshooting Notes
---------------------
If you see problems, look in var/log/syslog (or messages on some distros).
If you see problems, look in var/log/syslog (or messages on some distros).
Also, at Rackspace we have seen hints at drive failures by looking at error messages in /var/log/kern.log.
Also, at Rackspace we have seen hints at drive failures by looking at error messages in /var/log/kern.log.
There are more debugging hints and tips in the :doc:`admin_guide`.

10
doc/source/misc.rst
View File

@ -60,16 +60,6 @@ WSGI
:members:
:show-inheritance:
.. _client:
Client
======
.. automodule:: swift.common.client
:members:
:undoc-members:
:show-inheritance:
.. _direct_client:
Direct Client

6
doc/source/overview_container_sync.rst
View File

@ -76,6 +76,10 @@ setups -- for testing.
Using the ``swift`` tool to set up synchronized containers
----------------------------------------------------------
.. note::
The ``swift`` tool is available from the `python-swiftclient`_ library.
.. note::
You must be the account admin on the account to set synchronization targets
@ -145,6 +149,8 @@ You can also set up a chain of synced containers if you want more than two.
You'd point 1 -> 2, then 2 -> 3, and finally 3 -> 1 for three containers.
They'd all need to share the same secret synchronization key.
.. _`python-swiftclient`: http://github.com/openstack/python-swiftclient
-----------------------------------
Using curl (or other tools) instead
-----------------------------------

9
doc/source/overview_large_objects.rst
View File

@ -17,9 +17,10 @@ with the possibility of parallel uploads of the segments.
Using ``swift`` for Segmented Objects
-------------------------------------
The quickest way to try out this feature is use the included ``swift``
Swift Tool. You can use the ``-S`` option to specify the segment size
to use when splitting a large file. For example::
The quickest way to try out this feature is use the ``swift`` Swift Tool
included with the `python-swiftclient`_ library. You can use the ``-S``
option to specify the segment size to use when splitting a large file. For
example::
swift upload test_container -S 1073741824 large_file
@ -48,6 +49,8 @@ deletes and overwrites, etc. You can override this behavior with the
``--leave-segments`` option if desired; this is useful if you want to have
multiple versions of the same large object available.
.. _`python-swiftclient`: http://github.com/openstack/python-swiftclient
----------
Direct API
----------

1
setup.py
View File

@ -41,7 +41,6 @@ setup(
],
install_requires=[], # removed for better compat
scripts=[
'bin/swift',
'bin/swift-account-audit',
'bin/swift-account-auditor',
'bin/swift-account-reaper',

2
swift/common/bench.py
View File

@ -22,7 +22,7 @@ import eventlet.pools
from eventlet.green.httplib import CannotSendRequest
from swift.common.utils import TRUE_VALUES
from swift.common import client
import swiftclient as client
from swift.common import direct_client
from swift.common.http import HTTP_CONFLICT

953
swift/common/client.py
View File

@ -1,953 +0,0 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Cloud Files client library used internally
"""
import socket
from urllib import quote as _quote
from urlparse import urlparse, urlunparse, urljoin
try:
from eventlet.green.httplib import HTTPException, HTTPSConnection
except ImportError:
from httplib import HTTPException, HTTPSConnection
try:
from eventlet import sleep
except ImportError:
from time import sleep
try:
from swift.common.bufferedhttp \
import BufferedHTTPConnection as HTTPConnection
except ImportError:
try:
from eventlet.green.httplib import HTTPConnection
except ImportError:
from httplib import HTTPConnection
def quote(value, safe='/'):
"""
Patched version of urllib.quote that encodes utf8 strings before quoting
"""
if isinstance(value, unicode):
value = value.encode('utf8')
return _quote(value, safe)
# look for a real json parser first
try:
# simplejson is popular and pretty good
from simplejson import loads as json_loads
from simplejson import dumps as json_dumps
except ImportError:
# 2.6 will have a json module in the stdlib
from json import loads as json_loads
from json import dumps as json_dumps
class ClientException(Exception):
def __init__(self, msg, http_scheme='', http_host='', http_port='',
http_path='', http_query='', http_status=0, http_reason='',
http_device='', http_response_content=''):
Exception.__init__(self, msg)
self.msg = msg
self.http_scheme = http_scheme
self.http_host = http_host
self.http_port = http_port
self.http_path = http_path
self.http_query = http_query
self.http_status = http_status
self.http_reason = http_reason
self.http_device = http_device
self.http_response_content = http_response_content
def __str__(self):
a = self.msg
b = ''
if self.http_scheme:
b += '%s://' % self.http_scheme
if self.http_host:
b += self.http_host
if self.http_port:
b += ':%s' % self.http_port
if self.http_path:
b += self.http_path
if self.http_query:
b += '?%s' % self.http_query
if self.http_status:
if b:
b = '%s %s' % (b, self.http_status)
else:
b = str(self.http_status)
if self.http_reason:
if b:
b = '%s %s' % (b, self.http_reason)
else:
b = '- %s' % self.http_reason
if self.http_device:
if b:
b = '%s: device %s' % (b, self.http_device)
else:
b = 'device %s' % self.http_device
if self.http_response_content:
if len(self.http_response_content) <= 60:
b += ' %s' % self.http_response_content
else:
b += ' [first 60 chars of response] %s' % \
self.http_response_content[:60]
return b and '%s: %s' % (a, b) or a
def http_connection(url, proxy=None):
"""
Make an HTTPConnection or HTTPSConnection
:param url: url to connect to
:param proxy: proxy to connect through, if any; None by default; str of the
format 'http://127.0.0.1:8888' to set one
:returns: tuple of (parsed url, connection object)
:raises ClientException: Unable to handle protocol scheme
"""
parsed = urlparse(url)
proxy_parsed = urlparse(proxy) if proxy else None
if parsed.scheme == 'http':
conn = HTTPConnection((proxy_parsed if proxy else parsed).netloc)
elif parsed.scheme == 'https':
conn = HTTPSConnection((proxy_parsed if proxy else parsed).netloc)
else:
raise ClientException('Cannot handle protocol scheme %s for url %s' %
(parsed.scheme, repr(url)))
if proxy:
conn._set_tunnel(parsed.hostname, parsed.port)
return parsed, conn
def json_request(method, url, **kwargs):
"""Takes a request in json parse it and return in json"""
kwargs.setdefault('headers', {})
if 'body' in kwargs:
kwargs['headers']['Content-Type'] = 'application/json'
kwargs['body'] = json_dumps(kwargs['body'])
parsed, conn = http_connection(url)
conn.request(method, parsed.path, **kwargs)
resp = conn.getresponse()
body = resp.read()
if body:
try:
body = json_loads(body)
except ValueError:
body = None
if not body or resp.status < 200 or resp.status >= 300:
raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
http_host=conn.host,
http_port=conn.port,
http_path=parsed.path,
http_status=resp.status,
http_reason=resp.reason)
return resp, body
def _get_auth_v1_0(url, user, key, snet):
parsed, conn = http_connection(url)
conn.request('GET', parsed.path, '',
{'X-Auth-User': user, 'X-Auth-Key': key})
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port,
http_path=parsed.path, http_status=resp.status,
http_reason=resp.reason)
url = resp.getheader('x-storage-url')
if snet:
parsed = list(urlparse(url))
# Second item in the list is the netloc
netloc = parsed[1]
parsed[1] = 'snet-' + netloc
url = urlunparse(parsed)
return url, resp.getheader('x-storage-token',
resp.getheader('x-auth-token'))
def _get_auth_v2_0(url, user, tenant_name, key, snet):
body = {'auth': {'passwordCredentials':
{'password': key, 'username': user},
'tenantName': tenant_name}}
token_url = urljoin(url, "tokens")
resp, body = json_request("POST", token_url, body=body)
token_id = None
try:
url = None
catalogs = body['access']['serviceCatalog']
for service in catalogs:
if service['type'] == 'object-store':
url = service['endpoints'][0]['publicURL']
token_id = body['access']['token']['id']
if not url:
raise ClientException("There is no object-store endpoint " \
"on this auth server.")
except(KeyError, IndexError):
raise ClientException("Error while getting answers from auth server")
if snet:
parsed = list(urlparse(url))
# Second item in the list is the netloc
parsed[1] = 'snet-' + parsed[1]
url = urlunparse(parsed)
return url, token_id
def get_auth(url, user, key, snet=False, tenant_name=None, auth_version="1.0"):
"""
Get authentication/authorization credentials.
The snet parameter is used for Rackspace's ServiceNet internal network
implementation. In this function, it simply adds *snet-* to the beginning
of the host name for the returned storage URL. With Rackspace Cloud Files,
use of this network path causes no bandwidth charges but requires the
client to be running on Rackspace's ServiceNet network.
:param url: authentication/authorization URL
:param user: user to authenticate as
:param key: key or password for authorization
:param snet: use SERVICENET internal network (see above), default is False
:param auth_version: OpenStack auth version, default is 1.0
:param tenant_name: The tenant/account name, required when connecting
to a auth 2.0 system.
:returns: tuple of (storage URL, auth token)
:raises: ClientException: HTTP GET request to auth URL failed
"""
if auth_version in ["1.0", "1"]:
return _get_auth_v1_0(url, user, key, snet)
elif auth_version in ["2.0", "2"]:
if not tenant_name:
raise ClientException('No tenant specified')
return _get_auth_v2_0(url, user, tenant_name, key, snet)
else:
raise ClientException('Unknown auth_version %s specified.' %
auth_version)
def get_account(url, token, marker=None, limit=None, prefix=None,
http_conn=None, full_listing=False):
"""
Get a listing of containers for the account.
:param url: storage URL
:param token: auth token
:param marker: marker query
:param limit: limit query
:param prefix: prefix query
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:returns: a tuple of (response headers, a list of containers) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if not http_conn:
http_conn = http_connection(url)
if full_listing:
rv = get_account(url, token, marker, limit, prefix, http_conn)
listing = rv[1]
while listing:
marker = listing[-1]['name']
listing = \
get_account(url, token, marker, limit, prefix, http_conn)[1]
if listing:
rv[1].extend(listing)
return rv
parsed, conn = http_conn
qs = 'format=json'
if marker:
qs += '&marker=%s' % quote(marker)
if limit:
qs += '&limit=%d' % limit
if prefix:
qs += '&prefix=%s' % quote(prefix)
conn.request('GET', '%s?%s' % (parsed.path, qs), '',
{'X-Auth-Token': token})
resp = conn.getresponse()
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status < 200 or resp.status >= 300:
body = resp.read()
raise ClientException('Account GET failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port,
http_path=parsed.path, http_query=qs, http_status=resp.status,
http_reason=resp.reason, http_response_content=body)
if resp.status == 204:
resp.read()
return resp_headers, []
return resp_headers, json_loads(resp.read())
def head_account(url, token, http_conn=None):
"""
Get account stats.
:param url: storage URL
:param token: auth token
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
resp = conn.getresponse()
body = resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Account HEAD failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port,
http_path=parsed.path, http_status=resp.status,
http_reason=resp.reason, http_response_content=body)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def post_account(url, token, headers, http_conn=None):
"""
Update an account's metadata.
:param url: storage URL
:param token: auth token
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
headers['X-Auth-Token'] = token
conn.request('POST', parsed.path, '', headers)
resp = conn.getresponse()
body = resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Account POST failed',
http_scheme=parsed.scheme,
http_host=conn.host,
http_port=conn.port,
http_path=parsed.path,
http_status=resp.status,
http_reason=resp.reason,
http_response_content=body)
def get_container(url, token, container, marker=None, limit=None,
prefix=None, delimiter=None, http_conn=None,
full_listing=False):
"""
Get a listing of objects for the container.
:param url: storage URL
:param token: auth token
:param container: container name to get a listing for
:param marker: marker query
:param limit: limit query
:param prefix: prefix query
:param delimeter: string to delimit the queries on
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:returns: a tuple of (response headers, a list of objects) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if not http_conn:
http_conn = http_connection(url)
if full_listing:
rv = get_container(url, token, container, marker, limit, prefix,
delimiter, http_conn)
listing = rv[1]
while listing:
if not delimiter:
marker = listing[-1]['name']
else:
marker = listing[-1].get('name', listing[-1].get('subdir'))
listing = get_container(url, token, container, marker, limit,
prefix, delimiter, http_conn)[1]
if listing:
rv[1].extend(listing)
return rv
parsed, conn = http_conn
path = '%s/%s' % (parsed.path, quote(container))
qs = 'format=json'
if marker:
qs += '&marker=%s' % quote(marker)
if limit:
qs += '&limit=%d' % limit
if prefix:
qs += '&prefix=%s' % quote(prefix)
if delimiter:
qs += '&delimiter=%s' % quote(delimiter)
conn.request('GET', '%s?%s' % (path, qs), '', {'X-Auth-Token': token})
resp = conn.getresponse()
if resp.status < 200 or resp.status >= 300:
body = resp.read()
raise ClientException('Container GET failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_query=qs,
http_status=resp.status, http_reason=resp.reason,
http_response_content=body)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status == 204:
resp.read()
return resp_headers, []
return resp_headers, json_loads(resp.read())
def head_container(url, token, container, http_conn=None, headers=None):
"""
Get container stats.
:param url: storage URL
:param token: auth token
:param container: container name to get stats for
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
req_headers = {'X-Auth-Token': token}
if headers:
req_headers.update(headers)
conn.request('HEAD', path, '', req_headers)
resp = conn.getresponse()
body = resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Container HEAD failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason, http_response_content=body)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def put_container(url, token, container, headers=None, http_conn=None):
"""
Create a container
:param url: storage URL
:param token: auth token
:param container: container name to create
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP PUT request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
if not headers:
headers = {}
headers['X-Auth-Token'] = token
conn.request('PUT', path, '', headers)
resp = conn.getresponse()
body = resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Container PUT failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason, http_response_content=body)
def post_container(url, token, container, headers, http_conn=None):
"""
Update a container's metadata.
:param url: storage URL
:param token: auth token
:param container: container name to update
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
headers['X-Auth-Token'] = token
conn.request('POST', path, '', headers)
resp = conn.getresponse()
body = resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Container POST failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason, http_response_content=body)
def delete_container(url, token, container, http_conn=None):
"""
Delete a container
:param url: storage URL
:param token: auth token
:param container: container name to delete
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP DELETE request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
conn.request('DELETE', path, '', {'X-Auth-Token': token})
resp = conn.getresponse()
body = resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Container DELETE failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason, http_response_content=body)
def get_object(url, token, container, name, http_conn=None,
resp_chunk_size=None):
"""
Get an object
:param url: storage URL
:param token: auth token
:param container: container name that the object is in
:param name: object name to get
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:param resp_chunk_size: if defined, chunk size of data to read. NOTE: If
you specify a resp_chunk_size you must fully read
the object's contents before making another
request.
:returns: a tuple of (response headers, the object's contents) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
conn.request('GET', path, '', {'X-Auth-Token': token})
resp = conn.getresponse()
if resp.status < 200 or resp.status >= 300:
body = resp.read()
raise ClientException('Object GET failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port, http_path=path,
http_status=resp.status, http_reason=resp.reason,
http_response_content=body)
if resp_chunk_size:
def _object_body():
buf = resp.read(resp_chunk_size)
while buf:
yield buf
buf = resp.read(resp_chunk_size)
object_body = _object_body()
else:
object_body = resp.read()
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers, object_body
def head_object(url, token, container, name, http_conn=None):
"""
Get object info
:param url: storage URL
:param token: auth token
:param container: container name that the object is in
:param name: object name to get info for
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
conn.request('HEAD', path, '', {'X-Auth-Token': token})
resp = conn.getresponse()
body = resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Object HEAD failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port, http_path=path,
http_status=resp.status, http_reason=resp.reason,
http_response_content=body)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def put_object(url, token=None, container=None, name=None, contents=None,
content_length=None, etag=None, chunk_size=65536,
content_type=None, headers=None, http_conn=None, proxy=None):
"""
Put an object
:param url: storage URL
:param token: auth token; if None, no token will be sent
:param container: container name that the object is in; if None, the
container name is expected to be part of the url
:param name: object name to put; if None, the object name is expected to be
part of the url
:param contents: a string or a file like object to read object data from;
if None, a zero-byte put will be done
:param content_length: value to send as content-length header; also limits
the amount read from contents; if None, it will be
computed via the contents or chunked transfer
encoding will be used
:param etag: etag of contents; if None, no etag will be sent
:param chunk_size: chunk size of data to write; default 65536
:param content_type: value to send as content-type header; if None, no
content-type will be set (remote end will likely try
to auto-detect it)
:param headers: additional headers to include in the request, if any
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:param proxy: proxy to connect through, if any; None by default; str of the
format 'http://127.0.0.1:8888' to set one
:returns: etag from server response
:raises ClientException: HTTP PUT request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url, proxy=proxy)
path = parsed.path
if container:
path = '%s/%s' % (path.rstrip('/'), quote(container))
if name:
path = '%s/%s' % (path.rstrip('/'), quote(name))
if headers:
headers = dict(headers)
else:
headers = {}
if token:
headers['X-Auth-Token'] = token
if etag:
headers['ETag'] = etag.strip('"')
if content_length is not None:
headers['Content-Length'] = str(content_length)
else:
for n, v in headers.iteritems():
if n.lower() == 'content-length':
content_length = int(v)
if content_type is not None:
headers['Content-Type'] = content_type
if not contents:
headers['Content-Length'] = '0'
if hasattr(contents, 'read'):
conn.putrequest('PUT', path)
for header, value in headers.iteritems():
conn.putheader(header, value)
if content_length is None:
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
chunk = contents.read(chunk_size)
while chunk:
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = contents.read(chunk_size)
conn.send('0\r\n\r\n')
else:
conn.endheaders()
left = content_length
while left > 0:
size = chunk_size
if size > left:
size = left
chunk = contents.read(size)
conn.send(chunk)
left -= len(chunk)
else:
conn.request('PUT', path, contents, headers)
resp = conn.getresponse()
body = resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Object PUT failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port, http_path=path,
http_status=resp.status, http_reason=resp.reason,
http_response_content=body)
return resp.getheader('etag', '').strip('"')
def post_object(url, token, container, name, headers, http_conn=None):
"""
Update object metadata
:param url: storage URL
:param token: auth token
:param container: container name that the object is in
:param name: name of the object to update
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
headers['X-Auth-Token'] = token
conn.request('POST', path, '', headers)
resp = conn.getresponse()
body = resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Object POST failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port, http_path=path,
http_status=resp.status, http_reason=resp.reason,
http_response_content=body)
def delete_object(url, token=None, container=None, name=None, http_conn=None,
headers=None, proxy=None):
"""
Delete object
:param url: storage URL
:param token: auth token; if None, no token will be sent
:param container: container name that the object is in; if None, the
container name is expected to be part of the url
:param name: object name to delete; if None, the object name is expected to
be part of the url
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:param headers: additional headers to include in the request
:param proxy: proxy to connect through, if any; None by default; str of the
format 'http://127.0.0.1:8888' to set one
:raises ClientException: HTTP DELETE request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url, proxy=proxy)
path = parsed.path
if container:
path = '%s/%s' % (path.rstrip('/'), quote(container))
if name:
path = '%s/%s' % (path.rstrip('/'), quote(name))
if headers:
headers = dict(headers)
else:
headers = {}
if token:
headers['X-Auth-Token'] = token
conn.request('DELETE', path, '', headers)
resp = conn.getresponse()
body = resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Object DELETE failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason, http_response_content=body)
class Connection(object):
"""Convenience class to make requests that will also retry the request"""
def __init__(self, authurl, user, key, retries=5, preauthurl=None,
preauthtoken=None, snet=False, starting_backoff=1,
tenant_name=None,
auth_version="1"):
"""
:param authurl: authentication URL
:param user: user name to authenticate as
:param key: key/password to authenticate with
:param retries: Number of times to retry the request before failing
:param preauthurl: storage URL (if you have already authenticated)
:param preauthtoken: authentication token (if you have already
authenticated)
:param snet: use SERVICENET internal network default is False
:param auth_version: OpenStack auth version, default is 1.0
:param tenant_name: The tenant/account name, required when connecting
to a auth 2.0 system.
"""
self.authurl = authurl
self.user = user
self.key = key
self.retries = retries
self.http_conn = None
self.url = preauthurl
self.token = preauthtoken
self.attempts = 0
self.snet = snet
self.starting_backoff = starting_backoff
self.auth_version = auth_version
self.tenant_name = tenant_name
def get_auth(self):
return get_auth(self.authurl, self.user,
self.key, snet=self.snet,
tenant_name=self.tenant_name,
auth_version=self.auth_version)
def http_connection(self):
return http_connection(self.url)
def _retry(self, reset_func, func, *args, **kwargs):
self.attempts = 0
backoff = self.starting_backoff
while self.attempts <= self.retries:
self.attempts += 1
try:
if not self.url or not self.token:
self.url, self.token = self.get_auth()
self.http_conn = None
if not self.http_conn:
self.http_conn = self.http_connection()
kwargs['http_conn'] = self.http_conn
rv = func(self.url, self.token, *args, **kwargs)
return rv
except (socket.error, HTTPException):
if self.attempts > self.retries:
raise
self.http_conn = None
except ClientException, err:
if self.attempts > self.retries:
raise
if err.http_status == 401:
self.url = self.token = None
if self.attempts > 1:
raise
elif err.http_status == 408:
self.http_conn = None
elif 500 <= err.http_status <= 599:
pass
else:
raise
sleep(backoff)
backoff *= 2
if reset_func:
reset_func(func, *args, **kwargs)
def head_account(self):
"""Wrapper for :func:`head_account`"""
return self._retry(None, head_account)
def get_account(self, marker=None, limit=None, prefix=None,
full_listing=False):
"""Wrapper for :func:`get_account`"""
# TODO(unknown): With full_listing=True this will restart the entire
# listing with each retry. Need to make a better version that just
# retries where it left off.
return self._retry(None, get_account, marker=marker, limit=limit,
prefix=prefix, full_listing=full_listing)
def post_account(self, headers):
"""Wrapper for :func:`post_account`"""
return self._retry(None, post_account, headers)
def head_container(self, container):
"""Wrapper for :func:`head_container`"""
return self._retry(None, head_container, container)
def get_container(self, container, marker=None, limit=None, prefix=None,
delimiter=None, full_listing=False):
"""Wrapper for :func:`get_container`"""
# TODO(unknown): With full_listing=True this will restart the entire
# listing with each retry. Need to make a better version that just
# retries where it left off.
return self._retry(None, get_container, container, marker=marker,
limit=limit, prefix=prefix, delimiter=delimiter,
full_listing=full_listing)
def put_container(self, container, headers=None):
"""Wrapper for :func:`put_container`"""
return self._retry(None, put_container, container, headers=headers)
def post_container(self, container, headers):
"""Wrapper for :func:`post_container`"""
return self._retry(None, post_container, container, headers)
def delete_container(self, container):
"""Wrapper for :func:`delete_container`"""
return self._retry(None, delete_container, container)
def head_object(self, container, obj):
"""Wrapper for :func:`head_object`"""
return self._retry(None, head_object, container, obj)
def get_object(self, container, obj, resp_chunk_size=None):
"""Wrapper for :func:`get_object`"""
return self._retry(None, get_object, container, obj,
resp_chunk_size=resp_chunk_size)
def put_object(self, container, obj, contents, content_length=None,
etag=None, chunk_size=65536, content_type=None,
headers=None):
"""Wrapper for :func:`put_object`"""
def _default_reset(*args, **kwargs):
raise ClientException('put_object(%r, %r, ...) failure and no '
'ability to reset contents for reupload.' % (container, obj))
reset_func = _default_reset
tell = getattr(contents, 'tell', None)
seek = getattr(contents, 'seek', None)
if tell and seek:
orig_pos = tell()
reset_func = lambda *a, **k: seek(orig_pos)
elif not contents:
reset_func = lambda *a, **k: None
return self._retry(reset_func, put_object, container, obj, contents,
content_length=content_length, etag=etag, chunk_size=chunk_size,
content_type=content_type, headers=headers)
def post_object(self, container, obj, headers):
"""Wrapper for :func:`post_object`"""
return self._retry(None, post_object, container, obj, headers)
def delete_object(self, container, obj):
"""Wrapper for :func:`delete_object`"""
return self._retry(None, delete_object, container, obj)

2
swift/common/direct_client.py
View File

@ -26,7 +26,7 @@ from urllib import quote as _quote
from eventlet import sleep, Timeout
from swift.common.bufferedhttp import http_connect
from swift.common.client import ClientException, json_loads
from swiftclient import ClientException, json_loads
from swift.common.utils import normalize_timestamp
from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \
is_success, is_server_error

2
swift/container/sync.py
View File

@ -21,7 +21,7 @@ from eventlet import sleep, Timeout
import swift.common.db
from swift.container import server as container_server
from swift.common.client import ClientException, delete_object, put_object, \
from swiftclient import ClientException, delete_object, put_object, \
quote
from swift.common.direct_client import direct_get_object
from swift.common.ring import Ring

2
test/functionalnosetests/swift_testing.py
View File

@ -9,7 +9,7 @@ from ConfigParser import MissingSectionHeaderError
from test import get_config
from swift.common.client import get_auth, http_connection
from swiftclient import get_auth, http_connection
conf = get_config('func_test')

2
test/probe/common.py
View File

@ -19,7 +19,7 @@ from subprocess import call, Popen
from time import sleep
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.client import get_auth
from swiftclient import get_auth
from swift.common.ring import Ring

520
test/unit/common/test_client.py
View File

@ -1,520 +0,0 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO: More tests
import socket
import unittest
from urlparse import urlparse
# TODO: mock http connection class with more control over headers
from test.unit.proxy.test_server import fake_http_connect
from swift.common import client as c
class TestClientException(unittest.TestCase):
def test_is_exception(self):
self.assertTrue(issubclass(c.ClientException, Exception))
def test_format(self):
exc = c.ClientException('something failed')
self.assertTrue('something failed' in str(exc))
test_kwargs = (
'scheme',
'host',
'port',
'path',
'query',
'status',
'reason',
'device',
)
for value in test_kwargs:
kwargs = {
'http_%s' % value: value,
}
exc = c.ClientException('test', **kwargs)
self.assertTrue(value in str(exc))
class TestJsonImport(unittest.TestCase):
def tearDown(self):
try:
import json
except ImportError:
pass
else:
reload(json)
try:
import simplejson
except ImportError:
pass
else:
reload(simplejson)
def test_any(self):
self.assertTrue(hasattr(c, 'json_loads'))
def test_no_simplejson(self):
# break simplejson
try:
import simplejson
except ImportError:
# not installed, so we don't have to break it for these tests
pass
else:
delattr(simplejson, 'loads')
reload(c)
try:
from json import loads
except ImportError:
# this case is stested in _no_json
pass
else:
self.assertEquals(loads, c.json_loads)
class MockHttpTest(unittest.TestCase):
def setUp(self):
def fake_http_connection(*args, **kwargs):
_orig_http_connection = c.http_connection
return_read = kwargs.get('return_read')
def wrapper(url, proxy=None):
parsed, _conn = _orig_http_connection(url, proxy=proxy)
conn = fake_http_connect(*args, **kwargs)()
def request(*args, **kwargs):
return
conn.request = request
conn.has_been_read = False
_orig_read = conn.read
def read(*args, **kwargs):
conn.has_been_read = True
return _orig_read(*args, **kwargs)
conn.read = return_read or read
return parsed, conn
return wrapper
self.fake_http_connection = fake_http_connection
def tearDown(self):
reload(c)
class TestHttpHelpers(MockHttpTest):
def test_quote(self):
value = 'standard string'
self.assertEquals('standard%20string', c.quote(value))
value = u'\u0075nicode string'
self.assertEquals('unicode%20string', c.quote(value))
def test_http_connection(self):
url = 'http://www.test.com'
_junk, conn = c.http_connection(url)
self.assertTrue(isinstance(conn, c.HTTPConnection))
url = 'https://www.test.com'
_junk, conn = c.http_connection(url)
self.assertTrue(isinstance(conn, c.HTTPSConnection))
url = 'ftp://www.test.com'
self.assertRaises(c.ClientException, c.http_connection, url)
def test_json_request(self):
def read(*args, **kwargs):
body = {'a': '1',
'b': '2'}
return c.json_dumps(body)
c.http_connection = self.fake_http_connection(200, return_read=read)
url = 'http://www.test.com'
_junk, conn = c.json_request('GET', url, body={'username': 'user1',
'password': 'secure'})
self.assertTrue(type(conn) is dict)
# TODO: following tests are placeholders, need more tests, better coverage
class TestGetAuth(MockHttpTest):
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf')
self.assertEquals(url, None)
self.assertEquals(token, None)
def test_invalid_auth(self):
c.http_connection = self.fake_http_connection(200)
self.assertRaises(c.ClientException, c.get_auth,
'http://www.tests.com', 'asdf', 'asdf',
auth_version="foo")
def test_auth_v1(self):
c.http_connection = self.fake_http_connection(200)
url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
auth_version="1.0")
self.assertEquals(url, None)
self.assertEquals(token, None)
def test_auth_v2(self):
def read(*args, **kwargs):
acct_url = 'http://127.0.01/AUTH_FOO'
body = {'access': {'serviceCatalog':
[{u'endpoints': [{'publicURL': acct_url}],
'type': 'object-store'}],
'token': {'id': 'XXXXXXX'}}}
return c.json_dumps(body)
c.http_connection = self.fake_http_connection(200, return_read=read)
url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
tenant_name='asdf', auth_version="2.0")
self.assertTrue(url.startswith("http"))
self.assertTrue(token)
def test_auth_v2_no_tenant_name(self):
def read(*args, **kwargs):
acct_url = 'http://127.0.01/AUTH_FOO'
body = {'access': {'serviceCatalog':
[{u'endpoints': [{'publicURL': acct_url}],
'type': 'object-store'}],
'token': {'id': 'XXXXXXX'}}}
return c.json_dumps(body)
c.http_connection = self.fake_http_connection(200, return_read=read)
self.assertRaises(c.ClientException, c.get_auth,
'http://www.tests.com', 'asdf', 'asdf',
auth_version='2.0')
class TestGetAccount(MockHttpTest):
def test_no_content(self):
c.http_connection = self.fake_http_connection(204)
value = c.get_account('http://www.test.com', 'asdf')[1]
self.assertEquals(value, [])
class TestHeadAccount(MockHttpTest):
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
value = c.head_account('http://www.tests.com', 'asdf')
# TODO: Hmm. This doesn't really test too much as it uses a fake that
# always returns the same dict. I guess it "exercises" the code, so
# I'll leave it for now.
self.assertEquals(type(value), dict)
def test_server_error(self):
body = 'c' * 65
c.http_connection = self.fake_http_connection(500, body=body)
self.assertRaises(c.ClientException, c.head_account,
'http://www.tests.com', 'asdf')
try:
value = c.head_account('http://www.tests.com', 'asdf')
except c.ClientException as e:
new_body = "[first 60 chars of response] " + body[0:60]
self.assertEquals(e.__str__()[-89:], new_body)
class TestGetContainer(MockHttpTest):
def test_no_content(self):
c.http_connection = self.fake_http_connection(204)
value = c.get_container('http://www.test.com', 'asdf', 'asdf')[1]
self.assertEquals(value, [])
class TestHeadContainer(MockHttpTest):
def test_server_error(self):
body = 'c' * 60
c.http_connection = self.fake_http_connection(500, body=body)
self.assertRaises(c.ClientException, c.head_container,
'http://www.test.com', 'asdf', 'asdf',
)
try:
value = c.head_container('http://www.test.com', 'asdf', 'asdf')
except c.ClientException as e:
self.assertEquals(e.http_response_content, body)
class TestPutContainer(MockHttpTest):
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
value = c.put_container('http://www.test.com', 'asdf', 'asdf')
self.assertEquals(value, None)
def test_server_error(self):
body = 'c' * 60
c.http_connection = self.fake_http_connection(500, body=body)
self.assertRaises(c.ClientException, c.put_container,
'http://www.test.com', 'asdf', 'asdf',
)
try:
value = c.put_container('http://www.test.com', 'asdf', 'asdf')
except c.ClientException as e:
self.assertEquals(e.http_response_content, body)
class TestDeleteContainer(MockHttpTest):
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
value = c.delete_container('http://www.test.com', 'asdf', 'asdf')
self.assertEquals(value, None)
class TestGetObject(MockHttpTest):
def test_server_error(self):
c.http_connection = self.fake_http_connection(500)
self.assertRaises(c.ClientException, c.get_object,
'http://www.test.com', 'asdf', 'asdf', 'asdf')
class TestHeadObject(MockHttpTest):
def test_server_error(self):
c.http_connection = self.fake_http_connection(500)
self.assertRaises(c.ClientException, c.head_object,
'http://www.test.com', 'asdf', 'asdf', 'asdf')
class TestPutObject(MockHttpTest):
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf')
value = c.put_object(*args)
self.assertTrue(isinstance(value, basestring))
def test_server_error(self):
body = 'c' * 60
c.http_connection = self.fake_http_connection(500, body=body)
args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf')
self.assertRaises(c.ClientException, c.put_object, *args)
try:
value = c.put_object(*args)
except c.ClientException as e:
self.assertEquals(e.http_response_content, body)
class TestPostObject(MockHttpTest):
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', {})
value = c.post_object(*args)
def test_server_error(self):
body = 'c' * 60
c.http_connection = self.fake_http_connection(500, body=body)
args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', {})
self.assertRaises(c.ClientException, c.post_object, *args)
try:
value = c.post_object(*args)
except c.ClientException as e:
self.assertEquals(e.http_response_content, body)
class TestDeleteObject(MockHttpTest):
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
value = c.delete_object('http://www.test.com', 'asdf', 'asdf', 'asdf')
def test_server_error(self):
c.http_connection = self.fake_http_connection(500)
self.assertRaises(c.ClientException, c.delete_object,
'http://www.test.com', 'asdf', 'asdf', 'asdf')
class TestConnection(MockHttpTest):
def test_instance(self):
conn = c.Connection('http://www.test.com', 'asdf', 'asdf')
self.assertEquals(conn.retries, 5)
def test_retry(self):
c.http_connection = self.fake_http_connection(500)
def quick_sleep(*args):
pass
c.sleep = quick_sleep
conn = c.Connection('http://www.test.com', 'asdf', 'asdf')
self.assertRaises(c.ClientException, conn.head_account)
self.assertEquals(conn.attempts, conn.retries + 1)
def test_resp_read_on_server_error(self):
c.http_connection = self.fake_http_connection(500)
conn = c.Connection('http://www.test.com', 'asdf', 'asdf', retries=0)
def get_auth(*args, **kwargs):
return 'http://www.new.com', 'new'
conn.get_auth = get_auth
self.url, self.token = conn.get_auth()
method_signatures = (
(conn.head_account, []),
(conn.get_account, []),
(conn.head_container, ('asdf',)),
(conn.get_container, ('asdf',)),
(conn.put_container, ('asdf',)),
(conn.delete_container, ('asdf',)),
(conn.head_object, ('asdf', 'asdf')),
(conn.get_object, ('asdf', 'asdf')),
(conn.put_object, ('asdf', 'asdf', 'asdf')),
(conn.post_object, ('asdf', 'asdf', {})),
(conn.delete_object, ('asdf', 'asdf')),
)
for method, args in method_signatures:
self.assertRaises(c.ClientException, method, *args)
try:
self.assertTrue(conn.http_conn[1].has_been_read)
except AssertionError:
msg = '%s did not read resp on server error' % method.__name__
self.fail(msg)
except Exception, e:
raise e.__class__("%s - %s" % (method.__name__, e))
def test_reauth(self):
c.http_connection = self.fake_http_connection(401)
def get_auth(*args, **kwargs):
return 'http://www.new.com', 'new'
def swap_sleep(*args):
self.swap_sleep_called = True
c.get_auth = get_auth
c.http_connection = self.fake_http_connection(200)
c.sleep = swap_sleep
self.swap_sleep_called = False
conn = c.Connection('http://www.test.com', 'asdf', 'asdf',
preauthurl='http://www.old.com',
preauthtoken='old',
)
self.assertEquals(conn.attempts, 0)
self.assertEquals(conn.url, 'http://www.old.com')
self.assertEquals(conn.token, 'old')
value = conn.head_account()
self.assertTrue(self.swap_sleep_called)
self.assertEquals(conn.attempts, 2)
self.assertEquals(conn.url, 'http://www.new.com')
self.assertEquals(conn.token, 'new')
def test_reset_stream(self):
class LocalContents(object):
def __init__(self, tell_value=0):
self.already_read = False
self.seeks = []
self.tell_value = tell_value
def tell(self):
return self.tell_value
def seek(self, position):
self.seeks.append(position)
self.already_read = False
def read(self, size=-1):
if self.already_read:
return ''
else:
self.already_read = True
return 'abcdef'
class LocalConnection(object):
def putrequest(self, *args, **kwargs):
return
def putheader(self, *args, **kwargs):
return
def endheaders(self, *args, **kwargs):
return
def send(self, *args, **kwargs):
raise socket.error('oops')
def request(self, *args, **kwargs):
return
def getresponse(self, *args, **kwargs):
self.status = 200
return self
def getheader(self, *args, **kwargs):
return ''
def read(self, *args, **kwargs):
return ''
def local_http_connection(url, proxy=None):
parsed = urlparse(url)
return parsed, LocalConnection()
orig_conn = c.http_connection
try:
c.http_connection = local_http_connection
conn = c.Connection('http://www.example.com', 'asdf', 'asdf',
retries=1, starting_backoff=.0001)
contents = LocalContents()
exc = None
try:
conn.put_object('c', 'o', contents)
except socket.error, err:
exc = err
self.assertEquals(contents.seeks, [0])
self.assertEquals(str(exc), 'oops')
contents = LocalContents(tell_value=123)
exc = None
try:
conn.put_object('c', 'o', contents)
except socket.error, err:
exc = err
self.assertEquals(contents.seeks, [123])
self.assertEquals(str(exc), 'oops')
contents = LocalContents()
contents.tell = None
exc = None
try:
conn.put_object('c', 'o', contents)
except c.ClientException, err:
exc = err
self.assertEquals(contents.seeks, [])
self.assertEquals(str(exc), "put_object('c', 'o', ...) failure "
"and no ability to reset contents for reupload.")
finally:
c.http_connection = orig_conn
if __name__ == '__main__':
unittest.main()

2
test/unit/container/test_sync.py
View File

@ -19,7 +19,7 @@ import re
from test.unit import FakeLogger
from swift.container import sync
from swift.common import utils
from swift.common.client import ClientException
from swiftclient import ClientException
utils.HASH_PATH_SUFFIX = 'endcap'

1
test/unit/proxy/test_server.py
View File

@ -2898,7 +2898,6 @@ class TestObjectController(unittest.TestCase):
finally:
proxy_server.time.time = orig_time
def test_POST_non_int_delete_after(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',

3
tools/pip-requires
View File

@ -6,3 +6,6 @@ netifaces==0.6
pastedeploy==1.3.3
simplejson==2.0.9
xattr==0.4
# Install python-swiftclient from git
https://github.com/openstack/python-swiftclient/zipball/master#egg=python-swiftclient