From 41b91061436210afd1d22cad3c2ee1e038c23e3c Mon Sep 17 00:00:00 2001 From: Luis Pabon Date: Fri, 14 Jun 2013 00:20:41 -0400 Subject: [PATCH] Copy OpenStack Swift (Grizzly) Functional tests Copy the functional tests to our tree so that we can edit and skip any tests we know we are not going to support for this release Change-Id: I93a76550aaaa58de49ec9a7178a34e081b7b7cf0 Signed-off-by: Luis Pabon Reviewed-on: http://review.gluster.org/5211 --- test/functional/__init__.py | 0 test/functional/swift_test_client.py | 736 +++++++++ test/functional/tests.py | 1617 ++++++++++++++++++++ test/functionalnosetests/__init__.py | 0 test/functionalnosetests/swift_testing.py | 175 +++ test/functionalnosetests/test_account.py | 152 ++ test/functionalnosetests/test_container.py | 573 +++++++ test/functionalnosetests/test_object.py | 600 ++++++++ 8 files changed, 3853 insertions(+) create mode 100644 test/functional/__init__.py create mode 100644 test/functional/swift_test_client.py create mode 100644 test/functional/tests.py create mode 100644 test/functionalnosetests/__init__.py create mode 100644 test/functionalnosetests/swift_testing.py create mode 100755 test/functionalnosetests/test_account.py create mode 100755 test/functionalnosetests/test_container.py create mode 100755 test/functionalnosetests/test_object.py diff --git a/test/functional/__init__.py b/test/functional/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py new file mode 100644 index 0000000..a6d8aec --- /dev/null +++ b/test/functional/swift_test_client.py @@ -0,0 +1,736 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import httplib +import os +import random +import socket +import StringIO +import time +import urllib + +import simplejson as json + +from nose import SkipTest +from xml.dom import minidom +from swiftclient import get_auth + + +class AuthenticationFailed(Exception): + pass + + +class RequestError(Exception): + pass + + +class ResponseError(Exception): + def __init__(self, response): + self.status = response.status + self.reason = response.reason + Exception.__init__(self) + + def __str__(self): + return '%d: %s' % (self.status, self.reason) + + def __repr__(self): + return '%d: %s' % (self.status, self.reason) + + +def listing_empty(method): + for i in xrange(0, 6): + if len(method()) == 0: + return True + + time.sleep(2 ** i) + + return False + + +def listing_items(method): + marker = None + once = True + items = [] + + while once or items: + for i in items: + yield i + + if once or marker: + if marker: + items = method(parms={'marker': marker}) + else: + items = method() + + if len(items) == 10000: + marker = items[-1] + else: + marker = None + + once = False + else: + items = [] + + +class Connection(object): + def __init__(self, config): + for key in 'auth_host auth_port auth_ssl username password'.split(): + if key not in config: + raise SkipTest + + self.auth_host = config['auth_host'] + self.auth_port = int(config['auth_port']) + self.auth_ssl = config['auth_ssl'] in ('on', 'true', 'yes', '1') + self.auth_prefix = config.get('auth_prefix', '/') + self.auth_version = str(config.get('auth_version', '1')) + + self.account = config.get('account') + self.username = config['username'] + self.password = config['password'] + + self.storage_host = None + self.storage_port = None + + self.conn_class = None + + def get_account(self): + return Account(self, self.account) + + def authenticate(self, clone_conn=None): + if clone_conn: + self.conn_class = clone_conn.conn_class + self.storage_host = clone_conn.storage_host + self.storage_url = clone_conn.storage_url + self.storage_port = clone_conn.storage_port + self.storage_token = clone_conn.storage_token + return + + if self.auth_version == "1": + auth_path = '%sv1.0' % (self.auth_prefix) + if self.account: + auth_user = '%s:%s' % (self.account, self.username) + else: + auth_user = self.username + else: + auth_user = self.username + auth_path = self.auth_prefix + auth_scheme = 'https://' if self.auth_ssl else 'http://' + auth_netloc = "%s:%d" % (self.auth_host, self.auth_port) + auth_url = auth_scheme + auth_netloc + auth_path + + (storage_url, storage_token) = get_auth(auth_url, + auth_user, self.password, + snet=False, + tenant_name=self.account, + auth_version=self.auth_version, + os_options={}) + + if not (storage_url and storage_token): + raise AuthenticationFailed() + + x = storage_url.split('/') + + if x[0] == 'http:': + self.conn_class = httplib.HTTPConnection + self.storage_port = 80 + elif x[0] == 'https:': + self.conn_class = httplib.HTTPSConnection + self.storage_port = 443 + else: + raise ValueError('unexpected protocol %s' % (x[0])) + + self.storage_host = x[2].split(':')[0] + if ':' in x[2]: + self.storage_port = int(x[2].split(':')[1]) + self.storage_url = '/%s/%s' % (x[3], x[4]) + + self.storage_token = storage_token + + self.http_connect() + return self.storage_url, self.storage_token + + def http_connect(self): + self.connection = self.conn_class(self.storage_host, + port=self.storage_port) + #self.connection.set_debuglevel(3) + + def make_path(self, path=[], cfg={}): + if cfg.get('version_only_path'): + return '/' + self.storage_url.split('/')[1] + + if path: + quote = urllib.quote + if cfg.get('no_quote') or cfg.get('no_path_quote'): + quote = lambda x: x + return '%s/%s' % (self.storage_url, + '/'.join([quote(i) for i in path])) + else: + return self.storage_url + + def make_headers(self, hdrs, cfg={}): + headers = {} + + if not cfg.get('no_auth_token'): + headers['X-Auth-Token'] = self.storage_token + + if isinstance(hdrs, dict): + headers.update(hdrs) + return headers + + def make_request(self, method, path=[], data='', hdrs={}, parms={}, + cfg={}): + path = self.make_path(path, cfg=cfg) + headers = self.make_headers(hdrs, cfg=cfg) + if isinstance(parms, dict) and parms: + quote = urllib.quote + if cfg.get('no_quote') or cfg.get('no_parms_quote'): + quote = lambda x: x + query_args = ['%s=%s' % (quote(x), quote(str(y))) + for (x, y) in parms.items()] + path = '%s?%s' % (path, '&'.join(query_args)) + if not cfg.get('no_content_length'): + if cfg.get('set_content_length'): + headers['Content-Length'] = cfg.get('set_content_length') + else: + headers['Content-Length'] = len(data) + + def try_request(): + self.http_connect() + self.connection.request(method, path, data, headers) + return self.connection.getresponse() + + self.response = None + try_count = 0 + while try_count < 5: + try_count += 1 + + try: + self.response = try_request() + except httplib.HTTPException: + continue + + if self.response.status == 401: + self.authenticate() + continue + elif self.response.status == 503: + if try_count != 5: + time.sleep(5) + continue + + break + + if self.response: + return self.response.status + + raise RequestError('Unable to complete http request') + + def put_start(self, path, hdrs={}, parms={}, cfg={}, chunked=False): + self.http_connect() + + path = self.make_path(path, cfg) + headers = self.make_headers(hdrs, cfg=cfg) + + if chunked: + headers['Transfer-Encoding'] = 'chunked' + headers.pop('Content-Length', None) + + if isinstance(parms, dict) and parms: + quote = urllib.quote + if cfg.get('no_quote') or cfg.get('no_parms_quote'): + quote = lambda x: x + query_args = ['%s=%s' % (quote(x), quote(str(y))) + for (x, y) in parms.items()] + path = '%s?%s' % (path, '&'.join(query_args)) + + query_args = ['%s=%s' % (urllib.quote(x), + urllib.quote(str(y))) for (x, y) in parms.items()] + path = '%s?%s' % (path, '&'.join(query_args)) + + self.connection = self.conn_class(self.storage_host, + port=self.storage_port) + #self.connection.set_debuglevel(3) + self.connection.putrequest('PUT', path) + for key, value in headers.iteritems(): + self.connection.putheader(key, value) + self.connection.endheaders() + + def put_data(self, data, chunked=False): + if chunked: + self.connection.send('%x\r\n%s\r\n' % (len(data), data)) + else: + self.connection.send(data) + + def put_end(self, chunked=False): + if chunked: + self.connection.send('0\r\n\r\n') + + self.response = self.connection.getresponse() + self.connection.close() + return self.response.status + + +class Base: + def __str__(self): + return self.name + + def header_fields(self, fields): + headers = dict(self.conn.response.getheaders()) + ret = {} + for field in fields: + if field[1] not in headers: + raise ValueError("%s was not found in response header" % + (field[1])) + + try: + ret[field[0]] = int(headers[field[1]]) + except ValueError: + ret[field[0]] = headers[field[1]] + return ret + + +class Account(Base): + def __init__(self, conn, name): + self.conn = conn + self.name = str(name) + + def container(self, container_name): + return Container(self.conn, self.name, container_name) + + def containers(self, hdrs={}, parms={}, cfg={}): + format = parms.get('format', None) + if format not in [None, 'json', 'xml']: + raise RequestError('Invalid format: %s' % format) + if format is None and 'format' in parms: + del parms['format'] + + status = self.conn.make_request('GET', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) + if status == 200: + if format == 'json': + conts = json.loads(self.conn.response.read()) + for cont in conts: + cont['name'] = cont['name'].encode('utf-8') + return conts + elif format == 'xml': + conts = [] + tree = minidom.parseString(self.conn.response.read()) + for x in tree.getElementsByTagName('container'): + cont = {} + for key in ['name', 'count', 'bytes']: + cont[key] = x.getElementsByTagName(key)[0].\ + childNodes[0].nodeValue + conts.append(cont) + for cont in conts: + cont['name'] = cont['name'].encode('utf-8') + return conts + else: + lines = self.conn.response.read().split('\n') + if lines and not lines[-1]: + lines = lines[:-1] + return lines + elif status == 204: + return [] + + raise ResponseError(self.conn.response) + + def delete_containers(self): + for c in listing_items(self.containers): + cont = self.container(c) + if not cont.delete_recursive(): + return False + + return listing_empty(self.containers) + + def info(self, hdrs={}, parms={}, cfg={}): + if self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) != 204: + + raise ResponseError(self.conn.response) + + fields = [['object_count', 'x-account-object-count'], + ['container_count', 'x-account-container-count'], + ['bytes_used', 'x-account-bytes-used']] + + return self.header_fields(fields) + + @property + def path(self): + return [] + + +class Container(Base): + def __init__(self, conn, account, name): + self.conn = conn + self.account = str(account) + self.name = str(name) + + def create(self, hdrs={}, parms={}, cfg={}): + return self.conn.make_request('PUT', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) in (201, 202) + + def delete(self, hdrs={}, parms={}): + return self.conn.make_request('DELETE', self.path, hdrs=hdrs, + parms=parms) == 204 + + def delete_files(self): + for f in listing_items(self.files): + file = self.file(f) + if not file.delete(): + return False + + return listing_empty(self.files) + + def delete_recursive(self): + return self.delete_files() and self.delete() + + def file(self, file_name): + return File(self.conn, self.account, self.name, file_name) + + def files(self, hdrs={}, parms={}, cfg={}): + format = parms.get('format', None) + if format not in [None, 'json', 'xml']: + raise RequestError('Invalid format: %s' % format) + if format is None and 'format' in parms: + del parms['format'] + + status = self.conn.make_request('GET', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) + if status == 200: + if format == 'json': + files = json.loads(self.conn.response.read()) + + for file in files: + file['name'] = file['name'].encode('utf-8') + file['content_type'] = file['content_type'].encode('utf-8') + return files + elif format == 'xml': + files = [] + tree = minidom.parseString(self.conn.response.read()) + for x in tree.getElementsByTagName('object'): + file = {} + for key in ['name', 'hash', 'bytes', 'content_type', + 'last_modified']: + + file[key] = x.getElementsByTagName(key)[0].\ + childNodes[0].nodeValue + files.append(file) + + for file in files: + file['name'] = file['name'].encode('utf-8') + file['content_type'] = file['content_type'].encode('utf-8') + return files + else: + content = self.conn.response.read() + if content: + lines = content.split('\n') + if lines and not lines[-1]: + lines = lines[:-1] + return lines + else: + return [] + elif status == 204: + return [] + + raise ResponseError(self.conn.response) + + def info(self, hdrs={}, parms={}, cfg={}): + status = self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) + + if self.conn.response.status == 204: + fields = [['bytes_used', 'x-container-bytes-used'], + ['object_count', 'x-container-object-count']] + + return self.header_fields(fields) + + raise ResponseError(self.conn.response) + + @property + def path(self): + return [self.name] + + +class File(Base): + def __init__(self, conn, account, container, name): + self.conn = conn + self.account = str(account) + self.container = str(container) + self.name = str(name) + + self.chunked_write_in_progress = False + self.content_type = None + self.size = None + self.metadata = {} + + def make_headers(self, cfg={}): + headers = {} + if not cfg.get('no_content_length'): + if cfg.get('set_content_length'): + headers['Content-Length'] = cfg.get('set_content_length') + elif self.size: + headers['Content-Length'] = self.size + else: + headers['Content-Length'] = 0 + + if cfg.get('no_content_type'): + pass + elif self.content_type: + headers['Content-Type'] = self.content_type + else: + headers['Content-Type'] = 'application/octet-stream' + + for key in self.metadata: + headers['X-Object-Meta-' + key] = self.metadata[key] + + return headers + + @classmethod + def compute_md5sum(cls, data): + block_size = 4096 + + if isinstance(data, str): + data = StringIO.StringIO(data) + + checksum = hashlib.md5() + buff = data.read(block_size) + while buff: + checksum.update(buff) + buff = data.read(block_size) + data.seek(0) + return checksum.hexdigest() + + def copy(self, dest_cont, dest_file, hdrs={}, parms={}, cfg={}): + if 'destination' in cfg: + headers = {'Destination': cfg['destination']} + elif cfg.get('no_destination'): + headers = {} + else: + headers = {'Destination': '%s/%s' % (dest_cont, dest_file)} + headers.update(hdrs) + + if 'Destination' in headers: + headers['Destination'] = urllib.quote(headers['Destination']) + + return self.conn.make_request('COPY', self.path, hdrs=headers, + parms=parms) == 201 + + def delete(self, hdrs={}, parms={}): + if self.conn.make_request('DELETE', self.path, hdrs=hdrs, + parms=parms) != 204: + + raise ResponseError(self.conn.response) + + return True + + def info(self, hdrs={}, parms={}, cfg={}): + if self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms, cfg=cfg) != 200: + + raise ResponseError(self.conn.response) + + fields = [['content_length', 'content-length'], + ['content_type', 'content-type'], + ['last_modified', 'last-modified'], + ['etag', 'etag']] + + header_fields = self.header_fields(fields) + header_fields['etag'] = header_fields['etag'].strip('"') + return header_fields + + def initialize(self, hdrs={}, parms={}): + if not self.name: + return False + + status = self.conn.make_request('HEAD', self.path, hdrs=hdrs, + parms=parms) + if status == 404: + return False + elif (status < 200) or (status > 299): + raise ResponseError(self.conn.response) + + for hdr in self.conn.response.getheaders(): + if hdr[0].lower() == 'content-type': + self.content_type = hdr[1] + if hdr[0].lower().startswith('x-object-meta-'): + self.metadata[hdr[0][14:]] = hdr[1] + if hdr[0].lower() == 'etag': + self.etag = hdr[1].strip('"') + if hdr[0].lower() == 'content-length': + self.size = int(hdr[1]) + if hdr[0].lower() == 'last-modified': + self.last_modified = hdr[1] + + return True + + def load_from_filename(self, filename, callback=None): + fobj = open(filename, 'rb') + self.write(fobj, callback=callback) + fobj.close() + + @property + def path(self): + return [self.container, self.name] + + @classmethod + def random_data(cls, size=None): + if size is None: + size = random.randint(1, 32768) + fd = open('/dev/urandom', 'r') + data = fd.read(size) + fd.close() + return data + + def read(self, size=-1, offset=0, hdrs=None, buffer=None, + callback=None, cfg={}): + + if size > 0: + range = 'bytes=%d-%d' % (offset, (offset + size) - 1) + if hdrs: + hdrs['Range'] = range + else: + hdrs = {'Range': range} + + status = self.conn.make_request('GET', self.path, hdrs=hdrs, + cfg=cfg) + + if(status < 200) or (status > 299): + raise ResponseError(self.conn.response) + + for hdr in self.conn.response.getheaders(): + if hdr[0].lower() == 'content-type': + self.content_type = hdr[1] + + if hasattr(buffer, 'write'): + scratch = self.conn.response.read(8192) + transferred = 0 + + while len(scratch) > 0: + buffer.write(scratch) + transferred += len(scratch) + if callable(callback): + callback(transferred, self.size) + scratch = self.conn.response.read(8192) + return None + else: + return self.conn.response.read() + + def read_md5(self): + status = self.conn.make_request('GET', self.path) + + if(status < 200) or (status > 299): + raise ResponseError(self.conn.response) + + checksum = hashlib.md5() + + scratch = self.conn.response.read(8192) + while len(scratch) > 0: + checksum.update(scratch) + scratch = self.conn.response.read(8192) + + return checksum.hexdigest() + + def save_to_filename(self, filename, callback=None): + try: + fobj = open(filename, 'wb') + self.read(buffer=fobj, callback=callback) + finally: + fobj.close() + + def sync_metadata(self, metadata={}, cfg={}): + self.metadata.update(metadata) + + if self.metadata: + headers = self.make_headers(cfg=cfg) + if not cfg.get('no_content_length'): + if cfg.get('set_content_length'): + headers['Content-Length'] = \ + cfg.get('set_content_length') + else: + headers['Content-Length'] = 0 + + self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) + + if self.conn.response.status not in (201, 202): + raise ResponseError(self.conn.response) + + return True + + def chunked_write(self, data=None, hdrs={}, parms={}, cfg={}): + if data is not None and self.chunked_write_in_progress: + self.conn.put_data(data, True) + elif data is not None: + self.chunked_write_in_progress = True + + headers = self.make_headers(cfg=cfg) + headers.update(hdrs) + + self.conn.put_start(self.path, hdrs=headers, parms=parms, + cfg=cfg, chunked=True) + + self.conn.put_data(data, True) + elif self.chunked_write_in_progress: + self.chunked_write_in_progress = False + return self.conn.put_end(True) == 201 + else: + raise RuntimeError + + def write(self, data='', hdrs={}, parms={}, callback=None, cfg={}): + block_size = 2 ** 20 + + if isinstance(data, file): + try: + data.flush() + data.seek(0) + except IOError: + pass + self.size = int(os.fstat(data.fileno())[6]) + else: + data = StringIO.StringIO(data) + self.size = data.len + + headers = self.make_headers(cfg=cfg) + headers.update(hdrs) + + self.conn.put_start(self.path, hdrs=headers, parms=parms, cfg=cfg) + + transferred = 0 + buff = data.read(block_size) + try: + while len(buff) > 0: + self.conn.put_data(buff) + buff = data.read(block_size) + transferred += len(buff) + if callable(callback): + callback(transferred, self.size) + + self.conn.put_end() + except socket.timeout, err: + raise err + + if (self.conn.response.status < 200) or \ + (self.conn.response.status > 299): + raise ResponseError(self.conn.response) + + self.md5 = self.compute_md5sum(data) + + return True + + def write_random(self, size=None, hdrs={}, parms={}, cfg={}): + data = self.random_data(size) + if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg): + raise ResponseError(self.conn.response) + self.md5 = self.compute_md5sum(StringIO.StringIO(data)) + return data diff --git a/test/functional/tests.py b/test/functional/tests.py new file mode 100644 index 0000000..d6f8d70 --- /dev/null +++ b/test/functional/tests.py @@ -0,0 +1,1617 @@ +#!/usr/bin/python -u +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +import locale +import random +import StringIO +import time +import threading +import uuid +import unittest +from nose import SkipTest +from ConfigParser import ConfigParser + +from test import get_config +from test.functional.swift_test_client import Account, Connection, File, \ + ResponseError +from swift.common.constraints import MAX_FILE_SIZE, MAX_META_NAME_LENGTH, \ + MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ + MAX_OBJECT_NAME_LENGTH, CONTAINER_LISTING_LIMIT, ACCOUNT_LISTING_LIMIT, \ + MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH + +default_constraints = dict(( + ('max_file_size', MAX_FILE_SIZE), + ('max_meta_name_length', MAX_META_NAME_LENGTH), + ('max_meta_value_length', MAX_META_VALUE_LENGTH), + ('max_meta_count', MAX_META_COUNT), + ('max_meta_overall_size', MAX_META_OVERALL_SIZE), + ('max_object_name_length', MAX_OBJECT_NAME_LENGTH), + ('container_listing_limit', CONTAINER_LISTING_LIMIT), + ('account_listing_limit', ACCOUNT_LISTING_LIMIT), + ('max_account_name_length', MAX_ACCOUNT_NAME_LENGTH), + ('max_container_name_length', MAX_CONTAINER_NAME_LENGTH))) +constraints_conf = ConfigParser() +conf_exists = constraints_conf.read('/etc/swift/swift.conf') +# Constraints are set first from the test config, then from +# /etc/swift/swift.conf if it exists. If swift.conf doesn't exist, +# then limit test coverage. This allows SAIO tests to work fine but +# requires remote funtional testing to know something about the cluster +# that is being tested. +config = get_config('func_test') +for k in default_constraints: + if k in config: + # prefer what's in test.conf + config[k] = int(config[k]) + elif conf_exists: + # swift.conf exists, so use what's defined there (or swift defaults) + # This normally happens when the test is running locally to the cluster + # as in a SAIO. + config[k] = default_constraints[k] + else: + # .functests don't know what the constraints of the tested cluster are, + # so the tests can't reliably pass or fail. Therefore, skip those + # tests. + config[k] = '%s constraint is not defined' % k + +web_front_end = config.get('web_front_end', 'integral') +normalized_urls = config.get('normalized_urls', False) + +def load_constraint(name): + c = config[name] + if not isinstance(c, int): + raise SkipTest(c) + return c + +locale.setlocale(locale.LC_COLLATE, config.get('collate', 'C')) + + +def chunks(s, length=3): + i, j = 0, length + while i < len(s): + yield s[i:j] + i, j = j, j + length + + +def timeout(seconds, method, *args, **kwargs): + class TimeoutThread(threading.Thread): + def __init__(self, method, *args, **kwargs): + threading.Thread.__init__(self) + + self.method = method + self.args = args + self.kwargs = kwargs + self.exception = None + + def run(self): + try: + self.method(*self.args, **self.kwargs) + except Exception, e: + self.exception = e + + t = TimeoutThread(method, *args, **kwargs) + t.start() + t.join(seconds) + + if t.exception: + raise t.exception + + if t.isAlive(): + t._Thread__stop() + return True + return False + + +class Utils: + @classmethod + def create_ascii_name(cls, length=None): + return uuid.uuid4().hex + + @classmethod + def create_utf8_name(cls, length=None): + if length is None: + length = 15 + else: + length = int(length) + + utf8_chars = u'\uF10F\uD20D\uB30B\u9409\u8508\u5605\u3703\u1801'\ + u'\u0900\uF110\uD20E\uB30C\u940A\u8509\u5606\u3704'\ + u'\u1802\u0901\uF111\uD20F\uB30D\u940B\u850A\u5607'\ + u'\u3705\u1803\u0902\uF112\uD210\uB30E\u940C\u850B'\ + u'\u5608\u3706\u1804\u0903\u03A9\u2603' + return ''.join([random.choice(utf8_chars) + for x in xrange(length)]).encode('utf-8') + + create_name = create_ascii_name + + +class Base(unittest.TestCase): + def setUp(self): + cls = type(self) + if not cls.set_up: + cls.env.setUp() + cls.set_up = True + + def assert_body(self, body): + response_body = self.env.conn.response.read() + self.assert_(response_body == body, + 'Body returned: %s' % (response_body)) + + def assert_status(self, status_or_statuses): + self.assert_(self.env.conn.response.status == status_or_statuses or + (hasattr(status_or_statuses, '__iter__') and + self.env.conn.response.status in status_or_statuses), + 'Status returned: %d Expected: %s' % + (self.env.conn.response.status, status_or_statuses)) + + +class Base2(object): + def setUp(self): + Utils.create_name = Utils.create_utf8_name + super(Base2, self).setUp() + + def tearDown(self): + Utils.create_name = Utils.create_ascii_name + + +class TestAccountEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + cls.containers = [] + for i in range(10): + cont = cls.account.container(Utils.create_name()) + if not cont.create(): + raise ResponseError(cls.conn.response) + + cls.containers.append(cont) + + +class TestAccountDev(Base): + env = TestAccountEnv + set_up = False + + +class TestAccountDevUTF8(Base2, TestAccountDev): + set_up = False + + +class TestAccount(Base): + env = TestAccountEnv + set_up = False + + def testNoAuthToken(self): + self.assertRaises(ResponseError, self.env.account.info, + cfg={'no_auth_token': True}) + self.assert_status([401, 412]) + + self.assertRaises(ResponseError, self.env.account.containers, + cfg={'no_auth_token': True}) + self.assert_status([401, 412]) + + def testInvalidUTF8Path(self): + invalid_utf8 = Utils.create_utf8_name()[::-1] + container = self.env.account.container(invalid_utf8) + self.assert_(not container.create(cfg={'no_path_quote': True})) + self.assert_status(412) + self.assert_body('Invalid UTF8 or contains NULL') + + def testVersionOnlyPath(self): + self.env.account.conn.make_request('PUT', + cfg={'version_only_path': True}) + self.assert_status(412) + self.assert_body('Bad URL') + + def testInvalidPath(self): + was_url = self.env.account.conn.storage_url + if (normalized_urls): + self.env.account.conn.storage_url = '/' + else: + self.env.account.conn.storage_url = "/%s" % was_url + self.env.account.conn.make_request('GET') + try: + self.assert_status(404) + finally: + self.env.account.conn.storage_url = was_url + + def testPUT(self): + self.env.account.conn.make_request('PUT') + self.assert_status([403, 405]) + + def testAccountHead(self): + try_count = 0 + while try_count < 5: + try_count += 1 + + info = self.env.account.info() + for field in ['object_count', 'container_count', 'bytes_used']: + self.assert_(info[field] >= 0) + + if info['container_count'] == len(self.env.containers): + break + + if try_count < 5: + time.sleep(1) + + self.assertEquals(info['container_count'], len(self.env.containers)) + self.assert_status(204) + + def testContainerSerializedInfo(self): + container_info = {} + for container in self.env.containers: + info = {'bytes': 0} + info['count'] = random.randint(10, 30) + for i in range(info['count']): + file = container.file(Utils.create_name()) + bytes = random.randint(1, 32768) + file.write_random(bytes) + info['bytes'] += bytes + + container_info[container.name] = info + + for format in ['json', 'xml']: + for a in self.env.account.containers(parms={'format': format}): + self.assert_(a['count'] >= 0) + self.assert_(a['bytes'] >= 0) + + headers = dict(self.env.conn.response.getheaders()) + if format == 'json': + self.assertEquals(headers['content-type'], + 'application/json; charset=utf-8') + elif format == 'xml': + self.assertEquals(headers['content-type'], + 'application/xml; charset=utf-8') + + def testListingLimit(self): + limit = load_constraint('account_listing_limit') + for l in (1, 100, limit / 2, limit - 1, limit, limit + 1, limit * 2): + p = {'limit': l} + + if l <= limit: + self.assert_(len(self.env.account.containers(parms=p)) <= l) + self.assert_status(200) + else: + self.assertRaises(ResponseError, + self.env.account.containers, parms=p) + self.assert_status(412) + + def testContainerListing(self): + a = sorted([c.name for c in self.env.containers]) + + for format in [None, 'json', 'xml']: + b = self.env.account.containers(parms={'format': format}) + + if isinstance(b[0], dict): + b = [x['name'] for x in b] + + self.assertEquals(a, b) + + def testInvalidAuthToken(self): + hdrs = {'X-Auth-Token': 'bogus_auth_token'} + self.assertRaises(ResponseError, self.env.account.info, hdrs=hdrs) + self.assert_status(401) + + def testLastContainerMarker(self): + for format in [None, 'json', 'xml']: + containers = self.env.account.containers({'format': format}) + self.assertEquals(len(containers), len(self.env.containers)) + self.assert_status(200) + + containers = self.env.account.containers( + parms={'format': format, 'marker': containers[-1]}) + self.assertEquals(len(containers), 0) + if format is None: + self.assert_status(204) + else: + self.assert_status(200) + + def testMarkerLimitContainerList(self): + for format in [None, 'json', 'xml']: + for marker in ['0', 'A', 'I', 'R', 'Z', 'a', 'i', 'r', 'z', + 'abc123', 'mnop', 'xyz']: + + limit = random.randint(2, 9) + containers = self.env.account.containers( + parms={'format': format, 'marker': marker, 'limit': limit}) + self.assert_(len(containers) <= limit) + if containers: + if isinstance(containers[0], dict): + containers = [x['name'] for x in containers] + self.assert_(locale.strcoll(containers[0], marker) > 0) + + def testContainersOrderedByName(self): + for format in [None, 'json', 'xml']: + containers = self.env.account.containers( + parms={'format': format}) + if isinstance(containers[0], dict): + containers = [x['name'] for x in containers] + self.assertEquals(sorted(containers, cmp=locale.strcoll), + containers) + + +class TestAccountUTF8(Base2, TestAccount): + set_up = False + + +class TestAccountNoContainersEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + +class TestAccountNoContainers(Base): + env = TestAccountNoContainersEnv + set_up = False + + def testGetRequest(self): + for format in [None, 'json', 'xml']: + self.assert_(not self.env.account.containers( + parms={'format': format})) + + if format is None: + self.assert_status(204) + else: + self.assert_status(200) + + +class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers): + set_up = False + + +class TestContainerEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.file_count = 10 + cls.file_size = 128 + cls.files = list() + for x in range(cls.file_count): + file = cls.container.file(Utils.create_name()) + file.write_random(cls.file_size) + cls.files.append(file.name) + + +class TestContainerDev(Base): + env = TestContainerEnv + set_up = False + + +class TestContainerDevUTF8(Base2, TestContainerDev): + set_up = False + + +class TestContainer(Base): + env = TestContainerEnv + set_up = False + + def testContainerNameLimit(self): + limit = load_constraint('max_container_name_length') + + for l in (limit - 100, limit - 10, limit - 1, limit, + limit + 1, limit + 10, limit + 100): + cont = self.env.account.container('a' * l) + if l <= limit: + self.assert_(cont.create()) + self.assert_status(201) + else: + self.assert_(not cont.create()) + self.assert_status(400) + + def testFileThenContainerDelete(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + file = cont.file(Utils.create_name()) + self.assert_(file.write_random()) + + self.assert_(file.delete()) + self.assert_status(204) + self.assert_(file.name not in cont.files()) + + self.assert_(cont.delete()) + self.assert_status(204) + self.assert_(cont.name not in self.env.account.containers()) + + def testFileListingLimitMarkerPrefix(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + + files = sorted([Utils.create_name() for x in xrange(10)]) + for f in files: + file = cont.file(f) + self.assert_(file.write_random()) + + for i in xrange(len(files)): + f = files[i] + for j in xrange(1, len(files) - i): + self.assert_(cont.files(parms={'limit': j, 'marker': f}) == + files[i + 1: i + j + 1]) + self.assert_(cont.files(parms={'marker': f}) == files[i + 1:]) + self.assert_(cont.files(parms={'marker': f, 'prefix': f}) == []) + self.assert_(cont.files(parms={'prefix': f}) == [f]) + + def testPrefixAndLimit(self): + load_constraint('container_listing_limit') + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + + prefix_file_count = 10 + limit_count = 2 + prefixs = ['alpha/', 'beta/', 'kappa/'] + prefix_files = {} + + all_files = [] + for prefix in prefixs: + prefix_files[prefix] = [] + + for i in range(prefix_file_count): + file = cont.file(prefix + Utils.create_name()) + file.write() + prefix_files[prefix].append(file.name) + + for format in [None, 'json', 'xml']: + for prefix in prefixs: + files = cont.files(parms={'prefix': prefix}) + self.assertEquals(files, sorted(prefix_files[prefix])) + + for format in [None, 'json', 'xml']: + for prefix in prefixs: + files = cont.files(parms={'limit': limit_count, + 'prefix': prefix}) + self.assertEquals(len(files), limit_count) + + for file in files: + self.assert_(file.startswith(prefix)) + + def testCreate(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + self.assert_status(201) + self.assert_(cont.name in self.env.account.containers()) + + def testContainerFileListOnContainerThatDoesNotExist(self): + for format in [None, 'json', 'xml']: + container = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, container.files, + parms={'format': format}) + self.assert_status(404) + + def testUtf8Container(self): + valid_utf8 = Utils.create_utf8_name() + invalid_utf8 = valid_utf8[::-1] + container = self.env.account.container(valid_utf8) + self.assert_(container.create(cfg={'no_path_quote': True})) + self.assert_(container.name in self.env.account.containers()) + self.assertEquals(container.files(), []) + self.assert_(container.delete()) + + container = self.env.account.container(invalid_utf8) + self.assert_(not container.create(cfg={'no_path_quote': True})) + self.assert_status(412) + self.assertRaises(ResponseError, container.files, + cfg={'no_path_quote': True}) + self.assert_status(412) + + def testCreateOnExisting(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + self.assert_status(201) + self.assert_(cont.create()) + self.assert_status(202) + + def testSlashInName(self): + if Utils.create_name == Utils.create_utf8_name: + cont_name = list(unicode(Utils.create_name(), 'utf-8')) + else: + cont_name = list(Utils.create_name()) + + cont_name[random.randint(2, len(cont_name) - 2)] = '/' + cont_name = ''.join(cont_name) + + if Utils.create_name == Utils.create_utf8_name: + cont_name = cont_name.encode('utf-8') + + cont = self.env.account.container(cont_name) + self.assert_(not cont.create(cfg={'no_path_quote': True}), + 'created container with name %s' % (cont_name)) + self.assert_status(404) + self.assert_(cont.name not in self.env.account.containers()) + + def testDelete(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + self.assert_status(201) + self.assert_(cont.delete()) + self.assert_status(204) + self.assert_(cont.name not in self.env.account.containers()) + + def testDeleteOnContainerThatDoesNotExist(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(not cont.delete()) + self.assert_status(404) + + def testDeleteOnContainerWithFiles(self): + cont = self.env.account.container(Utils.create_name()) + self.assert_(cont.create()) + file = cont.file(Utils.create_name()) + file.write_random(self.env.file_size) + self.assert_(file.name in cont.files()) + self.assert_(not cont.delete()) + self.assert_status(409) + + def testFileCreateInContainerThatDoesNotExist(self): + file = File(self.env.conn, self.env.account, Utils.create_name(), + Utils.create_name()) + self.assertRaises(ResponseError, file.write) + self.assert_status(404) + + def testLastFileMarker(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files({'format': format}) + self.assertEquals(len(files), len(self.env.files)) + self.assert_status(200) + + files = self.env.container.files( + parms={'format': format, 'marker': files[-1]}) + self.assertEquals(len(files), 0) + + if format is None: + self.assert_status(204) + else: + self.assert_status(200) + + def testContainerFileList(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files(parms={'format': format}) + self.assert_status(200) + if isinstance(files[0], dict): + files = [x['name'] for x in files] + + for file in self.env.files: + self.assert_(file in files) + + for file in files: + self.assert_(file in self.env.files) + + def testMarkerLimitFileList(self): + for format in [None, 'json', 'xml']: + for marker in ['0', 'A', 'I', 'R', 'Z', 'a', 'i', 'r', 'z', + 'abc123', 'mnop', 'xyz']: + limit = random.randint(2, self.env.file_count - 1) + files = self.env.container.files(parms={'format': format, + 'marker': marker, + 'limit': limit}) + + if not files: + continue + + if isinstance(files[0], dict): + files = [x['name'] for x in files] + + self.assert_(len(files) <= limit) + if files: + if isinstance(files[0], dict): + files = [x['name'] for x in files] + self.assert_(locale.strcoll(files[0], marker) > 0) + + def testFileOrder(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files(parms={'format': format}) + if isinstance(files[0], dict): + files = [x['name'] for x in files] + self.assertEquals(sorted(files, cmp=locale.strcoll), files) + + def testContainerInfo(self): + info = self.env.container.info() + self.assert_status(204) + self.assertEquals(info['object_count'], self.env.file_count) + self.assertEquals(info['bytes_used'], + self.env.file_count * self.env.file_size) + + def testContainerInfoOnContainerThatDoesNotExist(self): + container = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, container.info) + self.assert_status(404) + + def testContainerFileListWithLimit(self): + for format in [None, 'json', 'xml']: + files = self.env.container.files(parms={'format': format, + 'limit': 2}) + self.assertEquals(len(files), 2) + + def testTooLongName(self): + cont = self.env.account.container('x' * 257) + self.assert_(not cont.create(), + 'created container with name %s' % (cont.name)) + self.assert_status(400) + + def testContainerExistenceCachingProblem(self): + cont = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, cont.files) + self.assert_(cont.create()) + cont.files() + + cont = self.env.account.container(Utils.create_name()) + self.assertRaises(ResponseError, cont.files) + self.assert_(cont.create()) + file = cont.file(Utils.create_name()) + file.write_random() + + +class TestContainerUTF8(Base2, TestContainer): + set_up = False + + +class TestContainerPathsEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + cls.file_size = 8 + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.files = [ + '/file1', + '/file A', + '/dir1/', + '/dir2/', + '/dir1/file2', + '/dir1/subdir1/', + '/dir1/subdir2/', + '/dir1/subdir1/file2', + '/dir1/subdir1/file3', + '/dir1/subdir1/file4', + '/dir1/subdir1/subsubdir1/', + '/dir1/subdir1/subsubdir1/file5', + '/dir1/subdir1/subsubdir1/file6', + '/dir1/subdir1/subsubdir1/file7', + '/dir1/subdir1/subsubdir1/file8', + '/dir1/subdir1/subsubdir2/', + '/dir1/subdir1/subsubdir2/file9', + '/dir1/subdir1/subsubdir2/file0', + 'file1', + 'dir1/', + 'dir2/', + 'dir1/file2', + 'dir1/subdir1/', + 'dir1/subdir2/', + 'dir1/subdir1/file2', + 'dir1/subdir1/file3', + 'dir1/subdir1/file4', + 'dir1/subdir1/subsubdir1/', + 'dir1/subdir1/subsubdir1/file5', + 'dir1/subdir1/subsubdir1/file6', + 'dir1/subdir1/subsubdir1/file7', + 'dir1/subdir1/subsubdir1/file8', + 'dir1/subdir1/subsubdir2/', + 'dir1/subdir1/subsubdir2/file9', + 'dir1/subdir1/subsubdir2/file0', + 'dir1/subdir with spaces/', + 'dir1/subdir with spaces/file B', + 'dir1/subdir+with{whatever/', + 'dir1/subdir+with{whatever/file D', + ] + + stored_files = set() + for f in cls.files: + file = cls.container.file(f) + if f.endswith('/'): + file.write(hdrs={'Content-Type': 'application/directory'}) + else: + file.write_random(cls.file_size, hdrs={'Content-Type': + 'application/directory'}) + if (normalized_urls): + nfile = '/'.join(filter(None, f.split('/'))) + if (f[-1] == '/'): + nfile += '/' + stored_files.add(nfile) + else: + stored_files.add(f) + cls.stored_files = sorted(stored_files) + + + + +class TestContainerPaths(Base): + env = TestContainerPathsEnv + set_up = False + + def testTraverseContainer(self): + found_files = [] + found_dirs = [] + + def recurse_path(path, count=0): + if count > 10: + raise ValueError('too deep recursion') + + for file in self.env.container.files(parms={'path': path}): + self.assert_(file.startswith(path)) + if file.endswith('/'): + recurse_path(file, count + 1) + found_dirs.append(file) + else: + found_files.append(file) + + recurse_path('') + for file in self.env.stored_files: + if file.startswith('/'): + self.assert_(file not in found_dirs) + self.assert_(file not in found_files) + elif file.endswith('/'): + self.assert_(file in found_dirs) + self.assert_(file not in found_files) + else: + self.assert_(file in found_files) + self.assert_(file not in found_dirs) + + found_files = [] + found_dirs = [] + recurse_path('/') + for file in self.env.stored_files: + if not file.startswith('/'): + self.assert_(file not in found_dirs) + self.assert_(file not in found_files) + elif file.endswith('/'): + self.assert_(file in found_dirs) + self.assert_(file not in found_files) + else: + self.assert_(file in found_files) + self.assert_(file not in found_dirs) + + def testContainerListing(self): + for format in (None, 'json', 'xml'): + files = self.env.container.files(parms={'format': format}) + + if isinstance(files[0], dict): + files = [str(x['name']) for x in files] + + self.assertEquals(files, self.env.stored_files) + + for format in ('json', 'xml'): + for file in self.env.container.files(parms={'format': format}): + self.assert_(int(file['bytes']) >= 0) + self.assert_('last_modified' in file) + if file['name'].endswith('/'): + self.assertEquals(file['content_type'], + 'application/directory') + + def testStructure(self): + def assert_listing(path, list): + files = self.env.container.files(parms={'path': path}) + self.assertEquals(sorted(list, cmp=locale.strcoll), files) + if not normalized_urls: + assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A']) + assert_listing('/dir1', + ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/']) + assert_listing('/dir1/', + ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/']) + assert_listing('/dir1/subdir1', + ['/dir1/subdir1/subsubdir2/', '/dir1/subdir1/file2', + '/dir1/subdir1/file3', '/dir1/subdir1/file4', + '/dir1/subdir1/subsubdir1/']) + assert_listing('/dir1/subdir2', []) + assert_listing('', ['file1', 'dir1/', 'dir2/']) + else: + assert_listing('', ['file1', 'dir1/', 'dir2/', 'file A']) + assert_listing('dir1', ['dir1/file2', 'dir1/subdir1/', + 'dir1/subdir2/', 'dir1/subdir with spaces/', + 'dir1/subdir+with{whatever/']) + assert_listing('dir1/subdir1', + ['dir1/subdir1/file4', 'dir1/subdir1/subsubdir2/', + 'dir1/subdir1/file2', 'dir1/subdir1/file3', + 'dir1/subdir1/subsubdir1/']) + assert_listing('dir1/subdir1/subsubdir1', + ['dir1/subdir1/subsubdir1/file7', + 'dir1/subdir1/subsubdir1/file5', + 'dir1/subdir1/subsubdir1/file8', + 'dir1/subdir1/subsubdir1/file6']) + assert_listing('dir1/subdir1/subsubdir1/', + ['dir1/subdir1/subsubdir1/file7', + 'dir1/subdir1/subsubdir1/file5', + 'dir1/subdir1/subsubdir1/file8', + 'dir1/subdir1/subsubdir1/file6']) + assert_listing('dir1/subdir with spaces/', + ['dir1/subdir with spaces/file B']) + + +class TestFileEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.file_size = 128 + + +class TestFileDev(Base): + env = TestFileEnv + set_up = False + + +class TestFileDevUTF8(Base2, TestFileDev): + set_up = False + + +class TestFile(Base): + env = TestFileEnv + set_up = False + + def testCopy(self): + # makes sure to test encoded characters" + source_filename = 'dealde%2Fl04 011e%204c8df/flash.png' + file = self.env.container.file(source_filename) + + metadata = {} + for i in range(1): + metadata[Utils.create_ascii_name()] = Utils.create_name() + + data = file.write_random() + file.sync_metadata(metadata) + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + # copy both from within and across containers + for cont in (self.env.container, dest_cont): + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file = self.env.container.file(source_filename) + file.copy('%s%s' % (prefix, cont), dest_filename) + + self.assert_(dest_filename in cont.files()) + + file = cont.file(dest_filename) + + self.assert_(data == file.read()) + self.assert_(file.initialize()) + self.assert_(metadata == file.metadata) + + def testCopy404s(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + for prefix in ('', '/'): + # invalid source container + source_cont = self.env.account.container(Utils.create_name()) + file = source_cont.file(source_filename) + self.assert_(not file.copy('%s%s' % (prefix, self.env.container), + Utils.create_name())) + self.assert_status(404) + + self.assert_(not file.copy('%s%s' % (prefix, dest_cont), + Utils.create_name())) + self.assert_status(404) + + # invalid source object + file = self.env.container.file(Utils.create_name()) + self.assert_(not file.copy('%s%s' % (prefix, self.env.container), + Utils.create_name())) + self.assert_status(404) + + self.assert_(not file.copy('%s%s' % (prefix, dest_cont), + Utils.create_name())) + self.assert_status(404) + + # invalid destination container + file = self.env.container.file(source_filename) + self.assert_(not file.copy('%s%s' % (prefix, Utils.create_name()), + Utils.create_name())) + + def testCopyNoDestinationHeader(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + file = self.env.container.file(source_filename) + self.assert_(not file.copy(Utils.create_name(), Utils.create_name(), + cfg={'no_destination': True})) + self.assert_status(412) + + def testCopyDestinationSlashProblems(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + # no slash + self.assert_(not file.copy(Utils.create_name(), Utils.create_name(), + cfg={'destination': Utils.create_name()})) + self.assert_status(412) + + def testCopyFromHeader(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + + metadata = {} + for i in range(1): + metadata[Utils.create_ascii_name()] = Utils.create_name() + file.metadata = metadata + + data = file.write_random() + + dest_cont = self.env.account.container(Utils.create_name()) + self.assert_(dest_cont.create()) + + # copy both from within and across containers + for cont in (self.env.container, dest_cont): + # copy both with and without initial slash + for prefix in ('', '/'): + dest_filename = Utils.create_name() + + file = cont.file(dest_filename) + file.write(hdrs={'X-Copy-From': '%s%s/%s' % (prefix, + self.env.container.name, source_filename)}) + + self.assert_(dest_filename in cont.files()) + + file = cont.file(dest_filename) + + self.assert_(data == file.read()) + self.assert_(file.initialize()) + self.assert_(metadata == file.metadata) + + def testCopyFromHeader404s(self): + source_filename = Utils.create_name() + file = self.env.container.file(source_filename) + file.write_random() + + for prefix in ('', '/'): + # invalid source container + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, + hdrs={'X-Copy-From': '%s%s/%s' % + (prefix, + Utils.create_name(), source_filename)}) + self.assert_status(404) + + # invalid source object + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, + hdrs={'X-Copy-From': '%s%s/%s' % + (prefix, + self.env.container.name, Utils.create_name())}) + self.assert_status(404) + + # invalid destination container + dest_cont = self.env.account.container(Utils.create_name()) + file = dest_cont.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, + hdrs={'X-Copy-From': '%s%s/%s' % + (prefix, + self.env.container.name, source_filename)}) + self.assert_status(404) + + def testNameLimit(self): + limit = load_constraint('max_object_name_length') + + for l in (1, 10, limit / 2, limit - 1, limit, limit + 1, limit * 2): + file = self.env.container.file('a' * l) + + if l <= limit: + self.assert_(file.write()) + self.assert_status(201) + else: + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + + def testQuestionMarkInName(self): + if Utils.create_name == Utils.create_ascii_name: + file_name = list(Utils.create_name()) + file_name[random.randint(2, len(file_name) - 2)] = '?' + file_name = "".join(file_name) + else: + file_name = Utils.create_name(6) + '?' + Utils.create_name(6) + + file = self.env.container.file(file_name) + self.assert_(file.write(cfg={'no_path_quote': True})) + self.assert_(file_name not in self.env.container.files()) + self.assert_(file_name.split('?')[0] in self.env.container.files()) + + def testDeleteThen404s(self): + file = self.env.container.file(Utils.create_name()) + self.assert_(file.write_random()) + self.assert_status(201) + + self.assert_(file.delete()) + self.assert_status(204) + + file.metadata = {Utils.create_ascii_name(): Utils.create_name()} + + for method in (file.info, file.read, file.sync_metadata, + file.delete): + self.assertRaises(ResponseError, method) + self.assert_status(404) + + def testBlankMetadataName(self): + file = self.env.container.file(Utils.create_name()) + file.metadata = {'': Utils.create_name()} + self.assertRaises(ResponseError, file.write_random) + self.assert_status(400) + + def testMetadataNumberLimit(self): + number_limit = load_constraint('max_meta_count') + size_limit = load_constraint('max_meta_overall_size') + + for i in (number_limit - 10, number_limit - 1, number_limit, + number_limit + 1, number_limit + 10, number_limit + 100): + + j = size_limit / (i * 2) + + size = 0 + metadata = {} + while len(metadata.keys()) < i: + key = Utils.create_ascii_name() + val = Utils.create_name() + + if len(key) > j: + key = key[:j] + val = val[:j] + + size += len(key) + len(val) + metadata[key] = val + + file = self.env.container.file(Utils.create_name()) + file.metadata = metadata + + if i <= number_limit: + self.assert_(file.write()) + self.assert_status(201) + self.assert_(file.sync_metadata()) + self.assert_status((201, 202)) + else: + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + file.metadata = {} + self.assert_(file.write()) + self.assert_status(201) + file.metadata = metadata + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(400) + + def testContentTypeGuessing(self): + file_types = {'wav': 'audio/x-wav', 'txt': 'text/plain', + 'zip': 'application/zip'} + + container = self.env.account.container(Utils.create_name()) + self.assert_(container.create()) + + for i in file_types.keys(): + file = container.file(Utils.create_name() + '.' + i) + file.write('', cfg={'no_content_type': True}) + + file_types_read = {} + for i in container.files(parms={'format': 'json'}): + file_types_read[i['name'].split('.')[1]] = i['content_type'] + + self.assertEquals(file_types, file_types_read) + + def testRangedGets(self): + file_length = 10000 + range_size = file_length / 10 + file = self.env.container.file(Utils.create_name()) + data = file.write_random(file_length) + + for i in range(0, file_length, range_size): + range_string = 'bytes=%d-%d' % (i, i + range_size - 1) + hdrs = {'Range': range_string} + self.assert_(data[i: i + range_size] == file.read(hdrs=hdrs), + range_string) + + range_string = 'bytes=-%d' % (i) + hdrs = {'Range': range_string} + if i == 0: + # RFC 2616 14.35.1 + # "If a syntactically valid byte-range-set includes ... at + # least one suffix-byte-range-spec with a NON-ZERO + # suffix-length, then the byte-range-set is satisfiable. + # Otherwise, the byte-range-set is unsatisfiable. + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(416) + else: + self.assertEquals(file.read(hdrs=hdrs), data[-i:]) + + range_string = 'bytes=%d-' % (i) + hdrs = {'Range': range_string} + self.assert_(file.read(hdrs=hdrs) == data[i - file_length:], + range_string) + + range_string = 'bytes=%d-%d' % (file_length + 1000, file_length + 2000) + hdrs = {'Range': range_string} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(416) + + range_string = 'bytes=%d-%d' % (file_length - 1000, file_length + 2000) + hdrs = {'Range': range_string} + self.assert_(file.read(hdrs=hdrs) == data[-1000:], range_string) + + hdrs = {'Range': '0-4'} + self.assert_(file.read(hdrs=hdrs) == data, range_string) + + # RFC 2616 14.35.1 + # "If the entity is shorter than the specified suffix-length, the + # entire entity-body is used." + range_string = 'bytes=-%d' % (file_length + 10) + hdrs = {'Range': range_string} + self.assert_(file.read(hdrs=hdrs) == data, range_string) + + def testRangedGetsWithLWSinHeader(self): + #Skip this test until webob 1.2 can tolerate LWS in Range header. + file_length = 10000 + range_size = file_length / 10 + file = self.env.container.file(Utils.create_name()) + data = file.write_random(file_length) + + for r in ('BYTES=0-999', 'bytes = 0-999', 'BYTES = 0 - 999', + 'bytes = 0 - 999', 'bytes=0 - 999', 'bytes=0-999 '): + + self.assert_(file.read(hdrs={'Range': r}) == data[0:1000]) + + def testFileSizeLimit(self): + limit = load_constraint('max_file_size') + tsecs = 3 + + for i in (limit - 100, limit - 10, limit - 1, limit, limit + 1, + limit + 10, limit + 100): + + file = self.env.container.file(Utils.create_name()) + + if i <= limit: + self.assert_(timeout(tsecs, file.write, + cfg={'set_content_length': i})) + else: + self.assertRaises(ResponseError, timeout, tsecs, + file.write, cfg={'set_content_length': i}) + + def testNoContentLengthForPut(self): + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.write, 'testing', + cfg={'no_content_length': True}) + self.assert_status(411) + + def testDelete(self): + file = self.env.container.file(Utils.create_name()) + file.write_random(self.env.file_size) + + self.assert_(file.name in self.env.container.files()) + self.assert_(file.delete()) + self.assert_(file.name not in self.env.container.files()) + + def testBadHeaders(self): + file_length = 100 + + # no content type on puts should be ok + file = self.env.container.file(Utils.create_name()) + file.write_random(file_length, cfg={'no_content_type': True}) + self.assert_status(201) + + # content length x + self.assertRaises(ResponseError, file.write_random, file_length, + hdrs={'Content-Length': 'X'}, + cfg={'no_content_length': True}) + self.assert_status(400) + + # bad request types + #for req in ('LICK', 'GETorHEAD_base', 'container_info', + # 'best_response'): + for req in ('LICK', 'GETorHEAD_base'): + self.env.account.conn.make_request(req) + self.assert_status(405) + + # bad range headers + self.assert_(len(file.read(hdrs={'Range': 'parsecs=8-12'})) == + file_length) + self.assert_status(200) + + def testMetadataLengthLimits(self): + key_limit = load_constraint('max_meta_name_length') + value_limit = load_constraint('max_meta_value_length') + lengths = [[key_limit, value_limit], [key_limit, value_limit + 1], + [key_limit + 1, value_limit], [key_limit, 0], + [key_limit, value_limit * 10], + [key_limit * 10, value_limit]] + + for l in lengths: + metadata = {'a' * l[0]: 'b' * l[1]} + file = self.env.container.file(Utils.create_name()) + file.metadata = metadata + + if l[0] <= key_limit and l[1] <= value_limit: + self.assert_(file.write()) + self.assert_status(201) + self.assert_(file.sync_metadata()) + else: + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + file.metadata = {} + self.assert_(file.write()) + self.assert_status(201) + file.metadata = metadata + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(400) + + def testEtagWayoff(self): + file = self.env.container.file(Utils.create_name()) + hdrs = {'etag': 'reallylonganddefinitelynotavalidetagvalue'} + self.assertRaises(ResponseError, file.write_random, hdrs=hdrs) + self.assert_status(422) + + def testFileCreate(self): + for i in range(10): + file = self.env.container.file(Utils.create_name()) + data = file.write_random() + self.assert_status(201) + self.assert_(data == file.read()) + self.assert_status(200) + + def testHead(self): + file_name = Utils.create_name() + content_type = Utils.create_name() + + file = self.env.container.file(file_name) + file.content_type = content_type + file.write_random(self.env.file_size) + + md5 = file.md5 + + file = self.env.container.file(file_name) + info = file.info() + + self.assert_status(200) + self.assertEquals(info['content_length'], self.env.file_size) + self.assertEquals(info['etag'], md5) + self.assertEquals(info['content_type'], content_type) + self.assert_('last_modified' in info) + + def testDeleteOfFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.delete) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.delete) + self.assert_status(404) + + def testHeadOnFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.info) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.info) + self.assert_status(404) + + def testMetadataOnPost(self): + file = self.env.container.file(Utils.create_name()) + file.write_random(self.env.file_size) + + for i in range(10): + metadata = {} + for i in range(10): + metadata[Utils.create_ascii_name()] = Utils.create_name() + + file.metadata = metadata + self.assert_(file.sync_metadata()) + self.assert_status((201, 202)) + + file = self.env.container.file(file.name) + self.assert_(file.initialize()) + self.assert_status(200) + self.assertEquals(file.metadata, metadata) + + def testGetContentType(self): + file_name = Utils.create_name() + content_type = Utils.create_name() + + file = self.env.container.file(file_name) + file.content_type = content_type + file.write_random() + + file = self.env.container.file(file_name) + file.read() + + self.assertEquals(content_type, file.content_type) + + def testGetOnFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.read) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + self.assertRaises(ResponseError, file.read) + self.assert_status(404) + + def testPostOnFileThatDoesNotExist(self): + # in container that exists + file = self.env.container.file(Utils.create_name()) + file.metadata['Field'] = 'Value' + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(404) + + # in container that does not exist + container = self.env.account.container(Utils.create_name()) + file = container.file(Utils.create_name()) + file.metadata['Field'] = 'Value' + self.assertRaises(ResponseError, file.sync_metadata) + self.assert_status(404) + + def testMetadataOnPut(self): + for i in range(10): + metadata = {} + for j in range(10): + metadata[Utils.create_ascii_name()] = Utils.create_name() + + file = self.env.container.file(Utils.create_name()) + file.metadata = metadata + file.write_random(self.env.file_size) + + file = self.env.container.file(file.name) + self.assert_(file.initialize()) + self.assert_status(200) + self.assertEquals(file.metadata, metadata) + + def testSerialization(self): + container = self.env.account.container(Utils.create_name()) + self.assert_(container.create()) + + files = [] + for i in (0, 1, 10, 100, 1000, 10000): + files.append({'name': Utils.create_name(), + 'content_type': Utils.create_name(), 'bytes': i}) + + write_time = time.time() + for f in files: + file = container.file(f['name']) + file.content_type = f['content_type'] + file.write_random(f['bytes']) + + f['hash'] = file.md5 + f['json'] = False + f['xml'] = False + write_time = time.time() - write_time + + for format in ['json', 'xml']: + for file in container.files(parms={'format': format}): + found = False + for f in files: + if f['name'] != file['name']: + continue + + self.assertEquals(file['content_type'], + f['content_type']) + self.assertEquals(int(file['bytes']), f['bytes']) + + d = datetime.strptime(file['last_modified'].split('.')[0], + "%Y-%m-%dT%H:%M:%S") + lm = time.mktime(d.timetuple()) + + if 'last_modified' in f: + self.assertEquals(f['last_modified'], lm) + else: + f['last_modified'] = lm + + f[format] = True + found = True + + self.assert_(found, 'Unexpected file %s found in ' + '%s listing' % (file['name'], format)) + + headers = dict(self.env.conn.response.getheaders()) + if format == 'json': + self.assertEquals(headers['content-type'], + 'application/json; charset=utf-8') + elif format == 'xml': + self.assertEquals(headers['content-type'], + 'application/xml; charset=utf-8') + + lm_diff = max([f['last_modified'] for f in files]) -\ + min([f['last_modified'] for f in files]) + self.assert_(lm_diff < write_time + 1, 'Diff in last ' + 'modified times should be less than time to write files') + + for f in files: + for format in ['json', 'xml']: + self.assert_(f[format], 'File %s not found in %s listing' + % (f['name'], format)) + + def testStackedOverwrite(self): + file = self.env.container.file(Utils.create_name()) + + for i in range(1, 11): + data = file.write_random(512) + file.write(data) + + self.assert_(file.read() == data) + + def testTooLongName(self): + file = self.env.container.file('x' * 1025) + self.assertRaises(ResponseError, file.write) + self.assert_status(400) + + def testZeroByteFile(self): + file = self.env.container.file(Utils.create_name()) + + self.assert_(file.write('')) + self.assert_(file.name in self.env.container.files()) + self.assert_(file.read() == '') + + def testEtagResponse(self): + file = self.env.container.file(Utils.create_name()) + + data = StringIO.StringIO(file.write_random(512)) + etag = File.compute_md5sum(data) + + headers = dict(self.env.conn.response.getheaders()) + self.assert_('etag' in headers.keys()) + + header_etag = headers['etag'].strip('"') + self.assertEquals(etag, header_etag) + + def testChunkedPut(self): + if (web_front_end == 'apache2'): + raise SkipTest() + data = File.random_data(10000) + etag = File.compute_md5sum(data) + + for i in (1, 10, 100, 1000): + file = self.env.container.file(Utils.create_name()) + + for j in chunks(data, i): + file.chunked_write(j) + + self.assert_(file.chunked_write()) + self.assert_(data == file.read()) + + info = file.info() + self.assertEquals(etag, info['etag']) + + +class TestFileUTF8(Base2, TestFile): + set_up = False + + +class TestFileComparisonEnv: + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + cls.account = Account(cls.conn, config.get('account', + config['username'])) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.file_count = 20 + cls.file_size = 128 + cls.files = list() + for x in range(cls.file_count): + file = cls.container.file(Utils.create_name()) + file.write_random(cls.file_size) + cls.files.append(file) + + cls.time_old = time.asctime(time.localtime(time.time() - 86400)) + cls.time_new = time.asctime(time.localtime(time.time() + 86400)) + + +class TestFileComparison(Base): + env = TestFileComparisonEnv + set_up = False + + def testIfMatch(self): + for file in self.env.files: + hdrs = {'If-Match': file.md5} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Match': 'bogus'} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + + def testIfNoneMatch(self): + for file in self.env.files: + hdrs = {'If-None-Match': 'bogus'} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-None-Match': file.md5} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(304) + + def testIfModifiedSince(self): + for file in self.env.files: + hdrs = {'If-Modified-Since': self.env.time_old} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Modified-Since': self.env.time_new} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(304) + + def testIfUnmodifiedSince(self): + for file in self.env.files: + hdrs = {'If-Unmodified-Since': self.env.time_new} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Unmodified-Since': self.env.time_old} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + + def testIfMatchAndUnmodified(self): + for file in self.env.files: + hdrs = {'If-Match': file.md5, + 'If-Unmodified-Since': self.env.time_new} + self.assert_(file.read(hdrs=hdrs)) + + hdrs = {'If-Match': 'bogus', + 'If-Unmodified-Since': self.env.time_new} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + + hdrs = {'If-Match': file.md5, + 'If-Unmodified-Since': self.env.time_old} + self.assertRaises(ResponseError, file.read, hdrs=hdrs) + self.assert_status(412) + + +class TestFileComparisonUTF8(Base2, TestFileComparison): + set_up = False + +if __name__ == '__main__': + unittest.main() diff --git a/test/functionalnosetests/__init__.py b/test/functionalnosetests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/functionalnosetests/swift_testing.py b/test/functionalnosetests/swift_testing.py new file mode 100644 index 0000000..023a753 --- /dev/null +++ b/test/functionalnosetests/swift_testing.py @@ -0,0 +1,175 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import errno +import os +import socket +import sys +from time import sleep +from nose import SkipTest +from ConfigParser import MissingSectionHeaderError + +from test import get_config + +from swiftclient import get_auth, http_connection, HTTPException + +conf = get_config('func_test') +web_front_end = conf.get('web_front_end', 'integral') +normalized_urls = conf.get('normalized_urls', False) + +# If no conf was read, we will fall back to old school env vars +swift_test_auth = os.environ.get('SWIFT_TEST_AUTH') +swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None] +swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None] +swift_test_tenant = ['', '', ''] +swift_test_perm = ['', '', ''] + +if conf: + swift_test_auth_version = str(conf.get('auth_version', '1')) + + swift_test_auth = 'http' + if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'): + swift_test_auth = 'https' + if 'auth_prefix' not in conf: + conf['auth_prefix'] = '/' + try: + swift_test_auth += \ + '://%(auth_host)s:%(auth_port)s%(auth_prefix)s' % conf + except KeyError: + pass # skip + + if swift_test_auth_version == "1": + swift_test_auth += 'v1.0' + + if 'account' in conf: + swift_test_user[0] = '%(account)s:%(username)s' % conf + else: + swift_test_user[0] = '%(username)s' % conf + swift_test_key[0] = conf['password'] + try: + swift_test_user[1] = '%s%s' % \ + ('%s:' % conf['account2'] if 'account2' in conf else '', + conf['username2']) + swift_test_key[1] = conf['password2'] + except KeyError, err: + pass # old conf, no second account tests can be run + try: + swift_test_user[2] = '%s%s' % ('%s:' % conf['account'] if 'account' + in conf else '', conf['username3']) + swift_test_key[2] = conf['password3'] + except KeyError, err: + pass # old conf, no third account tests can be run + + for _ in range(3): + swift_test_perm[_] = swift_test_user[_] + + else: + swift_test_user[0] = conf['username'] + swift_test_tenant[0] = conf['account'] + swift_test_key[0] = conf['password'] + swift_test_user[1] = conf['username2'] + swift_test_tenant[1] = conf['account2'] + swift_test_key[1] = conf['password2'] + swift_test_user[2] = conf['username3'] + swift_test_tenant[2] = conf['account'] + swift_test_key[2] = conf['password3'] + + for _ in range(3): + swift_test_perm[_] = swift_test_tenant[_] + ':' + swift_test_user[_] + +skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]) +if skip: + print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG' + +skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]]) +if not skip and skip2: + print >>sys.stderr, \ + 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + +skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]]) +if not skip and skip3: + print >>sys.stderr, \ + 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + + +class AuthError(Exception): + pass + + +class InternalServerError(Exception): + pass + + +url = [None, None, None] +token = [None, None, None] +parsed = [None, None, None] +conn = [None, None, None] + + +def retry(func, *args, **kwargs): + """ + You can use the kwargs to override the 'retries' (default: 5) and + 'use_account' (default: 1). + """ + global url, token, parsed, conn + retries = kwargs.get('retries', 5) + use_account = 1 + if 'use_account' in kwargs: + use_account = kwargs['use_account'] + del kwargs['use_account'] + use_account -= 1 + attempts = 0 + backoff = 1 + while attempts <= retries: + attempts += 1 + try: + if not url[use_account] or not token[use_account]: + url[use_account], token[use_account] = \ + get_auth(swift_test_auth, swift_test_user[use_account], + swift_test_key[use_account], + snet=False, + tenant_name=swift_test_tenant[use_account], + auth_version=swift_test_auth_version, + os_options={}) + parsed[use_account] = conn[use_account] = None + if not parsed[use_account] or not conn[use_account]: + parsed[use_account], conn[use_account] = \ + http_connection(url[use_account]) + return func(url[use_account], token[use_account], + parsed[use_account], conn[use_account], *args, **kwargs) + except (socket.error, HTTPException): + if attempts > retries: + raise + parsed[use_account] = conn[use_account] = None + except AuthError, err: + url[use_account] = token[use_account] = None + continue + except InternalServerError, err: + pass + if attempts <= retries: + sleep(backoff) + backoff *= 2 + raise Exception('No result after %s retries.' % retries) + + +def check_response(conn): + resp = conn.getresponse() + if resp.status == 401: + resp.read() + raise AuthError() + elif resp.status // 100 == 5: + resp.read() + raise InternalServerError() + return resp diff --git a/test/functionalnosetests/test_account.py b/test/functionalnosetests/test_account.py new file mode 100755 index 0000000..ae6e3c4 --- /dev/null +++ b/test/functionalnosetests/test_account.py @@ -0,0 +1,152 @@ +#!/usr/bin/python + +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from nose import SkipTest + +from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ + MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH + +from swift_testing import check_response, retry, skip + + +class TestAccount(unittest.TestCase): + + def test_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, value): + conn.request('POST', parsed.path, '', + {'X-Auth-Token': token, 'X-Account-Meta-Test': value}) + return check_response(conn) + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + def get(url, token, parsed, conn): + conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(post, '') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), None) + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), None) + resp = retry(post, 'Value') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') + + def test_multi_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, name, value): + conn.request('POST', parsed.path, '', + {'X-Auth-Token': token, name: value}) + return check_response(conn) + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(post, 'X-Account-Meta-One', '1') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-one'), '1') + resp = retry(post, 'X-Account-Meta-Two', '2') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-account-meta-one'), '1') + self.assertEquals(resp.getheader('x-account-meta-two'), '2') + + def test_bad_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path, '', headers) + return check_response(conn) + resp = retry(post, + {'X-Account-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(post, + {'X-Account-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + resp.read() + self.assertEquals(resp.status, 400) + + resp = retry(post, + {'X-Account-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(post, + {'X-Account-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + for x in xrange(MAX_META_COUNT): + headers['X-Account-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers = {} + for x in xrange(MAX_META_COUNT + 1): + headers['X-Account-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + header_value = 'k' * MAX_META_VALUE_LENGTH + size = 0 + x = 0 + while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: + size += 4 + MAX_META_VALUE_LENGTH + headers['X-Account-Meta-%04d' % x] = header_value + x += 1 + if MAX_META_OVERALL_SIZE - size > 1: + headers['X-Account-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size - 1) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers['X-Account-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/functionalnosetests/test_container.py b/test/functionalnosetests/test_container.py new file mode 100755 index 0000000..e92a86c --- /dev/null +++ b/test/functionalnosetests/test_container.py @@ -0,0 +1,573 @@ +#!/usr/bin/python + +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import unittest +from nose import SkipTest +from uuid import uuid4 + +from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ + MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH + +from swift_testing import check_response, retry, skip, skip2, skip3, \ + swift_test_perm, web_front_end + + +class TestContainer(unittest.TestCase): + + def setUp(self): + if skip: + raise SkipTest + self.name = uuid4().hex + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def tearDown(self): + if skip: + raise SkipTest + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name + '?format=json', + '', {'X-Auth-Token': token}) + return check_response(conn) + def delete(url, token, parsed, conn, obj): + conn.request('DELETE', + '/'.join([parsed.path, self.name, obj['name']]), '', + {'X-Auth-Token': token}) + return check_response(conn) + while True: + resp = retry(get) + body = resp.read() + self.assert_(resp.status // 100 == 2, resp.status) + objs = json.loads(body) + if not objs: + break + for obj in objs: + resp = retry(delete, obj) + resp.read() + self.assertEquals(resp.status, 204) + def delete(url, token, parsed, conn): + conn.request('DELETE', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_multi_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, name, value): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, name: value}) + return check_response(conn) + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(post, 'X-Container-Meta-One', '1') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-one'), '1') + resp = retry(post, 'X-Container-Meta-Two', '2') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-one'), '1') + self.assertEquals(resp.getheader('x-container-meta-two'), '2') + + def test_PUT_metadata(self): + if skip: + raise SkipTest + def put(url, token, parsed, conn, name, value): + conn.request('PUT', parsed.path + '/' + name, '', + {'X-Auth-Token': token, 'X-Container-Meta-Test': value}) + return check_response(conn) + def head(url, token, parsed, conn, name): + conn.request('HEAD', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + def get(url, token, parsed, conn, name): + conn.request('GET', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + def delete(url, token, parsed, conn, name): + conn.request('DELETE', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + name = uuid4().hex + resp = retry(put, name, 'Value') + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(head, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + resp = retry(get, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + + name = uuid4().hex + resp = retry(put, name, '') + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(head, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(get, name) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + + def test_POST_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, value): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Meta-Test': value}) + return check_response(conn) + def head(url, token, parsed, conn): + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), None) + resp = retry(post, 'Value') + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(head) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + resp = retry(get) + resp.read() + self.assert_(resp.status in (200, 204), resp.status) + self.assertEquals(resp.getheader('x-container-meta-test'), 'Value') + + def test_PUT_bad_metadata(self): + if skip: + raise SkipTest + def put(url, token, parsed, conn, name, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('PUT', parsed.path + '/' + name, '', headers) + return check_response(conn) + def delete(url, token, parsed, conn, name): + conn.request('DELETE', parsed.path + '/' + name, '', + {'X-Auth-Token': token}) + return check_response(conn) + name = uuid4().hex + resp = retry(put, name, + {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + resp = retry(put, name, + {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + name = uuid4().hex + resp = retry(put, name, + {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + resp = retry(put, name, + {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + name = uuid4().hex + headers = {} + for x in xrange(MAX_META_COUNT): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + headers = {} + for x in xrange(MAX_META_COUNT + 1): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + name = uuid4().hex + headers = {} + header_value = 'k' * MAX_META_VALUE_LENGTH + size = 0 + x = 0 + while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: + size += 4 + MAX_META_VALUE_LENGTH + headers['X-Container-Meta-%04d' % x] = header_value + x += 1 + if MAX_META_OVERALL_SIZE - size > 1: + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size - 1) + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 201) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 204) + name = uuid4().hex + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size) + resp = retry(put, name, headers) + resp.read() + self.assertEquals(resp.status, 400) + resp = retry(delete, name) + resp.read() + self.assertEquals(resp.status, 404) + + def test_POST_bad_metadata(self): + if skip: + raise SkipTest + def post(url, token, parsed, conn, extra_headers): + headers = {'X-Auth-Token': token} + headers.update(extra_headers) + conn.request('POST', parsed.path + '/' + self.name, '', headers) + return check_response(conn) + resp = retry(post, + {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(post, + {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) + resp.read() + self.assertEquals(resp.status, 400) + + resp = retry(post, + {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(post, + {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + for x in xrange(MAX_META_COUNT): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers = {} + for x in xrange(MAX_META_COUNT + 1): + headers['X-Container-Meta-%d' % x] = 'v' + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + headers = {} + header_value = 'k' * MAX_META_VALUE_LENGTH + size = 0 + x = 0 + while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: + size += 4 + MAX_META_VALUE_LENGTH + headers['X-Container-Meta-%04d' % x] = header_value + x += 1 + if MAX_META_OVERALL_SIZE - size > 1: + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size - 1) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 204) + headers['X-Container-Meta-k'] = \ + 'v' * (MAX_META_OVERALL_SIZE - size) + resp = retry(post, headers) + resp.read() + self.assertEquals(resp.status, 400) + + def test_public_container(self): + if skip: + raise SkipTest + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name) + return check_response(conn) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after '), err) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*,.rlistings'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get) + resp.read() + self.assertEquals(resp.status, 204) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after '), err) + + def test_cross_account_container(self): + if skip or skip2: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the second account + def get2(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container accessible by the second account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': swift_test_perm[1], + 'X-Container-Write': swift_test_perm[1]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now use the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # Make the container private again + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': '', + 'X-Container-Write': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can't access the container with the second account again + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + + def test_cross_account_public_container(self): + if skip or skip2: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the second account + def get2(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container completely public + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*,.rlistings'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now read the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # But we shouldn't be able to write with the second account + def put2(url, token, parsed, conn): + conn.request('PUT', first_account[0] + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Now make the container also writeable by the second account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Write': swift_test_perm[1]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can still read the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # And that we can now write with the second account + resp = retry(put2, use_account=2) + resp.read() + self.assertEquals(resp.status, 201) + + def test_nonadmin_user(self): + if skip or skip3: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the third account + def get3(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container accessible by the third account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': swift_test_perm[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now read the container with the third account + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + # But we shouldn't be able to write with the third account + def put3(url, token, parsed, conn): + conn.request('PUT', first_account[0] + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + # Now make the container also writeable by the third account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Write': swift_test_perm[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can still read the container with the third account + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + # And that we can now write with the third account + resp = retry(put3, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + + def test_long_name_content_type(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + container_name = 'X' * 2048 + conn.request('PUT', '%s/%s' % (parsed.path, + container_name), 'there', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 400) + self.assertEquals(resp.getheader('Content-Type'), + 'text/html; charset=UTF-8') + + def test_null_name(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/abc%%00def' % parsed.path, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + if (web_front_end == 'apache2'): + self.assertEquals(resp.status, 404) + else: + self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL') + self.assertEquals(resp.status, 412) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/functionalnosetests/test_object.py b/test/functionalnosetests/test_object.py new file mode 100755 index 0000000..168375d --- /dev/null +++ b/test/functionalnosetests/test_object.py @@ -0,0 +1,600 @@ +#!/usr/bin/python + +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from nose import SkipTest +from uuid import uuid4 + +from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ + MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH + +from swift_testing import check_response, retry, skip, skip3, \ + swift_test_perm, web_front_end +from test import get_config + + +class TestObject(unittest.TestCase): + + def setUp(self): + if skip: + raise SkipTest + self.container = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + self.obj = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + self.obj), 'test', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def tearDown(self): + if skip: + raise SkipTest + + def delete(url, token, parsed, conn, obj): + conn.request('DELETE', + '%s/%s/%s' % (parsed.path, self.container, obj), + '', {'X-Auth-Token': token}) + return check_response(conn) + + # get list of objects in container + def list(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, self.container), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(list) + object_listing = resp.read() + self.assertEquals(resp.status, 200) + + # iterate over object listing and delete all objects + for obj in object_listing.splitlines(): + resp = retry(delete, obj) + resp.read() + self.assertEquals(resp.status, 204) + + # delete the container + def delete(url, token, parsed, conn): + conn.request('DELETE', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_copy_object(self): + if skip: + raise SkipTest + + source = '%s/%s' % (self.container, self.obj) + dest = '%s/%s' % (self.container, 'test_copy') + + # get contents of source + def get_source(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, source), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get_source) + source_contents = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(source_contents, 'test') + + # copy source to dest with X-Copy-From + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s' % (parsed.path, dest), '', + {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Copy-From': source}) + return check_response(conn) + resp = retry(put) + contents = resp.read() + self.assertEquals(resp.status, 201) + + # contents of dest should be the same as source + def get_dest(url, token, parsed, conn): + conn.request('GET', + '%s/%s' % (parsed.path, dest), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get_dest) + dest_contents = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(dest_contents, source_contents) + + # delete the copy + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s' % (parsed.path, dest), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + # verify dest does not exist + resp = retry(get_dest) + resp.read() + self.assertEquals(resp.status, 404) + + # copy source to dest with COPY + def copy(url, token, parsed, conn): + conn.request('COPY', '%s/%s' % (parsed.path, source), '', + {'X-Auth-Token': token, + 'Destination': dest}) + return check_response(conn) + resp = retry(copy) + contents = resp.read() + self.assertEquals(resp.status, 201) + + # contents of dest should be the same as source + resp = retry(get_dest) + dest_contents = resp.read() + self.assertEquals(resp.status, 200) + self.assertEquals(dest_contents, source_contents) + + # delete the copy + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_public_object(self): + if skip: + raise SkipTest + + def get(url, token, parsed, conn): + conn.request('GET', + '%s/%s/%s' % (parsed.path, self.container, self.obj)) + return check_response(conn) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after ')) + + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get) + resp.read() + self.assertEquals(resp.status, 200) + + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token, 'X-Container-Read': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after ')) + + def test_private_object(self): + if skip or skip3: + raise SkipTest + + # Ensure we can't access the object with the third account + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/%s' % (parsed.path, self.container, + self.obj), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # create a shared container writable by account3 + shared_container = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s' % (parsed.path, + shared_container), '', + {'X-Auth-Token': token, + 'X-Container-Read': swift_test_perm[2], + 'X-Container-Write': swift_test_perm[2]}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # verify third account can not copy from private container + def copy(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, + shared_container, + 'private_object'), + '', {'X-Auth-Token': token, + 'Content-Length': '0', + 'X-Copy-From': '%s/%s' % (self.container, + self.obj)}) + return check_response(conn) + resp = retry(copy, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # verify third account can write "obj1" to shared container + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, shared_container, + 'obj1'), 'test', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + + # verify third account can copy "obj1" to shared container + def copy2(url, token, parsed, conn): + conn.request('COPY', '%s/%s/%s' % (parsed.path, + shared_container, + 'obj1'), + '', {'X-Auth-Token': token, + 'Destination': '%s/%s' % (shared_container, + 'obj1')}) + return check_response(conn) + resp = retry(copy2, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + + # verify third account STILL can not copy from private container + def copy3(url, token, parsed, conn): + conn.request('COPY', '%s/%s/%s' % (parsed.path, + self.container, + self.obj), + '', {'X-Auth-Token': token, + 'Destination': '%s/%s' % (shared_container, + 'private_object')}) + return check_response(conn) + resp = retry(copy3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # clean up "obj1" + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s/%s' % (parsed.path, shared_container, + 'obj1'), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + # clean up shared_container + def delete(url, token, parsed, conn): + conn.request('DELETE', + parsed.path + '/' + shared_container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_manifest(self): + if skip: + raise SkipTest + # Data for the object segments + segments1 = ['one', 'two', 'three', 'four', 'five'] + segments2 = ['six', 'seven', 'eight'] + segments3 = ['nine', 'ten', 'eleven'] + + # Upload the first set of segments + def put(url, token, parsed, conn, objnum): + conn.request('PUT', '%s/%s/segments1/%s' % (parsed.path, + self.container, str(objnum)), segments1[objnum], + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments1)): + resp = retry(put, objnum) + resp.read() + self.assertEquals(resp.status, 201) + + # Upload the manifest + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, + 'X-Object-Manifest': '%s/segments1/' % self.container, + 'Content-Type': 'text/jibberish', 'Content-Length': '0'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest (should get all the segments as the body) + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)) + self.assertEquals(resp.status, 200) + self.assertEquals(resp.getheader('content-type'), 'text/jibberish') + + # Get with a range at the start of the second segment + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, 'Range': + 'bytes=3-'}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1[1:])) + self.assertEquals(resp.status, 206) + + # Get with a range in the middle of the second segment + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, 'Range': + 'bytes=5-'}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)[5:]) + self.assertEquals(resp.status, 206) + + # Get with a full start and stop range + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, 'Range': + 'bytes=5-10'}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)[5:11]) + self.assertEquals(resp.status, 206) + + # Upload the second set of segments + def put(url, token, parsed, conn, objnum): + conn.request('PUT', '%s/%s/segments2/%s' % (parsed.path, + self.container, str(objnum)), segments2[objnum], + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments2)): + resp = retry(put, objnum) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest (should still be the first segments of course) + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments1)) + self.assertEquals(resp.status, 200) + + # Update the manifest + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, + 'X-Object-Manifest': '%s/segments2/' % self.container, + 'Content-Length': '0'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest (should be the second set of segments now) + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments2)) + self.assertEquals(resp.status, 200) + + if not skip3: + + # Ensure we can't access the manifest with the third account + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # Grant access to the third account + def post(url, token, parsed, conn): + conn.request('POST', '%s/%s' % (parsed.path, self.container), + '', {'X-Auth-Token': token, + 'X-Container-Read': swift_test_perm[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + + # The third account should be able to get the manifest now + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + self.assertEquals(resp.read(), ''.join(segments2)) + self.assertEquals(resp.status, 200) + + # Create another container for the third set of segments + acontainer = uuid4().hex + + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + acontainer, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Upload the third set of segments in the other container + def put(url, token, parsed, conn, objnum): + conn.request('PUT', '%s/%s/segments3/%s' % (parsed.path, + acontainer, str(objnum)), segments3[objnum], + {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments3)): + resp = retry(put, objnum) + resp.read() + self.assertEquals(resp.status, 201) + + # Update the manifest + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token, + 'X-Object-Manifest': '%s/segments3/' % acontainer, + 'Content-Length': '0'}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + # Get the manifest to ensure it's the third set of segments + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get) + self.assertEquals(resp.read(), ''.join(segments3)) + self.assertEquals(resp.status, 200) + + if not skip3: + + # Ensure we can't access the manifest with the third account + # (because the segments are in a protected container even if the + # manifest itself is not). + + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + + # Grant access to the third account + def post(url, token, parsed, conn): + conn.request('POST', '%s/%s' % (parsed.path, acontainer), + '', {'X-Auth-Token': token, + 'X-Container-Read': swift_test_perm[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + + # The third account should be able to get the manifest now + def get(url, token, parsed, conn): + conn.request('GET', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get, use_account=3) + self.assertEquals(resp.read(), ''.join(segments3)) + self.assertEquals(resp.status, 200) + + # Delete the manifest + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/manifest' % (parsed.path, + self.container), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the third set of segments + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/segments3/%s' % (parsed.path, + acontainer, str(objnum)), '', {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments3)): + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the second set of segments + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/segments2/%s' % (parsed.path, + self.container, str(objnum)), '', {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments2)): + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the first set of segments + def delete(url, token, parsed, conn, objnum): + conn.request('DELETE', '%s/%s/segments1/%s' % (parsed.path, + self.container, str(objnum)), '', {'X-Auth-Token': token}) + return check_response(conn) + for objnum in xrange(len(segments1)): + resp = retry(delete, objnum) + resp.read() + self.assertEquals(resp.status, 204) + + # Delete the extra container + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s' % (parsed.path, acontainer), '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_delete_content_type(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/hi' % (parsed.path, + self.container), 'there', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s/hi' % (parsed.path, self.container), + '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + self.assertEquals(resp.getheader('Content-Type'), + 'text/html; charset=UTF-8') + + def test_null_name(self): + if skip: + raise SkipTest + + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/abc%%00def' % (parsed.path, + self.container), 'test', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + if (web_front_end == 'apache2'): + self.assertEquals(resp.status, 404) + else: + self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL') + self.assertEquals(resp.status, 412) + + +if __name__ == '__main__': + unittest.main()