x/swift3
Code Issues Proposed changes

Draft: Change functional tests to the new structure.

Browse Source 
I change functional tests to the new structure using nosetests.

Existing functional tests has the following problems.
- File name becomes the nonsense sequence number.
- Large quantity of files will be located in one directory when we add tests because it becomes 1 file every 1 test.
- Because the system is such that it is more special than other products of OpenStack, it is hard to develop it.

I think that the problem mentioned above are solved by this patch.

Change-Id: Ic75858ca1894e19f12c37b25559354f65fb48fa6
This commit is contained in:
Masaki Tsukuda 2015-02-20 14:25:32 +09:00
parent 771453c60d
commit 1ac8f82abb
26 changed files with 551 additions and 1763 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
swift3/test/functional/001
View File

@ -1,12 +0,0 @@
#!/bin/bash
# GET Service
. ./common
S3USER=tester
_s3_put /abcdefghijklmnopqrstuvwxyz_01234
_s3_put /5ABCDEFGHIJKLMNOPQRSTUVWXYZ.5-6789
_s3_get / -D - | _filter_curl xml

29
swift3/test/functional/001.out
View File

@ -1,29 +0,0 @@
QA output created by 001
> s3curl --id tester -- -X PUT http://SWIFT_HOST/abcdefghijklmnopqrstuvwxyz_01234... 200
> s3curl --id tester -- -X PUT http://SWIFT_HOST/5ABCDEFGHIJKLMNOPQRSTUVWXYZ.5-6789... 200
> s3curl --id tester -- -X GET -D - http://SWIFT_HOST/... 200
HTTP/1.1 200 OK
Content-Length: LENGTH
Content-Type: application/xml
Date: DATE
x-amz-id-2: TXID
x-amz-request-id: TXID
X-Trans-Id: TXID
<?xml version='1.0' encoding='UTF-8'?>
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Owner>
<ID>TESTER</ID>
<DisplayName>TESTER</DisplayName>
</Owner>
<Buckets>
<Bucket>
<Name>5ABCDEFGHIJKLMNOPQRSTUVWXYZ.5-6789</Name>
<CreationDate>DATE</CreationDate>
</Bucket>
<Bucket>
<Name>abcdefghijklmnopqrstuvwxyz_01234</Name>
<CreationDate>DATE</CreationDate>
</Bucket>
</Buckets>
</ListAllMyBucketsResult>

14
swift3/test/functional/002
View File

@ -1,14 +0,0 @@
#!/bin/bash
# PUT Bucket
. ./common
echo '
<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<LocationConstraint>US</LocationConstraint>
</CreateBucketConfiguration>' > $tmp.default.xml
S3USER=tester
_s3_put /bucket -D - -T $tmp.default.xml | _filter_curl

13
swift3/test/functional/002.out
View File

@ -1,13 +0,0 @@
QA output created by 002
> s3curl --id tester -- -X PUT -D - -T /TMP.default.xml http://SWIFT_HOST/bucket... 200
HTTP/1.1 100 Continue
HTTP/1.1 200 OK
Content-Length: LENGTH
Content-Type: text/html; charset=UTF-8
Date: DATE
Location: /bucket
x-amz-id-2: TXID
x-amz-request-id: TXID
X-Trans-Id: TXID

16
swift3/test/functional/003
View File

@ -1,16 +0,0 @@
#!/bin/bash
# GET Bucket
. ./common
S3USER=tester
_s3_put /bucket
_s3_put /bucket/sample.jpg -T /dev/null
_s3_put /bucket/photos/2006/January/sample.jpg -T /dev/null
_s3_put /bucket/photos/2006/February/sample2.jpg -T /dev/null
_s3_put /bucket/photos/2006/February/sample3.jpg -T /dev/null
_s3_put /bucket/pho/2006/February/sample4.jpg -T /dev/null
_s3_get /bucket?delimiter=/\&prefix=photos/2006/ -D - | _filter_curl xml

31
swift3/test/functional/003.out
View File

@ -1,31 +0,0 @@
QA output created by 003
> s3curl --id tester -- -X PUT http://SWIFT_HOST/bucket... 200
> s3curl --id tester -- -X PUT -T /dev/null http://SWIFT_HOST/bucket/sample.jpg... 200
> s3curl --id tester -- -X PUT -T /dev/null http://SWIFT_HOST/bucket/photos/2006/January/sample.jpg... 200
> s3curl --id tester -- -X PUT -T /dev/null http://SWIFT_HOST/bucket/photos/2006/February/sample2.jpg... 200
> s3curl --id tester -- -X PUT -T /dev/null http://SWIFT_HOST/bucket/photos/2006/February/sample3.jpg... 200
> s3curl --id tester -- -X PUT -T /dev/null http://SWIFT_HOST/bucket/pho/2006/February/sample4.jpg... 200
> s3curl --id tester -- -X GET -D - http://SWIFT_HOST/bucket?delimiter=/&prefix=photos/2006/... 200
HTTP/1.1 200 OK
Content-Length: LENGTH
Content-Type: application/xml
Date: DATE
x-amz-id-2: TXID
x-amz-request-id: TXID
X-Trans-Id: TXID
<?xml version='1.0' encoding='UTF-8'?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>bucket</Name>
<Prefix>photos/2006/</Prefix>
<Marker/>
<MaxKeys>1000</MaxKeys>
<Delimiter>/</Delimiter>
<IsTruncated>false</IsTruncated>
<CommonPrefixes>
<Prefix>photos/2006/February/</Prefix>
</CommonPrefixes>
<CommonPrefixes>
<Prefix>photos/2006/January/</Prefix>
</CommonPrefixes>
</ListBucketResult>

0
swift3/test/functional/__init__.py Normal file
View File

508
swift3/test/functional/check
View File

@ -1,508 +0,0 @@
#!/bin/bash
#
# Copyright (C) 2009 Red Hat, Inc.
# Copyright (c) 2000-2002,2005-2006 Silicon Graphics, Inc. All Rights Reserved.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#
# Control script for QA
#
tmp=/tmp/$$
status=0
needwrap=true
try=0
n_bad=0
bad=""
notrun=""
interrupt=true
# by default don't output timestamps
timestamp=${TIMESTAMP:=false}
# generic initialization
export iam=check
cd $(readlink -f $(dirname $0))
# we need common.config
if ! . ./common.config
then
echo "$iam: failed to source common.config"
exit 1
fi
_setenvironment()
{
MSGVERB="text:action"
export MSGVERB
}
here=`pwd`
rm -f $here/$iam.out
_setenvironment
check=${check-true}
diff="diff -u"
verbose=false
group=false
xgroup=false
exit_on_err=false
showme=false
sortme=false
expunge=true
have_test_arg=false
randomize=false
rm -f $tmp.list $tmp.tmp $tmp.sed
for r
do
if $group
then
# arg after -g
group_list=`sed -n <group -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
s/ .*//p
}'`
if [ -z "$group_list" ]
then
echo "Group \"$r\" is empty or not defined?"
exit 1
fi
[ ! -s $tmp.list ] && touch $tmp.list
for t in $group_list
do
if grep -s "^$t\$" $tmp.list >/dev/null
then
:
else
echo "$t" >>$tmp.list
fi
done
group=false
continue
elif $xgroup
then
# arg after -x
[ ! -s $tmp.list ] && ls [0-9][0-9][0-9] [0-9][0-9][0-9][0-9] >$tmp.list 2>/dev/null
group_list=`sed -n <group -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
s/ .*//p
}'`
if [ -z "$group_list" ]
then
echo "Group \"$r\" is empty or not defined?"
exit 1
fi
numsed=0
rm -f $tmp.sed
for t in $group_list
do
if [ $numsed -gt 100 ]
then
sed -f $tmp.sed <$tmp.list >$tmp.tmp
mv $tmp.tmp $tmp.list
numsed=0
rm -f $tmp.sed
fi
echo "/^$t\$/d" >>$tmp.sed
numsed=`expr $numsed + 1`
done
sed -f $tmp.sed <$tmp.list >$tmp.tmp
mv $tmp.tmp $tmp.list
xgroup=false
continue
fi
xpand=true
case "$r"
in
-\? | -h | --help) # usage
echo "Usage: $0 [options] [testlist]"'
common options
-v verbose
check options
-xdiff graphical mode diff
-e exit immediately on test failure
-n show me, do not run tests
-T output timestamps
-r randomize test order
-keep-passed keep directories of passed tests
testlist options
-g group[,group...] include tests from these groups
-x group[,group...] exclude tests from these groups
NNN include test NNN
NNN-NNN include test range (eg. 012-021)
'
exit 0
;;
-g) # -g group ... pick from group file
group=true
xpand=false
;;
-xdiff) # graphical diff mode
xpand=false
if [ ! -z "$DISPLAY" ]
then
which xdiff >/dev/null 2>&1 && diff=xdiff
which gdiff >/dev/null 2>&1 && diff=gdiff
which tkdiff >/dev/null 2>&1 && diff=tkdiff
which xxdiff >/dev/null 2>&1 && diff=xxdiff
fi
;;
-e) # exit immediately on test failure
exit_on_err=true
xpand=false
;;
-n) # show me, don't do it
showme=true
xpand=false
;;
-r) # randomize test order
randomize=true
xpand=false
;;
-T) # turn on timestamp output
timestamp=true
xpand=false
;;
-v)
verbose=true
xpand=false
;;
-x) # -x group ... exclude from group file
xgroup=true
xpand=false
;;
'[0-9][0-9][0-9] [0-9][0-9][0-9][0-9]')
echo "No tests?"
status=1
exit $status
;;
[0-9]*-[0-9]*)
eval `echo $r | sed -e 's/^/start=/' -e 's/-/ end=/'`
;;
[0-9]*-)
eval `echo $r | sed -e 's/^/start=/' -e 's/-//'`
end=`echo [0-9][0-9][0-9] [0-9][0-9][0-9][0-9] | sed -e 's/\[0-9]//g' -e 's/ *$//' -e 's/.* //'`
if [ -z "$end" ]
then
echo "No tests in range \"$r\"?"
status=1
exit $status
fi
;;
*)
start=$r
end=$r
;;
esac
# get rid of leading 0s as can be interpreted as octal
start=`echo $start | sed 's/^0*//'`
end=`echo $end | sed 's/^0*//'`
if $xpand
then
have_test_arg=true
$AWK_PROG </dev/null '
BEGIN { for (t='$start'; t<='$end'; t++) printf "%03d\n",t }' \
| while read id
do
if grep -s "^$id " group >/dev/null
then
# in group file ... OK
echo $id >>$tmp.list
else
if [ -f expunged ] && $expunge && egrep "^$id([ ]|\$)" expunged >/dev/null
then
# expunged ... will be reported, but not run, later
echo $id >>$tmp.list
else
# oops
echo "$id - unknown test, ignored"
fi
fi
done
fi
done
if [ -s $tmp.list ]
then
# found some valid test numbers ... this is good
:
else
if $have_test_arg
then
# had test numbers, but none in group file ... do nothing
touch $tmp.list
else
# no test numbers, do everything from group file
sed -n -e '/^[0-9][0-9][0-9]*/s/[ ].*//p' <group >$tmp.list
fi
fi
# should be sort -n, but this did not work for Linux when this
# was ported from IRIX
#
list=`sort $tmp.list`
rm -f $tmp.list $tmp.tmp $tmp.sed
if $randomize
then
list=`echo $list | awk -f randomize.awk`
fi
# we need common.rc
if ! . ./common.rc
then
echo "check: failed to source common.rc"
exit 1
fi
_wallclock()
{
date "+%H %M %S" | $AWK_PROG '{ print $1*3600 + $2*60 + $3 }'
}
_timestamp()
{
now=`date "+%T"`
echo -n " [$now]"
}
_wrapup()
{
# for hangcheck ...
# remove files that were used by hangcheck
#
[ -f /tmp/check.pid ] && rm -rf /tmp/check.pid
[ -f /tmp/check.sts ] && rm -rf /tmp/check.sts
if $showme
then
:
elif $needwrap
then
if [ -f check.time -a -f $tmp.time ]
then
cat check.time $tmp.time \
| $AWK_PROG '
{ t[$1] = $2 }
END { if (NR > 0) {
for (i in t) print i " " t[i]
}
}' \
| sort -n >$tmp.out
mv $tmp.out check.time
fi
if [ -f $tmp.expunged ]
then
notrun=`wc -l <$tmp.expunged | sed -e 's/ *//g'`
try=`expr $try - $notrun`
list=`echo "$list" | sed -f $tmp.expunged`
fi
echo "" >>check.log
date >>check.log
echo $list | fmt | sed -e 's/^/ /' >>check.log
$interrupt && echo "Interrupted!" >>check.log
if [ ! -z "$notrun" ]
then
echo "Not run:$notrun"
echo "Not run:$notrun" >>check.log
fi
if [ ! -z "$n_bad" -a $n_bad != 0 ]
then
echo "Failures:$bad"
echo "Failed $n_bad of $try tests"
echo "Failures:$bad" | fmt >>check.log
echo "Failed $n_bad of $try tests" >>check.log
else
echo "Passed all $try tests"
echo "Passed all $try tests" >>check.log
fi
needwrap=false
fi
rm -f /tmp/*.out /tmp/*.err /tmp/*.time
rm -f /tmp/check.pid /tmp/check.sts
rm -f $tmp.*
}
trap "_wrapup; exit \$status" 0 1 2 3 15
# for hangcheck ...
# Save pid of check in a well known place, so that hangcheck can be sure it
# has the right pid (getting the pid from ps output is not reliable enough).
#
rm -rf /tmp/check.pid
echo $$ >/tmp/check.pid
# for hangcheck ...
# Save the status of check in a well known place, so that hangcheck can be
# sure to know where check is up to (getting test number from ps output is
# not reliable enough since the trace stuff has been introduced).
#
rm -rf /tmp/check.sts
echo "preamble" >/tmp/check.sts
# don't leave old full output behind on a clean run
rm -f check.full
[ -f check.time ] || touch check.time
FULL_HOST_DETAILS=`_full_platform_details`
cat <<EOF
PLATFORM -- $FULL_HOST_DETAILS
EOF
seq="check"
[ -n "$TESTS_REMAINING_LOG" ] && echo $list > $TESTS_REMAINING_LOG
for seq in $list
do
STORE="$WD/$seq"
err=false
echo -n "$seq"
if [ -n "$TESTS_REMAINING_LOG" ] ; then
sed -e "s/$seq//" -e 's/ / /' -e 's/^ *//' $TESTS_REMAINING_LOG > $TESTS_REMAINING_LOG.tmp
mv $TESTS_REMAINING_LOG.tmp $TESTS_REMAINING_LOG
sync
fi
if $showme
then
description=`sed -n '3s/^#//p' $seq`
echo " $description"
continue
elif [ -f expunged ] && $expunge && egrep "^$seq([ ]|\$)" expunged >/dev/null
then
echo " - expunged"
rm -f $seq.out.bad
echo "/^$seq\$/d" >>$tmp.expunged
elif [ ! -f $seq ]
then
echo " - no such test?"
echo "/^$seq\$/d" >>$tmp.expunged
else
# really going to try and run this one
#
rm -f $seq.out.bad
lasttime=`sed -n -e "/^$seq /s/.* //p" <check.time`
description=`sed -n '3s/^#//p' $seq`
if [ "X$lasttime" != X ]; then
echo -n " Last Used:${lasttime}s. $description"
else
echo -n " $description" # prettier output with timestamps.
fi
rm -f core $seq.notrun
# for hangcheck ...
echo "$seq" >/tmp/check.sts
start=`_wallclock`
$timestamp && echo -n " ["`date "+%T"`"]"
[ ! -x $seq ] && chmod u+x $seq # ensure we can run it
./$seq >$tmp.out 2>&1
sts=$?
$timestamp && _timestamp
stop=`_wallclock`
if [ -f core ]
then
echo -n " [dumped core]"
mv core $seq.core
err=true
fi
if [ -f $seq.notrun ]
then
$timestamp || echo -n " [not run] "
$timestamp && echo " [not run]" && echo -n " $seq -- "
cat $seq.notrun
notrun="$notrun $seq"
else
if [ $sts -ne 0 ]
then
echo -n " [failed, exit status $sts]"
err=true
fi
if [ ! -f $seq.out ]
then
echo " - no qualified output"
err=true
else
if diff $seq.out $tmp.out >/dev/null 2>&1
then
echo ""
if $err
then
:
else
echo "$seq `expr $stop - $start`" >>$tmp.time
fi
else
echo " - output mismatch (see $seq.out.bad)"
mv $tmp.out $seq.out.bad
$diff $seq.out $seq.out.bad
err=true
fi
fi
fi
fi
# come here for each test, except when $showme is true
#
[ -f $seq.notrun ] || try=`expr $try + 1`
if $err
then
bad="$bad $seq"
n_bad=`expr $n_bad + 1`
quick=false
if $exit_on_err
then
break
fi
fi
seq="after_$seq"
done
interrupt=false
status=`expr $n_bad`
exit

17
swift3/test/functional/common
View File

@ -1,17 +0,0 @@
set -e
seq=`basename $0`
echo "QA output created by $seq"
here=`pwd`
tmp=/tmp/$$
status=1 # failure is the default!
# get standard environment, filters and checks
. ./common.rc
. ./common.filter
. ./swift.rc
. ./s3.rc
_sw_setup
_s3_setup

111
swift3/test/functional/common.config
View File

@ -1,111 +0,0 @@
#!/bin/bash
#
# Copyright (C) 2009 Red Hat, Inc.
# Copyright (c) 2000-2003,2006 Silicon Graphics, Inc. All Rights Reserved.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#
# setup and check for config parameters
#
# all tests should use a common language setting to prevent golden
# output mismatches.
export LANG=C
PATH=".:$PATH"
HOST=`hostname -s`
HOSTOS=`uname -s`
export CHECK_OPTIONS=${CHECK_OPTIONS:="-g auto"}
export PWD=`pwd`
# $1 = prog to look for, $2* = default pathnames if not found in $PATH
set_prog_path()
{
p=`which $1 2> /dev/null`
if [ -n "$p" -a -x "$p" ]; then
echo $p
return 0
fi
p=$1
shift
for f; do
if [ -x $f ]; then
echo $f
return 0
fi
done
echo ""
return 1
}
_fatal()
{
echo "$*"
status=1
exit 1
}
export PERL_PROG="`set_prog_path perl`"
[ "$PERL_PROG" = "" ] && _fatal "perl not found"
export AWK_PROG="`set_prog_path awk`"
[ "$AWK_PROG" = "" ] && _fatal "awk not found"
export SED_PROG="`set_prog_path sed`"
[ "$SED_PROG" = "" ] && _fatal "sed not found"
export CURL_PROG="`set_prog_path curl`"
[ "$CURL_PROG" = "" ] && _fatal "curl not found"
if [ -z "$TEST_DIR" ]; then
TEST_DIR=`pwd`/scratch
fi
if [ ! -e "$TEST_DIR" ]; then
mkdir "$TEST_DIR"
fi
if [ ! -d "$TEST_DIR" ]; then
echo "common.config: Error: \$TEST_DIR ($TEST_DIR) is not a directory"
exit 1
fi
export TEST_DIR
export SWIFT_HOST=${SWIFT_HOST:-"localhost:8080"}
export KEYSTONE_HOST=${KEYSTONE_HOST:-"localhost:35357"}
export AUTH=${AUTH:-"keystone"}
export TENANT=${TENANT:-"test"}
export ADMIN_USER=${ADMIN_USER:-"admin"}
export ADMIN_PASS=${ADMIN_PASS:-"admin"}
export ADMIN_ACCESS_KEY=${ADMIN_ACCESS_KEY:-"${TENANT}:${ADMIN_USER}"}
export ADMIN_SECRET_KEY=${ADMIN_SECRET_KEY:-"${ADMIN_PASS}"}
export TESTER_USER=${TESTER_USER:-"tester"}
export TESTER_PASS=${TESTER_PASS:-"testing"}
export TESTER_ACCESS_KEY=${TESTER_ACCESS_KEY:-"${TENANT}:${TESTER_USER}"}
export TESTER_SECRET_KEY=${TESTER_SECRET_KEY:-"${TESTER_PASS}"}
export TESTER2_USER=${TESTER2_USER:-"tester2"}
export TESTER2_PASS=${TESTER2_PASS:-"testing2"}
export TESTER2_ACCESS_KEY=${TESTER2_ACCESS_KEY:-"${TENANT}:${TESTER2_USER}"}
export TESTER2_SECRET_KEY=${TESTER2_SECRET_KEY:-"${TESTER2_PASS}"}
# make sure this script returns success
/bin/true

205
swift3/test/functional/common.filter
View File

@ -1,205 +0,0 @@
#!/bin/bash
#
# Copyright (C) 2009 Red Hat, Inc.
# Copyright (c) 2000-2001 Silicon Graphics, Inc. All Rights Reserved.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#
# standard filters
#
# ctime(3) dates
#
_filter_date()
{
sed -e 's/[A-Z][a-z][a-z] [A-Za-z][a-z][a-z] *[0-9][0-9]* [0-9][0-9]:[0-9][0-9]:[0-9][0-9] [0-9][0-9][0-9][0-9]$/DATE/'
}
# ISO dates
_filter_iso_date()
{
sed -e 's/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]/DATE/g'
}
_filter_short_date()
{
sed -e 's/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]/DATE/g'
}
# remove trailing whitespace, some versions of sg3_utils do that
_filter_spaces()
{
sed -e 's/ *$//'
}
_filter_eol()
{
tr -d '\r\n'
}
_filter_nop()
{
cat
}
_filter_user()
{
sed "s/${TENANT}:${ADMIN_USER}/ADMIN_USER/g" | \
sed "s/${TENANT}:${TESTER_USER}/TESTER/g" | \
sed "s/${TENANT}:${TESTER2_USER}/TESTER2/g"
}
_filter_tenant()
{
sed -e 's/AUTH_[a-z0-9]*\>/TENANT/g'
}
_filter_timestamp()
{
sed -e 's/[0-9]\{10\}\.[0-9]\{5\}/TIMESTAMP/g'
}
_filter_host()
{
sed "s/$SWIFT_HOST/SWIFT_HOST/g"
}
_filter_s3_iso_date()
{
sed -e 's/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9][0-9][0-9]Z/DATE/g'
}
_filter_upload_id()
{
sed -e 's#<UploadId>[-_0-9a-zA-Z]*</#<UploadId>UPLOAD_ID</#g' \
-e 's#<NextUploadIdMarker>[-_0-9a-zA-Z]*</#<NextUploadIdMarker>UPLOAD_ID</#g' \
-e 's#uploadId=[-_0-9a-zA-Z]*#uploadId=UPLOAD_ID#g'
}
_filter_version_id()
{
sed -e 's#^x-amz-version-id: [-_0-9a-zA-Z]*#x-amz-version-id: VERSION_ID#g' \
-e 's#<VersionId>[-_0-9a-zA-Z]*</#<VersionId>VERSION_ID</#g' \
-e 's#<NextVersionIdMarker>[-_0-9a-zA-Z]*</#<NextVersionIdMarker>VERSION_ID</#g' \
-e 's#versionId=[-_0-9a-zA-Z]*#versionId=VERSION_ID#g'
}
_filter_txid()
{
sed -e 's/tx[a-z0-9]\{21\}-[a-z0-9]\{10\}\>/TXID/g'
}
_filter_etag()
{
sed -e 's/\<[a-z0-9]\{32\}\>/ETAG/g'
}
_filter_header_date()
{
sed -e 's/[A-Z][a-z][a-z], [0-9][0-9] [A-Z][a-z][a-z] [0-9]\{4\} [0-9][0-9]:[0-9][0-9]:[0-9][0-9] GMT/DATE/g'
}
_filter_header_content_length()
{
sed -e 's/^Content-Length: [0-9]*$/Content-Length: LENGTH/g'
}
_filter_header()
{
_filter_header_date | _filter_user | \
_filter_upload_id | _filter_version_id | _filter_txid | _filter_etag | \
_filter_timestamp | _filter_header_content_length
}
_filter_body()
{
local format=$1
local fmt_filter=""
case "$format"
in
xml)
fmt_filter=_xmlindent
;;
json)
fmt_filter=_jsonindent
;;
*)
fmt_filter=_filter_nop
;;
esac
$fmt_filter | _filter_user | _filter_s3_iso_date | \
_filter_host | _filter_tenant | _filter_upload_id | \
_filter_version_id | _filter_txid | _filter_etag | \
_filter_timestamp
}
_filter_curl()
{
local format=$1
local type=body
local status=""
local header=""
local body=""
while read line; do
line=$(echo -n $line | _filter_eol)
if [[ "$body" == "" && "$line" == HTTP/1.1* ]]; then
type=status
fi
case "$type"
in
status)
if [[ "$line" == *:* ]]; then
type=header
header="${header}${line}\n"
else
status="${status}${line}\n"
fi
;;
header)
if [ "$line" == "" ]; then
type=body
else
header="${header}${line}\n"
fi
;;
body)
body="${body}${line}\n"
;;
esac
done
body="${body}${line}"
echo -en $status
if [ "$header" != "" ]; then
echo -en $header | _filter_header | sort -f
echo
fi
echo -en $body | _filter_body $format
}
_filter_curl_command()
{
sed "s#$tmp#/TMP#g" | _filter_user | _filter_host | _filter_tenant | \
_filter_upload_id | _filter_version_id | _filter_txid | _filter_etag | \
_filter_timestamp
}
# make sure this script returns success
/bin/true

209
swift3/test/functional/common.rc
View File

@ -1,209 +0,0 @@
#!/bin/bash
#
# Copyright (C) 2009 Red Hat, Inc.
# Copyright (c) 2000-2006 Silicon Graphics, Inc. All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# we need common.config
if [ "$iam" != "check" ]
then
if ! . ./common.config
then
echo "$iam: failed to source common.config"
exit 1
fi
fi
# make sure we have a standard umask
umask 022
# check if run as root
#
_need_to_be_root()
{
local id=`id | $SED_PROG -e 's/(.*//' -e 's/.*=//'`
if [ "$id" -ne 0 ]
then
_notrun "you need to be root (not uid=$id) to run this test"
fi
}
# To remove directory successfully always, we have to rename it first
# so that new files are not created in the directory while we remove it.
_safe_remove()
{
local dir=$1
mv ${dir} ${dir}.tmp
rm -rf ${dir}.tmp
}
# Do a command, log it to $seq.full, optionally test return status
# and die if command fails. If called with one argument _do executes the
# command, logs it, and returns its exit status. With two arguments _do
# first prints the message passed in the first argument, and then "done"
# or "fail" depending on the return status of the command passed in the
# second argument. If the command fails and the variable _do_die_on_error
# is set to "always" or the two argument form is used and _do_die_on_error
# is set to "message_only" _do will print an error message to
# $seq.out and exit.
_do()
{
if [ $# -eq 1 ]; then
_cmd=$1
elif [ $# -eq 2 ]; then
_note=$1
_cmd=$2
echo -n "$_note... "
else
echo "Usage: _do [note] cmd" 1>&2
status=1; exit
fi
(eval "echo '---' \"$_cmd\"") >>$here/$seq.full
(eval "$_cmd") >$tmp._out 2>&1; ret=$?
cat $tmp._out >>$here/$seq.full
if [ $# -eq 2 ]; then
if [ $ret -eq 0 ]; then
echo "done"
else
echo "fail"
fi
fi
if [ $ret -ne 0 ] \
&& [ "$_do_die_on_error" = "always" \
-o \( $# -eq 2 -a "$_do_die_on_error" = "message_only" \) ]
then
[ $# -ne 2 ] && echo
eval "echo \"$_cmd\" failed \(returned $ret\): see $seq.full"
status=1; exit
fi
return $ret
}
# bail out, setting up .notrun file
#
_notrun()
{
echo "$*" >$seq.notrun
echo "$seq not run: $*"
status=0
exit
}
# just plain bail out
#
_fail()
{
echo "$*" | tee -a $here/$seq.full
echo "(see $seq.full for details)"
status=1
exit 1
}
# this test requires that a specified command (executable) exists
#
_require_command()
{
[ -n "`which $1`" ] || _notrun "$1 utility required, skipped this test"
[ -x "`which $1`" ] || _notrun "$1 utility required, skipped this test"
}
_full_platform_details()
{
os=`uname -s`
host=`hostname -s`
kernel=`uname -r`
platform=`uname -m`
echo "$os/$platform $host $kernel"
}
_die()
{
echo $@
exit 1
}
_random()
{
openssl enc -rc4 -pass pass:"$(date)" < /dev/zero 2>/dev/null
}
_one()
{
yes $'\xFF' | tr -d "\n"
}
_hq()
{
local name=$1
egrep -i "^$name: " | $AWK_PROG '{print $2}' | _filter_eol
}
_xmlindent()
{
./xmlindent.py
}
_xpath()
{
local path=$1
./xpath.py ${path}
}
_md5()
{
local file=$1
cat $file | openssl md5 -binary | base64 | _filter_eol
}
_etag()
{
local file=$1
md5sum $file | awk '{print $1}' | _filter_eol
}
_file_size()
{
local file=$1
wc -c $file | awk '{print $1}' | _filter_eol
}
_is_http_success()
{
local status=$1
[ $(($status / 100)) == 2 ]
}
_retry()
{
for n in `seq 5`; do
"$@" > /dev/null 2>&1 && return
echo "try again"
sleep 1
done
_die "FAILED: $@"
}
# make sure this script returns success
/bin/true

16
swift3/test/functional/group
View File

@ -1,16 +0,0 @@
#
# QA groups control file
#
# Defines test groups
# - do not start group names with a digit
# - test-group association are one line per test
# - each test can be part of multiple groups
#
# auto: tests that are run by defaul
# quick: test that take less than 30 seconds (normally)
# bucket: test bucket operations
# object: test object operations
#
001 auto quick
002 auto quick
003 auto quick

24
swift3/test/functional/run_test.sh
View File

@ -1,8 +1,22 @@
#!/bin/bash
# Copyright (c) 2014 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
cd $(readlink -f $(dirname $0))
. ./common.config
. ./swift3.config
CONF_DIR=$(readlink -f ./conf)
@ -17,7 +31,8 @@ if [ "$AUTH" == 'keystone' ]; then
elif [ "$AUTH" == 'tempauth' ]; then
MIDDLEWARE="tempauth"
else
_fatal "unknown auth: $AUTH"
echo "unknown auth: $AUTH"
exit 1
fi
for server in keystone swift proxy-server object-server container-server account-server; do
@ -78,7 +93,8 @@ _start()
done
cat ${TEST_DIR}/log/${name}.log
_fatal "Cannot start ${name}-server."
echo "Cannot start ${name}-server."
exit 1
}
_start account ./run_daemon.py account 6002 conf/account-server.conf -v
@ -90,7 +106,7 @@ _start proxy coverage run --branch --include=../../* --omit=./* \
./run_daemon.py proxy 8080 conf/proxy-server.conf -v
# run tests
./check "$@"
nosetests -v ./
rvalue=$?
# cleanup

108
swift3/test/functional/s3.rc
View File

@ -1,108 +0,0 @@
#!/bin/bash
#
# Copyright (c) 2014 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
_s3_setup()
{
echo "
%awsSecretAccessKeys = (
tester => {
id => '$TESTER_ACCESS_KEY',
key => '$TESTER_SECRET_KEY',
endpoints => '${SWIFT_HOST%:*}',
},
tester2 => {
id => '$TESTER2_ACCESS_KEY',
key => '$TESTER2_SECRET_KEY',
endpoints => '${SWIFT_HOST%:*}',
},
admin => {
id => '$ADMIN_ACCESS_KEY',
key => '$ADMIN_SECRET_KEY',
endpoints => '${SWIFT_HOST%:*}',
},
);" > .s3curl
chmod 600 .s3curl
}
_s3curl()
{
local tmp_file=$tmp.$RANDOM
local args=""
if [ "$S3USER" == "" ]; then
_die "S3USER is not defined."
fi
args="--id $S3USER"
if [ "$MD5" != "" ]; then
args="$args --contentMd5 $MD5"
fi
if [ "$CONTENT_TYPE" != "" ]; then
args="$args --contentType $CONTENT_TYPE"
fi
LANG=C ./s3curl.pl $args -- -s "$@" -w '%{http_code}' > $tmp_file
status=$(tail -c -3 $tmp_file)
echo "> s3curl $args -- $@... $status" | _filter_curl_command >&2
head -c -3 $tmp_file
_is_http_success $status
}
_s3_head()
{
local path=$1; shift
_s3curl -I -X HEAD "$@" http://${SWIFT_HOST}${path}
}
_s3_get()
{
local path=$1; shift
_s3curl -X GET "$@" http://${SWIFT_HOST}${path}
}
_s3_put()
{
local path=$1; shift
_s3curl -X PUT "$@" http://${SWIFT_HOST}${path}
}
_s3_post()
{
local path=$1; shift
_s3curl -X POST "$@" http://${SWIFT_HOST}${path}
}
_s3_delete()
{
local path=$1; shift
_s3curl -X DELETE "$@" http://${SWIFT_HOST}${path}
}
# make sure this script returns success
/bin/true

88
swift3/test/functional/s3_test_client.py Normal file
View File

@ -0,0 +1,88 @@
# Copyright (c) 2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from boto.s3.connection import S3Connection, OrdinaryCallingFormat
from swift3.response import NoSuchKey, NoSuchBucket
RETRY_COUNT = 3
class Connection(object):
"""
Manage Connection
"""
def __init__(self, aws_access_key=os.environ.get('ADMIN_ACCESS_KEY'),
aws_secret_key=os.environ.get('ADMIN_SECRET_KEY'),
user_id='%s:%s' % (os.environ.get('ADMIN_TENANT'),
os.environ.get('ADMIN_USER'))):
self.aws_access_key = aws_access_key
self.aws_secret_key = aws_secret_key
self.user_id = user_id
swift_host = os.environ.get('SWIFT_HOST').split(':')
self.host = swift_host[0]
self.port = int(swift_host[1]) if len(swift_host) == 2 else 80
self.conn = \
S3Connection(aws_access_key, aws_secret_key, is_secure=False,
host=self.host, port=self.port,
calling_format=OrdinaryCallingFormat())
def reset(self):
for i in range(RETRY_COUNT):
buckets = self.conn.get_all_buckets()
if not buckets:
break
for bucket in buckets:
for obj in bucket.list():
try:
bucket.delete_key(obj.name)
except (NoSuchKey):
pass
try:
self.conn.delete_bucket(bucket.name)
except (NoSuchBucket):
pass
def make_request(self, method, bucket='', obj='', headers=None, body='',
query=None):
response = \
self.conn.make_request(method, bucket=bucket, key=obj,
headers=headers, data=body,
query_args=query, sender=None,
override_num_retries=RETRY_COUNT,
retry_handler=None)
return response.status, dict(response.getheaders()), response.read()
def get_tester_connection():
"""
Return tester connection
"""
aws_access_key = os.environ.get('TESTER_ACCESS_KEY')
aws_secret_key = os.environ.get('TESTER_SECRET_KEY')
user_id = os.environ.get('TESTER_TENANT') + ':' + \
os.environ.get('TESTER_USER')
return Connection(aws_access_key, aws_secret_key, user_id)
def get_tester2_connection():
"""
Return tester2 connection
"""
aws_access_key = os.environ.get('TESTER2_ACCESS_KEY')
aws_secret_key = os.environ.get('TESTER2_SECRET_KEY')
user_id = os.environ.get('TESTER2_TENANT') + ':' + \
os.environ.get('TESTER2_USER')
return Connection(aws_access_key, aws_secret_key, user_id)

326
swift3/test/functional/s3curl.pl
View File

@ -1,326 +0,0 @@
#!/usr/bin/perl -w
# Copyright 2006-2010 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this
# file except in compliance with the License. A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
# for the specific language governing permissions and limitations under the License.
use strict;
use POSIX;
# you might need to use CPAN to get these modules.
# run perl -MCPAN -e "install <module>" to get them.
use FindBin;
use Getopt::Long qw(GetOptions);
use constant STAT_MODE => 2;
use constant STAT_UID => 4;
# begin customizing here
my $CURL = "curl";
# stop customizing here
my @endpoints = ();
my $cmdLineSecretKey;
my %awsSecretAccessKeys = ();
my $keyFriendlyName;
my $keyId;
my $secretKey;
my $contentType = "";
my $acl;
my $contentMD5 = "";
my $fileToPut;
my $createBucket;
my $doDelete;
my $doHead;
my $help;
my $debug = 0;
my $copySourceObject;
my $copySourceRange;
my $postBody;
my $print;
my $DOTFILENAME=".s3curl";
my $EXECFILE=$FindBin::Bin;
my $LOCALDOTFILE = $EXECFILE . "/" . $DOTFILENAME;
my $HOMEDOTFILE = $ENV{HOME} . "/" . $DOTFILENAME;
my $DOTFILE = -f $LOCALDOTFILE? $LOCALDOTFILE : $HOMEDOTFILE;
if (-f $DOTFILE) {
open(CONFIG, $DOTFILE) || die "can't open $DOTFILE: $!";
my @stats = stat(*CONFIG);
if (($stats[STAT_UID] != $<) || $stats[STAT_MODE] & 066) {
die "I refuse to read your credentials from $DOTFILE as this file is " .
"readable by, writable by or owned by someone else. Try " .
"chmod 600 $DOTFILE";
}
my @lines = <CONFIG>;
close CONFIG;
eval("@lines");
die "Failed to eval() file $DOTFILE:\n$@\n" if ($@);
}
GetOptions(
'id=s' => \$keyId,
'key=s' => \$cmdLineSecretKey,
'contentType=s' => \$contentType,
'acl=s' => \$acl,
'contentMd5=s' => \$contentMD5,
'put=s' => \$fileToPut,
'copySrc=s' => \$copySourceObject,
'copySrcRange=s' => \$copySourceRange,
'post:s' => \$postBody,
'delete' => \$doDelete,
'createBucket:s' => \$createBucket,
'head' => \$doHead,
'help' => \$help,
'debug' => \$debug,
'print' => \$print,
);
my $usage = <<USAGE;
Usage $0 --id friendly-name (or AWSAccessKeyId) [options] -- [curl-options] [URL]
options:
--key SecretAccessKey id/key are AWSAcessKeyId and Secret (unsafe)
--contentType text/plain set content-type header
--acl public-read use a 'canned' ACL (x-amz-acl header)
--contentMd5 content_md5 add x-amz-content-md5 header
--put <filename> PUT request (from the provided local file)
--post [<filename>] POST request (optional local file)
--copySrc bucket/key Copy from this source key
--copySrcRange {startIndex}-{endIndex}
--createBucket [<region>] create-bucket with optional location constraint
--head HEAD request
--debug enable debug logging
--print print command instead of executing it
common curl options:
-H 'x-amz-acl: public-read' another way of using canned ACLs
-v verbose logging
USAGE
die $usage if $help || !defined $keyId;
if ($cmdLineSecretKey) {
printCmdlineSecretWarning();
sleep 5;
$secretKey = $cmdLineSecretKey;
} else {
my $keyinfo = $awsSecretAccessKeys{$keyId};
die "I don't know about key with friendly name $keyId. " .
"Do you need to set it up in $DOTFILE?"
unless defined $keyinfo;
$keyId = $keyinfo->{id};
$secretKey = $keyinfo->{key};
@endpoints = split /,/, $keyinfo->{endpoints};
}
my $method = "";
if (defined $fileToPut or defined $createBucket or defined $copySourceObject) {
$method = "PUT";
} elsif (defined $doDelete) {
$method = "DELETE";
} elsif (defined $doHead) {
$method = "HEAD";
} elsif (defined $postBody) {
$method = "POST";
} else {
$method = "GET";
}
my $resource;
my $host;
my %xamzHeaders;
$xamzHeaders{'x-amz-acl'}=$acl if (defined $acl);
$xamzHeaders{'x-amz-copy-source'}=$copySourceObject if (defined $copySourceObject);
$xamzHeaders{'x-amz-copy-source-range'}="bytes=$copySourceRange" if (defined $copySourceRange);
# try to understand curl args
for (my $i=0; $i<@ARGV; $i++) {
my $arg = $ARGV[$i];
# resource name
if ($arg =~ /https?:\/\/([^\/:?]+)(?::(\d+))?([^?]*)(?:\?(\S+))?/) {
$host = $1 if !$host;
my $port = defined $2 ? $2 : "";
my $requestURI = $3;
my $query = defined $4 ? $4 : "";
debug("Found the url: host=$host; port=$port; uri=$requestURI; query=$query;");
if (length $requestURI) {
$resource = $requestURI;
} else {
$resource = "/";
}
my @attributes = ();
for my $attribute ("acl", "location", "logging", "notification", "lifecycle",
"partNumber", "policy", "requestPayment", "response-cache-control",
"response-content-disposition", "response-content-encoding", "response-content-language",
"response-content-type", "response-expires", "torrent", "delete", "restore",
"uploadId", "uploads", "versionId", "versioning", "versions", "website") {
if ($query =~ /(?:^|&)($attribute(?:=[^&]*)?)(?:&|$)/) {
push @attributes, uri_unescape($1);
}
}
if (@attributes) {
$resource .= "?" . join("&", @attributes);
}
# handle virtual hosted requests
getResourceToSign($host, \$resource);
}
elsif ($arg =~ /\-X/) {
# mainly for DELETE
$method = $ARGV[++$i];
}
elsif ($arg =~ /\-H/) {
my $header = $ARGV[++$i];
#check for host: and x-amz*
if ($header =~ /^[Hh][Oo][Ss][Tt]:(.+)$/) {
$host = $1;
}
elsif ($header =~ /^([Xx]-[Aa][Mm][Zz]-.+?): *(.+)$/) {
my $name = lc $1;
my $value = $2;
# merge with existing values
if (exists $xamzHeaders{$name}) {
$value = $xamzHeaders{$name} . "," . $value;
}
$xamzHeaders{$name} = $value;
}
}
}
die "Couldn't find resource by digging through your curl command line args!"
unless defined $resource;
my $xamzHeadersToSign = "";
foreach (sort (keys %xamzHeaders)) {
my $headerValue = $xamzHeaders{$_};
$xamzHeadersToSign .= "$_:$headerValue\n";
}
my $httpDate = POSIX::strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime );
my $stringToSign = "$method\n$contentMD5\n$contentType\n$httpDate\n$xamzHeadersToSign$resource";
debug("StringToSign='" . $stringToSign . "'");
my $signature = `echo -n "$stringToSign" | openssl dgst -sha1 -hmac "$secretKey" -binary | base64`;
chomp($signature);
debug("signature='" . $signature. "'");
my @args = ();
push @args, ("-H", "Date: $httpDate");
push @args, ("-H", "Authorization: AWS $keyId:$signature");
push @args, ("-H", "x-amz-acl: $acl") if (defined $acl);
push @args, ("-L");
push @args, ("-H", "content-type: $contentType") if (length $contentType);
push @args, ("-H", "Content-MD5: $contentMD5") if (length $contentMD5);
push @args, ("-T", $fileToPut) if (defined $fileToPut);
push @args, ("-X", "DELETE") if (defined $doDelete);
push @args, ("-X", "POST") if(defined $postBody);
push @args, ("-I") if (defined $doHead);
if (defined $createBucket) {
# createBucket is a special kind of put from stdin. Reason being, curl mangles the Request-URI
# to include the local filename when you use -T and it decides there is no remote filename (bucket PUT)
my $data="";
if (length($createBucket)>0) {
$data="<CreateBucketConfiguration><LocationConstraint>$createBucket</LocationConstraint></CreateBucketConfiguration>";
}
push @args, ("--data-binary", $data);
push @args, ("-X", "PUT");
} elsif (defined $copySourceObject) {
# copy operation is a special kind of PUT operation where the resource to put
# is specified in the header
push @args, ("-X", "PUT");
push @args, ("-H", "x-amz-copy-source: $copySourceObject");
} elsif (defined $postBody) {
if (length($postBody)>0) {
push @args, ("-T", $postBody);
}
}
push @args, @ARGV;
debug("exec $CURL " . join (" ", @args));
if (defined($print)) {
print join(" ", $CURL, @args, "\n");
exit(0)
}
exec($CURL, @args) or die "can't exec program: $!";
sub debug {
my ($str) = @_;
$str =~ s/\n/\\n/g;
print STDERR "s3curl: $str\n" if ($debug);
}
sub getResourceToSign {
my ($host, $resourceToSignRef) = @_;
for my $ep (@endpoints) {
if ($host =~ /(.*)\.$ep/) { # vanity subdomain case
my $vanityBucket = $1;
$$resourceToSignRef = "/$vanityBucket".$$resourceToSignRef;
debug("vanity endpoint signing case");
return;
}
elsif ($host eq $ep) {
debug("ordinary endpoint signing case");
return;
}
}
# cname case
$$resourceToSignRef = "/$host".$$resourceToSignRef;
debug("cname endpoint signing case");
}
sub printCmdlineSecretWarning {
print STDERR <<END_WARNING;
WARNING: It isn't safe to put your AWS secret access key on the
command line! The recommended key management system is to store
your AWS secret access keys in a file owned by, and only readable
by you.
For example:
\%awsSecretAccessKeys = (
# personal account
personal => {
id => '1ME55KNV6SBTR7EXG0R2',
key => 'zyMrlZUKeG9UcYpwzlPko/+Ciu0K2co0duRM3fhi',
},
# corporate account
company => {
id => '1ATXQ3HHA59CYF1CVS02',
key => 'WQY4SrSS95pJUT95V6zWea01gBKBCL6PI0cdxeH8',
},
);
\$ chmod 600 $DOTFILE
Will sleep and continue despite this problem.
Please set up $DOTFILE for future requests.
END_WARNING
}
sub uri_unescape {
my ($input) = @_;
$input =~ s/\%([A-Fa-f0-9]{2})/pack('C', hex($1))/seg;
debug("replaced string: " . $input);
return ($input);
}

15
swift3/test/functional/setup_keystone
View File

@ -1,3 +1,18 @@
# Copyright (c) 2014 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
export OS_AUTH_URL=http://localhost:35357/v2.0
export OS_TENANT_NAME=admin
export OS_USERNAME=admin

112
swift3/test/functional/swift.rc
View File

@ -1,112 +0,0 @@
#!/bin/bash
#
# Copyright (c) 2014 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
_sw_setup()
{
case "$AUTH"
in
tempauth)
export TENANT_ID="AUTH_${TENANT}"
export TOKEN_ID=$($CURL_PROG -s -I -X GET \
-H "x-storage-user: ${TENANT}:${ADMIN_USER}" \
-H "x-storage-pass: ${ADMIN_PASS}" \
http://${SWIFT_HOST}/auth/v1.0 | \
_hq x-storage-token)
;;
keystone)
local xml="<auth tenantName=\"${TENANT}\">
<passwordCredentials username=\"${ADMIN_USER}\"
password=\"${ADMIN_PASS}\"/></auth>"
local res=$($CURL_PROG -s -d "$xml" \
-H 'Content-type: application/xml' \
-H 'Accept: application/xml' \
http://${KEYSTONE_HOST}/v2.0/tokens)
export TENANT_ID="AUTH_$(echo $res | _xpath '/access/token/tenant/@id')"
export TOKEN_ID=$(echo $res | _xpath '/access/token/@id')
;;
*)
_die "unknown auth, $AUTH"
;;
esac
local c
local o
# remove user data
for c in $(_sw_get / 2>/dev/null); do
_retry _sw_post /$c -H "x-versions-location: $c" # disable versioning
for o in $(_sw_get /$c 2>/dev/null); do
_retry _sw_delete /$c/$o
done
_retry _sw_delete /$c
done
}
_swcurl()
{
local tmp_file=$tmp.$RANDOM
$CURL_PROG -s -H "x-storage-token: ${TOKEN_ID}" "$@" -w '%{http_code}' \
> $tmp_file
status=$(tail -c -3 $tmp_file)
echo "> curl $@... $status" | _filter_curl_command >&2
head -c -3 $tmp_file
_is_http_success $status
}
_sw_head()
{
local path=$1; shift
_swcurl -I -X HEAD "$@" http://${SWIFT_HOST}/v1/${TENANT_ID}${path}
}
_sw_get()
{
local path=$1; shift
_swcurl -X GET "$@" http://${SWIFT_HOST}/v1/${TENANT_ID}${path}
}
_sw_put()
{
local path=$1; shift
_swcurl -X PUT "$@" http://${SWIFT_HOST}/v1/${TENANT_ID}${path}
}
_sw_post()
{
local path=$1; shift
_swcurl -X POST "$@" http://${SWIFT_HOST}/v1/${TENANT_ID}${path}
}
_sw_delete()
{
local path=$1; shift
_swcurl -X DELETE "$@" http://${SWIFT_HOST}/v1/${TENANT_ID}${path}
}
# make sure this script returns success
/bin/true

42
swift3/test/functional/swift3.config Normal file
View File

@ -0,0 +1,42 @@
#!/bin/bash
# Copyright (c) 2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
export LANG=C
export PWD=`pwd`
export TEST_DIR=$PWD/'scratch'
export SWIFT_HOST=${SWIFT_HOST:-"localhost:8080"}
export KEYSTONE_HOST=${KEYSTONE_HOST:-"localhost:35357"}
export AUTH=${AUTH:-"keystone"}
export ADMIN_TENANT=${ADMIN_TENANT:-"test"}
export ADMIN_USER=${ADMIN_USER:-"admin"}
export ADMIN_PASS=${ADMIN_PASS:-"admin"}
export ADMIN_ACCESS_KEY=${ADMIN_ACCESS_KEY:-"${ADMIN_TENANT}:${ADMIN_USER}"}
export ADMIN_SECRET_KEY=${ADMIN_SECRET_KEY:-"${ADMIN_PASS}"}
export TESTER_TENANT=${TESTER_TENANT:-"test"}
export TESTER_USER=${TESTER_USER:-"tester"}
export TESTER_PASS=${TESTER_PASS:-"testing"}
export TESTER_ACCESS_KEY=${TESTER_ACCESS_KEY:-"${TESTER_TENANT}:${TESTER_USER}"}
export TESTER_SECRET_KEY=${TESTER_SECRET_KEY:-"${TESTER_PASS}"}
export TESTER2_TENANT=${TESTER2_TENANT:-"test"}
export TESTER2_USER=${TESTER2_USER:-"tester2"}
export TESTER2_PASS=${TESTER2_PASS:-"testing2"}
export TESTER2_ACCESS_KEY=${TESTER2_ACCESS_KEY:-"${TESTER2_TENANT}:${TESTER2_USER}"}
export TESTER2_SECRET_KEY=${TESTER2_SECRET_KEY:-"${TESTER2_PASS}"}

167
swift3/test/functional/test_bucket.py Normal file
View File

@ -0,0 +1,167 @@
# Copyright (c) 2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from swift3.test.functional.s3_test_client import get_tester_connection,\
Connection
from swift3.test.functional.utils import get_error_code,\
assert_common_response_headers
from swift3.etree import fromstring
from swift3.cfg import CONF
class TestSwift3Bucket(unittest.TestCase):
def setUp(self):
self.conn = get_tester_connection()
self.conn.reset()
def test_bucket(self):
bucket = 'bucket'
# PUT Bucket
status, headers, body = self.conn.make_request('PUT', bucket)
self.assertEquals(status, 200)
assert_common_response_headers(self, headers)
self.assertEquals(headers['location'], '/' + bucket)
self.assertEquals(headers['content-length'], '0')
# GET Bucket(Without Object)
status, headers, body = self.conn.make_request('GET', bucket)
self.assertEquals(status, 200)
assert_common_response_headers(self, headers)
self.assertTrue(headers['content-type'] is not None)
self.assertEquals(headers['content-length'], str(len(body)))
# TODO; requires consideration
# self.assertEquasl(headers['transfer-encoding'], 'chunked')
elem = fromstring(body, 'ListBucketResult')
self.assertEquals(elem.find('Name').text, bucket)
self.assertEquals(elem.find('Prefix').text, None)
self.assertEquals(elem.find('Marker').text, None)
self.assertEquals(elem.find('MaxKeys').text,
str(CONF.max_bucket_listing))
self.assertEquals(elem.find('IsTruncated').text, 'false')
objects = elem.findall('./Contents')
self.assertEquals(list(objects), [])
# GET Bucket(With Object)
req_objects = ('object', 'object2')
for obj in req_objects:
self.conn.make_request('PUT', bucket, obj)
status, headers, body = self.conn.make_request('GET', bucket)
self.assertEquals(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEquals(elem.find('Name').text, bucket)
self.assertEquals(elem.find('Prefix').text, None)
self.assertEquals(elem.find('Marker').text, None)
self.assertEquals(elem.find('MaxKeys').text,
str(CONF.max_bucket_listing))
self.assertEquals(elem.find('IsTruncated').text, 'false')
resp_objects = elem.findall('./Contents')
self.assertEquals(len(list(resp_objects)), 2)
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
self.assertTrue(o.find('LastModified').text is not None)
self.assertTrue(o.find('ETag').text is not None)
self.assertTrue(o.find('Size').text is not None)
self.assertTrue(o.find('StorageClass').text is not None)
self.assertTrue(o.find('Owner/ID').text, self.conn.user_id)
self.assertTrue(o.find('Owner/DisplayName').text,
self.conn.user_id)
# HEAD Bucket
status, headers, body = self.conn.make_request('HEAD', bucket)
self.assertEquals(status, 200)
assert_common_response_headers(self, headers)
self.assertTrue(headers['content-type'] is not None)
self.assertEquals(headers['content-length'], str(len(body)))
# TODO; requires consideration
# self.assertEquasl(headers['transfer-encoding'], 'chunked')
# DELETE Bucket
for obj in req_objects:
self.conn.make_request('DELETE', bucket, obj)
status, headers, body = self.conn.make_request('DELETE', bucket)
self.assertEquals(status, 204)
assert_common_response_headers(self, headers)
def test_put_bucket_error(self):
status, headers, body = \
self.conn.make_request('PUT', 'bucket+invalid')
self.assertEquals(get_error_code(body), 'InvalidBucketName')
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = auth_error_conn.make_request('PUT', 'bucket')
self.assertEquals(get_error_code(body), 'SignatureDoesNotMatch')
self.conn.make_request('PUT', 'bucket')
status, headers, body = self.conn.make_request('PUT', 'bucket')
self.assertEquals(get_error_code(body), 'BucketAlreadyExists')
self.conn.make_request('DELETE', 'bucket')
def test_get_bucket_error(self):
self.conn.make_request('PUT', 'bucket')
status, headers, body = \
self.conn.make_request('GET', 'bucket+invalid')
self.assertEquals(get_error_code(body), 'InvalidBucketName')
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = auth_error_conn.make_request('GET', 'bucket')
self.assertEquals(get_error_code(body), 'SignatureDoesNotMatch')
status, headers, body = self.conn.make_request('GET', 'nothing')
self.assertEquals(get_error_code(body), 'NoSuchBucket')
self.conn.make_request('DELETE', 'bucket')
def test_head_bucket_error(self):
self.conn.make_request('PUT', 'bucket')
status, headers, body = \
self.conn.make_request('HEAD', 'bucket+invalid')
self.assertEquals(status, 400)
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('HEAD', 'bucket')
self.assertEquals(status, 403)
status, headers, body = self.conn.make_request('HEAD', 'nothing')
self.assertEquals(status, 404)
self.conn.make_request('DELETE', 'bucket')
def test_delete_bucket_error(self):
status, headers, body = \
self.conn.make_request('DELETE', 'bucket+invalid')
self.assertEquals(get_error_code(body), 'InvalidBucketName')
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('DELETE', 'bucket')
self.assertEquals(get_error_code(body), 'SignatureDoesNotMatch')
status, headers, body = self.conn.make_request('DELETE', 'bucket')
self.assertEquals(get_error_code(body), 'NoSuchBucket')
if __name__ == '__main__':
unittest.main()

138
swift3/test/functional/test_object.py Normal file
View File

@ -0,0 +1,138 @@
# Copyright (c) 2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from swift3.test.functional.s3_test_client import get_tester_connection,\
Connection
from swift3.test.functional.utils import get_error_code,\
assert_common_response_headers
class TestSwift3Object(unittest.TestCase):
def setUp(self):
self.conn = get_tester_connection()
self.conn.reset()
self.bucket = 'bucket'
self.conn.make_request('PUT', self.bucket)
def test_object(self):
obj = 'object'
contents = 'abc123'
# PUT Object
status, headers, body = \
self.conn.make_request('PUT', self.bucket, obj, body=contents)
self.assertEquals(status, 200)
assert_common_response_headers(self, headers)
self.assertTrue(headers['etag'] is not None)
self.assertEquals(headers['content-length'], '0')
# GET Object
status, headers, body = \
self.conn.make_request('GET', self.bucket, obj)
self.assertEquals(status, 200)
assert_common_response_headers(self, headers)
self.assertTrue(headers['last-modified'] is not None)
self.assertTrue(headers['etag'] is not None)
self.assertTrue(headers['content-type'] is not None)
self.assertEquals(headers['content-length'], str(len(contents)))
# HEAD Object
status, headers, body = \
self.conn.make_request('HEAD', self.bucket, obj)
self.assertEquals(status, 200)
assert_common_response_headers(self, headers)
self.assertTrue(headers['last-modified'] is not None)
self.assertTrue(headers['etag'] is not None)
self.assertTrue(headers['content-type'] is not None)
self.assertEquals(headers['content-length'], str(len(contents)))
# DELETE Object
status, headers, body = \
self.conn.make_request('DELETE', self.bucket, obj)
self.assertEquals(status, 204)
assert_common_response_headers(self, headers)
def test_put_object_error(self):
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('PUT', self.bucket, 'object')
self.assertEquals(get_error_code(body), 'SignatureDoesNotMatch')
status, headers, body = \
self.conn.make_request('PUT', 'bucket2', 'object')
self.assertEquals(get_error_code(body), 'NoSuchBucket')
def test_get_object_error(self):
obj = 'object'
self.conn.make_request('PUT', self.bucket, obj)
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('GET', self.bucket, obj)
self.assertEquals(get_error_code(body), 'SignatureDoesNotMatch')
status, headers, body = \
self.conn.make_request('GET', self.bucket, 'invalid')
self.assertEquals(get_error_code(body), 'NoSuchKey')
status, headers, body = self.conn.make_request('GET', 'invalid', obj)
# TODO; requires consideration
# self.assertEquals(get_error_code(body), 'NoSuchBucket')
self.assertEquals(get_error_code(body), 'NoSuchKey')
def test_head_object_error(self):
obj = 'object'
self.conn.make_request('PUT', self.bucket, obj)
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('HEAD', self.bucket, obj)
self.assertEquals(status, 403)
status, headers, body = \
self.conn.make_request('HEAD', self.bucket, 'invalid')
self.assertEquals(status, 404)
status, headers, body = \
self.conn.make_request('HEAD', 'invalid', obj)
self.assertEquals(status, 404)
def test_delete_object_error(self):
obj = 'object'
self.conn.make_request('PUT', self.bucket, obj)
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('DELETE', self.bucket, obj)
self.assertEquals(get_error_code(body), 'SignatureDoesNotMatch')
status, headers, body = \
self.conn.make_request('DELETE', self.bucket, 'invalid')
self.assertEquals(get_error_code(body), 'NoSuchKey')
status, headers, body = \
self.conn.make_request('DELETE', 'invalid', obj)
# TODO; requires consideration
# self.assertEquals(get_error_code(body), 'NoSuchBucket')
self.assertEquals(get_error_code(body), 'NoSuchKey')
if __name__ == '__main__':
unittest.main()

67
swift3/test/functional/test_service.py Normal file
View File

@ -0,0 +1,67 @@
# Copyright (c) 2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from swift3.test.functional.s3_test_client import get_tester_connection,\
Connection
from swift3.test.functional.utils import get_error_code,\
assert_common_response_headers
from swift3.etree import fromstring
class TestSwift3Service(unittest.TestCase):
def setUp(self):
self.conn = get_tester_connection()
self.conn.reset()
def test_service(self):
# GET Service(without bucket)
status, headers, body = self.conn.make_request('GET')
self.assertEquals(status, 200)
assert_common_response_headers(self, headers)
self.assertTrue(headers['content-type'] is not None)
# TODO; requires consideration
# self.assertEquasl(headers['transfer-encoding'], 'chunked')
elem = fromstring(body, 'ListAllMyBucketsResult')
buckets = elem.findall('./Buckets/Bucket')
self.assertEquals(list(buckets), [])
owner = elem.find('Owner')
self.assertEquals(self.conn.user_id, owner.find('ID').text)
self.assertEquals(self.conn.user_id, owner.find('DisplayName').text)
# GET Service(with Bucket)
req_buckets = ('bucket', 'bucket2')
for bucket in req_buckets:
self.conn.make_request('PUT', bucket)
status, headers, body = self.conn.make_request('GET')
self.assertEquals(status, 200)
elem = fromstring(body, 'ListAllMyBucketsResult')
resp_buckets = elem.findall('./Buckets/Bucket')
self.assertEquals(len(list(resp_buckets)), 2)
for b in resp_buckets:
self.assertTrue(b.find('Name').text in req_buckets)
self.assertTrue(b.find('CreationDate') is not None)
def test_service_error(self):
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = auth_error_conn.make_request('GET')
self.assertEquals(get_error_code(body), 'SignatureDoesNotMatch')
if __name__ == '__main__':
unittest.main()

20
swift3/test/functional/xpath.py → swift3/test/functional/utils.py Executable file → Normal file
View File

@ -1,5 +1,4 @@
#!/usr/bin/env python
# Copyright (c) 2014 OpenStack Foundation
# Copyright (c) 2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -14,10 +13,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from swift3.etree import fromstring
if __name__ == '__main__':
elem = fromstring(sys.stdin.read())
print elem.xpath(sys.argv[1])[0]
def assert_common_response_headers(self, headers):
self.assertTrue(headers['x-amz-id-2'] is not None)
self.assertTrue(headers['x-amz-request-id'] is not None)
self.assertTrue(headers['date'] is not None)
# TODO; requires consideration
# self.assertTrue(headers['server'] is not None)
def get_error_code(body):
elem = fromstring(body, 'Error')
return elem.find('Code').text

25
swift3/test/functional/xmlindent.py
View File

@ -1,25 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2014 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from lxml.etree import fromstring, tostring
if __name__ == '__main__':
xml = sys.stdin.read().replace('\n', '')
elem = fromstring(xml)
print tostring(elem, xml_declaration=True, encoding='UTF-8',
pretty_print=True),

1
test-requirements.txt
View File

@ -6,3 +6,4 @@ coverage
mock
pylint
python-openstackclient
boto