diff --git a/bin/swauth-add-account b/bin/swauth-add-account index 32aceffc7b..fe18b5a72d 100755 --- a/bin/swauth-add-account +++ b/bin/swauth-add-account @@ -18,9 +18,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-add-user b/bin/swauth-add-user index a844ed2a37..045dc0a766 100755 --- a/bin/swauth-add-user +++ b/bin/swauth-add-user @@ -18,9 +18,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-delete-account b/bin/swauth-delete-account index c46e5e3b91..3d98f6ec4e 100755 --- a/bin/swauth-delete-account +++ b/bin/swauth-delete-account @@ -18,9 +18,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-delete-user b/bin/swauth-delete-user index 5ee162437c..ede076dd5b 100755 --- a/bin/swauth-delete-user +++ b/bin/swauth-delete-user @@ -18,9 +18,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-list b/bin/swauth-list index 7433e3ddfd..85a7633966 100755 --- a/bin/swauth-list +++ b/bin/swauth-list @@ -22,9 +22,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-prep b/bin/swauth-prep index 5a931ae1d0..3d2cb7d3eb 100755 --- a/bin/swauth-prep +++ b/bin/swauth-prep @@ -18,9 +18,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swauth-set-account-service b/bin/swauth-set-account-service index 32eb06dc6b..054e4cfc4b 100755 --- a/bin/swauth-set-account-service +++ b/bin/swauth-set-account-service @@ -22,9 +22,9 @@ import gettext from optparse import OptionParser from os.path import basename from sys import argv, exit -from urlparse import urlparse from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import urlparse if __name__ == '__main__': diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index c448bea5ca..41293f7d37 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -235,10 +235,17 @@ swift-ring-builder add z-:/_ print 'Invalid add value: %s' % argv[3] exit(EXIT_ERROR) i = 1 - while i < len(rest) and rest[i] in '0123456789.': + if rest[i] == '[': + while i < len(rest) and rest[i] != ']': + i += 1 + ip = rest[2:i] i += 1 - ip = rest[1:i] - rest = rest[i:] + rest = rest[i:] + else: + while i < len(rest) and rest[i] in '0123456789.': + i += 1 + ip = rest[1:i] + rest = rest[i:] if not rest.startswith(':'): print 'Invalid add value: %s' % argv[3] diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index fda7d0d034..2d85f19508 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -68,13 +68,13 @@ use = egg:swift#swauth # auth_prefix = /auth/ # Cluster strings are of the format name:url where name is a short name for the # Swift cluster and url is the url to the proxy server(s) for the cluster. -# default_swift_cluster = local:http://127.0.0.1:8080/v1 +# default_swift_cluster = local#http://127.0.0.1:8080/v1 # You may also use the format name::url::url where the first url is the one # given to users to access their account (public url) and the second is the one # used by swauth itself to create and delete accounts (private url). This is # useful when a load balancer url should be used by users, but swauth itself is # behind the load balancer. Example: -# default_swift_cluster = local::https://public.com:8080/v1::http://private.com:8080/v1 +# default_swift_cluster = local##https://public.com:8080/v1##http://private.com:8080/v1 # token_life = 86400 # node_timeout = 10 # Highly recommended to change this. diff --git a/swift/auth/server.py b/swift/auth/server.py index a0bd31ccda..967f853291 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -20,7 +20,6 @@ from contextlib import contextmanager from time import gmtime, strftime, time from urllib import unquote, quote from uuid import uuid4 -from urlparse import urlparse from hashlib import md5, sha1 import hmac import base64 @@ -32,7 +31,7 @@ from webob.exc import HTTPBadRequest, HTTPConflict, HTTPForbidden, \ from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.db import get_db_connection -from swift.common.utils import get_logger, split_path +from swift.common.utils import get_logger, split_path, urlparse class AuthController(object): diff --git a/swift/common/bench.py b/swift/common/bench.py index 4abafeb947..169497ef13 100644 --- a/swift/common/bench.py +++ b/swift/common/bench.py @@ -16,13 +16,12 @@ import uuid import time import random -from urlparse import urlparse from contextlib import contextmanager import eventlet.pools from eventlet.green.httplib import CannotSendRequest -from swift.common.utils import TRUE_VALUES +from swift.common.utils import TRUE_VALUES, urlparse from swift.common import client from swift.common import direct_client diff --git a/swift/common/middleware/acl.py b/swift/common/middleware/acl.py index f6784953ac..f08780eedb 100644 --- a/swift/common/middleware/acl.py +++ b/swift/common/middleware/acl.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from urlparse import urlparse +from swift.common.utils import urlparse def clean_acl(name, value): diff --git a/swift/common/middleware/swauth.py b/swift/common/middleware/swauth.py index 961f3a3ba4..568b00fb35 100644 --- a/swift/common/middleware/swauth.py +++ b/swift/common/middleware/swauth.py @@ -21,7 +21,6 @@ from httplib import HTTPConnection, HTTPSConnection from time import gmtime, strftime, time from traceback import format_exc from urllib import quote, unquote -from urlparse import urlparse from uuid import uuid4 from eventlet.timeout import Timeout @@ -32,7 +31,7 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ from swift.common.bufferedhttp import http_connect_raw as http_connect from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed -from swift.common.utils import cache_from_env, get_logger, split_path +from swift.common.utils import cache_from_env, get_logger, split_path, urlparse class Swauth(object): @@ -61,23 +60,23 @@ class Swauth(object): self.auth_prefix += '/' self.auth_account = '%s.auth' % self.reseller_prefix self.default_swift_cluster = conf.get('default_swift_cluster', - 'local:http://127.0.0.1:8080/v1') + 'local#http://127.0.0.1:8080/v1') # This setting is a little messy because of the options it has to # provide. The basic format is cluster_name:url, such as the default - # value of local:http://127.0.0.1:8080/v1. But, often the url given to + # value of local#http://127.0.0.1:8080/v1. But, often the url given to # the user needs to be different than the url used by Swauth to # create/delete accounts. So there's a more complex format of # cluster_name::url::url, such as - # local::https://public.com:8080/v1::http://private.com:8080/v1. + # local##https://public.com:8080/v1##http://private.com:8080/v1. # The double colon is what sets the two apart. - if '::' in self.default_swift_cluster: + if '##' in self.default_swift_cluster: self.dsc_name, self.dsc_url, self.dsc_url2 = \ - self.default_swift_cluster.split('::', 2) + self.default_swift_cluster.split('##', 2) self.dsc_url = self.dsc_url.rstrip('/') self.dsc_url2 = self.dsc_url2.rstrip('/') else: self.dsc_name, self.dsc_url = \ - self.default_swift_cluster.split(':', 1) + self.default_swift_cluster.split('#', 1) self.dsc_url = self.dsc_url2 = self.dsc_url.rstrip('/') self.dsc_parsed = urlparse(self.dsc_url) if self.dsc_parsed.scheme not in ('http', 'https'): diff --git a/swift/common/utils.py b/swift/common/utils.py index 299980493a..05b15e99fa 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -34,6 +34,7 @@ from ConfigParser import ConfigParser, NoSectionError, NoOptionError from optparse import OptionParser from tempfile import mkstemp import cPickle as pickle +from urlparse import urlparse as stdlib_urlparse, ParseResult import eventlet from eventlet import greenio, GreenPool, sleep, Timeout, listen @@ -845,3 +846,35 @@ def ratelimit_sleep(running_time, max_rate, incr_by=1): elif running_time - now > time_per_request: eventlet.sleep((running_time - now) / clock_accuracy) return running_time + time_per_request + + +class ModifiedParseResult(ParseResult): + "Parse results class for urlparse." + + @property + def hostname(self): + netloc = self.netloc.split('@', 1)[-1] + if netloc.startswith('['): + return netloc[1:].split(']')[0] + elif ':' in netloc: + return netloc.rsplit(':')[0] + return netloc + + @property + def port(self): + netloc = self.netloc.split('@', 1)[-1] + if netloc.startswith('['): + netloc = netloc.rsplit(']')[1] + if ':' in netloc: + return int(netloc.rsplit(':')[1]) + return None + + +def urlparse(url): + """ + urlparse augmentation. + This is necessary because urlparse can't handle RFC 2732 URLs. + + :param url: URL to parse. + """ + return ModifiedParseResult(*stdlib_urlparse(url)) diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 9450bcf439..cedc4b2c8b 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -68,11 +68,15 @@ def get_socket(conf, default_port=8080): """ bind_addr = (conf.get('bind_ip', '0.0.0.0'), int(conf.get('bind_port', default_port))) + address_family = [addr[0] for addr in socket.getaddrinfo(bind_addr[0], + bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM) + if addr[0] in (socket.AF_INET, socket.AF_INET6)][0] sock = None retry_until = time.time() + 30 while not sock and time.time() < retry_until: try: - sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096))) + sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)), + family=address_family) if 'cert_file' in conf: sock = ssl.wrap_socket(sock, certfile=conf['cert_file'], keyfile=conf['key_file']) diff --git a/swift/container/server.py b/swift/container/server.py index 7ba375ce33..1ffba8a909 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -88,7 +88,7 @@ class ContainerController(object): account_partition = req.headers.get('X-Account-Partition') account_device = req.headers.get('X-Account-Device') if all([account_host, account_partition, account_device]): - account_ip, account_port = account_host.split(':') + account_ip, account_port = account_host.rsplit(':', 1) new_path = '/' + '/'.join([account, container]) info = broker.get_info() account_headers = {'x-put-timestamp': info['put_timestamp'], diff --git a/swift/obj/server.py b/swift/obj/server.py index 4afc38057d..f20b40d57a 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -294,7 +294,7 @@ class ObjectController(object): full_path = '/%s/%s/%s' % (account, container, obj) try: with ConnectionTimeout(self.conn_timeout): - ip, port = host.split(':') + ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): diff --git a/test/unit/common/middleware/test_swauth.py b/test/unit/common/middleware/test_swauth.py index 00c010b9dc..a6edab9c2c 100644 --- a/test/unit/common/middleware/test_swauth.py +++ b/test/unit/common/middleware/test_swauth.py @@ -151,21 +151,21 @@ class TestAuth(unittest.TestCase): app = FakeApp() self.assertRaises(Exception, auth.filter_factory({ 'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:badscheme://host/path'}), app) + 'default_swift_cluster': 'local#badscheme://host/path'}), app) ath = auth.filter_factory({'super_admin_key': 'supertest'})(app) self.assertEquals(ath.default_swift_cluster, - 'local:http://127.0.0.1:8080/v1') + 'local#http://127.0.0.1:8080/v1') ath = auth.filter_factory({'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:http://host/path'})(app) + 'default_swift_cluster': 'local#http://host/path'})(app) self.assertEquals(ath.default_swift_cluster, - 'local:http://host/path') + 'local#http://host/path') ath = auth.filter_factory({'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:https://host/path/'})(app) + 'default_swift_cluster': 'local#https://host/path/'})(app) self.assertEquals(ath.dsc_url, 'https://host/path') self.assertEquals(ath.dsc_url2, 'https://host/path') ath = auth.filter_factory({'super_admin_key': 'supertest', 'default_swift_cluster': - 'local::https://host/path/::http://host2/path2/'})(app) + 'local##https://host/path/##http://host2/path2/'})(app) self.assertEquals(ath.dsc_url, 'https://host/path') self.assertEquals(ath.dsc_url2, 'http://host2/path2') @@ -2882,7 +2882,7 @@ class TestAuth(unittest.TestCase): def test_get_conn_default_https(self): local_auth = auth.filter_factory({'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:https://1.2.3.4/v1'})(FakeApp()) + 'default_swift_cluster': 'local#https://1.2.3.4/v1'})(FakeApp()) conn = local_auth.get_conn() self.assertEquals(conn.__class__, auth.HTTPSConnection) self.assertEquals(conn.host, '1.2.3.4') @@ -2890,7 +2890,7 @@ class TestAuth(unittest.TestCase): def test_get_conn_overridden(self): local_auth = auth.filter_factory({'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:https://1.2.3.4/v1'})(FakeApp()) + 'default_swift_cluster': 'local#https://1.2.3.4/v1'})(FakeApp()) conn = \ local_auth.get_conn(urlparsed=auth.urlparse('http://5.6.7.8/v1')) self.assertEquals(conn.__class__, auth.HTTPConnection) @@ -2899,7 +2899,7 @@ class TestAuth(unittest.TestCase): def test_get_conn_overridden_https(self): local_auth = auth.filter_factory({'super_admin_key': 'supertest', - 'default_swift_cluster': 'local:http://1.2.3.4/v1'})(FakeApp()) + 'default_swift_cluster': 'local#http://1.2.3.4/v1'})(FakeApp()) conn = \ local_auth.get_conn(urlparsed=auth.urlparse('https://5.6.7.8/v1')) self.assertEquals(conn.__class__, auth.HTTPSConnection) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 1f5a94edd5..b9e8a3f81b 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -477,6 +477,27 @@ log_name = yarr''' total += i self.assertTrue(abs(50 - (time.time() - start) * 100) < 10) + def test_urlparse(self): + parsed = utils.urlparse('http://127.0.0.1/') + self.assertEquals(parsed.scheme, 'http') + self.assertEquals(parsed.hostname, '127.0.0.1') + self.assertEquals(parsed.path, '/') + + parsed = utils.urlparse('http://127.0.0.1:8080/') + self.assertEquals(parsed.port, 8080) + + parsed = utils.urlparse('https://127.0.0.1/') + self.assertEquals(parsed.scheme, 'https') + + parsed = utils.urlparse('http://[::1]/') + self.assertEquals(parsed.hostname, '::1') + + parsed = utils.urlparse('http://[::1]:8080/') + self.assertEquals(parsed.hostname, '::1') + self.assertEquals(parsed.port, 8080) + + parsed = utils.urlparse('www.example.com') + self.assertEquals(parsed.hostname, '') if __name__ == '__main__': unittest.main()