openstack/swift
Code Issues Proposed changes

Changes to support deployment on IPv6

Browse Source
This commit is contained in:
Michael Barton 2011-02-11 17:27:05 +00:00 committed by Tarmac
commit b2e0b926a4
17 changed files with 119 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/swauth-add-account
View File

@ -18,9 +18,9 @@ import gettext
from optparse import OptionParser
from os.path import basename
from sys import argv, exit
from urlparse import urlparse
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.utils import urlparse
if __name__ == '__main__':

2
bin/swauth-add-user
View File

@ -18,9 +18,9 @@ import gettext
from optparse import OptionParser
from os.path import basename
from sys import argv, exit
from urlparse import urlparse
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.utils import urlparse
if __name__ == '__main__':

2
bin/swauth-delete-account
View File

@ -18,9 +18,9 @@ import gettext
from optparse import OptionParser
from os.path import basename
from sys import argv, exit
from urlparse import urlparse
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.utils import urlparse
if __name__ == '__main__':

2
bin/swauth-delete-user
View File

@ -18,9 +18,9 @@ import gettext
from optparse import OptionParser
from os.path import basename
from sys import argv, exit
from urlparse import urlparse
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.utils import urlparse
if __name__ == '__main__':

2
bin/swauth-list
View File

@ -22,9 +22,9 @@ import gettext
from optparse import OptionParser
from os.path import basename
from sys import argv, exit
from urlparse import urlparse
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.utils import urlparse
if __name__ == '__main__':

2
bin/swauth-prep
View File

@ -18,9 +18,9 @@ import gettext
from optparse import OptionParser
from os.path import basename
from sys import argv, exit
from urlparse import urlparse
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.utils import urlparse
if __name__ == '__main__':

2
bin/swauth-set-account-service
View File

@ -22,9 +22,9 @@ import gettext
from optparse import OptionParser
from os.path import basename
from sys import argv, exit
from urlparse import urlparse
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.utils import urlparse
if __name__ == '__main__':

58
bin/swift-ring-builder
View File

@ -48,6 +48,8 @@ The <search-value> can be of the form:
/sdb1 Matches devices with the device name sdb1
_shiny Matches devices with shiny in the meta data
_"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data
[::1] Matches devices in any zone with the ip ::1
z1-[::1]:5678 Matches devices in zone 1 with the ip ::1 and port 5678
Most specific example:
d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8"
Nerd explanation:
@ -76,6 +78,13 @@ The <search-value> can be of the form:
i += 1
match.append(('ip', search_value[:i]))
search_value = search_value[i:]
elif len(search_value) and search_value[0] == '[':
i = 1
while i < len(search_value) and search_value[i] != ']':
i += 1
i += 1
match.append(('ip', search_value[:i].lstrip('[').rstrip(']')))
search_value = search_value[i:]
if search_value.startswith(':'):
i = 1
while i < len(search_value) and search_value[i].isdigit():
@ -110,6 +119,16 @@ The <search-value> can be of the form:
return devs
def format_device(dev):
"""
Format a device for display.
"""
if ':' in dev['ip']:
return 'd%(id)sz%(zone)s-[%(ip)s]:%(port)s/%(device)s_"%(meta)s"' % dev
else:
return 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev
class Commands:
def unknown():
@ -235,10 +254,18 @@ swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
print 'Invalid add value: %s' % argv[3]
exit(EXIT_ERROR)
i = 1
while i < len(rest) and rest[i] in '0123456789.':
if rest[i] == '[':
i += 1
ip = rest[1:i]
rest = rest[i:]
while i < len(rest) and rest[i] != ']':
i += 1
i += 1
ip = rest[1:i].lstrip('[').rstrip(']')
rest = rest[i:]
else:
while i < len(rest) and rest[i] in '0123456789.':
i += 1
ip = rest[1:i]
rest = rest[i:]
if not rest.startswith(':'):
print 'Invalid add value: %s' % argv[3]
@ -279,8 +306,12 @@ swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
builder.add_dev({'id': next_dev_id, 'zone': zone, 'ip': ip,
'port': port, 'device': device_name, 'weight': weight,
'meta': meta})
print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \
(zone, ip, port, device_name, meta, weight, next_dev_id)
if ':' in ip:
print 'Device z%s-[%s]:%s/%s_"%s" with %s weight got id %s' % \
(zone, ip, port, device_name, meta, weight, next_dev_id)
else:
print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \
(zone, ip, port, device_name, meta, weight, next_dev_id)
pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
exit(EXIT_RING_UNCHANGED)
@ -342,6 +373,13 @@ swift-ring-builder <builder_file> set_info <search-value>
i += 1
change.append(('ip', change_value[:i]))
change_value = change_value[i:]
elif len(change_value) and change_value[0] == '[':
i = 1
while i < len(change_value) and change_value[i] != ']':
i += 1
i += 1
change.append(('ip', change_value[:i].lstrip('[').rstrip(']')))
change_value = change_value[i:]
if change_value.startswith(':'):
i = 1
while i < len(change_value) and change_value[i].isdigit():
@ -366,15 +404,13 @@ swift-ring-builder <builder_file> set_info <search-value>
if len(devs) > 1:
print 'Matched more than one device:'
for dev in devs:
print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \
'"%(meta)s"' % dev
print ' %s' % format_device(dev)
if raw_input('Are you sure you want to update the info for '
'these %s devices? (y/N) ' % len(devs)) != 'y':
print 'Aborting device modifications'
exit(EXIT_ERROR)
for dev in devs:
orig_dev_string = \
'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev
orig_dev_string = format_device(dev)
test_dev = dict(dev)
for key, value in change:
test_dev[key] = value
@ -390,9 +426,7 @@ swift-ring-builder <builder_file> set_info <search-value>
exit(EXIT_ERROR)
for key, value in change:
dev[key] = value
new_dev_string = \
'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev
print 'Device %s is now %s' % (orig_dev_string, new_dev_string)
print 'Device %s is now %s' % (orig_dev_string, format_device(dev))
pickle.dump(builder, open(argv[1], 'wb'), protocol=2)
exit(EXIT_RING_UNCHANGED)

3
swift/auth/server.py
View File

@ -20,7 +20,6 @@ from contextlib import contextmanager
from time import gmtime, strftime, time
from urllib import unquote, quote
from uuid import uuid4
from urlparse import urlparse
from hashlib import md5, sha1
import hmac
import base64
@ -32,7 +31,7 @@ from webob.exc import HTTPBadRequest, HTTPConflict, HTTPForbidden, \
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.db import get_db_connection
from swift.common.utils import get_logger, split_path
from swift.common.utils import get_logger, split_path, urlparse
class AuthController(object):

3
swift/common/bench.py
View File

@ -16,13 +16,12 @@
import uuid
import time
import random
from urlparse import urlparse
from contextlib import contextmanager
import eventlet.pools
from eventlet.green.httplib import CannotSendRequest
from swift.common.utils import TRUE_VALUES
from swift.common.utils import TRUE_VALUES, urlparse
from swift.common import client
from swift.common import direct_client

2
swift/common/middleware/acl.py
View File

@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from urlparse import urlparse
from swift.common.utils import urlparse
def clean_acl(name, value):

3
swift/common/middleware/swauth.py
View File

@ -21,7 +21,6 @@ from httplib import HTTPConnection, HTTPSConnection
from time import gmtime, strftime, time
from traceback import format_exc
from urllib import quote, unquote
from urlparse import urlparse
from uuid import uuid4
from hashlib import md5, sha1
import hmac
@ -36,7 +35,7 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
from swift.common.utils import cache_from_env, get_logger, split_path
from swift.common.utils import cache_from_env, get_logger, split_path, urlparse
class Swauth(object):

33
swift/common/utils.py
View File

@ -34,6 +34,7 @@ from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from optparse import OptionParser
from tempfile import mkstemp
import cPickle as pickle
from urlparse import urlparse as stdlib_urlparse, ParseResult
import eventlet
from eventlet import greenio, GreenPool, sleep, Timeout, listen
@ -861,3 +862,35 @@ def ratelimit_sleep(running_time, max_rate, incr_by=1, rate_buffer=5):
elif running_time - now > time_per_request:
eventlet.sleep((running_time - now) / clock_accuracy)
return running_time + time_per_request
class ModifiedParseResult(ParseResult):
"Parse results class for urlparse."
@property
def hostname(self):
netloc = self.netloc.split('@', 1)[-1]
if netloc.startswith('['):
return netloc[1:].split(']')[0]
elif ':' in netloc:
return netloc.rsplit(':')[0]
return netloc
@property
def port(self):
netloc = self.netloc.split('@', 1)[-1]
if netloc.startswith('['):
netloc = netloc.rsplit(']')[1]
if ':' in netloc:
return int(netloc.rsplit(':')[1])
return None
def urlparse(url):
"""
urlparse augmentation.
This is necessary because urlparse can't handle RFC 2732 URLs.
:param url: URL to parse.
"""
return ModifiedParseResult(*stdlib_urlparse(url))

6
swift/common/wsgi.py
View File

@ -68,11 +68,15 @@ def get_socket(conf, default_port=8080):
"""
bind_addr = (conf.get('bind_ip', '0.0.0.0'),
int(conf.get('bind_port', default_port)))
address_family = [addr[0] for addr in socket.getaddrinfo(bind_addr[0],
bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM)
if addr[0] in (socket.AF_INET, socket.AF_INET6)][0]
sock = None
retry_until = time.time() + 30
while not sock and time.time() < retry_until:
try:
sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)))
sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)),
family=address_family)
if 'cert_file' in conf:
sock = ssl.wrap_socket(sock, certfile=conf['cert_file'],
keyfile=conf['key_file'])

2
swift/container/server.py
View File

@ -89,7 +89,7 @@ class ContainerController(object):
account_partition = req.headers.get('X-Account-Partition')
account_device = req.headers.get('X-Account-Device')
if all([account_host, account_partition, account_device]):
account_ip, account_port = account_host.split(':')
account_ip, account_port = account_host.rsplit(':', 1)
new_path = '/' + '/'.join([account, container])
info = broker.get_info()
account_headers = {'x-put-timestamp': info['put_timestamp'],

2
swift/obj/server.py
View File

@ -301,7 +301,7 @@ class ObjectController(object):
full_path = '/%s/%s/%s' % (account, container, obj)
try:
with ConnectionTimeout(self.conn_timeout):
ip, port = host.split(':')
ip, port = host.rsplit(':', 1)
conn = http_connect(ip, port, contdevice, partition, op,
full_path, headers_out)
with Timeout(self.node_timeout):

22
test/unit/common/test_utils.py
View File

@ -626,6 +626,28 @@ log_name = yarr'''
total += i
self.assertTrue(abs(50 - (time.time() - start) * 100) < 10)
def test_urlparse(self):
parsed = utils.urlparse('http://127.0.0.1/')
self.assertEquals(parsed.scheme, 'http')
self.assertEquals(parsed.hostname, '127.0.0.1')
self.assertEquals(parsed.path, '/')
parsed = utils.urlparse('http://127.0.0.1:8080/')
self.assertEquals(parsed.port, 8080)
parsed = utils.urlparse('https://127.0.0.1/')
self.assertEquals(parsed.scheme, 'https')
parsed = utils.urlparse('http://[::1]/')
self.assertEquals(parsed.hostname, '::1')
parsed = utils.urlparse('http://[::1]:8080/')
self.assertEquals(parsed.hostname, '::1')
self.assertEquals(parsed.port, 8080)
parsed = utils.urlparse('www.example.com')
self.assertEquals(parsed.hostname, '')
def test_ratelimit_sleep_with_sleep(self):
running_time = 0
start = time.time()