openstack/swift
Code Issues Proposed changes

merged with trunk

Browse Source
This commit is contained in:
John Dickinson 2010-08-26 16:29:47 -07:00
commit 6a5f09a19d
57 changed files with 889 additions and 765 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
bin/swift-account-auditor
View File

@ -24,31 +24,19 @@ from swift.common import utils
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: account-auditor CONFIG_FILE [once]"
print "Usage: swift-account-auditor CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('account-server'))
if c.has_section('account-auditor'):
auditor_conf = dict(c.items('account-auditor'))
else:
print "Unable to find account-auditor config section in %s." % \
sys.argv[1]
sys.exit(1)
logger = utils.get_logger(auditor_conf, 'account-auditor')
conf = utils.readconf(sys.argv[1], 'account-auditor')
logger = utils.get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(conf.get('user', 'swift'))
try:
os.setsid()
@ -62,7 +50,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
auditor = AccountAuditor(server_conf, auditor_conf)
auditor = AccountAuditor(conf)
if once:
auditor.audit_once()
else:

20
bin/swift-account-reaper
View File

@ -29,26 +29,14 @@ if __name__ == '__main__':
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('account-server'))
if c.has_section('account-reaper'):
reaper_conf = dict(c.items('account-reaper'))
else:
print "Unable to find account-reaper config section in %s." % \
sys.argv[1]
sys.exit(1)
logger = utils.get_logger(reaper_conf, 'account-reaper')
conf = utils.readconf(sys.argv[1], 'account-reaper')
logger = utils.get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(conf.get('user', 'swift'))
try:
os.setsid()
@ -62,7 +50,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
reaper = AccountReaper(server_conf, reaper_conf)
reaper = AccountReaper(conf)
if once:
reaper.reap_once()
else:

22
bin/swift-account-replicator
View File

@ -32,26 +32,14 @@ if __name__ == '__main__':
optlist, args = getopt.getopt(sys.argv[1:], '', ['once'])
if not args:
print "Usage: account-replicator <--once> CONFIG_FILE [once]"
print "Usage: swift-account-replicator <--once> CONFIG_FILE [once]"
sys.exit()
c = ConfigParser()
if not c.read(args[0]):
print "Unable to read config file."
sys.exit(1)
once = len(args) > 1 and args[1] == 'once'
server_conf = dict(c.items('account-server'))
if c.has_section('account-replicator'):
replicator_conf = dict(c.items('account-replicator'))
else:
print "Unable to find account-replicator config section in %s." % \
args[0]
sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift'))
conf = utils.readconf(sys.argv[1], 'account-replicator')
utils.drop_privileges(conf.get('user', 'swift'))
if once or '--once' in [opt[0] for opt in optlist]:
AccountReplicator(server_conf, replicator_conf).replicate_once()
AccountReplicator(conf).replicate_once()
else:
AccountReplicator(server_conf, replicator_conf).replicate_forever()
AccountReplicator(conf).replicate_forever()

11
bin/swift-account-server
View File

@ -14,17 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
import sys
from swift.common.wsgi import run_wsgi
from swift.account.server import AccountController
if __name__ == '__main__':
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
conf = dict(c.items('account-server'))
run_wsgi(AccountController, conf, default_port=6002)
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'account-server', default_port=6002)

2
bin/swift-auth-create-account
View File

@ -33,7 +33,7 @@ if __name__ == '__main__':
c = ConfigParser()
if not c.read(f):
exit('Unable to read conf file: %s' % f)
conf = dict(c.items('auth-server'))
conf = dict(c.items('app:auth-server'))
host = conf.get('bind_ip', '127.0.0.1')
port = int(conf.get('bind_port', 11000))
ssl = conf.get('cert_file') is not None

2
bin/swift-auth-recreate-accounts
View File

@ -28,7 +28,7 @@ if __name__ == '__main__':
c = ConfigParser()
if not c.read(f):
exit('Unable to read conf file: %s' % f)
conf = dict(c.items('auth-server'))
conf = dict(c.items('app:auth-server'))
host = conf.get('bind_ip', '127.0.0.1')
port = int(conf.get('bind_port', 11000))
ssl = conf.get('cert_file') is not None

12
bin/swift-auth-server
View File

@ -14,17 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
import sys
from swift.common.wsgi import run_wsgi
from swift.auth.server import AuthController
if __name__ == '__main__':
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
conf = dict(c.items('auth-server'))
run_wsgi(AuthController, conf, default_port=11000)
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'auth-server', default_port=11000)

22
bin/swift-container-auditor
View File

@ -24,31 +24,19 @@ from swift.common import utils
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: container-auditor CONFIG_FILE [once]"
print "Usage: swift-container-auditor CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('container-server'))
if c.has_section('container-auditor'):
auditor_conf = dict(c.items('container-auditor'))
else:
print "Unable to find container-auditor config section in %s." % \
sys.argv[1]
sys.exit(1)
logger = utils.get_logger(auditor_conf, 'container-auditor')
conf = utils.readconf(sys.argv[1], 'container-auditor')
logger = utils.get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(conf.get('user', 'swift'))
try:
os.setsid()
@ -62,7 +50,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
auditor = ContainerAuditor(server_conf, auditor_conf)
auditor = ContainerAuditor(conf)
if once:
auditor.audit_once()
else:

24
bin/swift-container-replicator
View File

@ -32,26 +32,14 @@ if __name__ == '__main__':
optlist, args = getopt.getopt(sys.argv[1:], '', ['once'])
if not args:
print "Usage: container-replicator <--once> CONFIG_FILE [once]"
sys.exit()
c = ConfigParser()
if not c.read(args[0]):
print "Unable to read config file."
print "Usage: swift-container-replicator <--once> CONFIG_FILE [once]"
sys.exit(1)
once = len(args) > 1 and args[1] == 'once'
server_conf = dict(c.items('container-server'))
if c.has_section('container-replicator'):
replicator_conf = dict(c.items('container-replicator'))
else:
print "Unable to find container-replicator config section in %s." % \
args[0]
sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift'))
conf = utils.readconf(args[0], 'container-replicator')
utils.drop_privileges(conf.get('user', 'swift'))
if once or '--once' in [opt[0] for opt in optlist]:
ContainerReplicator(server_conf, replicator_conf).replicate_once()
ContainerReplicator(conf).replicate_once()
else:
ContainerReplicator(server_conf, replicator_conf).replicate_forever()
ContainerReplicator(conf).replicate_forever()

12
bin/swift-container-server
View File

@ -14,17 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
import sys
from swift.common.wsgi import run_wsgi
from swift.container.server import ContainerController
if __name__ == '__main__':
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
conf = dict(c.items('container-server'))
run_wsgi(ContainerController, conf, default_port=6001)
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'container-server', default_port=6001)

21
bin/swift-container-updater
View File

@ -24,25 +24,12 @@ from swift.common import utils
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: container-updater CONFIG_FILE [once]"
print "Usage: swift-container-updater CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('container-server'))
if c.has_section('container-updater'):
updater_conf = dict(c.items('container-updater'))
else:
print "Unable to find container-updater config section in %s." % \
sys.argv[1]
sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift'))
conf = utils.readconf(sys.argv[1], 'container-updater')
utils.drop_privileges(conf.get('user', 'swift'))
try:
os.setsid()
@ -56,7 +43,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
updater = ContainerUpdater(server_conf, updater_conf)
updater = ContainerUpdater(conf)
if once:
updater.update_once_single_threaded()
else:

24
bin/swift-object-auditor
View File

@ -17,38 +17,24 @@
import os
import signal
import sys
from ConfigParser import ConfigParser
from swift.obj.auditor import ObjectAuditor
from swift.common import utils
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: object-auditor CONFIG_FILE [once]"
print "Usage: swift-object-auditor CONFIG_FILE [once]"
sys.exit()
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('object-server'))
if c.has_section('object-auditor'):
auditor_conf = dict(c.items('object-auditor'))
else:
print "Unable to find object-auditor config section in %s." % \
sys.argv[1]
sys.exit(1)
logger = utils.get_logger(auditor_conf, 'object-auditor')
conf = utils.readconf(sys.argv[1], 'object-auditor')
logger = utils.get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = utils.LoggerFileObject(logger)
utils.drop_privileges(server_conf.get('user', 'swift'))
utils.drop_privileges(conf.get('user', 'swift'))
try:
os.setsid()
@ -62,7 +48,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
auditor = ObjectAuditor(server_conf, auditor_conf)
auditor = ObjectAuditor(conf)
if once:
auditor.audit_once()
else:

35
bin/swift-object-replicator
View File

@ -15,7 +15,6 @@
# limitations under the License.
import sys
from ConfigParser import ConfigParser
import logging
import time
@ -23,42 +22,18 @@ from eventlet import sleep, hubs
hubs.use_hub('poll')
from swift.obj.replicator import ObjectReplicator
from swift.common.utils import get_logger, drop_privileges, LoggerFileObject
from swift.common.utils import get_logger, drop_privileges, LoggerFileObject, \
readconf
TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes'))
def read_configs(conf_file):
c = ConfigParser()
if not c.read(conf_file):
print "Unable to read config file: %s" % conf_file
sys.exit(1)
conf = dict(c.items('object-server'))
repl_conf = dict(c.items('object-replicator'))
if not repl_conf:
sys.exit()
conf['replication_concurrency'] = repl_conf.get('concurrency',1)
conf['vm_test_mode'] = repl_conf.get('vm_test_mode', 'no')
conf['daemonize'] = repl_conf.get('daemonize', 'yes')
conf['run_pause'] = repl_conf.get('run_pause', '30')
conf['log_facility'] = repl_conf.get('log_facility', 'LOG_LOCAL1')
conf['log_level'] = repl_conf.get('log_level', 'INFO')
conf['timeout'] = repl_conf.get('timeout', '5')
conf['stats_interval'] = repl_conf.get('stats_interval', '3600')
conf['reclaim_age'] = int(repl_conf.get('reclaim_age', 86400))
return conf
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: object-replicator CONFIG_FILE [once]"
sys.exit()
try:
conf = read_configs(sys.argv[1])
except:
print "Problem reading the config. Aborting object replication."
print "Usage: swift-object-replicator CONFIG_FILE [once]"
sys.exit()
conf = readconf(sys.argv[1], "object-replicator")
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
logger = get_logger(conf, 'object-replicator')
logger = get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda *exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)

12
bin/swift-object-server
View File

@ -14,17 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
import sys
from swift.common.wsgi import run_wsgi
from swift.obj.server import ObjectController
if __name__ == '__main__':
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
conf = dict(c.items('object-server'))
run_wsgi(ObjectController, conf, default_port=6000)
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'object-server', default_port=6000)

23
bin/swift-object-updater
View File

@ -17,32 +17,19 @@
import os
import signal
import sys
from ConfigParser import ConfigParser
from swift.obj.updater import ObjectUpdater
from swift.common import utils
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: object-updater CONFIG_FILE [once]"
sys.exit()
print "Usage: swift-object-updater CONFIG_FILE [once]"
sys.exit(1)
once = len(sys.argv) > 2 and sys.argv[2] == 'once'
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
server_conf = dict(c.items('object-server'))
if c.has_section('object-updater'):
updater_conf = dict(c.items('object-updater'))
else:
print "Unable to find object-updater config section in %s." % \
sys.argv[1]
sys.exit(1)
utils.drop_privileges(server_conf.get('user', 'swift'))
conf = utils.readconf(sys.argv[1], 'object-updater')
utils.drop_privileges(conf.get('user', 'swift'))
try:
os.setsid()
@ -56,7 +43,7 @@ if __name__ == '__main__':
signal.signal(signal.SIGTERM, kill_children)
updater = ObjectUpdater(server_conf, updater_conf)
updater = ObjectUpdater(conf)
if once:
updater.update_once_single_threaded()
else:

32
bin/swift-proxy-server
View File

@ -14,37 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
import os
import sys
from swift.common.wsgi import run_wsgi
from swift.common.auth import DevAuthMiddleware
from swift.common.memcached import MemcacheRing
from swift.common.utils import get_logger
from swift.proxy.server import Application
if __name__ == '__main__':
c = ConfigParser()
if not c.read(sys.argv[1]):
print "Unable to read config file."
sys.exit(1)
conf = dict(c.items('proxy-server'))
if c.has_section('auth-server'):
auth_conf = dict(c.items('auth-server'))
else:
auth_conf = {}
swift_dir = conf.get('swift_dir', '/etc/swift')
m, c = auth_conf.get('class',
'swift.common.auth.DevAuthMiddleware').rsplit('.', 1)
m = __import__(m, fromlist=[c])
authware = m.__dict__[c]
memcache = MemcacheRing([s.strip() for s in
conf.get('memcache_servers', '127.0.0.1:11211').split(',')
if s.strip()])
logger = get_logger(conf, 'proxy')
app = Application(conf, memcache, logger)
# Wrap the app with auth
app = authware(app, auth_conf, memcache, logger)
run_wsgi(app, conf, logger=logger, default_port=80)
if len(sys.argv) != 2:
sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
run_wsgi(sys.argv[1], 'proxy-server', default_port=8080)

342
doc/source/deployment_guide.rst
View File

@ -130,6 +130,14 @@ swift-ring-builder with no options will display help text with available
commands and options. More information on how the ring works internally
can be found in the :doc:`Ring Overview <overview_ring>`.
----------------------------
General Server Configuration
----------------------------
Swift uses paste.deploy to manage server configurations. Default configuration
options are set in the `[DEFAULT]` section, and any options specified there
can be overridden in any of the other sections.
---------------------------
Object Server Configuration
---------------------------
@ -139,7 +147,7 @@ etc/object-server.conf-sample in the source code repository.
The following configuration options are available:
[object-server]
[DEFAULT]
================== ========== =============================================
Option Default Description
@ -152,65 +160,79 @@ mount_check true Weather or not check if the devices are
bind_ip 0.0.0.0 IP Address for server to bind to
bind_port 6000 Port for server to bind to
workers 1 Number of workers to fork
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
log_requests True Weather or not to log each request
user swift User to run as
node_timeout 3 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
network_chunk_size 65536 Size of chunks to read/write over the
network
disk_chunk_size 65536 Size of chunks to read/write to disk
max_upload_time 86400 Maximum time allowed to upload an object
slow 0 If > 0, Minimum time in seconds for a PUT
or DELETE request to complete
================== ========== =============================================
[object-server]
================== ============= ===========================================
Option Default Description
------------------ ------------- -------------------------------------------
use paste.deploy entry point for the object
server. For most cases, this should be
`egg:swift#object`.
log_name object-server Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
log_requests True Weather or not to log each request
user swift User to run as
node_timeout 3 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
network_chunk_size 65536 Size of chunks to read/write over the
network
disk_chunk_size 65536 Size of chunks to read/write to disk
max_upload_time 86400 Maximum time allowed to upload an object
slow 0 If > 0, Minimum time in seconds for a PUT
or DELETE request to complete
================== ============= ===========================================
[object-replicator]
================== ========== ===========================================
Option Default Description
------------------ ---------- -------------------------------------------
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
daemonize yes Weather or not to run replication as a
daemon
run_pause 30 Time in seconds to wait between replication
passes
concurrency 1 Number of replication workers to spawn
timeout 5 Timeout value sent to rsync --timeout and
--contimeout options
stats_interval 3600 Interval in seconds between logging
replication statistics
reclaim_age 604800 Time elapsed in seconds before an object
can be reclaimed
================== ========== ===========================================
================== ================= =======================================
Option Default Description
------------------ ----------------- ---------------------------------------
log_name object-replicator Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
daemonize yes Weather or not to run replication as a
daemon
run_pause 30 Time in seconds to wait between
replication passes
concurrency 1 Number of replication workers to spawn
timeout 5 Timeout value sent to rsync --timeout
and --contimeout options
stats_interval 3600 Interval in seconds between logging
replication statistics
reclaim_age 604800 Time elapsed in seconds before an
object can be reclaimed
================== ================= =======================================
[object-updater]
================== ========== ===========================================
Option Default Description
------------------ ---------- -------------------------------------------
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 300 Minimum time for a pass to take
concurrency 1 Number of updater workers to spawn
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
slowdown 0.01 Time in seconds to wait between objects
================== ========== ===========================================
================== ============== ==========================================
Option Default Description
------------------ -------------- ------------------------------------------
log_name object-updater Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 300 Minimum time for a pass to take
concurrency 1 Number of updater workers to spawn
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
slowdown 0.01 Time in seconds to wait between objects
================== ============== ==========================================
[object-auditor]
================== ========== ===========================================
Option Default Description
------------------ ---------- -------------------------------------------
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
================== ========== ===========================================
================== ============== ==========================================
Option Default Description
------------------ -------------- ------------------------------------------
log_name object-auditor Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
================== ============== ==========================================
------------------------------
Container Server Configuration
@ -221,13 +243,11 @@ etc/container-server.conf-sample in the source code repository.
The following configuration options are available:
[container-server]
[DEFAULT]
================== ========== ============================================
Option Default Description
------------------ ---------- --------------------------------------------
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
swift_dir /etc/swift Swift configuration directory
devices /srv/node Parent irectory of where devices are mounted
mount_check true Weather or not check if the devices are
@ -237,52 +257,71 @@ bind_ip 0.0.0.0 IP Address for server to bind to
bind_port 6001 Port for server to bind to
workers 1 Number of workers to fork
user swift User to run as
node_timeout 3 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
================== ========== ============================================
[container-server]
================== ================ ========================================
Option Default Description
------------------ ---------------- ----------------------------------------
use paste.deploy entry point for the
container server. For most cases, this
should be `egg:swift#container`.
log_name container-server Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
node_timeout 3 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
================== ================ ========================================
[container-replicator]
================== ========== ===========================================
Option Default Description
------------------ ---------- -------------------------------------------
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
================== ==================== ====================================
Option Default Description
------------------ -------------------- ------------------------------------
log_name container-replicator Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
per_diff 1000
concurrency 8 Number of replication workers to spawn
run_pause 30 Time in seconds to wait between replication
passes
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
reclaim_age 604800 Time elapsed in seconds before a container
can be reclaimed
================== ========== ===========================================
concurrency 8 Number of replication workers to
spawn
run_pause 30 Time in seconds to wait between
replication passes
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external
services
reclaim_age 604800 Time elapsed in seconds before a
container can be reclaimed
================== ==================== ====================================
[container-updater]
================== ========== ===========================================
Option Default Description
------------------ ---------- -------------------------------------------
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 300 Minimum time for a pass to take
concurrency 4 Number of updater workers to spawn
node_timeout 3 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
slowdown 0.01 Time in seconds to wait between containers
================== ========== ===========================================
================== ================= =======================================
Option Default Description
------------------ ----------------- ---------------------------------------
log_name container-updater Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 300 Minimum time for a pass to take
concurrency 4 Number of updater workers to spawn
node_timeout 3 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
slowdown 0.01 Time in seconds to wait between
containers
================== ================= =======================================
[container-auditor]
================== ========== ===========================================
Option Default Description
------------------ ---------- -------------------------------------------
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
================== ========== ===========================================
================== ================= =======================================
Option Default Description
------------------ ----------------- ---------------------------------------
log_name container-auditor Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
================== ================= =======================================
----------------------------
Account Server Configuration
@ -293,13 +332,11 @@ etc/account-server.conf-sample in the source code repository.
The following configuration options are available:
[account-server]
[DEFAULT]
================== ========== =============================================
Option Default Description
------------------ ---------- ---------------------------------------------
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
swift_dir /etc/swift Swift configuration directory
devices /srv/node Parent directory or where devices are mounted
mount_check true Weather or not check if the devices are
@ -311,71 +348,99 @@ workers 1 Number of workers to fork
user swift User to run as
================== ========== =============================================
[account-server]
================== ============== ==========================================
Option Default Description
------------------ -------------- ------------------------------------------
use paste.deploy entry point for the account
server. For most cases, this should be
`egg:swift#account`.
log_name account-server Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
================== ============== ==========================================
[account-replicator]
================== ========== ===========================================
Option Default Description
------------------ ---------- -------------------------------------------
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
================== ================== ======================================
Option Default Description
------------------ ------------------ --------------------------------------
log_name account-replicator Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
per_diff 1000
concurrency 8 Number of replication workers to spawn
run_pause 30 Time in seconds to wait between replication
passes
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
reclaim_age 604800 Time elapsed in seconds before a account
can be reclaimed
================== ========== ===========================================
concurrency 8 Number of replication workers to spawn
run_pause 30 Time in seconds to wait between
replication passes
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
reclaim_age 604800 Time elapsed in seconds before an
account can be reclaimed
================== ================== ======================================
[account-auditor]
==================== ========== ===========================================
Option Default Description
-------------------- ---------- -------------------------------------------
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
max_container_count 100 Maximum containers randomly picked for
a given account audit
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
==================== ========== ===========================================
==================== =============== =======================================
Option Default Description
-------------------- --------------- ---------------------------------------
log_name account-auditor Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
interval 1800 Minimum time for a pass to take
max_container_count 100 Maximum containers randomly picked for
a given account audit
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
==================== =============== =======================================
[account-reaper]
================== ========== ===========================================
Option Default Description
------------------ ---------- -------------------------------------------
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
concurrency 25 Number of replication workers to spawn
interval 3600 Minimum time for a pass to take
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
================== ========== ===========================================
================== =============== =========================================
Option Default Description
------------------ --------------- -----------------------------------------
log_name account-auditor Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
concurrency 25 Number of replication workers to spawn
interval 3600 Minimum time for a pass to take
node_timeout 10 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
================== =============== =========================================
--------------------------
Proxy Server Configuration
--------------------------
[DEFAULT]
============================ =============== =============================
Option Default Description
---------------------------- --------------- -----------------------------
bind_ip 0.0.0.0 IP Address for server to
bind to
bind_port 80 Port for server to bind to
swift_dir /etc/swift Swift configuration directory
workers 1 Number of workers to fork
user swift User to run as
cert_file Path to the ssl .crt
key_file Path to the ssl .key
============================ =============== =============================
[proxy-server]
============================ =============== =============================
Option Default Description
---------------------------- --------------- -----------------------------
use paste.deploy entry point for
the proxy server. For most
cases, this should be
`egg:swift#proxy`.
log_name proxy-server Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Log level
bind_ip 0.0.0.0 IP Address for server to
bind to
bind_port 80 Port for server to bind to
cert_file Path to the ssl .crt
key_file Path to the ssl .key
swift_dir /etc/swift Swift configuration directory
log_headers True If True, log headers in each
request
workers 1 Number of workers to fork
user swift User to run as
recheck_account_existence 60 Cache timeout in seconds to
send memcached for account
existance
@ -412,16 +477,21 @@ rate_limit_account_blacklist Comma separated list of
completly
============================ =============== =============================
[auth-server]
[auth]
============ =================================== ========================
Option Default Description
------------ ----------------------------------- ------------------------
class swift.common.auth.DevAuthMiddleware Auth wsgi middleware
to use
use paste.deploy entry point
to use for auth. To
use the swift dev auth,
set to:
`egg:swift#auth`
ip 127.0.0.1 IP address of auth
server
port 11000 Port of auth server
ssl False If True, use SSL to
connect to auth
node_timeout 10 Request timeout
============ =================================== ========================

128
doc/source/development_saio.rst
View File

@ -39,7 +39,7 @@ good idea what to do on other environments.
#. `apt-get install curl gcc bzr memcached python-configobj
python-coverage python-dev python-nose python-setuptools python-simplejson
python-xattr sqlite3 xfsprogs python-webob python-eventlet
python-greenlet`
python-greenlet python-pastedeploy`
#. Install anything else you want, like screen, ssh, vim, etc.
#. `fdisk /dev/sdb` (set up a single partition)
#. `mkfs.xfs -i size=1024 /dev/sdb1`
@ -168,24 +168,51 @@ good idea what to do on other environments.
#. `. ~/.bashrc`
#. Create `/etc/swift/auth-server.conf`::
[auth-server]
default_cluster_url = http://127.0.0.1:8080/v1
[DEFAULT]
user = <your-user-name>
[pipeline:main]
pipeline = auth-server
[app:auth-server]
use = egg:swift#auth
default_cluster_url = http://127.0.0.1:8080/v1
#. Create `/etc/swift/proxy-server.conf`::
[proxy-server]
[DEFAULT]
bind_port = 8080
user = <your-user-name>
[pipeline:main]
pipeline = healthcheck cache auth proxy-server
[app:proxy-server]
use = egg:swift#proxy
[filter:auth]
use = egg:swift#auth
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:cache]
use = egg:swift#memcache
#. Create `/etc/swift/account-server/1.conf`::
[account-server]
[DEFAULT]
devices = /srv/1/node
mount_check = false
bind_port = 6012
user = <your-user-name>
[pipeline:main]
pipeline = account-server
[app:account-server]
use = egg:swift#account
[account-replicator]
vm_test_mode = yes
@ -195,12 +222,18 @@ good idea what to do on other environments.
#. Create `/etc/swift/account-server/2.conf`::
[account-server]
[DEFAULT]
devices = /srv/2/node
mount_check = false
bind_port = 6022
user = <your-user-name>
[pipeline:main]
pipeline = account-server
[app:account-server]
use = egg:swift#account
[account-replicator]
vm_test_mode = yes
@ -210,12 +243,18 @@ good idea what to do on other environments.
#. Create `/etc/swift/account-server/3.conf`::
[account-server]
[DEFAULT]
devices = /srv/3/node
mount_check = false
bind_port = 6032
user = <your-user-name>
[pipeline:main]
pipeline = account-server
[app:account-server]
use = egg:swift#account
[account-replicator]
vm_test_mode = yes
@ -225,12 +264,18 @@ good idea what to do on other environments.
#. Create `/etc/swift/account-server/4.conf`::
[account-server]
[DEFAULT]
devices = /srv/4/node
mount_check = false
bind_port = 6042
user = <your-user-name>
[pipeline:main]
pipeline = account-server
[app:account-server]
use = egg:swift#account
[account-replicator]
vm_test_mode = yes
@ -240,12 +285,18 @@ good idea what to do on other environments.
#. Create `/etc/swift/container-server/1.conf`::
[container-server]
[DEFAULT]
devices = /srv/1/node
mount_check = false
bind_port = 6011
user = <your-user-name>
[pipeline:main]
pipeline = container-server
[app:container-server]
use = egg:swift#container
[container-replicator]
vm_test_mode = yes
@ -255,12 +306,18 @@ good idea what to do on other environments.
#. Create `/etc/swift/container-server/2.conf`::
[container-server]
[DEFAULT]
devices = /srv/2/node
mount_check = false
bind_port = 6021
user = <your-user-name>
[pipeline:main]
pipeline = container-server
[app:container-server]
use = egg:swift#container
[container-replicator]
vm_test_mode = yes
@ -268,14 +325,21 @@ good idea what to do on other environments.
[container-auditor]
#. Create `/etc/swift/container-server/3.conf`::
[container-server]
[DEFAULT]
devices = /srv/3/node
mount_check = false
bind_port = 6031
user = <your-user-name>
[pipeline:main]
pipeline = container-server
[app:container-server]
use = egg:swift#container
[container-replicator]
vm_test_mode = yes
@ -283,14 +347,21 @@ good idea what to do on other environments.
[container-auditor]
#. Create `/etc/swift/container-server/4.conf`::
[container-server]
[DEFAULT]
devices = /srv/4/node
mount_check = false
bind_port = 6041
user = <your-user-name>
[pipeline:main]
pipeline = container-server
[app:container-server]
use = egg:swift#container
[container-replicator]
vm_test_mode = yes
@ -298,14 +369,21 @@ good idea what to do on other environments.
[container-auditor]
#. Create `/etc/swift/object-server/1.conf`::
[object-server]
[DEFAULT]
devices = /srv/1/node
mount_check = false
bind_port = 6010
user = <your-user-name>
[pipeline:main]
pipeline = object-server
[app:object-server]
use = egg:swift#object
[object-replicator]
vm_test_mode = yes
@ -315,12 +393,18 @@ good idea what to do on other environments.
#. Create `/etc/swift/object-server/2.conf`::
[object-server]
[DEFAULT]
devices = /srv/2/node
mount_check = false
bind_port = 6020
user = <your-user-name>
[pipeline:main]
pipeline = object-server
[app:object-server]
use = egg:swift#object
[object-replicator]
vm_test_mode = yes
@ -330,12 +414,18 @@ good idea what to do on other environments.
#. Create `/etc/swift/object-server/3.conf`::
[object-server]
[DEFAULT]
devices = /srv/3/node
mount_check = false
bind_port = 6030
user = <your-user-name>
[pipeline:main]
pipeline = object-server
[app:object-server]
use = egg:swift#object
[object-replicator]
vm_test_mode = yes
@ -345,12 +435,18 @@ good idea what to do on other environments.
#. Create `/etc/swift/object-server/4.conf`::
[object-server]
[DEFAULT]
devices = /srv/4/node
mount_check = false
bind_port = 6040
user = <your-user-name>
[pipeline:main]
pipeline = object-server
[app:object-server]
use = egg:swift#object
[object-replicator]
vm_test_mode = yes

4
doc/source/misc.rst
View File

@ -38,7 +38,7 @@ Utils
Auth
====
.. automodule:: swift.common.auth
.. automodule:: swift.common.middleware.auth
:members:
:show-inheritance:
@ -85,7 +85,7 @@ Buffered HTTP
Healthcheck
===========
.. automodule:: swift.common.healthcheck
.. automodule:: swift.common.middleware.healthcheck
:members:
:show-inheritance:

22
etc/account-server.conf-sample
View File

@ -1,15 +1,24 @@
[account-server]
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[DEFAULT]
# bind_ip = 0.0.0.0
# bind_port = 6002
# workers = 1
# user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[pipeline:main]
pipeline = account-server
[app:account-server]
use = egg:swift#account
# log_name = account-server
# log_facility = LOG_LOCAL0
# log_level = INFO
# user = swift
[account-replicator]
# log_name = account-replicator
# vm_test_mode = no
# log_facility = LOG_LOCAL0
# log_level = INFO
# per_diff = 1000
@ -26,6 +35,7 @@
# reclaim_age = 86400
[account-stats]
# log_name = account-stats
# cf_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31
# container_name = account_stats
# proxy_server_conf = /etc/swift/proxy-server.conf
@ -33,6 +43,7 @@
# log_level = INFO
[account-auditor]
# log_name = account-auditor
# Will audit, at most, 1 account per device per interval
# interval = 1800
# Maximum containers randomly picked for a given account audit
@ -43,6 +54,7 @@
# log_level = INFO
[account-reaper]
# log_name = account-reaper
# concurrency = 25
# interval = 3600
# node_timeout = 10

25
etc/auth-server.conf-sample
View File

@ -1,15 +1,22 @@
[auth-server]
# swift_dir = /etc/swift
[DEFAULT]
# bind_ip = 0.0.0.0
# bind_port = 11000
# log_facility = LOG_LOCAL0
# log_level = INFO
# workers = 1
# reseller_prefix = AUTH
# default_cluster_url = http://127.0.0.1:9000/v1
# token_life = 86400
# log_headers = False
# user = swift
# swift_dir = /etc/swift
# cert_file = Default is no cert; format is path like /etc/swift/auth.crt
# key_file = Default is no key; format is path like /etc/swift/auth.key
[pipeline:main]
pipeline = auth-server
[app:auth-server]
use = egg:swift#auth
# log_name = auth-server
# log_facility = LOG_LOCAL0
# log_level = INFO
# reseller_prefix = AUTH
# default_cluster_url = http://127.0.0.1:8080/v1
# token_life = 86400
# log_headers = False
# node_timeout = 10
user = swift

27
etc/container-server.conf-sample
View File

@ -1,19 +1,26 @@
[container-server]
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[DEFAULT]
# bind_ip = 0.0.0.0
# bind_port = 6001
# workers = 1
# user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[pipeline:main]
pipeline = container-server
[app:container-server]
use = egg:swift#container
# log_name = container-server
# log_facility = LOG_LOCAL0
# log_level = INFO
# user = swift
# node_timeout = 3
# conn_timeout = 0.5
[container-replicator]
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_name = container-replicator
# vm_test_mode = no
# per_diff = 1000
# concurrency = 8
# run_pause = 30
@ -23,21 +30,19 @@
# reclaim_age = 604800
[container-updater]
# log_name = container-updater
# interval = 300
# concurrency = 4
# node_timeout = 3
# conn_timeout = 0.5
# slowdown will sleep that amount between containers
# slowdown = 0.01
# log_facility = LOG_LOCAL0
# log_level = INFO
[container-auditor]
# log_name = container-auditor
# Will audit, at most, 1 container per device per interval
# interval = 1800
# Maximum objects randomly picked for a given container audit
# max_object_count = 100
# node_timeout = 10
# conn_timeout = 0.5
# log_facility = LOG_LOCAL0
# log_level = INFO

31
etc/object-server.conf-sample
View File

@ -1,24 +1,31 @@
[object-server]
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[DEFAULT]
# bind_ip = 0.0.0.0
# bind_port = 6000
# workers = 1
# user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
[pipeline:main]
pipeline = object-server
[app:object-server]
use = egg:swift#object
# log_name = object-server
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_requests = True
# user = swift
# node_timeout = 3
# conn_timeout = 0.5
# network_chunk_size = 8192
# disk_chunk_size = 32768
# network_chunk_size = 65536
# disk_chunk_size = 65536
# max_upload_time = 86400
# slow = 1
[object-replicator]
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_name = object-replicator
# vm_test_mode = no
# daemonize = on
# run_pause = 30
# concurrency = 1
@ -28,19 +35,17 @@
# reclaim_age = 604800
[object-updater]
# log_name = object-updater
# interval = 300
# concurrency = 1
# node_timeout = 10
# conn_timeout = 0.5
# slowdown will sleep that amount between objects
# slowdown = 0.01
# log_facility = LOG_LOCAL0
# log_level = INFO
[object-auditor]
# log_name = object-auditor
# Will audit, at most, 1 object per device per interval
# interval = 1800
# node_timeout = 10
# conn_timeout = 0.5
# log_facility = LOG_LOCAL0
# log_level = INFO

31
etc/proxy-server.conf-sample
View File

@ -1,21 +1,25 @@
[proxy-server]
[DEFAULT]
# bind_ip = 0.0.0.0
# bind_port = 80
# swift_dir = /etc/swift
# workers = 1
# user = swift
# cert_file = /etc/swift/proxy.crt
# key_file = /etc/swift/proxy.key
# swift_dir = /etc/swift
[pipeline:main]
pipeline = healthcheck cache auth proxy
[app:proxy]
use = egg:swift#proxy
# log_name = proxy-server
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_headers = False
# workers = 1
# user = swift
# recheck_account_existence = 60
# recheck_container_existence = 60
# object_chunk_size = 8192
# client_chunk_size = 8192
# Default for memcache_servers is below, but you can specify multiple servers
# with the format: 10.1.2.3:11211,10.1.2.4:11211
# memcache_servers = 127.0.0.1:11211
# node_timeout = 10
# client_timeout = 60
# conn_timeout = 0.5
@ -31,9 +35,18 @@
# rate_limit_account_whitelist = acct1,acct2,etc
# rate_limit_account_blacklist = acct3,acct4,etc
# [auth-server]
# class = swift.common.auth.DevAuthMiddleware
[filter:auth]
use = egg:swift#auth
# ip = 127.0.0.1
# port = 11000
# ssl = false
# node_timeout = 10
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:cache]
use = egg:swift#memcache
# Default for memcache_servers is below, but you can specify multiple servers
# with the format: 10.1.2.3:11211,10.1.2.4:11211
# memcache_servers = 127.0.0.1:11211

53
setup.py
View File

@ -55,21 +55,38 @@ setup(
'Operating System :: POSIX :: Linux',
'Programming Language :: Python :: 2.6',
'Environment :: No Input/Output (Daemon)',
],
install_requires=[], # removed for better compat
scripts=['bin/st', 'bin/swift-account-auditor',
'bin/swift-account-audit', 'bin/swift-account-reaper',
'bin/swift-account-replicator', 'bin/swift-account-server',
'bin/swift-auth-create-account',
'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server',
'bin/swift-container-auditor',
'bin/swift-container-replicator',
'bin/swift-container-server', 'bin/swift-container-updater',
'bin/swift-drive-audit', 'bin/swift-get-nodes',
'bin/swift-init', 'bin/swift-object-auditor',
'bin/swift-object-info',
'bin/swift-object-replicator',
'bin/swift-object-server',
'bin/swift-object-updater', 'bin/swift-proxy-server',
'bin/swift-ring-builder', 'bin/swift-stats-populate',
'bin/swift-stats-report'])
],
install_requires=[], # removed for better compat
scripts=[
'bin/st', 'bin/swift-account-auditor',
'bin/swift-account-audit', 'bin/swift-account-reaper',
'bin/swift-account-replicator', 'bin/swift-account-server',
'bin/swift-auth-create-account',
'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server',
'bin/swift-container-auditor',
'bin/swift-container-replicator',
'bin/swift-container-server', 'bin/swift-container-updater',
'bin/swift-drive-audit', 'bin/swift-get-nodes',
'bin/swift-init', 'bin/swift-object-auditor',
'bin/swift-object-info',
'bin/swift-object-replicator',
'bin/swift-object-server',
'bin/swift-object-updater', 'bin/swift-proxy-server',
'bin/swift-ring-builder', 'bin/swift-stats-populate',
'bin/swift-stats-report'
],
entry_points={
'paste.app_factory': [
'proxy=swift.proxy.server:app_factory',
'object=swift.obj.server:app_factory',
'container=swift.container.server:app_factory',
'account=swift.account.server:app_factory',
'auth=swift.auth.server:app_factory',
],
'paste.filter_factory': [
'auth=swift.common.middleware.auth:filter_factory',
'healthcheck=swift.common.middleware.healthcheck:filter_factory',
'memcache=swift.common.middleware.memcache:filter_factory',
],
},
)

18
swift/account/auditor.py
View File

@ -35,19 +35,19 @@ class AuditException(Exception):
class AccountAuditor(object):
"""Audit accounts."""
def __init__(self, server_conf, auditor_conf):
self.logger = get_logger(auditor_conf, 'account-auditor')
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf, 'account-auditor')
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(auditor_conf.get('interval', 1800))
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 1800))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.container_ring = None
self.node_timeout = int(auditor_conf.get('node_timeout', 10))
self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.max_container_count = \
int(auditor_conf.get('max_container_count', 100))
int(conf.get('max_container_count', 100))
self.container_passes = 0
self.container_failures = 0
self.container_errors = 0

20
swift/account/reaper.py
View File

@ -50,25 +50,23 @@ class AccountReaper(object):
configuration parameters.
"""
log_name = 'account-reaper'
def __init__(self, server_conf, reaper_conf):
self.logger = get_logger(reaper_conf, self.log_name)
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf)
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(reaper_conf.get('interval', 3600))
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 3600))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz')
self.account_ring = None
self.container_ring = None
self.object_ring = None
self.node_timeout = int(reaper_conf.get('node_timeout', 10))
self.conn_timeout = float(reaper_conf.get('conn_timeout', 0.5))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.myips = whataremyips()
self.concurrency = int(reaper_conf.get('concurrency', 25))
self.concurrency = int(conf.get('concurrency', 25))
self.container_concurrency = self.object_concurrency = \
sqrt(self.concurrency)
self.container_pool = GreenPool(size=self.container_concurrency)

14
swift/account/server.py
View File

@ -33,7 +33,6 @@ from swift.common.utils import get_logger, get_param, hash_path, \
normalize_timestamp, split_path, storage_directory
from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
check_mount, check_float, check_xml_encodable
from swift.common.healthcheck import healthcheck
from swift.common.db_replicator import ReplicatorRpc
@ -42,10 +41,9 @@ DATADIR = 'accounts'
class AccountController(object):
"""WSGI controller for the account server."""
log_name = 'account'
def __init__(self, conf):
self.logger = get_logger(conf, self.log_name)
self.logger = get_logger(conf)
self.root = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
@ -296,9 +294,7 @@ class AccountController(object):
def __call__(self, env, start_response):
start_time = time.time()
req = Request(env)
if req.path_info == '/healthcheck':
return healthcheck(req)(env, start_response)
elif not check_xml_encodable(req.path_info):
if not check_xml_encodable(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8')
else:
try:
@ -331,3 +327,9 @@ class AccountController(object):
else:
self.logger.info(log_message)
return res(env, start_response)
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI account server apps"""
conf = global_conf.copy()
conf.update(local_conf)
return AccountController(conf)

11
swift/auth/server.py
View File

@ -89,13 +89,11 @@ class AuthController(object):
configuration parameters.
"""
log_name = 'auth'
def __init__(self, conf, ring=None):
self.logger = get_logger(conf, self.log_name)
self.logger = get_logger(conf)
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.default_cluster_url = \
conf.get('default_cluster_url', 'http://127.0.0.1:9000/v1')
conf.get('default_cluster_url', 'http://127.0.0.1:8080/v1')
self.token_life = int(conf.get('token_life', 86400))
self.log_headers = conf.get('log_headers') == 'True'
if ring:
@ -500,3 +498,8 @@ class AuthController(object):
def __call__(self, env, start_response):
""" Used by the eventlet.wsgi.server """
return self.handleREST(env, start_response)
def app_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
return AuthController(conf)

26
swift/common/db_replicator.py
View File

@ -89,28 +89,28 @@ class Replicator(object):
Implements the logic for directing db replication.
"""
def __init__(self, server_conf, replicator_conf):
def __init__(self, conf):
self.logger = \
get_logger(replicator_conf, '%s-replicator' % self.server_type)
get_logger(conf)
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
sys.stdout = sys.stderr = LoggerFileObject(self.logger)
self.root = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
self.root = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.port = int(server_conf.get('bind_port', self.default_port))
concurrency = int(replicator_conf.get('concurrency', 8))
self.port = int(conf.get('bind_port', self.default_port))
concurrency = int(conf.get('concurrency', 8))
self.cpool = GreenPool(size=concurrency)
swift_dir = server_conf.get('swift_dir', '/etc/swift')
swift_dir = conf.get('swift_dir', '/etc/swift')
self.ring = ring.Ring(os.path.join(swift_dir, self.ring_file))
self.per_diff = int(replicator_conf.get('per_diff', 1000))
self.run_pause = int(replicator_conf.get('run_pause', 30))
self.vm_test_mode = replicator_conf.get(
self.per_diff = int(conf.get('per_diff', 1000))
self.run_pause = int(conf.get('run_pause', 30))
self.vm_test_mode = conf.get(
'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1')
self.node_timeout = int(replicator_conf.get('node_timeout', 10))
self.conn_timeout = float(replicator_conf.get('conn_timeout', 0.5))
self.reclaim_age = float(replicator_conf.get('reclaim_age', 86400 * 7))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7))
self._zero_stats()
def _zero_stats(self):

29
swift/common/healthcheck.py
View File

@ -1,29 +0,0 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from webob import Response
class HealthCheckController(object):
"""Basic controller used for monitoring."""
def __init__(self, *args, **kwargs):
pass
@classmethod
def GET(self, req):
return Response(request=req, body="OK", content_type="text/plain")
healthcheck = HealthCheckController.GET

0
swift/common/middleware/__init__.py Normal file
View File

47
swift/common/auth.py → swift/common/middleware/auth.py
View File

@ -21,6 +21,8 @@ from eventlet.timeout import Timeout
from swift.common.utils import split_path
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.utils import get_logger, cache_from_env
from swift.common.memcached import MemcacheRing
class DevAuthMiddleware(object):
@ -28,10 +30,13 @@ class DevAuthMiddleware(object):
Auth Middleware that uses the dev auth server
"""
def __init__(self, app, conf, memcache_client, logger):
def __init__(self, app, conf, memcache_client=None, logger=None):
self.app = app
self.memcache_client = memcache_client
self.logger = logger
if logger is None:
self.logger = get_logger(conf)
else:
self.logger = logger
self.conf = conf
self.auth_host = conf.get('ip', '127.0.0.1')
self.auth_port = int(conf.get('port', 11000))
@ -40,23 +45,24 @@ class DevAuthMiddleware(object):
self.timeout = int(conf.get('node_timeout', 10))
def __call__(self, env, start_response):
if self.memcache_client is None:
self.memcache_client = cache_from_env(env)
req = Request(env)
if req.path != '/healthcheck':
if 'x-storage-token' in req.headers and \
'x-auth-token' not in req.headers:
req.headers['x-auth-token'] = req.headers['x-storage-token']
if 'x-storage-token' in req.headers and \
'x-auth-token' not in req.headers:
req.headers['x-auth-token'] = req.headers['x-storage-token']
try:
version, account, container, obj = split_path(req.path, 1, 4, True)
if account is None:
return HTTPPreconditionFailed(request=req, body='Bad URL')(
env, start_response)
if not req.headers.get('x-auth-token'):
return HTTPPreconditionFailed(request=req,
body='Missing Auth Token')(env, start_response)
if account is None:
return HTTPPreconditionFailed(
request=req, body='Bad URL')(env, start_response)
if not self.auth(account, req.headers['x-auth-token']):
return HTTPUnauthorized(request=req)(env, start_response)
except ValueError, e:
version = account = container = obj = None
if account is None:
return HTTPPreconditionFailed(request=req, body='Bad URL')(
env, start_response)
if not req.headers.get('x-auth-token'):
return HTTPPreconditionFailed(request=req,
body='Missing Auth Token')(env, start_response)
if not self.auth(account, req.headers['x-auth-token']):
return HTTPUnauthorized(request=req)(env, start_response)
# If we get here, then things should be good.
return self.app(env, start_response)
@ -97,3 +103,10 @@ class DevAuthMiddleware(object):
val = (now, validated)
self.memcache_client.set(key, val, timeout=validated)
return True
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def auth_filter(app):
return DevAuthMiddleware(app, conf)
return auth_filter

43
swift/common/middleware/healthcheck.py Normal file
View File

@ -0,0 +1,43 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from webob import Request, Response
class HealthCheckMiddleware(object):
"""
Healthcheck middleware used for monitoring.
If the path is /healthcheck, it will respond with "OK" in the body
"""
def __init__(self, app, *args, **kwargs):
self.app = app
def GET(self, req):
"""Returns a 200 response with "OK" in the body."""
return Response(request=req, body="OK", content_type="text/plain")
def __call__(self, env, start_response):
req = Request(env)
if req.path == '/healthcheck':
return self.GET(req)(env, start_response)
else:
return self.app(env, start_response)
def filter_factory(global_conf, **local_conf):
def healthcheck_filter(app):
return HealthCheckMiddleware(app)
return healthcheck_filter

36
swift/common/middleware/memcache.py Normal file
View File

@ -0,0 +1,36 @@
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from swift.common.memcached import MemcacheRing
class MemcacheMiddleware(object):
"""
Caching middleware that manages caching in swift.
"""
def __init__(self, app, conf):
self.app = app
self.memcache = MemcacheRing([s.strip() for s in
conf.get('memcache_servers', '127.0.0.1:11211').split(',')
if s.strip()])
def __call__(self, env, start_response):
env['swift.cache'] = self.memcache
return self.app(env, start_response)
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def cache_filter(app):
return MemcacheMiddleware(app, conf)
return cache_filter

35
swift/common/utils.py
View File

@ -31,6 +31,7 @@ import ctypes
import ctypes.util
import fcntl
import struct
from ConfigParser import ConfigParser
import eventlet
from eventlet import greenio, GreenPool, sleep, Timeout, listen
@ -323,7 +324,7 @@ class NamedLogger(object):
call('%s %s: %s' % (self.server, msg, emsg), *args)
def get_logger(conf, name):
def get_logger(conf, name=None):
"""
Get the current system logger using config settings.
@ -331,6 +332,7 @@ def get_logger(conf, name):
log_facility = LOG_LOCAL0
log_level = INFO
log_name = swift
:param conf: Configuration dict to read settings from
:param name: Name of the logger
@ -342,8 +344,11 @@ def get_logger(conf, name):
if conf is None:
root_logger.setLevel(logging.INFO)
return NamedLogger(root_logger, name)
if name is None:
name = conf.get('log_name', 'swift')
get_logger.handler = SysLogHandler(address='/dev/log',
facility=getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'),
facility=getattr(SysLogHandler,
conf.get('log_facility', 'LOG_LOCAL0'),
SysLogHandler.LOG_LOCAL0))
root_logger.addHandler(get_logger.handler)
root_logger.setLevel(
@ -513,3 +518,29 @@ def unlink_older_than(path, mtime):
os.unlink(fpath)
except OSError:
pass
def item_from_env(env, item_name):
item = env.get(item_name, None)
if item is None:
logging.error("ERROR: %s could not be found in env!" % item_name)
return item
def cache_from_env(env):
return item_from_env(env, 'swift.cache')
def readconf(conf, section_name, log_name=None):
c = ConfigParser()
if not c.read(conf):
print "Unable to read config file %s" % conf
sys.exit(1)
if c.has_section(section_name):
conf = dict(c.items(section_name))
else:
print "Unable to find %s config section in %s" % (section_name, conf)
sys.exit(1)
if "log_name" not in conf:
if log_name is not None:
conf['log_name'] = log_name
else:
conf['log_name'] = section_name
return conf

22
swift/common/wsgi.py
View File

@ -24,6 +24,7 @@ import mimetools
import eventlet
from eventlet import greenio, GreenPool, sleep, wsgi, listen
from paste.deploy import loadapp, appconfig
# Hook to ensure connection resets don't blow up our servers.
# Remove with next release of Eventlet that has it in the set already.
@ -58,19 +59,27 @@ def monkey_patch_mimetools():
# We might be able to pull pieces of this out to test, but right now it seems
# like more work than it's worth.
def run_wsgi(app, conf, *args, **kwargs): # pragma: no cover
def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover
"""
Loads common settings from conf, then instantiates app and runs
the server using the specified number of workers.
:param app: WSGI callable
:param conf: Configuration dictionary
:param conf_file: Path to paste.deploy style configuration file
:param app_section: App name from conf file to load config from
"""
try:
conf = appconfig('config:%s' % conf_file, name=app_section)
log_name = conf.get('log_name', app_section)
app = loadapp('config:%s' % conf_file,
global_conf={'log_name':log_name})
except Exception, e:
print "Error trying to load config %s: %s" % (conf_file, e)
return
if 'logger' in kwargs:
logger = kwargs['logger']
else:
logger = get_logger(conf, app.log_name)
logger = get_logger(conf, log_name)
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
@ -103,9 +112,6 @@ def run_wsgi(app, conf, *args, **kwargs): # pragma: no cover
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600)
worker_count = int(conf.get('workers', '1'))
drop_privileges(conf.get('user', 'swift'))
if isinstance(app, type):
# Instantiate app if it hasn't been already
app = app(conf, *args)
def run_server():
wsgi.HttpProtocol.default_request_version = "HTTP/1.0"

18
swift/container/auditor.py
View File

@ -36,20 +36,20 @@ class AuditException(Exception):
class ContainerAuditor(object):
"""Audit containers."""
def __init__(self, server_conf, auditor_conf):
self.logger = get_logger(auditor_conf, 'container-auditor')
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf)
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(auditor_conf.get('interval', 1800))
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 1800))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.account_ring = None
self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz')
self.object_ring = None
self.node_timeout = int(auditor_conf.get('node_timeout', 10))
self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5))
self.max_object_count = int(auditor_conf.get('max_object_count', 100))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.max_object_count = int(conf.get('max_object_count', 100))
self.account_passes = 0
self.account_failures = 0
self.account_errors = 0

15
swift/container/server.py
View File

@ -35,7 +35,6 @@ from swift.common.utils import get_logger, get_param, hash_path, \
from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
check_mount, check_float, check_xml_encodable
from swift.common.bufferedhttp import http_connect
from swift.common.healthcheck import healthcheck
from swift.common.exceptions import ConnectionTimeout
from swift.common.db_replicator import ReplicatorRpc
@ -45,10 +44,8 @@ DATADIR = 'containers'
class ContainerController(object):
"""WSGI Controller for the container server."""
log_name = 'container'
def __init__(self, conf):
self.logger = get_logger(conf, self.log_name)
self.logger = get_logger(conf)
self.root = conf.get('devices', '/srv/node/')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
@ -384,9 +381,7 @@ class ContainerController(object):
def __call__(self, env, start_response):
start_time = time.time()
req = Request(env)
if req.path_info == '/healthcheck':
return healthcheck(req)(env, start_response)
elif not check_xml_encodable(req.path_info):
if not check_xml_encodable(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8')
else:
try:
@ -415,3 +410,9 @@ class ContainerController(object):
else:
self.logger.info(log_message)
return res(env, start_response)
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI container server apps"""
conf = global_conf.copy()
conf.update(local_conf)
return ContainerController(conf)

20
swift/container/updater.py
View File

@ -33,19 +33,19 @@ from swift.common.utils import get_logger, whataremyips
class ContainerUpdater(object):
"""Update container information in account listings."""
def __init__(self, server_conf, updater_conf):
self.logger = get_logger(updater_conf, 'container-updater')
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf, 'container-updater')
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(updater_conf.get('interval', 300))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 300))
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.account_ring = None
self.concurrency = int(updater_conf.get('concurrency', 4))
self.slowdown = float(updater_conf.get('slowdown', 0.01))
self.node_timeout = int(updater_conf.get('node_timeout', 3))
self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5))
self.concurrency = int(conf.get('concurrency', 4))
self.slowdown = float(conf.get('slowdown', 0.01))
self.node_timeout = int(conf.get('node_timeout', 3))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.no_changes = 0
self.successes = 0
self.failures = 0

16
swift/obj/auditor.py
View File

@ -33,17 +33,17 @@ from swift.common.exceptions import AuditException
class ObjectAuditor(object):
"""Audit objects."""
def __init__(self, server_conf, auditor_conf):
self.logger = get_logger(auditor_conf, 'object-auditor')
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf)
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
self.interval = int(auditor_conf.get('interval', 1800))
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 1800))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.container_ring = None
self.node_timeout = int(auditor_conf.get('node_timeout', 10))
self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.passes = 0
self.quarantines = 0
self.errors = 0

6
swift/obj/replicator.py
View File

@ -213,9 +213,9 @@ class ObjectReplicator(object):
'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1')
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.port = int(conf.get('bind_port', 6000))
self.concurrency = int(conf.get('replication_concurrency', 1))
self.timeout = conf['timeout']
self.stats_interval = int(conf['stats_interval'])
self.concurrency = int(conf.get('concurrency', 1))
self.timeout = conf.get('timeout', '5')
self.stats_interval = int(conf.get('stats_interval', '3600'))
self.object_ring = Ring(join(self.swift_dir, 'object.ring.gz'))
self.ring_check_interval = int(conf.get('ring_check_interval', 15))
self.next_check = time.time() + self.ring_check_interval

19
swift/obj/server.py
View File

@ -36,9 +36,8 @@ from xattr import getxattr, setxattr
from eventlet import sleep, Timeout
from swift.common.utils import mkdirs, normalize_timestamp, \
storage_directory, hash_path, get_logger, renamer, fallocate, \
split_path, drop_buffer_cache
from swift.common.healthcheck import healthcheck
storage_directory, hash_path, renamer, fallocate, \
split_path, drop_buffer_cache, get_logger
from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_object_creation, check_mount, \
check_float, check_xml_encodable
@ -242,8 +241,6 @@ class DiskFile(object):
class ObjectController(object):
"""Implements the WSGI application for the Swift Object Server."""
log_name = 'object'
def __init__(self, conf):
"""
Creates a new WSGI application for the Swift Object Server. An
@ -251,7 +248,7 @@ class ObjectController(object):
<source-dir>/etc/object-server.conf-sample or
/etc/swift/object-server.conf-sample.
"""
self.logger = get_logger(conf, self.log_name)
self.logger = get_logger(conf)
self.devices = conf.get('devices', '/srv/node/')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
@ -560,9 +557,7 @@ class ObjectController(object):
"""WSGI Application entry point for the Swift Object Server."""
start_time = time.time()
req = Request(env)
if req.path_info == '/healthcheck':
return healthcheck(req)(env, start_response)
elif not check_xml_encodable(req.path_info):
if not check_xml_encodable(req.path_info):
res = HTTPPreconditionFailed(body='Invalid UTF8')
else:
try:
@ -596,3 +591,9 @@ class ObjectController(object):
if slow > 0:
sleep(slow)
return res(env, start_response)
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI object server apps"""
conf = global_conf.copy()
conf.update(local_conf)
return ObjectController(conf)

20
swift/obj/updater.py
View File

@ -32,19 +32,19 @@ from swift.obj.server import ASYNCDIR
class ObjectUpdater(object):
"""Update object information in container listings."""
def __init__(self, server_conf, updater_conf):
self.logger = get_logger(updater_conf, 'object-updater')
self.devices = server_conf.get('devices', '/srv/node')
self.mount_check = server_conf.get('mount_check', 'true').lower() in \
def __init__(self, conf):
self.logger = get_logger(conf)
self.devices = conf.get('devices', '/srv/node')
self.mount_check = conf.get('mount_check', 'true').lower() in \
('true', 't', '1', 'on', 'yes', 'y')
swift_dir = server_conf.get('swift_dir', '/etc/swift')
self.interval = int(updater_conf.get('interval', 300))
swift_dir = conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 300))
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.container_ring = None
self.concurrency = int(updater_conf.get('concurrency', 1))
self.slowdown = float(updater_conf.get('slowdown', 0.01))
self.node_timeout = int(updater_conf.get('node_timeout', 10))
self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5))
self.concurrency = int(conf.get('concurrency', 1))
self.slowdown = float(conf.get('slowdown', 0.01))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.successes = 0
self.failures = 0

33
swift/proxy/server.py
View File

@ -31,9 +31,9 @@ from webob.exc import HTTPBadRequest, HTTPMethodNotAllowed, \
from webob import Request, Response
from swift.common.ring import Ring
from swift.common.utils import get_logger, normalize_timestamp, split_path
from swift.common.utils import get_logger, normalize_timestamp, split_path, \
cache_from_env
from swift.common.bufferedhttp import http_connect
from swift.common.healthcheck import HealthCheckController
from swift.common.constraints import check_object_creation, check_metadata, \
MAX_FILE_SIZE, check_xml_encodable
from swift.common.exceptions import ChunkReadTimeout, \
@ -1032,14 +1032,12 @@ class AccountController(Controller):
class BaseApplication(object):
"""Base WSGI application for the proxy server"""
log_name = 'base_application'
def __init__(self, conf, memcache, logger=None, account_ring=None,
def __init__(self, conf, memcache=None, logger=None, account_ring=None,
container_ring=None, object_ring=None):
if logger:
self.logger = logger
if logger is None:
self.logger = get_logger(conf)
else:
self.logger = get_logger(conf, self.log_name)
self.logger = logger
if conf is None:
conf = {}
swift_dir = conf.get('swift_dir', '/etc/swift')
@ -1093,8 +1091,6 @@ class BaseApplication(object):
return ContainerController, d
elif account and not container and not obj:
return AccountController, d
elif version and version == 'healthcheck':
return HealthCheckController, d
return None, d
def __call__(self, env, start_response):
@ -1106,6 +1102,8 @@ class BaseApplication(object):
:param start_response: WSGI callable
"""
try:
if self.memcache is None:
self.memcache = cache_from_env(env)
req = self.update_request(Request(env))
if 'eventlet.posthooks' in env:
env['eventlet.posthooks'].append(
@ -1149,13 +1147,6 @@ class BaseApplication(object):
controller, path_parts = self.get_controller(req.path)
except ValueError:
return HTTPNotFound(request=req)
if controller == HealthCheckController:
controller = controller(self, **path_parts)
controller.trans_id = req.headers.get('x-cf-trans-id', '-')
if req.method == 'GET':
return controller.GET(req)
return HTTPMethodNotAllowed(request=req)
if not check_xml_encodable(req.path_info):
return HTTPPreconditionFailed(request=req, body='Invalid UTF8')
if not controller:
@ -1187,8 +1178,6 @@ class BaseApplication(object):
class Application(BaseApplication):
"""WSGI application for the proxy server."""
log_name = 'proxy'
def handle_request(self, req):
"""
Wraps the BaseApplication.handle_request and logs the request.
@ -1273,3 +1262,9 @@ class Application(BaseApplication):
return Response(status='498 Rate Limited',
body='Slow down', request=req)
return None
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI proxy apps."""
conf = global_conf.copy()
conf.update(local_conf)
return Application(conf)

27
test/unit/account/test_server.py
View File

@ -894,33 +894,6 @@ class TestAccountController(unittest.TestCase):
listing.append(node2.firstChild.nodeValue)
self.assertEquals(listing, ['sub.1.0', 'sub.1.1', 'sub.1.2'])
def test_healthcheck(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
""" Sends args to outbuf """
outbuf.writelines(args)
self.controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '/healthcheck',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'CONTENT_LENGTH': '0',
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http',
'wsgi.input': inbuf,
'wsgi.errors': errbuf,
'wsgi.multithread': False,
'wsgi.multiprocess': False,
'wsgi.run_once': False},
start_response)
self.assertEquals(errbuf.getvalue(), '')
self.assertEquals(outbuf.getvalue()[:4], '200 ')
def test_through_call(self):
inbuf = StringIO()
errbuf = StringIO()

2
test/unit/auth/test_server.py
View File

@ -75,7 +75,7 @@ class TestAuthServer(unittest.TestCase):
'auth_server')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
self.conf = {'swift_dir': self.testdir}
self.conf = {'swift_dir': self.testdir, 'log_name': 'auth'}
self.controller = auth_server.AuthController(self.conf, FakeRing())
def tearDown(self):

0
test/unit/common/middleware/__init__.py Normal file
View File

2
test/unit/common/test_auth.py → test/unit/common/middleware/test_auth.py
View File

@ -23,7 +23,7 @@ from contextlib import contextmanager
import eventlet
from webob import Request
from swift.common import auth
from swift.common.middleware import auth
# mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))

24
test/unit/common/test_healthcheck.py → test/unit/common/middleware/test_healthcheck.py
View File

@ -17,17 +17,29 @@ import unittest
from webob import Request
from swift.common import healthcheck
from swift.common.middleware import healthcheck
class FakeApp(object):
def __call__(self, env, start_response):
return "FAKE APP"
def start_response(*args):
pass
class TestHealthCheck(unittest.TestCase):
def test_healthcheck(self):
controller = healthcheck.HealthCheckController()
req = Request.blank('/any/path', environ={'REQUEST_METHOD': 'GET'})
resp = controller.GET(req)
self.assertEquals(resp.status_int, 200)
def setUp(self):
self.app = healthcheck.HealthCheckMiddleware(FakeApp())
def test_healthcheck(self):
req = Request.blank('/healthcheck', environ={'REQUEST_METHOD': 'GET'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, ['OK'])
def test_healtcheck_pass(self):
req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, 'FAKE APP')
if __name__ == '__main__':
unittest.main()

42
test/unit/common/middleware/test_memcache.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from webob import Request
from swift.common.middleware import memcache
from swift.common.memcached import MemcacheRing
class FakeApp(object):
def __call__(self, env, start_response):
return env
def start_response(*args):
pass
class TestCacheMiddleware(unittest.TestCase):
def setUp(self):
self.app = memcache.MemcacheMiddleware(FakeApp(), {})
def test_cache_middleware(self):
req = Request.blank('/something', environ={'REQUEST_METHOD': 'GET'})
resp = self.app(req.environ, start_response)
self.assertTrue('swift.cache' in resp)
self.assertTrue(isinstance(resp['swift.cache'], MemcacheRing))
if __name__ == '__main__':
unittest.main()

16
test/unit/common/test_db_replicator.py
View File

@ -139,7 +139,7 @@ class TestDBReplicator(unittest.TestCase):
self.assertEquals(conn.replicate(1, 2, 3), None)
def test_rsync_file(self):
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
with _mock_process(-1):
fake_device = {'ip': '127.0.0.1', 'device': 'sda1'}
self.assertEquals(False,
@ -150,13 +150,13 @@ class TestDBReplicator(unittest.TestCase):
replicator._rsync_file('/some/file', 'remote:/some/file'))
def test_rsync_db(self):
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
replicator._rsync_file = lambda *args: True
fake_device = {'ip': '127.0.0.1', 'device': 'sda1'}
replicator._rsync_db(FakeBroker(), fake_device, ReplHttp(), 'abcd')
def test_in_sync(self):
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
self.assertEquals(replicator._in_sync(
{'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'},
{'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'},
@ -171,16 +171,16 @@ class TestDBReplicator(unittest.TestCase):
FakeBroker(), -1)), False)
def test_replicate_once(self):
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
replicator.replicate_once()
def test_usync(self):
fake_http = ReplHttp()
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
replicator._usync_db(0, FakeBroker(), fake_http, '12345', '67890')
def test_repl_to_node(self):
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
fake_node = {'ip': '127.0.0.1', 'device': 'sda1', 'port': 1000}
fake_info = {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b',
'created_at': 100, 'put_timestamp': 0,
@ -193,13 +193,13 @@ class TestDBReplicator(unittest.TestCase):
def test_stats(self):
# I'm not sure how to test that this logs the right thing,
# but we can at least make sure it gets covered.
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
replicator._zero_stats()
replicator._report_stats()
def test_replicate_object(self):
db_replicator.lock_parent_directory = lock_parent_directory
replicator = TestReplicator({}, {})
replicator = TestReplicator({})
replicator._replicate_object('0', 'file', 'node_id')

27
test/unit/container/test_server.py
View File

@ -654,33 +654,6 @@ class TestContainerController(unittest.TestCase):
{"name":"US/TX","hash":"x","bytes":0,"content_type":"text/plain",
"last_modified":"1970-01-01T00:00:01"}])
def test_healthcheck(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
""" Sends args to outbuf """
outbuf.writelines(args)
self.controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '/healthcheck',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'CONTENT_LENGTH': '0',
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http',
'wsgi.input': inbuf,
'wsgi.errors': errbuf,
'wsgi.multithread': False,
'wsgi.multiprocess': False,
'wsgi.run_once': False},
start_response)
self.assertEquals(errbuf.getvalue(), '')
self.assertEquals(outbuf.getvalue()[:4], '200 ')
def test_through_call(self):
inbuf = StringIO()
errbuf = StringIO()

36
test/unit/container/test_updater.py
View File

@ -61,10 +61,14 @@ class TestContainerUpdater(unittest.TestCase):
rmtree(self.testdir, ignore_errors=1)
def test_creation(self):
cu = container_updater.ContainerUpdater(
{'devices': self.devices_dir, 'mount_check': 'false',
'swift_dir': self.testdir},
{'interval': '1', 'concurrency': '2', 'node_timeout': '5'})
cu = container_updater.ContainerUpdater({
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
'interval': '1',
'concurrency': '2',
'node_timeout': '5',
})
self.assert_(hasattr(cu, 'logger'))
self.assert_(cu.logger is not None)
self.assertEquals(cu.devices, self.devices_dir)
@ -74,10 +78,14 @@ class TestContainerUpdater(unittest.TestCase):
self.assert_(cu.get_account_ring() is not None)
def test_update_once_single_threaded(self):
cu = container_updater.ContainerUpdater(
{'devices': self.devices_dir, 'mount_check': 'false',
'swift_dir': self.testdir},
{'interval': '1', 'concurrency': '1', 'node_timeout': '15'})
cu = container_updater.ContainerUpdater({
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
'interval': '1',
'concurrency': '1',
'node_timeout': '15',
})
cu.update_once_single_threaded()
containers_dir = os.path.join(self.sda1, container_server.DATADIR)
os.mkdir(containers_dir)
@ -152,10 +160,14 @@ class TestContainerUpdater(unittest.TestCase):
self.assertEquals(info['reported_bytes_used'], 3)
def test_unicode(self):
cu = container_updater.ContainerUpdater(
{'devices': self.devices_dir, 'mount_check': 'false',
'swift_dir': self.testdir},
{'interval': '1', 'concurrency': '1', 'node_timeout': '15'})
cu = container_updater.ContainerUpdater({
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
'interval': '1',
'concurrency': '1',
'node_timeout': '15',
})
containers_dir = os.path.join(self.sda1, container_server.DATADIR)
os.mkdir(containers_dir)
subdir = os.path.join(containers_dir, 'subdir')

27
test/unit/obj/test_server.py
View File

@ -768,33 +768,6 @@ class TestObjectController(unittest.TestCase):
timestamp + '.ts')
self.assert_(os.path.isfile(objfile))
def test_healthcheck(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
""" Sends args to outbuf """
outbuf.writelines(args)
self.object_controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '/healthcheck',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'CONTENT_LENGTH': '0',
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http',
'wsgi.input': inbuf,
'wsgi.errors': errbuf,
'wsgi.multithread': False,
'wsgi.multiprocess': False,
'wsgi.run_once': False},
start_response)
self.assertEquals(errbuf.getvalue(), '')
self.assertEquals(outbuf.getvalue()[:4], '200 ')
def test_call(self):
""" Test swift.object_server.ObjectController.__call__ """
inbuf = StringIO()

24
test/unit/obj/test_updater.py
View File

@ -53,10 +53,14 @@ class TestObjectUpdater(unittest.TestCase):
rmtree(self.testdir, ignore_errors=1)
def test_creation(self):
cu = object_updater.ObjectUpdater(
{'devices': self.devices_dir, 'mount_check': 'false',
'swift_dir': self.testdir},
{'interval': '1', 'concurrency': '2', 'node_timeout': '5'})
cu = object_updater.ObjectUpdater({
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
'interval': '1',
'concurrency': '2',
'node_timeout': '5',
})
self.assert_(hasattr(cu, 'logger'))
self.assert_(cu.logger is not None)
self.assertEquals(cu.devices, self.devices_dir)
@ -66,10 +70,14 @@ class TestObjectUpdater(unittest.TestCase):
self.assert_(cu.get_container_ring() is not None)
def test_update_once_single_threaded(self):
cu = object_updater.ObjectUpdater(
{'devices': self.devices_dir, 'mount_check': 'false',
'swift_dir': self.testdir},
{'interval': '1', 'concurrency': '1', 'node_timeout': '15'})
cu = object_updater.ObjectUpdater({
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
'interval': '1',
'concurrency': '1',
'node_timeout': '15',
})
cu.update_once_single_threaded()
async_dir = os.path.join(self.sda1, object_server.ASYNCDIR)
os.mkdir(async_dir)

20
test/unit/proxy/test_server.py
View File

@ -1105,17 +1105,6 @@ class TestObjectController(unittest.TestCase):
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
try:
# healthcheck test
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /healthcheck HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nContent-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
body = fd.read()
self.assertEquals(body, 'OK')
# Check bad version
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
@ -1134,15 +1123,6 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 404'
self.assertEquals(headers[:len(exp)], exp)
# Check bad method
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('LICK /healthcheck HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nContent-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 405'
self.assertEquals(headers[:len(exp)], exp)
# Check blacklist
prosrv.rate_limit_blacklist = ['a']
sock = connect_tcp(('localhost', prolis.getsockname()[1]))