From 2c596c0a0fbb0a1ec74092278d9f8844b90bf760 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Fri, 20 Aug 2010 00:42:38 +0000 Subject: [PATCH 01/15] Initial commit of middleware refactor --- bin/swift-account-auditor | 9 ++-- bin/swift-account-reaper | 9 ++-- bin/swift-account-replicator | 9 ++-- bin/swift-account-server | 11 ++-- bin/swift-auth-server | 11 ++-- bin/swift-container-auditor | 9 ++-- bin/swift-container-replicator | 9 ++-- bin/swift-container-server | 11 ++-- bin/swift-container-updater | 7 ++- bin/swift-object-auditor | 9 ++-- bin/swift-object-replicator | 39 +++++--------- bin/swift-object-server | 12 ++--- bin/swift-object-updater | 7 ++- bin/swift-proxy-server | 32 ++---------- etc/account-server.conf-sample | 23 ++++++--- etc/auth-server.conf-sample | 17 +++++-- etc/container-server.conf-sample | 28 +++++----- etc/object-server.conf-sample | 34 +++++++------ etc/proxy-server.conf-sample | 38 +++++++++----- setup.py | 51 ++++++++++++------- swift/account/auditor.py | 18 +++---- swift/account/reaper.py | 20 ++++---- swift/account/server.py | 16 +++--- swift/auth/server.py | 9 ++-- swift/common/db_replicator.py | 26 +++++----- swift/common/healthcheck.py | 29 ----------- swift/common/middleware/__init__.py | 0 swift/common/{ => middleware}/auth.py | 45 +++++++++------- swift/common/middleware/cache.py | 34 +++++++++++++ swift/common/middleware/healthcheck.py | 43 ++++++++++++++++ swift/common/utils.py | 13 ++++- swift/common/wsgi.py | 18 ++++--- swift/container/auditor.py | 18 +++---- swift/container/server.py | 17 ++++--- swift/container/updater.py | 20 ++++---- swift/obj/auditor.py | 16 +++--- swift/obj/replicator.py | 2 +- swift/obj/server.py | 19 +++---- swift/obj/updater.py | 20 ++++---- swift/proxy/server.py | 33 +++++------- test/unit/account/test_server.py | 27 ---------- test/unit/auth/test_server.py | 2 +- test/unit/common/middleware/__init__.py | 0 .../unit/common/{ => middleware}/test_auth.py | 2 +- test/unit/common/middleware/test_cache.py | 42 +++++++++++++++ .../{ => middleware}/test_healthcheck.py | 24 ++++++--- test/unit/common/test_db_replicator.py | 16 +++--- test/unit/container/test_server.py | 27 ---------- test/unit/container/test_updater.py | 36 ++++++++----- test/unit/obj/test_server.py | 27 ---------- test/unit/obj/test_updater.py | 24 ++++++--- test/unit/proxy/test_server.py | 20 -------- 52 files changed, 535 insertions(+), 503 deletions(-) delete mode 100644 swift/common/healthcheck.py create mode 100644 swift/common/middleware/__init__.py rename swift/common/{ => middleware}/auth.py (68%) create mode 100644 swift/common/middleware/cache.py create mode 100644 swift/common/middleware/healthcheck.py create mode 100644 test/unit/common/middleware/__init__.py rename test/unit/common/{ => middleware}/test_auth.py (99%) create mode 100644 test/unit/common/middleware/test_cache.py rename test/unit/common/{ => middleware}/test_healthcheck.py (54%) diff --git a/bin/swift-account-auditor b/bin/swift-account-auditor index 2bbb73c23f..cba89e9e63 100755 --- a/bin/swift-account-auditor +++ b/bin/swift-account-auditor @@ -34,21 +34,20 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('account-server')) if c.has_section('account-auditor'): - auditor_conf = dict(c.items('account-auditor')) + conf = dict(c.items('account-auditor')) else: print "Unable to find account-auditor config section in %s." % \ sys.argv[1] sys.exit(1) - logger = utils.get_logger(auditor_conf, 'account-auditor') + logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) sys.stdout = sys.stderr = utils.LoggerFileObject(logger) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) try: os.setsid() @@ -62,7 +61,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - auditor = AccountAuditor(server_conf, auditor_conf) + auditor = AccountAuditor(conf) if once: auditor.audit_once() else: diff --git a/bin/swift-account-reaper b/bin/swift-account-reaper index d62e43add8..c5619d0785 100755 --- a/bin/swift-account-reaper +++ b/bin/swift-account-reaper @@ -34,21 +34,20 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('account-server')) if c.has_section('account-reaper'): - reaper_conf = dict(c.items('account-reaper')) + conf = dict(c.items('account-reaper')) else: print "Unable to find account-reaper config section in %s." % \ sys.argv[1] sys.exit(1) - logger = utils.get_logger(reaper_conf, 'account-reaper') + logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) sys.stdout = sys.stderr = utils.LoggerFileObject(logger) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) try: os.setsid() @@ -62,7 +61,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - reaper = AccountReaper(server_conf, reaper_conf) + reaper = AccountReaper(conf) if once: reaper.reap_once() else: diff --git a/bin/swift-account-replicator b/bin/swift-account-replicator index 3e47eaa4f1..284c751598 100755 --- a/bin/swift-account-replicator +++ b/bin/swift-account-replicator @@ -41,17 +41,16 @@ if __name__ == '__main__': sys.exit(1) once = len(args) > 1 and args[1] == 'once' - server_conf = dict(c.items('account-server')) if c.has_section('account-replicator'): - replicator_conf = dict(c.items('account-replicator')) + conf = dict(c.items('account-replicator')) else: print "Unable to find account-replicator config section in %s." % \ args[0] sys.exit(1) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) if once or '--once' in [opt[0] for opt in optlist]: - AccountReplicator(server_conf, replicator_conf).replicate_once() + AccountReplicator(conf).replicate_once() else: - AccountReplicator(server_conf, replicator_conf).replicate_forever() + AccountReplicator(conf).replicate_forever() diff --git a/bin/swift-account-server b/bin/swift-account-server index d5e8021049..6be5706ea4 100755 --- a/bin/swift-account-server +++ b/bin/swift-account-server @@ -14,17 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ConfigParser import ConfigParser import sys from swift.common.wsgi import run_wsgi -from swift.account.server import AccountController if __name__ == '__main__': - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - conf = dict(c.items('account-server')) - run_wsgi(AccountController, conf, default_port=6002) + if len(sys.argv) != 2: + print "Usage: %s CONFIG_FILE" + run_wsgi(sys.argv[1], default_port=6002) diff --git a/bin/swift-auth-server b/bin/swift-auth-server index 775530e078..5f32912a9c 100755 --- a/bin/swift-auth-server +++ b/bin/swift-auth-server @@ -14,17 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ConfigParser import ConfigParser import sys from swift.common.wsgi import run_wsgi -from swift.auth.server import AuthController if __name__ == '__main__': - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - conf = dict(c.items('auth-server')) - run_wsgi(AuthController, conf, default_port=11000) + if len(sys.argv) != 2: + print "Usage: %s CONFIG_FILE" + run_wsgi(sys.argv[1], default_port=11000) diff --git a/bin/swift-container-auditor b/bin/swift-container-auditor index 1ff1682c52..6dfdc4592d 100755 --- a/bin/swift-container-auditor +++ b/bin/swift-container-auditor @@ -34,21 +34,20 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('container-server')) if c.has_section('container-auditor'): - auditor_conf = dict(c.items('container-auditor')) + conf = dict(c.items('container-auditor')) else: print "Unable to find container-auditor config section in %s." % \ sys.argv[1] sys.exit(1) - logger = utils.get_logger(auditor_conf, 'container-auditor') + logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) sys.stdout = sys.stderr = utils.LoggerFileObject(logger) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(onf.get('user', 'swift')) try: os.setsid() @@ -62,7 +61,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - auditor = ContainerAuditor(server_conf, auditor_conf) + auditor = ContainerAuditor(conf) if once: auditor.audit_once() else: diff --git a/bin/swift-container-replicator b/bin/swift-container-replicator index db2ffc8b20..663cb51db7 100755 --- a/bin/swift-container-replicator +++ b/bin/swift-container-replicator @@ -41,17 +41,16 @@ if __name__ == '__main__': sys.exit(1) once = len(args) > 1 and args[1] == 'once' - server_conf = dict(c.items('container-server')) if c.has_section('container-replicator'): - replicator_conf = dict(c.items('container-replicator')) + conf = dict(c.items('container-replicator')) else: print "Unable to find container-replicator config section in %s." % \ args[0] sys.exit(1) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) if once or '--once' in [opt[0] for opt in optlist]: - ContainerReplicator(server_conf, replicator_conf).replicate_once() + ContainerReplicator(conf).replicate_once() else: - ContainerReplicator(server_conf, replicator_conf).replicate_forever() + ContainerReplicator(conf).replicate_forever() diff --git a/bin/swift-container-server b/bin/swift-container-server index e0ad813f46..f707eb091c 100755 --- a/bin/swift-container-server +++ b/bin/swift-container-server @@ -14,17 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ConfigParser import ConfigParser import sys from swift.common.wsgi import run_wsgi -from swift.container.server import ContainerController if __name__ == '__main__': - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - conf = dict(c.items('container-server')) - run_wsgi(ContainerController, conf, default_port=6001) + if len(sys.argv) != 2: + print "Usage: %s CONFIG_FILE" + run_wsgi(sys.argv[1], default_port=6001) diff --git a/bin/swift-container-updater b/bin/swift-container-updater index c37ba3680a..7c23115123 100755 --- a/bin/swift-container-updater +++ b/bin/swift-container-updater @@ -34,15 +34,14 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('container-server')) if c.has_section('container-updater'): - updater_conf = dict(c.items('container-updater')) + conf = dict(c.items('container-updater')) else: print "Unable to find container-updater config section in %s." % \ sys.argv[1] sys.exit(1) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) try: os.setsid() @@ -56,7 +55,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - updater = ContainerUpdater(server_conf, updater_conf) + updater = ContainerUpdater(conf) if once: updater.update_once_single_threaded() else: diff --git a/bin/swift-object-auditor b/bin/swift-object-auditor index 5d0e54e661..e995ad8725 100755 --- a/bin/swift-object-auditor +++ b/bin/swift-object-auditor @@ -34,21 +34,20 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('object-server')) if c.has_section('object-auditor'): - auditor_conf = dict(c.items('object-auditor')) + conf = dict(c.items('object-auditor')) else: print "Unable to find object-auditor config section in %s." % \ sys.argv[1] sys.exit(1) - logger = utils.get_logger(auditor_conf, 'object-auditor') + logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) sys.stdout = sys.stderr = utils.LoggerFileObject(logger) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) try: os.setsid() @@ -62,7 +61,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - auditor = ObjectAuditor(server_conf, auditor_conf) + auditor = ObjectAuditor(conf) if once: auditor.audit_once() else: diff --git a/bin/swift-object-replicator b/bin/swift-object-replicator index 0de0b5d64a..6a2da4b4f3 100755 --- a/bin/swift-object-replicator +++ b/bin/swift-object-replicator @@ -27,38 +27,23 @@ from swift.common.utils import get_logger, drop_privileges, LoggerFileObject TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes')) -def read_configs(conf_file): - c = ConfigParser() - if not c.read(conf_file): - print "Unable to read config file: %s" % conf_file - sys.exit(1) - conf = dict(c.items('object-server')) - repl_conf = dict(c.items('object-replicator')) - if not repl_conf: - sys.exit() - conf['replication_concurrency'] = repl_conf.get('concurrency',1) - conf['vm_test_mode'] = repl_conf.get('vm_test_mode', 'no') - conf['daemonize'] = repl_conf.get('daemonize', 'yes') - conf['run_pause'] = repl_conf.get('run_pause', '30') - conf['log_facility'] = repl_conf.get('log_facility', 'LOG_LOCAL1') - conf['log_level'] = repl_conf.get('log_level', 'INFO') - conf['timeout'] = repl_conf.get('timeout', '5') - conf['stats_interval'] = repl_conf.get('stats_interval', '3600') - conf['reclaim_age'] = int(repl_conf.get('reclaim_age', 86400)) - - return conf - if __name__ == '__main__': if len(sys.argv) < 2: print "Usage: object-replicator CONFIG_FILE [once]" sys.exit() - try: - conf = read_configs(sys.argv[1]) - except: - print "Problem reading the config. Aborting object replication." - sys.exit() + c = ConfigParser() + if not c.read(conf_file): + print "Unable to read config file: %s" % conf_file + sys.exit(1) + if c.has_section('object-replicator'): + conf = dict(c.items('object-replicator')) + else: + print "Unable to find object-replicator config section in %s" % \ + sys.argv[1] + sys.exit(1) + once = len(sys.argv) > 2 and sys.argv[2] == 'once' - logger = get_logger(conf, 'object-replicator') + logger = get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) diff --git a/bin/swift-object-server b/bin/swift-object-server index c0603b96a7..ce74b17195 100755 --- a/bin/swift-object-server +++ b/bin/swift-object-server @@ -14,17 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ConfigParser import ConfigParser import sys from swift.common.wsgi import run_wsgi -from swift.obj.server import ObjectController if __name__ == '__main__': - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - conf = dict(c.items('object-server')) - run_wsgi(ObjectController, conf, default_port=6000) - + if len(sys.argv) != 2: + print "Usage: %s CONFIG_FILE" + run_wsgi(sys.argv[1], default_port=6000) diff --git a/bin/swift-object-updater b/bin/swift-object-updater index 595840a079..a9512a4e76 100755 --- a/bin/swift-object-updater +++ b/bin/swift-object-updater @@ -34,15 +34,14 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('object-server')) if c.has_section('object-updater'): - updater_conf = dict(c.items('object-updater')) + conf = dict(c.items('object-updater')) else: print "Unable to find object-updater config section in %s." % \ sys.argv[1] sys.exit(1) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) try: os.setsid() @@ -56,7 +55,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - updater = ObjectUpdater(server_conf, updater_conf) + updater = ObjectUpdater(conf) if once: updater.update_once_single_threaded() else: diff --git a/bin/swift-proxy-server b/bin/swift-proxy-server index 0ba14e2357..c56df2e074 100755 --- a/bin/swift-proxy-server +++ b/bin/swift-proxy-server @@ -14,37 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ConfigParser import ConfigParser -import os import sys from swift.common.wsgi import run_wsgi -from swift.common.auth import DevAuthMiddleware -from swift.common.memcached import MemcacheRing -from swift.common.utils import get_logger -from swift.proxy.server import Application if __name__ == '__main__': - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - conf = dict(c.items('proxy-server')) - if c.has_section('auth-server'): - auth_conf = dict(c.items('auth-server')) - else: - auth_conf = {} - swift_dir = conf.get('swift_dir', '/etc/swift') - m, c = auth_conf.get('class', - 'swift.common.auth.DevAuthMiddleware').rsplit('.', 1) - m = __import__(m, fromlist=[c]) - authware = m.__dict__[c] - - memcache = MemcacheRing([s.strip() for s in - conf.get('memcache_servers', '127.0.0.1:11211').split(',') - if s.strip()]) - logger = get_logger(conf, 'proxy') - app = Application(conf, memcache, logger) - # Wrap the app with auth - app = authware(app, auth_conf, memcache, logger) - run_wsgi(app, conf, logger=logger, default_port=80) + if len(sys.argv) != 2: + print "Usage: %s CONFIG_FILE" + run_wsgi(sys.argv[1], default_port=80) diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index 3ded523ee2..303ab0e00b 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -1,15 +1,23 @@ -[account-server] -# swift_dir = /etc/swift -# devices = /srv/node -# mount_check = true +[DEFAULT] +log_name = account +# log_facility = LOG_LOCAL0 +# log_level = INFO # bind_ip = 0.0.0.0 # bind_port = 6002 # workers = 1 -# log_facility = LOG_LOCAL0 -# log_level = INFO # user = swift +# swift_dir = /etc/swift +# devices = /srv/node +# mount_check = true + +[pipeline:main] +pipeline = account-server + +[app:account-server] +use = egg:swift#account [account-replicator] +log_name = account-replicator # log_facility = LOG_LOCAL0 # log_level = INFO # per_diff = 1000 @@ -26,6 +34,7 @@ # reclaim_age = 86400 [account-stats] +log_name = account-stats # cf_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31 # container_name = account_stats # proxy_server_conf = /etc/swift/proxy-server.conf @@ -33,6 +42,7 @@ # log_level = INFO [account-auditor] +log_name = account-auditor # Will audit, at most, 1 account per device per interval # interval = 1800 # Maximum containers randomly picked for a given account audit @@ -43,6 +53,7 @@ # log_level = INFO [account-reaper] +log_name = account-reaper # concurrency = 25 # interval = 3600 # node_timeout = 10 diff --git a/etc/auth-server.conf-sample b/etc/auth-server.conf-sample index 29bcc63f1a..61c1ffafd9 100644 --- a/etc/auth-server.conf-sample +++ b/etc/auth-server.conf-sample @@ -1,15 +1,22 @@ -[auth-server] -# swift_dir = /etc/swift +[DEFAULT] +log_name = auth # bind_ip = 0.0.0.0 # bind_port = 11000 # log_facility = LOG_LOCAL0 # log_level = INFO # workers = 1 +# user = swift +# swift_dir = /etc/swift +# cert_file = Default is no cert; format is path like /etc/swift/auth.crt +# key_file = Default is no key; format is path like /etc/swift/auth.key + +[pipeline:main] +pipeline = auth-server + +[app:auth-server] +use = egg:swift#auth # reseller_prefix = AUTH # default_cluster_url = http://127.0.0.1:9000/v1 # token_life = 86400 # log_headers = False -# cert_file = Default is no cert; format is path like /etc/swift/auth.crt -# key_file = Default is no key; format is path like /etc/swift/auth.key # node_timeout = 10 -user = swift diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 086d4d8a0f..ccad08c710 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -1,19 +1,25 @@ -[container-server] -# swift_dir = /etc/swift -# devices = /srv/node -# mount_check = true +[DEFAULT] +log_name = container +# log_facility = LOG_LOCAL0 +# log_level = INFO # bind_ip = 0.0.0.0 # bind_port = 6001 # workers = 1 -# log_facility = LOG_LOCAL0 -# log_level = INFO # user = swift +# swift_dir = /etc/swift +# devices = /srv/node +# mount_check = true + +[pipeline:main] +pipeline = container-server + +[app:container-server] +use = egg:swift#container # node_timeout = 3 # conn_timeout = 0.5 [container-replicator] -# log_facility = LOG_LOCAL0 -# log_level = INFO +log_name = container-replicator # per_diff = 1000 # concurrency = 8 # run_pause = 30 @@ -23,21 +29,19 @@ # reclaim_age = 604800 [container-updater] +log_name = container-updater # interval = 300 # concurrency = 4 # node_timeout = 3 # conn_timeout = 0.5 # slowdown will sleep that amount between containers # slowdown = 0.01 -# log_facility = LOG_LOCAL0 -# log_level = INFO [container-auditor] +log_name = container-auditor # Will audit, at most, 1 container per device per interval # interval = 1800 # Maximum objects randomly picked for a given container audit # max_object_count = 100 # node_timeout = 10 # conn_timeout = 0.5 -# log_facility = LOG_LOCAL0 -# log_level = INFO diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index cce9ce39a1..81a35c5cb7 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -1,24 +1,30 @@ -[object-server] -# swift_dir = /etc/swift -# devices = /srv/node -# mount_check = true +[DEFAULT] +log_name = object +# log_facility = LOG_LOCAL0 +# log_level = INFO # bind_ip = 0.0.0.0 # bind_port = 6000 # workers = 1 -# log_facility = LOG_LOCAL0 -# log_level = INFO -# log_requests = True # user = swift +# swift_dir = /etc/swift +# devices = /srv/node +# mount_check = true + +[pipeline:main] +pipeline = object-server + +[app:object-server] +use = egg:swift#object +# log_requests = True # node_timeout = 3 # conn_timeout = 0.5 -# network_chunk_size = 8192 -# disk_chunk_size = 32768 +# network_chunk_size = 65536 +# disk_chunk_size = 65536 # max_upload_time = 86400 # slow = 1 [object-replicator] -# log_facility = LOG_LOCAL0 -# log_level = INFO +log_name = object-replicator # daemonize = on # run_pause = 30 # concurrency = 1 @@ -28,19 +34,17 @@ # reclaim_age = 604800 [object-updater] +log_name = object-updater # interval = 300 # concurrency = 1 # node_timeout = 10 # conn_timeout = 0.5 # slowdown will sleep that amount between objects # slowdown = 0.01 -# log_facility = LOG_LOCAL0 -# log_level = INFO [object-auditor] +log_name = object-auditor # Will audit, at most, 1 object per device per interval # interval = 1800 # node_timeout = 10 # conn_timeout = 0.5 -# log_facility = LOG_LOCAL0 -# log_level = INFO diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 33d612c724..d8bbc8f76b 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -1,21 +1,26 @@ -[proxy-server] -# bind_ip = 0.0.0.0 -# bind_port = 80 -# cert_file = /etc/swift/proxy.crt -# key_file = /etc/swift/proxy.key -# swift_dir = /etc/swift +[DEFAULT] +log_name = proxy # log_facility = LOG_LOCAL0 # log_level = INFO -# log_headers = False +# bind_ip = 0.0.0.0 +# bind_port = 80 # workers = 1 # user = swift +# cert_file = /etc/swift/proxy.crt +# key_file = /etc/swift/proxy.key + +[pipeline:main] +pipeline = healthcheck cache auth proxy + +[app:proxy] +use = egg:swift#proxy +filter-with = auth +# swift_dir = /etc/swift +# log_headers = False # recheck_account_existence = 60 # recheck_container_existence = 60 # object_chunk_size = 8192 # client_chunk_size = 8192 -# Default for memcache_servers is below, but you can specify multiple servers -# with the format: 10.1.2.3:11211,10.1.2.4:11211 -# memcache_servers = 127.0.0.1:11211 # node_timeout = 10 # client_timeout = 60 # conn_timeout = 0.5 @@ -31,8 +36,17 @@ # rate_limit_account_whitelist = acct1,acct2,etc # rate_limit_account_blacklist = acct3,acct4,etc -# [auth-server] -# class = swift.common.auth.DevAuthMiddleware +[filter:auth] +use = egg:swift#auth # ip = 127.0.0.1 # port = 11000 # node_timeout = 10 + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:cache] +use = egg:swift#cache +# Default for memcache_servers is below, but you can specify multiple servers +# with the format: 10.1.2.3:11211,10.1.2.4:11211 +# memcache_servers = 127.0.0.1:11211 diff --git a/setup.py b/setup.py index 1c711b764c..02dd389495 100644 --- a/setup.py +++ b/setup.py @@ -55,21 +55,38 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python :: 2.6', 'Environment :: No Input/Output (Daemon)', - ], + ], install_requires=[], # removed for better compat - scripts=['bin/st', 'bin/swift-account-auditor', - 'bin/swift-account-audit', 'bin/swift-account-reaper', - 'bin/swift-account-replicator', 'bin/swift-account-server', - 'bin/swift-auth-create-account', - 'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server', - 'bin/swift-container-auditor', - 'bin/swift-container-replicator', - 'bin/swift-container-server', 'bin/swift-container-updater', - 'bin/swift-drive-audit', 'bin/swift-get-nodes', - 'bin/swift-init', 'bin/swift-object-auditor', - 'bin/swift-object-info', - 'bin/swift-object-replicator', - 'bin/swift-object-server', - 'bin/swift-object-updater', 'bin/swift-proxy-server', - 'bin/swift-ring-builder', 'bin/swift-stats-populate', - 'bin/swift-stats-report']) + scripts=[ + 'bin/st', 'bin/swift-account-auditor', + 'bin/swift-account-audit', 'bin/swift-account-reaper', + 'bin/swift-account-replicator', 'bin/swift-account-server', + 'bin/swift-auth-create-account', + 'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server', + 'bin/swift-container-auditor', + 'bin/swift-container-replicator', + 'bin/swift-container-server', 'bin/swift-container-updater', + 'bin/swift-drive-audit', 'bin/swift-get-nodes', + 'bin/swift-init', 'bin/swift-object-auditor', + 'bin/swift-object-info', + 'bin/swift-object-replicator', + 'bin/swift-object-server', + 'bin/swift-object-updater', 'bin/swift-proxy-server', + 'bin/swift-ring-builder', 'bin/swift-stats-populate', + 'bin/swift-stats-report' + ], + entry_points={ + 'paste.app_factory' : [ + 'proxy=swift.proxy.server:app_factory', + 'object=swift.obj.server:app_factory', + 'container=swift.container.server:app_factory', + 'account=swift.account.server:app_factory', + 'auth=swift.auth.server:app_factory', + ], + 'paste.filter_factory' : [ + 'auth=swift.common.middleware.auth:filter_factory', + 'healthcheck=swift.common.middleware.healthcheck:filter_factory', + 'cache=swift.common.middleware.cache:filter_factory', + ], + }, + ) diff --git a/swift/account/auditor.py b/swift/account/auditor.py index a21ed0849b..249d9f436d 100644 --- a/swift/account/auditor.py +++ b/swift/account/auditor.py @@ -35,19 +35,19 @@ class AuditException(Exception): class AccountAuditor(object): """Audit accounts.""" - def __init__(self, server_conf, auditor_conf): - self.logger = get_logger(auditor_conf, 'account-auditor') - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf, 'account-auditor') + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.interval = int(auditor_conf.get('interval', 1800)) - swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 1800)) + swift_dir = conf.get('swift_dir', '/etc/swift') self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.container_ring = None - self.node_timeout = int(auditor_conf.get('node_timeout', 10)) - self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.max_container_count = \ - int(auditor_conf.get('max_container_count', 100)) + int(conf.get('max_container_count', 100)) self.container_passes = 0 self.container_failures = 0 self.container_errors = 0 diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 93b54e7525..f747e3ce09 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -50,25 +50,23 @@ class AccountReaper(object): configuration parameters. """ - log_name = 'account-reaper' - - def __init__(self, server_conf, reaper_conf): - self.logger = get_logger(reaper_conf, self.log_name) - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf) + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.interval = int(reaper_conf.get('interval', 3600)) - swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 3600)) + swift_dir = conf.get('swift_dir', '/etc/swift') self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz') self.account_ring = None self.container_ring = None self.object_ring = None - self.node_timeout = int(reaper_conf.get('node_timeout', 10)) - self.conn_timeout = float(reaper_conf.get('conn_timeout', 0.5)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.myips = whataremyips() - self.concurrency = int(reaper_conf.get('concurrency', 25)) + self.concurrency = int(conf.get('concurrency', 25)) self.container_concurrency = self.object_concurrency = \ sqrt(self.concurrency) self.container_pool = GreenPool(size=self.container_concurrency) diff --git a/swift/account/server.py b/swift/account/server.py index 6e5dd7472e..7194cc9f95 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -31,10 +31,9 @@ from xml.sax import saxutils from swift.common.db import AccountBroker from swift.common.utils import get_param, split_path, storage_directory, \ - hash_path + hash_path, get_logger from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_xml_encodable -from swift.common.healthcheck import healthcheck from swift.common.db_replicator import ReplicatorRpc @@ -43,10 +42,9 @@ DATADIR = 'accounts' class AccountController(object): """WSGI controller for the account server.""" - log_name = 'account' def __init__(self, conf): - self.logger = get_logger(conf, self.log_name) + self.logger = get_logger(conf) self.root = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') @@ -253,9 +251,7 @@ class AccountController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - if req.path_info == '/healthcheck': - return healthcheck(req)(env, start_response) - elif not check_xml_encodable(req.path_info): + if not check_xml_encodable(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: @@ -288,3 +284,9 @@ class AccountController(object): else: self.logger.info(log_message) return res(env, start_response) + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI account server apps""" + conf = global_conf.copy() + conf.update(local_conf) + return AccountController(conf) diff --git a/swift/auth/server.py b/swift/auth/server.py index 676b6fc6d9..48b40ab50f 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -89,10 +89,8 @@ class AuthController(object): configuration parameters. """ - log_name = 'auth' - def __init__(self, conf, ring=None): - self.logger = get_logger(conf, self.log_name) + self.logger = get_logger(conf) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.default_cluster_url = \ conf.get('default_cluster_url', 'http://127.0.0.1:9000/v1') @@ -500,3 +498,8 @@ class AuthController(object): def __call__(self, env, start_response): """ Used by the eventlet.wsgi.server """ return self.handleREST(env, start_response) + +def app_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + return AuthController(conf) diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 5f416cce31..ce66a95fb0 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -89,28 +89,28 @@ class Replicator(object): Implements the logic for directing db replication. """ - def __init__(self, server_conf, replicator_conf): + def __init__(self, conf): self.logger = \ - get_logger(replicator_conf, '%s-replicator' % self.server_type) + get_logger(conf) # log uncaught exceptions sys.excepthook = lambda * exc_info: \ self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) sys.stdout = sys.stderr = LoggerFileObject(self.logger) - self.root = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + self.root = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.port = int(server_conf.get('bind_port', self.default_port)) - concurrency = int(replicator_conf.get('concurrency', 8)) + self.port = int(conf.get('bind_port', self.default_port)) + concurrency = int(conf.get('concurrency', 8)) self.cpool = GreenPool(size=concurrency) - swift_dir = server_conf.get('swift_dir', '/etc/swift') + swift_dir = conf.get('swift_dir', '/etc/swift') self.ring = ring.Ring(os.path.join(swift_dir, self.ring_file)) - self.per_diff = int(replicator_conf.get('per_diff', 1000)) - self.run_pause = int(replicator_conf.get('run_pause', 30)) - self.vm_test_mode = replicator_conf.get( + self.per_diff = int(conf.get('per_diff', 1000)) + self.run_pause = int(conf.get('run_pause', 30)) + self.vm_test_mode = conf.get( 'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1') - self.node_timeout = int(replicator_conf.get('node_timeout', 10)) - self.conn_timeout = float(replicator_conf.get('conn_timeout', 0.5)) - self.reclaim_age = float(replicator_conf.get('reclaim_age', 86400 * 7)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) + self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7)) self._zero_stats() def _zero_stats(self): diff --git a/swift/common/healthcheck.py b/swift/common/healthcheck.py deleted file mode 100644 index 0e18f240a2..0000000000 --- a/swift/common/healthcheck.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2010 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from webob import Response - - -class HealthCheckController(object): - """Basic controller used for monitoring.""" - - def __init__(self, *args, **kwargs): - pass - - @classmethod - def GET(self, req): - return Response(request=req, body="OK", content_type="text/plain") - -healthcheck = HealthCheckController.GET diff --git a/swift/common/middleware/__init__.py b/swift/common/middleware/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/swift/common/auth.py b/swift/common/middleware/auth.py similarity index 68% rename from swift/common/auth.py rename to swift/common/middleware/auth.py index 6830b60ba7..13d0e426a2 100644 --- a/swift/common/auth.py +++ b/swift/common/middleware/auth.py @@ -21,6 +21,8 @@ from eventlet.timeout import Timeout from swift.common.utils import split_path from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import get_logger, cache_from_env +from swift.common.memcached import MemcacheRing class DevAuthMiddleware(object): @@ -28,33 +30,37 @@ class DevAuthMiddleware(object): Auth Middleware that uses the dev auth server """ - def __init__(self, app, conf, memcache_client, logger): + def __init__(self, app, conf, memcache_client=None, logger=None): self.app = app self.memcache_client = memcache_client - self.logger = logger + if logger is None: + self.logger = get_logger(conf) + else: + self.logger = logger self.conf = conf self.auth_host = conf.get('ip', '127.0.0.1') self.auth_port = int(conf.get('port', 11000)) self.timeout = int(conf.get('node_timeout', 10)) def __call__(self, env, start_response): + if self.memcache_client is None: + self.memcache_client = cache_from_env(env) req = Request(env) - if req.path != '/healthcheck': - if 'x-storage-token' in req.headers and \ - 'x-auth-token' not in req.headers: - req.headers['x-auth-token'] = req.headers['x-storage-token'] + if 'x-storage-token' in req.headers and \ + 'x-auth-token' not in req.headers: + req.headers['x-auth-token'] = req.headers['x-storage-token'] + try: version, account, container, obj = split_path(req.path, 1, 4, True) - if account is None: - return HTTPPreconditionFailed(request=req, body='Bad URL')( - env, start_response) - if not req.headers.get('x-auth-token'): - return HTTPPreconditionFailed(request=req, - body='Missing Auth Token')(env, start_response) - if account is None: - return HTTPPreconditionFailed( - request=req, body='Bad URL')(env, start_response) - if not self.auth(account, req.headers['x-auth-token']): - return HTTPUnauthorized(request=req)(env, start_response) + except ValueError, e: + version = account = container = obj = None + if account is None: + return HTTPPreconditionFailed(request=req, body='Bad URL')( + env, start_response) + if not req.headers.get('x-auth-token'): + return HTTPPreconditionFailed(request=req, + body='Missing Auth Token')(env, start_response) + if not self.auth(account, req.headers['x-auth-token']): + return HTTPUnauthorized(request=req)(env, start_response) # If we get here, then things should be good. return self.app(env, start_response) @@ -95,3 +101,8 @@ class DevAuthMiddleware(object): val = (now, validated) self.memcache_client.set(key, val, timeout=validated) return True + +def filter_factory(global_conf, **local_conf): + def auth_filter(app): + return DevAuthMiddleware(app, local_conf) + return auth_filter diff --git a/swift/common/middleware/cache.py b/swift/common/middleware/cache.py new file mode 100644 index 0000000000..a0dfc81b86 --- /dev/null +++ b/swift/common/middleware/cache.py @@ -0,0 +1,34 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from swift.common.memcached import MemcacheRing + +class CacheMiddleware(object): + """ + Caching middleware that manages caching in swift. + """ + def __init__(self, app, conf): + self.app = app + self.memcache = MemcacheRing([s.strip() for s in + conf.get('memcache_servers', '127.0.0.1:11211').split(',') + if s.strip()]) + + def __call__(self, env, start_response): + env['swift.cache'] = self.memcache + return self.app(env, start_response) + +def filter_factory(global_conf, **local_conf): + def cache_filter(app): + return CacheMiddleware(app, local_conf) + return cache_filter diff --git a/swift/common/middleware/healthcheck.py b/swift/common/middleware/healthcheck.py new file mode 100644 index 0000000000..3bfc745bc2 --- /dev/null +++ b/swift/common/middleware/healthcheck.py @@ -0,0 +1,43 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from webob import Request, Response + + +class HealthCheckMiddleware(object): + """ + Healthcheck middleware used for monitoring. + + If the path is /healthcheck, it will respond with "OK" in the body + """ + + def __init__(self, app, *args, **kwargs): + self.app = app + + def GET(self, req): + """Returns a 200 response with "OK" in the body.""" + return Response(request=req, body="OK", content_type="text/plain") + + def __call__(self, env, start_response): + req = Request(env) + if req.path == '/healthcheck': + return self.GET(req)(env, start_response) + else: + return self.app(env, start_response) + +def filter_factory(global_conf, **local_conf): + def healthcheck_filter(app): + return HealthCheckMiddleware(app) + return healthcheck_filter diff --git a/swift/common/utils.py b/swift/common/utils.py index dab0b9d255..e4962f2367 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -323,7 +323,7 @@ class NamedLogger(object): call('%s %s: %s' % (self.server, msg, emsg), *args) -def get_logger(conf, name): +def get_logger(conf, name=None): """ Get the current system logger using config settings. @@ -342,6 +342,8 @@ def get_logger(conf, name): if conf is None: root_logger.setLevel(logging.INFO) return NamedLogger(root_logger, name) + if name is None: + name = conf.get('log_name', 'swift') get_logger.handler = SysLogHandler(address='/dev/log', facility=getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'), SysLogHandler.LOG_LOCAL0)) @@ -513,3 +515,12 @@ def unlink_older_than(path, mtime): os.unlink(fpath) except OSError: pass + +def item_from_env(env, item_name): + item = env.get(item_name, None) + if item is None: + logging.error("ERROR: %s could not be found in env!" % item_name) + return item + +def cache_from_env(env): + return item_from_env(env, 'swift.cache') diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index f0045e0393..2944bd3bdf 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -24,6 +24,7 @@ import mimetools import eventlet from eventlet import greenio, GreenPool, sleep, wsgi, listen +from paste.deploy import loadapp, appconfig # Hook to ensure connection resets don't blow up our servers. # Remove with next release of Eventlet that has it in the set already. @@ -58,18 +59,24 @@ def monkey_patch_mimetools(): # We might be able to pull pieces of this out to test, but right now it seems # like more work than it's worth. -def run_wsgi(app, conf, *args, **kwargs): # pragma: no cover +def run_wsgi(conf_file, *args, **kwargs): # pragma: no cover """ Loads common settings from conf, then instantiates app and runs the server using the specified number of workers. - :param app: WSGI callable - :param conf: Configuration dictionary + :param conf_file: Path to paste.deploy style configuration file """ + + try: + app = loadapp('config:%s' % conf_file) + conf = appconfig('config:%s' % conf_file) + except Exception, e: + print "Error trying to load config %s: %s" % (conf_file, e) + return if 'logger' in kwargs: logger = kwargs['logger'] else: - logger = get_logger(conf, app.log_name) + logger = get_logger(conf) # log uncaught exceptions sys.excepthook = lambda * exc_info: \ @@ -103,9 +110,6 @@ def run_wsgi(app, conf, *args, **kwargs): # pragma: no cover sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) worker_count = int(conf.get('workers', '1')) drop_privileges(conf.get('user', 'swift')) - if isinstance(app, type): - # Instantiate app if it hasn't been already - app = app(conf, *args) def run_server(): wsgi.HttpProtocol.default_request_version = "HTTP/1.0" diff --git a/swift/container/auditor.py b/swift/container/auditor.py index b507b0c57d..a41bf5358c 100644 --- a/swift/container/auditor.py +++ b/swift/container/auditor.py @@ -36,20 +36,20 @@ class AuditException(Exception): class ContainerAuditor(object): """Audit containers.""" - def __init__(self, server_conf, auditor_conf): - self.logger = get_logger(auditor_conf, 'container-auditor') - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf) + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.interval = int(auditor_conf.get('interval', 1800)) - swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 1800)) + swift_dir = conf.get('swift_dir', '/etc/swift') self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') self.account_ring = None self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz') self.object_ring = None - self.node_timeout = int(auditor_conf.get('node_timeout', 10)) - self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) - self.max_object_count = int(auditor_conf.get('max_object_count', 100)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) + self.max_object_count = int(conf.get('max_object_count', 100)) self.account_passes = 0 self.account_failures = 0 self.account_errors = 0 diff --git a/swift/container/server.py b/swift/container/server.py index 57775ee661..fab7275980 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -31,11 +31,10 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ from swift.common.db import ContainerBroker from swift.common.utils import get_logger, get_param, hash_path, \ - storage_directory, split_path + storage_directory, split_path, get_logger from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ check_mount, check_float, check_xml_encodable from swift.common.bufferedhttp import http_connect -from swift.common.healthcheck import healthcheck from swift.common.exceptions import ConnectionTimeout from swift.common.db_replicator import ReplicatorRpc @@ -45,10 +44,8 @@ DATADIR = 'containers' class ContainerController(object): """WSGI Controller for the container server.""" - log_name = 'container' - def __init__(self, conf): - self.logger = get_logger(conf, self.log_name) + self.logger = get_logger(conf) self.root = conf.get('devices', '/srv/node/') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') @@ -347,9 +344,7 @@ class ContainerController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - if req.path_info == '/healthcheck': - return healthcheck(req)(env, start_response) - elif not check_xml_encodable(req.path_info): + if not check_xml_encodable(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: @@ -378,3 +373,9 @@ class ContainerController(object): else: self.logger.info(log_message) return res(env, start_response) + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI container server apps""" + conf = global_conf.copy() + conf.update(local_conf) + return ContainerController(conf) diff --git a/swift/container/updater.py b/swift/container/updater.py index a0db68cfa6..003d06b130 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -33,19 +33,19 @@ from swift.common.utils import get_logger, whataremyips class ContainerUpdater(object): """Update container information in account listings.""" - def __init__(self, server_conf, updater_conf): - self.logger = get_logger(updater_conf, 'container-updater') - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf, 'container-updater') + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - swift_dir = server_conf.get('swift_dir', '/etc/swift') - self.interval = int(updater_conf.get('interval', 300)) + swift_dir = conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 300)) self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') self.account_ring = None - self.concurrency = int(updater_conf.get('concurrency', 4)) - self.slowdown = float(updater_conf.get('slowdown', 0.01)) - self.node_timeout = int(updater_conf.get('node_timeout', 3)) - self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5)) + self.concurrency = int(conf.get('concurrency', 4)) + self.slowdown = float(conf.get('slowdown', 0.01)) + self.node_timeout = int(conf.get('node_timeout', 3)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.no_changes = 0 self.successes = 0 self.failures = 0 diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 62484ffb0b..d9e2145d92 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -33,17 +33,17 @@ from swift.common.exceptions import AuditException class ObjectAuditor(object): """Audit objects.""" - def __init__(self, server_conf, auditor_conf): - self.logger = get_logger(auditor_conf, 'object-auditor') - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf) + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.interval = int(auditor_conf.get('interval', 1800)) - swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 1800)) + swift_dir = conf.get('swift_dir', '/etc/swift') self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.container_ring = None - self.node_timeout = int(auditor_conf.get('node_timeout', 10)) - self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.passes = 0 self.quarantines = 0 self.errors = 0 diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index de3bea3972..8848748a94 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -215,7 +215,7 @@ class ObjectReplicator(object): 'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1') self.swift_dir = conf.get('swift_dir', '/etc/swift') self.port = int(conf.get('bind_port', 6000)) - self.concurrency = int(conf.get('replication_concurrency', 1)) + self.concurrency = int(conf.get('concurrency', 1)) self.timeout = conf['timeout'] self.stats_interval = int(conf['stats_interval']) self.object_ring = Ring(join(self.swift_dir, 'object.ring.gz')) diff --git a/swift/obj/server.py b/swift/obj/server.py index 39c2ca2276..c76f44f331 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -36,9 +36,8 @@ from xattr import getxattr, setxattr from eventlet import sleep, Timeout from swift.common.utils import mkdirs, normalize_timestamp, \ - storage_directory, hash_path, get_logger, renamer, fallocate, \ - split_path, drop_buffer_cache -from swift.common.healthcheck import healthcheck + storage_directory, hash_path, renamer, fallocate, \ + split_path, drop_buffer_cache, get_logger from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, check_mount, \ check_float, check_xml_encodable @@ -242,8 +241,6 @@ class DiskFile(object): class ObjectController(object): """Implements the WSGI application for the Swift Object Server.""" - log_name = 'object' - def __init__(self, conf): """ Creates a new WSGI application for the Swift Object Server. An @@ -251,7 +248,7 @@ class ObjectController(object): /etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ - self.logger = get_logger(conf, self.log_name) + self.logger = get_logger(conf) self.devices = conf.get('devices', '/srv/node/') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') @@ -560,9 +557,7 @@ class ObjectController(object): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) - if req.path_info == '/healthcheck': - return healthcheck(req)(env, start_response) - elif not check_xml_encodable(req.path_info): + if not check_xml_encodable(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: @@ -596,3 +591,9 @@ class ObjectController(object): if slow > 0: sleep(slow) return res(env, start_response) + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI object server apps""" + conf = global_conf.copy() + conf.update(local_conf) + return ObjectController(conf) diff --git a/swift/obj/updater.py b/swift/obj/updater.py index e974364090..7fa9182823 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -32,19 +32,19 @@ from swift.obj.server import ASYNCDIR class ObjectUpdater(object): """Update object information in container listings.""" - def __init__(self, server_conf, updater_conf): - self.logger = get_logger(updater_conf, 'object-updater') - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf) + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - swift_dir = server_conf.get('swift_dir', '/etc/swift') - self.interval = int(updater_conf.get('interval', 300)) + swift_dir = conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 300)) self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.container_ring = None - self.concurrency = int(updater_conf.get('concurrency', 1)) - self.slowdown = float(updater_conf.get('slowdown', 0.01)) - self.node_timeout = int(updater_conf.get('node_timeout', 10)) - self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5)) + self.concurrency = int(conf.get('concurrency', 1)) + self.slowdown = float(conf.get('slowdown', 0.01)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.successes = 0 self.failures = 0 diff --git a/swift/proxy/server.py b/swift/proxy/server.py index b1efd6ea6f..9ba79e13a8 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -31,9 +31,9 @@ from webob.exc import HTTPBadRequest, HTTPMethodNotAllowed, \ from webob import Request, Response from swift.common.ring import Ring -from swift.common.utils import get_logger, normalize_timestamp, split_path +from swift.common.utils import get_logger, normalize_timestamp, split_path, \ + cache_from_env from swift.common.bufferedhttp import http_connect -from swift.common.healthcheck import HealthCheckController from swift.common.constraints import check_object_creation, check_metadata, \ MAX_FILE_SIZE, check_xml_encodable from swift.common.exceptions import ChunkReadTimeout, \ @@ -47,7 +47,7 @@ def update_headers(response, headers): Helper function to update headers in the response. :param response: webob.Response object - :param headers: dictionary headers + :parm headers: dictionary headers """ if hasattr(headers, 'items'): headers = headers.items() @@ -930,14 +930,12 @@ class AccountController(Controller): class BaseApplication(object): """Base WSGI application for the proxy server""" - log_name = 'base_application' - - def __init__(self, conf, memcache, logger=None, account_ring=None, + def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, object_ring=None): - if logger: - self.logger = logger + if logger is None: + self.logger = get_logger(conf) else: - self.logger = get_logger(conf, self.log_name) + self.logger = logger if conf is None: conf = {} swift_dir = conf.get('swift_dir', '/etc/swift') @@ -991,8 +989,6 @@ class BaseApplication(object): return ContainerController, d elif account and not container and not obj: return AccountController, d - elif version and version == 'healthcheck': - return HealthCheckController, d return None, d def __call__(self, env, start_response): @@ -1004,6 +1000,8 @@ class BaseApplication(object): :param start_response: WSGI callable """ try: + if self.memcache is None: + self.memcache = cache_from_env(env) req = self.update_request(Request(env)) if 'eventlet.posthooks' in env: env['eventlet.posthooks'].append( @@ -1048,13 +1046,6 @@ class BaseApplication(object): controller, path_parts = self.get_controller(req.path) except ValueError: return HTTPNotFound(request=req) - if controller == HealthCheckController: - controller = controller(self, **path_parts) - controller.trans_id = req.headers.get('x-cf-trans-id', '-') - if req.method == 'GET': - return controller.GET(req) - return HTTPMethodNotAllowed(request=req) - if not check_xml_encodable(req.path_info): return HTTPPreconditionFailed(request=req, body='Invalid UTF8') if not controller: @@ -1086,8 +1077,6 @@ class BaseApplication(object): class Application(BaseApplication): """WSGI application for the proxy server.""" - log_name = 'proxy' - def handle_request(self, req): """ Wraps the BaseApplication.handle_request and logs the request. @@ -1181,3 +1170,7 @@ class Application(BaseApplication): return Response(status='498 Rate Limited', body='Slow down', request=req) return None + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI proxy apps.""" + return Application(local_conf) diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index 84ef0ed897..c4f445345d 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -795,33 +795,6 @@ class TestAccountController(unittest.TestCase): listing.append(node2.firstChild.nodeValue) self.assertEquals(listing, ['sub.1.0', 'sub.1.1', 'sub.1.2']) - def test_healthcheck(self): - inbuf = StringIO() - errbuf = StringIO() - outbuf = StringIO() - - def start_response(*args): - """ Sends args to outbuf """ - outbuf.writelines(args) - - self.controller.__call__({'REQUEST_METHOD': 'GET', - 'SCRIPT_NAME': '', - 'PATH_INFO': '/healthcheck', - 'SERVER_NAME': '127.0.0.1', - 'SERVER_PORT': '8080', - 'SERVER_PROTOCOL': 'HTTP/1.0', - 'CONTENT_LENGTH': '0', - 'wsgi.version': (1, 0), - 'wsgi.url_scheme': 'http', - 'wsgi.input': inbuf, - 'wsgi.errors': errbuf, - 'wsgi.multithread': False, - 'wsgi.multiprocess': False, - 'wsgi.run_once': False}, - start_response) - self.assertEquals(errbuf.getvalue(), '') - self.assertEquals(outbuf.getvalue()[:4], '200 ') - def test_through_call(self): inbuf = StringIO() errbuf = StringIO() diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py index 9bb23a925d..31495401f2 100644 --- a/test/unit/auth/test_server.py +++ b/test/unit/auth/test_server.py @@ -75,7 +75,7 @@ class TestAuthServer(unittest.TestCase): 'auth_server') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) - self.conf = {'swift_dir': self.testdir} + self.conf = {'swift_dir': self.testdir, 'log_name': 'auth'} self.controller = auth_server.AuthController(self.conf, FakeRing()) def tearDown(self): diff --git a/test/unit/common/middleware/__init__.py b/test/unit/common/middleware/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit/common/test_auth.py b/test/unit/common/middleware/test_auth.py similarity index 99% rename from test/unit/common/test_auth.py rename to test/unit/common/middleware/test_auth.py index cf35827522..cd5a02c91d 100644 --- a/test/unit/common/test_auth.py +++ b/test/unit/common/middleware/test_auth.py @@ -23,7 +23,7 @@ from contextlib import contextmanager import eventlet from webob import Request -from swift.common import auth +from swift.common.middleware import auth # mocks logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) diff --git a/test/unit/common/middleware/test_cache.py b/test/unit/common/middleware/test_cache.py new file mode 100644 index 0000000000..a83233e2eb --- /dev/null +++ b/test/unit/common/middleware/test_cache.py @@ -0,0 +1,42 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from webob import Request + +from swift.common.middleware import cache +from swift.common.memcached import MemcacheRing + +class FakeApp(object): + def __call__(self, env, start_response): + return env + +def start_response(*args): + pass + +class TestCacheMiddleware(unittest.TestCase): + + def setUp(self): + self.app = cache.CacheMiddleware(FakeApp(), {}) + + def test_cache_middleware(self): + req = Request.blank('/something', environ={'REQUEST_METHOD': 'GET'}) + resp = self.app(req.environ, start_response) + self.assertTrue('swift.cache' in resp) + self.assertTrue(isinstance(resp['swift.cache'], MemcacheRing)) + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_healthcheck.py b/test/unit/common/middleware/test_healthcheck.py similarity index 54% rename from test/unit/common/test_healthcheck.py rename to test/unit/common/middleware/test_healthcheck.py index 3256d474c0..fc1a2a198a 100644 --- a/test/unit/common/test_healthcheck.py +++ b/test/unit/common/middleware/test_healthcheck.py @@ -17,17 +17,29 @@ import unittest from webob import Request -from swift.common import healthcheck +from swift.common.middleware import healthcheck +class FakeApp(object): + def __call__(self, env, start_response): + return "FAKE APP" + +def start_response(*args): + pass class TestHealthCheck(unittest.TestCase): - def test_healthcheck(self): - controller = healthcheck.HealthCheckController() - req = Request.blank('/any/path', environ={'REQUEST_METHOD': 'GET'}) - resp = controller.GET(req) - self.assertEquals(resp.status_int, 200) + def setUp(self): + self.app = healthcheck.HealthCheckMiddleware(FakeApp()) + def test_healthcheck(self): + req = Request.blank('/healthcheck', environ={'REQUEST_METHOD': 'GET'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, ['OK']) + + def test_healtcheck_pass(self): + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, 'FAKE APP') if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py index bd247772f0..981deab184 100644 --- a/test/unit/common/test_db_replicator.py +++ b/test/unit/common/test_db_replicator.py @@ -139,7 +139,7 @@ class TestDBReplicator(unittest.TestCase): self.assertEquals(conn.post(1, 2, 3), None) def test_rsync_file(self): - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) with _mock_process(-1): fake_device = {'ip': '127.0.0.1', 'device': 'sda1'} self.assertEquals(False, @@ -150,13 +150,13 @@ class TestDBReplicator(unittest.TestCase): replicator._rsync_file('/some/file', 'remote:/some/file')) def test_rsync_db(self): - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) replicator._rsync_file = lambda *args: True fake_device = {'ip': '127.0.0.1', 'device': 'sda1'} replicator._rsync_db(FakeBroker(), fake_device, PostReplHttp(), 'abcd') def test_in_sync(self): - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) self.assertEquals(replicator._in_sync( {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'}, {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'}, @@ -171,16 +171,16 @@ class TestDBReplicator(unittest.TestCase): FakeBroker(), -1)), False) def test_replicate_once(self): - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) replicator.replicate_once() def test_usync(self): fake_http = PostReplHttp() - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) replicator._usync_db(0, FakeBroker(), fake_http, '12345', '67890') def test_repl_to_node(self): - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) fake_node = {'ip': '127.0.0.1', 'device': 'sda1', 'port': 1000} fake_info = {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b', 'created_at': 100, 'put_timestamp': 0, @@ -192,13 +192,13 @@ class TestDBReplicator(unittest.TestCase): def test_stats(self): # I'm not sure how to test that this logs the right thing, # but we can at least make sure it gets covered. - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) replicator._zero_stats() replicator._report_stats() def test_replicate_object(self): db_replicator.lock_parent_directory = lock_parent_directory - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) replicator._replicate_object('0', 'file', 'node_id') diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index b38c5489e4..08b15e4203 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -551,33 +551,6 @@ class TestContainerController(unittest.TestCase): {"name":"US/TX","hash":"x","bytes":0,"content_type":"text/plain", "last_modified":"1970-01-01T00:00:01"}]) - def test_healthcheck(self): - inbuf = StringIO() - errbuf = StringIO() - outbuf = StringIO() - - def start_response(*args): - """ Sends args to outbuf """ - outbuf.writelines(args) - - self.controller.__call__({'REQUEST_METHOD': 'GET', - 'SCRIPT_NAME': '', - 'PATH_INFO': '/healthcheck', - 'SERVER_NAME': '127.0.0.1', - 'SERVER_PORT': '8080', - 'SERVER_PROTOCOL': 'HTTP/1.0', - 'CONTENT_LENGTH': '0', - 'wsgi.version': (1, 0), - 'wsgi.url_scheme': 'http', - 'wsgi.input': inbuf, - 'wsgi.errors': errbuf, - 'wsgi.multithread': False, - 'wsgi.multiprocess': False, - 'wsgi.run_once': False}, - start_response) - self.assertEquals(errbuf.getvalue(), '') - self.assertEquals(outbuf.getvalue()[:4], '200 ') - def test_through_call(self): inbuf = StringIO() errbuf = StringIO() diff --git a/test/unit/container/test_updater.py b/test/unit/container/test_updater.py index a4e88eba43..108334363c 100644 --- a/test/unit/container/test_updater.py +++ b/test/unit/container/test_updater.py @@ -61,10 +61,14 @@ class TestContainerUpdater(unittest.TestCase): rmtree(self.testdir, ignore_errors=1) def test_creation(self): - cu = container_updater.ContainerUpdater( - {'devices': self.devices_dir, 'mount_check': 'false', - 'swift_dir': self.testdir}, - {'interval': '1', 'concurrency': '2', 'node_timeout': '5'}) + cu = container_updater.ContainerUpdater({ + 'devices': self.devices_dir, + 'mount_check': 'false', + 'swift_dir': self.testdir, + 'interval': '1', + 'concurrency': '2', + 'node_timeout': '5', + }) self.assert_(hasattr(cu, 'logger')) self.assert_(cu.logger is not None) self.assertEquals(cu.devices, self.devices_dir) @@ -74,10 +78,14 @@ class TestContainerUpdater(unittest.TestCase): self.assert_(cu.get_account_ring() is not None) def test_update_once_single_threaded(self): - cu = container_updater.ContainerUpdater( - {'devices': self.devices_dir, 'mount_check': 'false', - 'swift_dir': self.testdir}, - {'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) + cu = container_updater.ContainerUpdater({ + 'devices': self.devices_dir, + 'mount_check': 'false', + 'swift_dir': self.testdir, + 'interval': '1', + 'concurrency': '1', + 'node_timeout': '15', + }) cu.update_once_single_threaded() containers_dir = os.path.join(self.sda1, container_server.DATADIR) os.mkdir(containers_dir) @@ -152,10 +160,14 @@ class TestContainerUpdater(unittest.TestCase): self.assertEquals(info['reported_bytes_used'], 3) def test_unicode(self): - cu = container_updater.ContainerUpdater( - {'devices': self.devices_dir, 'mount_check': 'false', - 'swift_dir': self.testdir}, - {'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) + cu = container_updater.ContainerUpdater({ + 'devices': self.devices_dir, + 'mount_check': 'false', + 'swift_dir': self.testdir, + 'interval': '1', + 'concurrency': '1', + 'node_timeout': '15', + }) containers_dir = os.path.join(self.sda1, container_server.DATADIR) os.mkdir(containers_dir) subdir = os.path.join(containers_dir, 'subdir') diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 316f5763e4..17308c614a 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -768,33 +768,6 @@ class TestObjectController(unittest.TestCase): timestamp + '.ts') self.assert_(os.path.isfile(objfile)) - def test_healthcheck(self): - inbuf = StringIO() - errbuf = StringIO() - outbuf = StringIO() - - def start_response(*args): - """ Sends args to outbuf """ - outbuf.writelines(args) - - self.object_controller.__call__({'REQUEST_METHOD': 'GET', - 'SCRIPT_NAME': '', - 'PATH_INFO': '/healthcheck', - 'SERVER_NAME': '127.0.0.1', - 'SERVER_PORT': '8080', - 'SERVER_PROTOCOL': 'HTTP/1.0', - 'CONTENT_LENGTH': '0', - 'wsgi.version': (1, 0), - 'wsgi.url_scheme': 'http', - 'wsgi.input': inbuf, - 'wsgi.errors': errbuf, - 'wsgi.multithread': False, - 'wsgi.multiprocess': False, - 'wsgi.run_once': False}, - start_response) - self.assertEquals(errbuf.getvalue(), '') - self.assertEquals(outbuf.getvalue()[:4], '200 ') - def test_call(self): """ Test swift.object_server.ObjectController.__call__ """ inbuf = StringIO() diff --git a/test/unit/obj/test_updater.py b/test/unit/obj/test_updater.py index e5de8df4fd..6c7aa70c6c 100644 --- a/test/unit/obj/test_updater.py +++ b/test/unit/obj/test_updater.py @@ -53,10 +53,14 @@ class TestObjectUpdater(unittest.TestCase): rmtree(self.testdir, ignore_errors=1) def test_creation(self): - cu = object_updater.ObjectUpdater( - {'devices': self.devices_dir, 'mount_check': 'false', - 'swift_dir': self.testdir}, - {'interval': '1', 'concurrency': '2', 'node_timeout': '5'}) + cu = object_updater.ObjectUpdater({ + 'devices': self.devices_dir, + 'mount_check': 'false', + 'swift_dir': self.testdir, + 'interval': '1', + 'concurrency': '2', + 'node_timeout': '5', + }) self.assert_(hasattr(cu, 'logger')) self.assert_(cu.logger is not None) self.assertEquals(cu.devices, self.devices_dir) @@ -66,10 +70,14 @@ class TestObjectUpdater(unittest.TestCase): self.assert_(cu.get_container_ring() is not None) def test_update_once_single_threaded(self): - cu = object_updater.ObjectUpdater( - {'devices': self.devices_dir, 'mount_check': 'false', - 'swift_dir': self.testdir}, - {'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) + cu = object_updater.ObjectUpdater({ + 'devices': self.devices_dir, + 'mount_check': 'false', + 'swift_dir': self.testdir, + 'interval': '1', + 'concurrency': '1', + 'node_timeout': '15', + }) cu.update_once_single_threaded() async_dir = os.path.join(self.sda1, object_server.ASYNCDIR) os.mkdir(async_dir) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index eb3b4254da..2aad3a26fc 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1100,17 +1100,6 @@ class TestObjectController(unittest.TestCase): obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl) obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl) try: - # healthcheck test - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /healthcheck HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nContent-Length: 0\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEquals(headers[:len(exp)], exp) - body = fd.read() - self.assertEquals(body, 'OK') # Check bad version sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() @@ -1129,15 +1118,6 @@ class TestObjectController(unittest.TestCase): headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 404' self.assertEquals(headers[:len(exp)], exp) - # Check bad method - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('LICK /healthcheck HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nContent-Length: 0\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 405' - self.assertEquals(headers[:len(exp)], exp) # Check blacklist prosrv.rate_limit_blacklist = ['a'] sock = connect_tcp(('localhost', prolis.getsockname()[1])) From 386eb29b1f29bb82da0b62efa491e53692ee3273 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Fri, 20 Aug 2010 02:19:50 +0000 Subject: [PATCH 02/15] Updated docs (including SAIO) to refelct the paste.deploy config changes --- doc/source/deployment_guide.rst | 95 ++++++++++++----- doc/source/development_saio.rst | 176 +++++++++++++++++++++++++++++--- doc/source/misc.rst | 4 +- etc/proxy-server.conf-sample | 3 +- 4 files changed, 234 insertions(+), 44 deletions(-) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 8c222b18ac..0d805ce806 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -130,6 +130,14 @@ swift-ring-builder with no options will display help text with available commands and options. More information on how the ring works internally can be found in the :doc:`Ring Overview `. +---------------------------- +General Server Configuration +---------------------------- + +Swift uses paste.deploy to manage server configurations. Default configuration +options are set in the `[DEFAULT]` section, and any options specidifed there +can be overriden in any of the other sections. + --------------------------- Object Server Configuration --------------------------- @@ -139,11 +147,14 @@ etc/object-server.conf-sample in the source code repository. The following configuration options are available: -[object-server] +[DEFAULT] ================== ========== ============================================= Option Default Description ------------------ ---------- --------------------------------------------- +log_name swift Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level swift_dir /etc/swift Swift configuration directory devices /srv/node Parent directory of where devices are mounted mount_check true Weather or not check if the devices are @@ -152,8 +163,16 @@ mount_check true Weather or not check if the devices are bind_ip 0.0.0.0 IP Address for server to bind to bind_port 6000 Port for server to bind to workers 1 Number of workers to fork -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level +================== ========== ============================================= + +[object-server] + +================== ========== ============================================= +Option Default Description +------------------ ---------- --------------------------------------------- +use paste.deploy entrypoint for the object + server. For most cases, this should be + `egg:swfit#object`. log_requests True Weather or not to log each request user swift User to run as node_timeout 3 Request timeout to external services @@ -171,8 +190,6 @@ slow 0 If > 0, Minimum time in seconds for a PUT ================== ========== =========================================== Option Default Description ------------------ ---------- ------------------------------------------- -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level daemonize yes Weather or not to run replication as a daemon run_pause 30 Time in seconds to wait between replication @@ -191,8 +208,6 @@ reclaim_age 604800 Time elapsed in seconds before an object ================== ========== =========================================== Option Default Description ------------------ ---------- ------------------------------------------- -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level interval 300 Minimum time for a pass to take concurrency 1 Number of updater workers to spawn node_timeout 10 Request timeout to external services @@ -205,8 +220,6 @@ slowdown 0.01 Time in seconds to wait between objects ================== ========== =========================================== Option Default Description ------------------ ---------- ------------------------------------------- -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level interval 1800 Minimum time for a pass to take node_timeout 10 Request timeout to external services conn_timeout 0.5 Connection timeout to external services @@ -221,11 +234,12 @@ etc/container-server.conf-sample in the source code repository. The following configuration options are available: -[container-server] +[DEFAULT] ================== ========== ============================================ Option Default Description ------------------ ---------- -------------------------------------------- +log_name swift Label used when logging log_facility LOG_LOCAL0 Syslog log facility log_level INFO Logging level swift_dir /etc/swift Swift configuration directory @@ -237,6 +251,16 @@ bind_ip 0.0.0.0 IP Address for server to bind to bind_port 6001 Port for server to bind to workers 1 Number of workers to fork user swift User to run as +================== ========== ============================================ + +[container-server] + +================== ========== ============================================ +Option Default Description +------------------ ---------- -------------------------------------------- +use paste.deploy entrypoint for the container + server. For most cases, this should be + `egg:swfit#container`. node_timeout 3 Request timeout to external services conn_timeout 0.5 Connection timeout to external services ================== ========== ============================================ @@ -246,8 +270,6 @@ conn_timeout 0.5 Connection timeout to external services ================== ========== =========================================== Option Default Description ------------------ ---------- ------------------------------------------- -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level per_diff 1000 concurrency 8 Number of replication workers to spawn run_pause 30 Time in seconds to wait between replication @@ -263,8 +285,6 @@ reclaim_age 604800 Time elapsed in seconds before a container ================== ========== =========================================== Option Default Description ------------------ ---------- ------------------------------------------- -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level interval 300 Minimum time for a pass to take concurrency 4 Number of updater workers to spawn node_timeout 3 Request timeout to external services @@ -277,8 +297,6 @@ slowdown 0.01 Time in seconds to wait between containers ================== ========== =========================================== Option Default Description ------------------ ---------- ------------------------------------------- -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level interval 1800 Minimum time for a pass to take node_timeout 10 Request timeout to external services conn_timeout 0.5 Connection timeout to external services @@ -293,11 +311,12 @@ etc/account-server.conf-sample in the source code repository. The following configuration options are available: -[account-server] +[DEFAULT] ================== ========== ============================================= Option Default Description ------------------ ---------- --------------------------------------------- +log_name swift Label used when logging log_facility LOG_LOCAL0 Syslog log facility log_level INFO Logging level swift_dir /etc/swift Swift configuration directory @@ -311,6 +330,16 @@ workers 1 Number of workers to fork user swift User to run as ================== ========== ============================================= +[account-server] + +================== ========== ============================================= +Option Default Description +------------------ ---------- --------------------------------------------- +use paste.deploy entrypoint for the account + server. For most cases, this should be + `egg:swfit#account`. +================== ========== ============================================= + [account-replicator] ================== ========== =========================================== @@ -359,7 +388,7 @@ conn_timeout 0.5 Connection timeout to external services Proxy Server Configuration -------------------------- -[proxy-server] +[DEFAULT] ============================ =============== ============================= Option Default Description @@ -369,13 +398,24 @@ log_level INFO Log level bind_ip 0.0.0.0 IP Address for server to bind to bind_port 80 Port for server to bind to -cert_file Path to the ssl .crt -key_file Path to the ssl .key swift_dir /etc/swift Swift configuration directory -log_headers True If True, log headers in each - request workers 1 Number of workers to fork user swift User to run as +cert_file Path to the ssl .crt +key_file Path to the ssl .key +============================ =============== ============================= + +[proxy-server] + +============================ =============== ============================= +Option Default Description +---------------------------- --------------- ----------------------------- +use paste.deploy entrypoint for + the proxy server. For most + cases, this should be + `egg:swift#proxy`. +log_headers True If True, log headers in each + request recheck_account_existence 60 Cache timeout in seconds to send memcached for account existance @@ -412,16 +452,21 @@ rate_limit_account_blacklist Comma separated list of completly ============================ =============== ============================= -[auth-server] +[auth] ============ =================================== ======================== Option Default Description ------------ ----------------------------------- ------------------------ -class swift.common.auth.DevAuthMiddleware Auth wsgi middleware - to use +use paste.deploy entrypoint + to use for auth. To + use the swift def auth, + set to: + `egg:swift#auth` ip 127.0.0.1 IP address of auth server port 11000 Port of auth server +ssl False If True, use SSL to + connect to auth node_timeout 10 Request timeout ============ =================================== ======================== diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 88fd423c8f..107d78e36f 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -164,195 +164,341 @@ good idea what to do on other environments. #. `. ~/.bashrc` #. Create `/etc/swift/auth-server.conf`:: - [auth-server] - default_cluster_url = http://127.0.0.1:8080/v1 + [DEFAULT] + log_name = auth user = + [pipeline:main] + pipeline = auth-server + + [app:auth-server] + use = egg:swift#auth + default_cluster_url = http://127.0.0.1:8080/v1 + #. Create `/etc/swift/proxy-server.conf`:: - [proxy-server] + [DEFAULT] + log_name = proxy bind_port = 8080 user = + [pipeline:main] + pipeline = healthcheck cache auth proxy + + [app:proxy] + use = egg:swift#proxy + + [filter:auth] + use = egg:swift#auth + + [filter:healthcheck] + use = egg:swift#healthcheck + + [filter:cache] + use = egg:swift#cache + #. Create `/etc/swift/account-server/1.conf`:: - [account-server] + [DEFAULT] + log_name = account devices = /srv/1/node mount_check = false bind_port = 6012 user = + [pipeline:main] + pipeline = account-server + + [app:account-server] + use = egg:swift#account + [account-replicator] + log_name = account-replicator vm_test_mode = yes [account-auditor] + log_name = account-auditor [account-reaper] + log_name = account-reaper #. Create `/etc/swift/account-server/2.conf`:: - [account-server] + [DEFAULT] + log_name = account devices = /srv/2/node mount_check = false bind_port = 6022 user = + [pipeline:main] + pipeline = account-server + + [app:account-server] + use = egg:swift#account + [account-replicator] + log_name = account-replicator vm_test_mode = yes [account-auditor] + log_name = account-auditor [account-reaper] + log_name = account-reaper #. Create `/etc/swift/account-server/3.conf`:: - [account-server] + [DEFAULT] + log_name = account devices = /srv/3/node mount_check = false bind_port = 6032 user = + [pipeline:main] + pipeline = account-server + + [app:account-server] + use = egg:swift#account + [account-replicator] + log_name = account-replicator vm_test_mode = yes [account-auditor] + log_name = account-auditor [account-reaper] + log_name = account-reaper #. Create `/etc/swift/account-server/4.conf`:: - [account-server] + [DEFAULT] + log_name = account devices = /srv/4/node mount_check = false bind_port = 6042 user = + [pipeline:main] + pipeline = account-server + + [app:account-server] + use = egg:swift#account + [account-replicator] + log_name = account-replicator vm_test_mode = yes [account-auditor] + log_name = account-auditor [account-reaper] + log_name = account-reaper #. Create `/etc/swift/container-server/1.conf`:: - [container-server] + [DEFAULT] + log_name = container devices = /srv/1/node mount_check = false bind_port = 6011 user = + [pipeline:main] + pipeline = container-server + + [app:container-server] + use = egg:swift#container + [container-replicator] + log_name = container-replicator vm_test_mode = yes [container-updater] + log_name = container-updater [container-auditor] + log_name = container-auditor #. Create `/etc/swift/container-server/2.conf`:: - [container-server] + [DEFAULT] + log_name = container devices = /srv/2/node mount_check = false bind_port = 6021 user = + [pipeline:main] + pipeline = container-server + + [app:container-server] + use = egg:swift#container + [container-replicator] + log_name = container-replicator vm_test_mode = yes [container-updater] + log_name = container-updater [container-auditor] + log_name = container-auditor + #. Create `/etc/swift/container-server/3.conf`:: - [container-server] + [DEFAULT] + log_name = container devices = /srv/3/node mount_check = false bind_port = 6031 user = + [pipeline:main] + pipeline = container-server + + [app:container-server] + use = egg:swift#container + [container-replicator] + log_name = container-replicator vm_test_mode = yes [container-updater] + log_name = container-updater [container-auditor] + log_name = container-auditor + #. Create `/etc/swift/container-server/4.conf`:: - [container-server] + [DEFAULT] + log_name = container devices = /srv/4/node mount_check = false bind_port = 6041 user = + [pipeline:main] + pipeline = container-server + + [app:container-server] + use = egg:swift#container + [container-replicator] + log_name = container-replicator vm_test_mode = yes [container-updater] + log_name = container-updater [container-auditor] + log_name = container-auditor + #. Create `/etc/swift/object-server/1.conf`:: - [object-server] + [DEFAULT] + log_name = object devices = /srv/1/node mount_check = false bind_port = 6010 user = + [pipeline:main] + pipeline = object-server + + [app:object-server] + use = egg:swift#object + [object-replicator] + log_name = object-replicator vm_test_mode = yes [object-updater] + log_name = object-updater [object-auditor] + log_name = object-auditor #. Create `/etc/swift/object-server/2.conf`:: - [object-server] + [DEFAULT] + log_name = object devices = /srv/2/node mount_check = false bind_port = 6020 user = + [pipeline:main] + pipeline = object-server + + [app:object-server] + use = egg:swift#object + [object-replicator] + log_name = object-replicator vm_test_mode = yes [object-updater] + log_name = object-updater [object-auditor] + log_name = object-auditor #. Create `/etc/swift/object-server/3.conf`:: - [object-server] + [DEFAULT] + log_name = object devices = /srv/3/node mount_check = false bind_port = 6030 user = + [pipeline:main] + pipeline = object-server + + [app:object-server] + use = egg:swift#object + [object-replicator] + log_name = object-replicator vm_test_mode = yes [object-updater] + log_name = object-updater [object-auditor] + log_name = object-auditor #. Create `/etc/swift/object-server/4.conf`:: - [object-server] + [DEFAULT] + log_name = object devices = /srv/4/node mount_check = false bind_port = 6040 user = + [pipeline:main] + pipeline = object-server + + [app:object-server] + use = egg:swift#object + [object-replicator] + log_name = object-replicator vm_test_mode = yes [object-updater] + log_name = object-updater [object-auditor] + log_name = object-auditor #. Create `~/bin/resetswift`:: diff --git a/doc/source/misc.rst b/doc/source/misc.rst index c45dcf0fc3..bae61a699a 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -38,7 +38,7 @@ Utils Auth ==== -.. automodule:: swift.common.auth +.. automodule:: swift.common.middleware.auth :members: :show-inheritance: @@ -85,7 +85,7 @@ Buffered HTTP Healthcheck =========== -.. automodule:: swift.common.healthcheck +.. automodule:: swift.common.middleware.healthcheck :members: :show-inheritance: diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 8b50760a49..6ebc2ed451 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -4,6 +4,7 @@ log_name = proxy # log_level = INFO # bind_ip = 0.0.0.0 # bind_port = 80 +# swift_dir = /etc/swift # workers = 1 # user = swift # cert_file = /etc/swift/proxy.crt @@ -14,8 +15,6 @@ pipeline = healthcheck cache auth proxy [app:proxy] use = egg:swift#proxy -filter-with = auth -# swift_dir = /etc/swift # log_headers = False # recheck_account_existence = 60 # recheck_container_existence = 60 From ca1cd5aa705b1eaad8b05d1383c37a1bef9a677a Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Thu, 19 Aug 2010 21:28:24 -0500 Subject: [PATCH 03/15] Fixed a couple of doc typos --- doc/source/deployment_guide.rst | 6 +++--- doc/source/development_saio.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 0d805ce806..c8b1b810b5 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -172,7 +172,7 @@ Option Default Description ------------------ ---------- --------------------------------------------- use paste.deploy entrypoint for the object server. For most cases, this should be - `egg:swfit#object`. + `egg:swift#object`. log_requests True Weather or not to log each request user swift User to run as node_timeout 3 Request timeout to external services @@ -260,7 +260,7 @@ Option Default Description ------------------ ---------- -------------------------------------------- use paste.deploy entrypoint for the container server. For most cases, this should be - `egg:swfit#container`. + `egg:swift#container`. node_timeout 3 Request timeout to external services conn_timeout 0.5 Connection timeout to external services ================== ========== ============================================ @@ -337,7 +337,7 @@ Option Default Description ------------------ ---------- --------------------------------------------- use paste.deploy entrypoint for the account server. For most cases, this should be - `egg:swfit#account`. + `egg:swift#account`. ================== ========== ============================================= [account-replicator] diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 107d78e36f..00e3ae3df1 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -35,7 +35,7 @@ good idea what to do on other environments. #. `apt-get install curl gcc bzr memcached python-configobj python-coverage python-dev python-nose python-setuptools python-simplejson python-xattr sqlite3 xfsprogs python-webob python-eventlet - python-greenlet` + python-greenlet python-pastedeploy` #. Install anything else you want, like screen, ssh, vim, etc. #. `fdisk /dev/sdb` (set up a single partition) #. `mkfs.xfs -i size=1024 /dev/sdb1` From cf35829e78054d22fe37b3bcb07dd2a53015c483 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Fri, 20 Aug 2010 15:28:02 +0000 Subject: [PATCH 04/15] Fixed a couple of typos, and some config things I missed in the object-replicator --- bin/swift-container-auditor | 2 +- bin/swift-object-replicator | 4 ++-- swift/obj/replicator.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/swift-container-auditor b/bin/swift-container-auditor index 6dfdc4592d..b001350039 100755 --- a/bin/swift-container-auditor +++ b/bin/swift-container-auditor @@ -47,7 +47,7 @@ if __name__ == '__main__': logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) sys.stdout = sys.stderr = utils.LoggerFileObject(logger) - utils.drop_privileges(onf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) try: os.setsid() diff --git a/bin/swift-object-replicator b/bin/swift-object-replicator index 6a2da4b4f3..2f9c4a63e8 100755 --- a/bin/swift-object-replicator +++ b/bin/swift-object-replicator @@ -32,8 +32,8 @@ if __name__ == '__main__': print "Usage: object-replicator CONFIG_FILE [once]" sys.exit() c = ConfigParser() - if not c.read(conf_file): - print "Unable to read config file: %s" % conf_file + if not c.read(sys.argv[1]): + print "Unable to read config file" sys.exit(1) if c.has_section('object-replicator'): conf = dict(c.items('object-replicator')) diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 8848748a94..d59a3ceb4f 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -216,8 +216,8 @@ class ObjectReplicator(object): self.swift_dir = conf.get('swift_dir', '/etc/swift') self.port = int(conf.get('bind_port', 6000)) self.concurrency = int(conf.get('concurrency', 1)) - self.timeout = conf['timeout'] - self.stats_interval = int(conf['stats_interval']) + self.timeout = conf.get('timeout', '5') + self.stats_interval = int(conf.get('stats_interval', '3600')) self.object_ring = Ring(join(self.swift_dir, 'object.ring.gz')) self.ring_check_interval = int(conf.get('ring_check_interval', 15)) self.next_check = time.time() + self.ring_check_interval From c62707ae72ebeb7da99e7622f889a66ae5d7fc79 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 24 Aug 2010 13:41:58 +0000 Subject: [PATCH 05/15] Refactored logging configuration so that it has sane defaults --- bin/swift-account-auditor | 15 +- bin/swift-account-reaper | 13 +- bin/swift-account-replicator | 15 +- bin/swift-account-server | 4 +- bin/swift-auth-server | 4 +- bin/swift-container-auditor | 15 +- bin/swift-container-replicator | 17 +- bin/swift-container-server | 5 +- bin/swift-container-updater | 16 +- bin/swift-object-auditor | 17 +- bin/swift-object-replicator | 18 +- bin/swift-object-server | 4 +- bin/swift-object-updater | 18 +- bin/swift-proxy-server | 4 +- doc/source/deployment_guide.rst | 297 +++++++++++++++++-------------- etc/account-server.conf-sample | 14 +- etc/auth-server.conf-sample | 6 +- etc/container-server.conf-sample | 12 +- etc/object-server.conf-sample | 12 +- etc/proxy-server.conf-sample | 6 +- swift/common/middleware/auth.py | 4 +- swift/common/middleware/cache.py | 4 +- swift/common/utils.py | 22 ++- swift/common/wsgi.py | 12 +- swift/proxy/server.py | 4 +- 25 files changed, 254 insertions(+), 304 deletions(-) diff --git a/bin/swift-account-auditor b/bin/swift-account-auditor index cba89e9e63..681adc6a26 100755 --- a/bin/swift-account-auditor +++ b/bin/swift-account-auditor @@ -24,23 +24,12 @@ from swift.common import utils if __name__ == '__main__': if len(sys.argv) < 2: - print "Usage: account-auditor CONFIG_FILE [once]" + print "Usage: swift-account-auditor CONFIG_FILE [once]" sys.exit() once = len(sys.argv) > 2 and sys.argv[2] == 'once' - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - - if c.has_section('account-auditor'): - conf = dict(c.items('account-auditor')) - else: - print "Unable to find account-auditor config section in %s." % \ - sys.argv[1] - sys.exit(1) - + conf = utils.readconf(sys.argv[1], 'account-auditor') logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ diff --git a/bin/swift-account-reaper b/bin/swift-account-reaper index c5619d0785..444d19a09d 100755 --- a/bin/swift-account-reaper +++ b/bin/swift-account-reaper @@ -29,18 +29,7 @@ if __name__ == '__main__': once = len(sys.argv) > 2 and sys.argv[2] == 'once' - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - - if c.has_section('account-reaper'): - conf = dict(c.items('account-reaper')) - else: - print "Unable to find account-reaper config section in %s." % \ - sys.argv[1] - sys.exit(1) - + conf = utils.readconf(sys.argv[1], 'account-reaper') logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ diff --git a/bin/swift-account-replicator b/bin/swift-account-replicator index 284c751598..18bea931a6 100755 --- a/bin/swift-account-replicator +++ b/bin/swift-account-replicator @@ -32,22 +32,11 @@ if __name__ == '__main__': optlist, args = getopt.getopt(sys.argv[1:], '', ['once']) if not args: - print "Usage: account-replicator <--once> CONFIG_FILE [once]" + print "Usage: swift-account-replicator <--once> CONFIG_FILE [once]" sys.exit() - c = ConfigParser() - if not c.read(args[0]): - print "Unable to read config file." - sys.exit(1) once = len(args) > 1 and args[1] == 'once' - - if c.has_section('account-replicator'): - conf = dict(c.items('account-replicator')) - else: - print "Unable to find account-replicator config section in %s." % \ - args[0] - sys.exit(1) - + conf = utils.readconf(sys.argv[1], 'account-replicator') utils.drop_privileges(conf.get('user', 'swift')) if once or '--once' in [opt[0] for opt in optlist]: AccountReplicator(conf).replicate_once() diff --git a/bin/swift-account-server b/bin/swift-account-server index 6be5706ea4..8b9a1ec68c 100755 --- a/bin/swift-account-server +++ b/bin/swift-account-server @@ -20,6 +20,6 @@ from swift.common.wsgi import run_wsgi if __name__ == '__main__': if len(sys.argv) != 2: - print "Usage: %s CONFIG_FILE" - run_wsgi(sys.argv[1], default_port=6002) + print "Usage: %s CONFIG_FILE" % sys.argv[0] + run_wsgi(sys.argv[1], 'account-server', default_port=6002) diff --git a/bin/swift-auth-server b/bin/swift-auth-server index 5f32912a9c..62e1d8b592 100755 --- a/bin/swift-auth-server +++ b/bin/swift-auth-server @@ -20,6 +20,6 @@ from swift.common.wsgi import run_wsgi if __name__ == '__main__': if len(sys.argv) != 2: - print "Usage: %s CONFIG_FILE" - run_wsgi(sys.argv[1], default_port=11000) + print "Usage: %s CONFIG_FILE" % sys.argv[0] + run_wsgi(sys.argv[1], 'auth-server', default_port=11000) diff --git a/bin/swift-container-auditor b/bin/swift-container-auditor index b001350039..3f22fbf698 100755 --- a/bin/swift-container-auditor +++ b/bin/swift-container-auditor @@ -24,23 +24,12 @@ from swift.common import utils if __name__ == '__main__': if len(sys.argv) < 2: - print "Usage: container-auditor CONFIG_FILE [once]" + print "Usage: swift-container-auditor CONFIG_FILE [once]" sys.exit() once = len(sys.argv) > 2 and sys.argv[2] == 'once' - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - - if c.has_section('container-auditor'): - conf = dict(c.items('container-auditor')) - else: - print "Unable to find container-auditor config section in %s." % \ - sys.argv[1] - sys.exit(1) - + conf = utils.readconf(sys.argv[1], 'container-auditor') logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ diff --git a/bin/swift-container-replicator b/bin/swift-container-replicator index 663cb51db7..6e8ae0f41d 100755 --- a/bin/swift-container-replicator +++ b/bin/swift-container-replicator @@ -32,22 +32,11 @@ if __name__ == '__main__': optlist, args = getopt.getopt(sys.argv[1:], '', ['once']) if not args: - print "Usage: container-replicator <--once> CONFIG_FILE [once]" - sys.exit() - - c = ConfigParser() - if not c.read(args[0]): - print "Unable to read config file." + print "Usage: swift-container-replicator <--once> CONFIG_FILE [once]" sys.exit(1) + once = len(args) > 1 and args[1] == 'once' - - if c.has_section('container-replicator'): - conf = dict(c.items('container-replicator')) - else: - print "Unable to find container-replicator config section in %s." % \ - args[0] - sys.exit(1) - + conf = utils.readconf(args[0], 'container-replicator') utils.drop_privileges(conf.get('user', 'swift')) if once or '--once' in [opt[0] for opt in optlist]: ContainerReplicator(conf).replicate_once() diff --git a/bin/swift-container-server b/bin/swift-container-server index f707eb091c..02201041b1 100755 --- a/bin/swift-container-server +++ b/bin/swift-container-server @@ -20,6 +20,5 @@ from swift.common.wsgi import run_wsgi if __name__ == '__main__': if len(sys.argv) != 2: - print "Usage: %s CONFIG_FILE" - run_wsgi(sys.argv[1], default_port=6001) - + print "Usage: %s CONFIG_FILE" % sys.argv[0] + run_wsgi(sys.argv[1], 'container-server', default_port=6001) diff --git a/bin/swift-container-updater b/bin/swift-container-updater index 7c23115123..c582d15577 100755 --- a/bin/swift-container-updater +++ b/bin/swift-container-updater @@ -24,23 +24,11 @@ from swift.common import utils if __name__ == '__main__': if len(sys.argv) < 2: - print "Usage: container-updater CONFIG_FILE [once]" + print "Usage: swift-container-updater CONFIG_FILE [once]" sys.exit() once = len(sys.argv) > 2 and sys.argv[2] == 'once' - - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - - if c.has_section('container-updater'): - conf = dict(c.items('container-updater')) - else: - print "Unable to find container-updater config section in %s." % \ - sys.argv[1] - sys.exit(1) - + conf = readconf(sys.argv[1], 'container-updater') utils.drop_privileges(conf.get('user', 'swift')) try: diff --git a/bin/swift-object-auditor b/bin/swift-object-auditor index e995ad8725..a80065414a 100755 --- a/bin/swift-object-auditor +++ b/bin/swift-object-auditor @@ -17,30 +17,17 @@ import os import signal import sys -from ConfigParser import ConfigParser from swift.obj.auditor import ObjectAuditor from swift.common import utils if __name__ == '__main__': if len(sys.argv) < 2: - print "Usage: object-auditor CONFIG_FILE [once]" + print "Usage: swift-object-auditor CONFIG_FILE [once]" sys.exit() once = len(sys.argv) > 2 and sys.argv[2] == 'once' - - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - - if c.has_section('object-auditor'): - conf = dict(c.items('object-auditor')) - else: - print "Unable to find object-auditor config section in %s." % \ - sys.argv[1] - sys.exit(1) - + conf = utils.readconf(sys.argv[1], 'object-auditor') logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ diff --git a/bin/swift-object-replicator b/bin/swift-object-replicator index 2f9c4a63e8..c01c96aff4 100755 --- a/bin/swift-object-replicator +++ b/bin/swift-object-replicator @@ -15,7 +15,6 @@ # limitations under the License. import sys -from ConfigParser import ConfigParser import logging import time @@ -23,25 +22,16 @@ from eventlet import sleep, hubs hubs.use_hub('poll') from swift.obj.replicator import ObjectReplicator -from swift.common.utils import get_logger, drop_privileges, LoggerFileObject +from swift.common.utils import get_logger, drop_privileges, LoggerFileObject, \ + readconf TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes')) if __name__ == '__main__': if len(sys.argv) < 2: - print "Usage: object-replicator CONFIG_FILE [once]" + print "Usage: swift-object-replicator CONFIG_FILE [once]" sys.exit() - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file" - sys.exit(1) - if c.has_section('object-replicator'): - conf = dict(c.items('object-replicator')) - else: - print "Unable to find object-replicator config section in %s" % \ - sys.argv[1] - sys.exit(1) - + conf = readconf(sys.argv[1], "object-replicator") once = len(sys.argv) > 2 and sys.argv[2] == 'once' logger = get_logger(conf) # log uncaught exceptions diff --git a/bin/swift-object-server b/bin/swift-object-server index ce74b17195..58b5120bd8 100755 --- a/bin/swift-object-server +++ b/bin/swift-object-server @@ -20,5 +20,5 @@ from swift.common.wsgi import run_wsgi if __name__ == '__main__': if len(sys.argv) != 2: - print "Usage: %s CONFIG_FILE" - run_wsgi(sys.argv[1], default_port=6000) + print "Usage: %s CONFIG_FILE" % sys.argv[0] + run_wsgi(sys.argv[1], 'object-server', default_port=6000) diff --git a/bin/swift-object-updater b/bin/swift-object-updater index a9512a4e76..375e63ec83 100755 --- a/bin/swift-object-updater +++ b/bin/swift-object-updater @@ -17,30 +17,18 @@ import os import signal import sys -from ConfigParser import ConfigParser from swift.obj.updater import ObjectUpdater from swift.common import utils if __name__ == '__main__': if len(sys.argv) < 2: - print "Usage: object-updater CONFIG_FILE [once]" - sys.exit() + print "Usage: swift-object-updater CONFIG_FILE [once]" + sys.exit(1) once = len(sys.argv) > 2 and sys.argv[2] == 'once' - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - - if c.has_section('object-updater'): - conf = dict(c.items('object-updater')) - else: - print "Unable to find object-updater config section in %s." % \ - sys.argv[1] - sys.exit(1) - + conf = utils.readconf(sys.argv[1], 'object-updater') utils.drop_privileges(conf.get('user', 'swift')) try: diff --git a/bin/swift-proxy-server b/bin/swift-proxy-server index c56df2e074..2ed1b5adf1 100755 --- a/bin/swift-proxy-server +++ b/bin/swift-proxy-server @@ -20,5 +20,5 @@ from swift.common.wsgi import run_wsgi if __name__ == '__main__': if len(sys.argv) != 2: - print "Usage: %s CONFIG_FILE" - run_wsgi(sys.argv[1], default_port=80) + print "Usage: %s CONFIG_FILE" % sys.argv[0] + run_wsgi(sys.argv[1], 'proxy-server', default_port=80) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index c8b1b810b5..e723ddf3ea 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -152,9 +152,6 @@ The following configuration options are available: ================== ========== ============================================= Option Default Description ------------------ ---------- --------------------------------------------- -log_name swift Label used when logging -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level swift_dir /etc/swift Swift configuration directory devices /srv/node Parent directory of where devices are mounted mount_check true Weather or not check if the devices are @@ -167,63 +164,75 @@ workers 1 Number of workers to fork [object-server] -================== ========== ============================================= -Option Default Description ------------------- ---------- --------------------------------------------- -use paste.deploy entrypoint for the object - server. For most cases, this should be - `egg:swift#object`. -log_requests True Weather or not to log each request -user swift User to run as -node_timeout 3 Request timeout to external services -conn_timeout 0.5 Connection timeout to external services -network_chunk_size 65536 Size of chunks to read/write over the - network -disk_chunk_size 65536 Size of chunks to read/write to disk -max_upload_time 86400 Maximum time allowed to upload an object -slow 0 If > 0, Minimum time in seconds for a PUT - or DELETE request to complete -================== ========== ============================================= +================== ============= =========================================== +Option Default Description +------------------ ------------- ------------------------------------------- +use paste.deploy entrypoint for the object + server. For most cases, this should be + `egg:swift#object`. +log_name object-server Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +log_requests True Weather or not to log each request +user swift User to run as +node_timeout 3 Request timeout to external services +conn_timeout 0.5 Connection timeout to external services +network_chunk_size 65536 Size of chunks to read/write over the + network +disk_chunk_size 65536 Size of chunks to read/write to disk +max_upload_time 86400 Maximum time allowed to upload an object +slow 0 If > 0, Minimum time in seconds for a PUT + or DELETE request to complete +================== ============= =========================================== [object-replicator] -================== ========== =========================================== -Option Default Description ------------------- ---------- ------------------------------------------- -daemonize yes Weather or not to run replication as a - daemon -run_pause 30 Time in seconds to wait between replication - passes -concurrency 1 Number of replication workers to spawn -timeout 5 Timeout value sent to rsync --timeout and - --contimeout options -stats_interval 3600 Interval in seconds between logging - replication statistics -reclaim_age 604800 Time elapsed in seconds before an object - can be reclaimed -================== ========== =========================================== +================== ================= ======================================= +Option Default Description +------------------ ----------------- --------------------------------------- +log_name object-replicator Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +daemonize yes Weather or not to run replication as a + daemon +run_pause 30 Time in seconds to wait between + replication passes +concurrency 1 Number of replication workers to spawn +timeout 5 Timeout value sent to rsync --timeout + and --contimeout options +stats_interval 3600 Interval in seconds between logging + replication statistics +reclaim_age 604800 Time elapsed in seconds before an + object can be reclaimed +================== ================= ======================================= [object-updater] -================== ========== =========================================== -Option Default Description ------------------- ---------- ------------------------------------------- -interval 300 Minimum time for a pass to take -concurrency 1 Number of updater workers to spawn -node_timeout 10 Request timeout to external services -conn_timeout 0.5 Connection timeout to external services -slowdown 0.01 Time in seconds to wait between objects -================== ========== =========================================== +================== ============== ========================================== +Option Default Description +------------------ -------------- ------------------------------------------ +log_name object-updater Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +interval 300 Minimum time for a pass to take +concurrency 1 Number of updater workers to spawn +node_timeout 10 Request timeout to external services +conn_timeout 0.5 Connection timeout to external services +slowdown 0.01 Time in seconds to wait between objects +================== ============== ========================================== [object-auditor] -================== ========== =========================================== -Option Default Description ------------------- ---------- ------------------------------------------- -interval 1800 Minimum time for a pass to take -node_timeout 10 Request timeout to external services -conn_timeout 0.5 Connection timeout to external services -================== ========== =========================================== +================== ============== ========================================== +Option Default Description +------------------ -------------- ------------------------------------------ +log_name object-auditor Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +interval 1800 Minimum time for a pass to take +node_timeout 10 Request timeout to external services +conn_timeout 0.5 Connection timeout to external services +================== ============== ========================================== ------------------------------ Container Server Configuration @@ -239,9 +248,6 @@ The following configuration options are available: ================== ========== ============================================ Option Default Description ------------------ ---------- -------------------------------------------- -log_name swift Label used when logging -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level swift_dir /etc/swift Swift configuration directory devices /srv/node Parent irectory of where devices are mounted mount_check true Weather or not check if the devices are @@ -255,52 +261,67 @@ user swift User to run as [container-server] -================== ========== ============================================ -Option Default Description ------------------- ---------- -------------------------------------------- -use paste.deploy entrypoint for the container - server. For most cases, this should be - `egg:swift#container`. -node_timeout 3 Request timeout to external services -conn_timeout 0.5 Connection timeout to external services -================== ========== ============================================ +================== ================ ======================================== +Option Default Description +------------------ ---------------- ---------------------------------------- +use paste.deploy entrypoint for the container + server. For most cases, this should be + `egg:swift#container`. +log_name container-server Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +node_timeout 3 Request timeout to external services +conn_timeout 0.5 Connection timeout to external services +================== ================ ======================================== [container-replicator] -================== ========== =========================================== -Option Default Description ------------------- ---------- ------------------------------------------- +================== ==================== ==================================== +Option Default Description +------------------ -------------------- ------------------------------------ +log_name container-replicator Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level per_diff 1000 -concurrency 8 Number of replication workers to spawn -run_pause 30 Time in seconds to wait between replication - passes -node_timeout 10 Request timeout to external services -conn_timeout 0.5 Connection timeout to external services -reclaim_age 604800 Time elapsed in seconds before a container - can be reclaimed -================== ========== =========================================== +concurrency 8 Number of replication workers to + spawn +run_pause 30 Time in seconds to wait between + replication passes +node_timeout 10 Request timeout to external services +conn_timeout 0.5 Connection timeout to external + services +reclaim_age 604800 Time elapsed in seconds before a + container can be reclaimed +================== ==================== ==================================== [container-updater] -================== ========== =========================================== -Option Default Description ------------------- ---------- ------------------------------------------- -interval 300 Minimum time for a pass to take -concurrency 4 Number of updater workers to spawn -node_timeout 3 Request timeout to external services -conn_timeout 0.5 Connection timeout to external services -slowdown 0.01 Time in seconds to wait between containers -================== ========== =========================================== +================== ================= ======================================= +Option Default Description +------------------ ----------------- --------------------------------------- +log_name container-updater Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +interval 300 Minimum time for a pass to take +concurrency 4 Number of updater workers to spawn +node_timeout 3 Request timeout to external services +conn_timeout 0.5 Connection timeout to external services +slowdown 0.01 Time in seconds to wait between + containers +================== ================= ======================================= [container-auditor] -================== ========== =========================================== -Option Default Description ------------------- ---------- ------------------------------------------- -interval 1800 Minimum time for a pass to take -node_timeout 10 Request timeout to external services -conn_timeout 0.5 Connection timeout to external services -================== ========== =========================================== +================== ================= ======================================= +Option Default Description +------------------ ----------------- --------------------------------------- +log_name container-auditor Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +interval 1800 Minimum time for a pass to take +node_timeout 10 Request timeout to external services +conn_timeout 0.5 Connection timeout to external services +================== ================= ======================================= ---------------------------- Account Server Configuration @@ -316,9 +337,6 @@ The following configuration options are available: ================== ========== ============================================= Option Default Description ------------------ ---------- --------------------------------------------- -log_name swift Label used when logging -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level swift_dir /etc/swift Swift configuration directory devices /srv/node Parent directory or where devices are mounted mount_check true Weather or not check if the devices are @@ -332,57 +350,63 @@ user swift User to run as [account-server] -================== ========== ============================================= -Option Default Description ------------------- ---------- --------------------------------------------- -use paste.deploy entrypoint for the account - server. For most cases, this should be - `egg:swift#account`. -================== ========== ============================================= +================== ============== ========================================== +Option Default Description +------------------ -------------- ------------------------------------------ +use paste.deploy entrypoint for the account + server. For most cases, this should be + `egg:swift#account`. +log_name account-server Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +================== ============== ========================================== [account-replicator] -================== ========== =========================================== -Option Default Description ------------------- ---------- ------------------------------------------- -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level +================== ================== ====================================== +Option Default Description +------------------ ------------------ -------------------------------------- +log_name account-replicator Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level per_diff 1000 -concurrency 8 Number of replication workers to spawn -run_pause 30 Time in seconds to wait between replication - passes -node_timeout 10 Request timeout to external services -conn_timeout 0.5 Connection timeout to external services -reclaim_age 604800 Time elapsed in seconds before a account - can be reclaimed -================== ========== =========================================== +concurrency 8 Number of replication workers to spawn +run_pause 30 Time in seconds to wait between + replication passes +node_timeout 10 Request timeout to external services +conn_timeout 0.5 Connection timeout to external services +reclaim_age 604800 Time elapsed in seconds before an + account can be reclaimed +================== ================== ====================================== [account-auditor] -==================== ========== =========================================== -Option Default Description --------------------- ---------- ------------------------------------------- -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level -interval 1800 Minimum time for a pass to take -max_container_count 100 Maximum containers randomly picked for - a given account audit -node_timeout 10 Request timeout to external services -conn_timeout 0.5 Connection timeout to external services -==================== ========== =========================================== +==================== =============== ======================================= +Option Default Description +-------------------- --------------- --------------------------------------- +log_name account-auditor Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +interval 1800 Minimum time for a pass to take +max_container_count 100 Maximum containers randomly picked for + a given account audit +node_timeout 10 Request timeout to external services +conn_timeout 0.5 Connection timeout to external services +==================== =============== ======================================= [account-reaper] -================== ========== =========================================== -Option Default Description ------------------- ---------- ------------------------------------------- -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Logging level -concurrency 25 Number of replication workers to spawn -interval 3600 Minimum time for a pass to take -node_timeout 10 Request timeout to external services -conn_timeout 0.5 Connection timeout to external services -================== ========== =========================================== +================== =============== ========================================= +Option Default Description +------------------ --------------- ----------------------------------------- +log_name account-auditor Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Logging level +concurrency 25 Number of replication workers to spawn +interval 3600 Minimum time for a pass to take +node_timeout 10 Request timeout to external services +conn_timeout 0.5 Connection timeout to external services +================== =============== ========================================= -------------------------- Proxy Server Configuration @@ -393,8 +417,6 @@ Proxy Server Configuration ============================ =============== ============================= Option Default Description ---------------------------- --------------- ----------------------------- -log_facility LOG_LOCAL0 Syslog log facility -log_level INFO Log level bind_ip 0.0.0.0 IP Address for server to bind to bind_port 80 Port for server to bind to @@ -414,6 +436,9 @@ use paste.deploy entrypoint for the proxy server. For most cases, this should be `egg:swift#proxy`. +log_name proxy-server Label used when logging +log_facility LOG_LOCAL0 Syslog log facility +log_level INFO Log level log_headers True If True, log headers in each request recheck_account_existence 60 Cache timeout in seconds to diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index 303ab0e00b..2c072285f5 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -1,7 +1,4 @@ [DEFAULT] -log_name = account -# log_facility = LOG_LOCAL0 -# log_level = INFO # bind_ip = 0.0.0.0 # bind_port = 6002 # workers = 1 @@ -15,9 +12,12 @@ pipeline = account-server [app:account-server] use = egg:swift#account +# log_name = account-server +# log_facility = LOG_LOCAL0 +# log_level = INFO [account-replicator] -log_name = account-replicator +# log_name = account-replicator # log_facility = LOG_LOCAL0 # log_level = INFO # per_diff = 1000 @@ -34,7 +34,7 @@ log_name = account-replicator # reclaim_age = 86400 [account-stats] -log_name = account-stats +# log_name = account-stats # cf_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31 # container_name = account_stats # proxy_server_conf = /etc/swift/proxy-server.conf @@ -42,7 +42,7 @@ log_name = account-stats # log_level = INFO [account-auditor] -log_name = account-auditor +# log_name = account-auditor # Will audit, at most, 1 account per device per interval # interval = 1800 # Maximum containers randomly picked for a given account audit @@ -53,7 +53,7 @@ log_name = account-auditor # log_level = INFO [account-reaper] -log_name = account-reaper +# log_name = account-reaper # concurrency = 25 # interval = 3600 # node_timeout = 10 diff --git a/etc/auth-server.conf-sample b/etc/auth-server.conf-sample index 61c1ffafd9..ae02fb07da 100644 --- a/etc/auth-server.conf-sample +++ b/etc/auth-server.conf-sample @@ -1,9 +1,6 @@ [DEFAULT] -log_name = auth # bind_ip = 0.0.0.0 # bind_port = 11000 -# log_facility = LOG_LOCAL0 -# log_level = INFO # workers = 1 # user = swift # swift_dir = /etc/swift @@ -15,6 +12,9 @@ pipeline = auth-server [app:auth-server] use = egg:swift#auth +# log_name = auth-server +# log_facility = LOG_LOCAL0 +# log_level = INFO # reseller_prefix = AUTH # default_cluster_url = http://127.0.0.1:9000/v1 # token_life = 86400 diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index ccad08c710..f069543f96 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -1,7 +1,4 @@ [DEFAULT] -log_name = container -# log_facility = LOG_LOCAL0 -# log_level = INFO # bind_ip = 0.0.0.0 # bind_port = 6001 # workers = 1 @@ -15,11 +12,14 @@ pipeline = container-server [app:container-server] use = egg:swift#container +# log_name = container-server +# log_facility = LOG_LOCAL0 +# log_level = INFO # node_timeout = 3 # conn_timeout = 0.5 [container-replicator] -log_name = container-replicator +# log_name = container-replicator # per_diff = 1000 # concurrency = 8 # run_pause = 30 @@ -29,7 +29,7 @@ log_name = container-replicator # reclaim_age = 604800 [container-updater] -log_name = container-updater +# log_name = container-updater # interval = 300 # concurrency = 4 # node_timeout = 3 @@ -38,7 +38,7 @@ log_name = container-updater # slowdown = 0.01 [container-auditor] -log_name = container-auditor +# log_name = container-auditor # Will audit, at most, 1 container per device per interval # interval = 1800 # Maximum objects randomly picked for a given container audit diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index 81a35c5cb7..61756a6194 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -1,7 +1,4 @@ [DEFAULT] -log_name = object -# log_facility = LOG_LOCAL0 -# log_level = INFO # bind_ip = 0.0.0.0 # bind_port = 6000 # workers = 1 @@ -15,6 +12,9 @@ pipeline = object-server [app:object-server] use = egg:swift#object +# log_name = object-server +# log_facility = LOG_LOCAL0 +# log_level = INFO # log_requests = True # node_timeout = 3 # conn_timeout = 0.5 @@ -24,7 +24,7 @@ use = egg:swift#object # slow = 1 [object-replicator] -log_name = object-replicator +# log_name = object-replicator # daemonize = on # run_pause = 30 # concurrency = 1 @@ -34,7 +34,7 @@ log_name = object-replicator # reclaim_age = 604800 [object-updater] -log_name = object-updater +# log_name = object-updater # interval = 300 # concurrency = 1 # node_timeout = 10 @@ -43,7 +43,7 @@ log_name = object-updater # slowdown = 0.01 [object-auditor] -log_name = object-auditor +# log_name = object-auditor # Will audit, at most, 1 object per device per interval # interval = 1800 # node_timeout = 10 diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 6ebc2ed451..e81cea65a4 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -1,7 +1,4 @@ [DEFAULT] -log_name = proxy -# log_facility = LOG_LOCAL0 -# log_level = INFO # bind_ip = 0.0.0.0 # bind_port = 80 # swift_dir = /etc/swift @@ -15,6 +12,9 @@ pipeline = healthcheck cache auth proxy [app:proxy] use = egg:swift#proxy +# log_name = proxy-server +# log_facility = LOG_LOCAL0 +# log_level = INFO # log_headers = False # recheck_account_existence = 60 # recheck_container_existence = 60 diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index d35b992418..eb920fad41 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -105,6 +105,8 @@ class DevAuthMiddleware(object): return True def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) def auth_filter(app): - return DevAuthMiddleware(app, local_conf) + return DevAuthMiddleware(app, conf) return auth_filter diff --git a/swift/common/middleware/cache.py b/swift/common/middleware/cache.py index a0dfc81b86..0199261751 100644 --- a/swift/common/middleware/cache.py +++ b/swift/common/middleware/cache.py @@ -29,6 +29,8 @@ class CacheMiddleware(object): return self.app(env, start_response) def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) def cache_filter(app): - return CacheMiddleware(app, local_conf) + return CacheMiddleware(app, conf) return cache_filter diff --git a/swift/common/utils.py b/swift/common/utils.py index e4962f2367..920d54d644 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -31,6 +31,7 @@ import ctypes import ctypes.util import fcntl import struct +from ConfigParser import ConfigParser import eventlet from eventlet import greenio, GreenPool, sleep, Timeout, listen @@ -331,6 +332,7 @@ def get_logger(conf, name=None): log_facility = LOG_LOCAL0 log_level = INFO + log_name = swift :param conf: Configuration dict to read settings from :param name: Name of the logger @@ -345,7 +347,8 @@ def get_logger(conf, name=None): if name is None: name = conf.get('log_name', 'swift') get_logger.handler = SysLogHandler(address='/dev/log', - facility=getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'), + facility=getattr(SysLogHandler, + conf.get('log_facility', 'LOG_LOCAL0'), SysLogHandler.LOG_LOCAL0)) root_logger.addHandler(get_logger.handler) root_logger.setLevel( @@ -524,3 +527,20 @@ def item_from_env(env, item_name): def cache_from_env(env): return item_from_env(env, 'swift.cache') + +def readconf(conf, section_name, log_name=None): + c = ConfigParser() + if not c.read(conf): + print "Unable to read config file %s" % conf + sys.exit(1) + if c.has_section(section_name): + conf = dict(c.items(section_name)) + else: + print "Unable to find %s config section in %s" % (section_name, conf) + sys.exit(1) + if "log_name" not in conf: + if log_name is not None: + conf['log_name'] = log_name + else: + conf['log_name'] = section_name + return conf diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 2944bd3bdf..13c16aad7f 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -59,25 +59,27 @@ def monkey_patch_mimetools(): # We might be able to pull pieces of this out to test, but right now it seems # like more work than it's worth. -def run_wsgi(conf_file, *args, **kwargs): # pragma: no cover +def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover """ Loads common settings from conf, then instantiates app and runs the server using the specified number of workers. :param conf_file: Path to paste.deploy style configuration file + :param app_section: App name from conf file to load config from """ try: - app = loadapp('config:%s' % conf_file) - conf = appconfig('config:%s' % conf_file) + conf = appconfig('config:%s' % conf_file, name=app_section) + log_name = conf.get('log_name', app_section) + app = loadapp('config:%s' % conf_file, + global_conf={'log_name':log_name}) except Exception, e: print "Error trying to load config %s: %s" % (conf_file, e) return if 'logger' in kwargs: logger = kwargs['logger'] else: - logger = get_logger(conf) - + logger = get_logger(conf, log_name) # log uncaught exceptions sys.excepthook = lambda * exc_info: \ logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 82b642ab2c..5cfb23fbdc 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1264,5 +1264,7 @@ class Application(BaseApplication): return None def app_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) """paste.deploy app factory for creating WSGI proxy apps.""" - return Application(local_conf) + return Application(conf) From 183e0623e77f9f52187103d7b1f636b55957e627 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 24 Aug 2010 13:51:36 +0000 Subject: [PATCH 06/15] Fixed doc typos and pep 8 cleanup on setup.py --- doc/source/deployment_guide.rst | 20 ++++++++++---------- setup.py | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index e723ddf3ea..8ba9d660c0 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -135,8 +135,8 @@ General Server Configuration ---------------------------- Swift uses paste.deploy to manage server configurations. Default configuration -options are set in the `[DEFAULT]` section, and any options specidifed there -can be overriden in any of the other sections. +options are set in the `[DEFAULT]` section, and any options specified there +can be overridden in any of the other sections. --------------------------- Object Server Configuration @@ -167,7 +167,7 @@ workers 1 Number of workers to fork ================== ============= =========================================== Option Default Description ------------------ ------------- ------------------------------------------- -use paste.deploy entrypoint for the object +use paste.deploy entry point for the object server. For most cases, this should be `egg:swift#object`. log_name object-server Label used when logging @@ -264,9 +264,9 @@ user swift User to run as ================== ================ ======================================== Option Default Description ------------------ ---------------- ---------------------------------------- -use paste.deploy entrypoint for the container - server. For most cases, this should be - `egg:swift#container`. +use paste.deploy entry point for the + container server. For most cases, this + should be `egg:swift#container`. log_name container-server Label used when logging log_facility LOG_LOCAL0 Syslog log facility log_level INFO Logging level @@ -353,7 +353,7 @@ user swift User to run as ================== ============== ========================================== Option Default Description ------------------ -------------- ------------------------------------------ -use paste.deploy entrypoint for the account +use paste.deploy entry point for the account server. For most cases, this should be `egg:swift#account`. log_name account-server Label used when logging @@ -432,7 +432,7 @@ key_file Path to the ssl .key ============================ =============== ============================= Option Default Description ---------------------------- --------------- ----------------------------- -use paste.deploy entrypoint for +use paste.deploy entry point for the proxy server. For most cases, this should be `egg:swift#proxy`. @@ -482,9 +482,9 @@ rate_limit_account_blacklist Comma separated list of ============ =================================== ======================== Option Default Description ------------ ----------------------------------- ------------------------ -use paste.deploy entrypoint +use paste.deploy entry point to use for auth. To - use the swift def auth, + use the swift dev auth, set to: `egg:swift#auth` ip 127.0.0.1 IP address of auth diff --git a/setup.py b/setup.py index 02dd389495..931442b1da 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( 'Programming Language :: Python :: 2.6', 'Environment :: No Input/Output (Daemon)', ], - install_requires=[], # removed for better compat + install_requires=[], # removed for better compat scripts=[ 'bin/st', 'bin/swift-account-auditor', 'bin/swift-account-audit', 'bin/swift-account-reaper', @@ -76,14 +76,14 @@ setup( 'bin/swift-stats-report' ], entry_points={ - 'paste.app_factory' : [ + 'paste.app_factory': [ 'proxy=swift.proxy.server:app_factory', 'object=swift.obj.server:app_factory', 'container=swift.container.server:app_factory', 'account=swift.account.server:app_factory', 'auth=swift.auth.server:app_factory', ], - 'paste.filter_factory' : [ + 'paste.filter_factory': [ 'auth=swift.common.middleware.auth:filter_factory', 'healthcheck=swift.common.middleware.healthcheck:filter_factory', 'cache=swift.common.middleware.cache:filter_factory', From 7bbbf3dbefcecdccd5fba71bda5406a6ab24d27e Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 24 Aug 2010 13:58:32 +0000 Subject: [PATCH 07/15] Renamed the cache middleware to memcache middleware --- etc/proxy-server.conf-sample | 2 +- setup.py | 2 +- swift/common/middleware/{cache.py => memcache.py} | 4 ++-- .../common/middleware/{test_cache.py => test_memcache.py} | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) rename swift/common/middleware/{cache.py => memcache.py} (93%) rename test/unit/common/middleware/{test_cache.py => test_memcache.py} (91%) diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index e81cea65a4..04fb01fa68 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -46,7 +46,7 @@ use = egg:swift#auth use = egg:swift#healthcheck [filter:cache] -use = egg:swift#cache +use = egg:swift#memcache # Default for memcache_servers is below, but you can specify multiple servers # with the format: 10.1.2.3:11211,10.1.2.4:11211 # memcache_servers = 127.0.0.1:11211 diff --git a/setup.py b/setup.py index 931442b1da..2a3084e110 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ setup( 'paste.filter_factory': [ 'auth=swift.common.middleware.auth:filter_factory', 'healthcheck=swift.common.middleware.healthcheck:filter_factory', - 'cache=swift.common.middleware.cache:filter_factory', + 'memcache=swift.common.middleware.memcache:filter_factory', ], }, ) diff --git a/swift/common/middleware/cache.py b/swift/common/middleware/memcache.py similarity index 93% rename from swift/common/middleware/cache.py rename to swift/common/middleware/memcache.py index 0199261751..8b7030eafa 100644 --- a/swift/common/middleware/cache.py +++ b/swift/common/middleware/memcache.py @@ -14,7 +14,7 @@ from swift.common.memcached import MemcacheRing -class CacheMiddleware(object): +class MemcacheMiddleware(object): """ Caching middleware that manages caching in swift. """ @@ -32,5 +32,5 @@ def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def cache_filter(app): - return CacheMiddleware(app, conf) + return MemcacheMiddleware(app, conf) return cache_filter diff --git a/test/unit/common/middleware/test_cache.py b/test/unit/common/middleware/test_memcache.py similarity index 91% rename from test/unit/common/middleware/test_cache.py rename to test/unit/common/middleware/test_memcache.py index a83233e2eb..a6f9336fec 100644 --- a/test/unit/common/middleware/test_cache.py +++ b/test/unit/common/middleware/test_memcache.py @@ -17,7 +17,7 @@ import unittest from webob import Request -from swift.common.middleware import cache +from swift.common.middleware import memcache from swift.common.memcached import MemcacheRing class FakeApp(object): @@ -30,7 +30,7 @@ def start_response(*args): class TestCacheMiddleware(unittest.TestCase): def setUp(self): - self.app = cache.CacheMiddleware(FakeApp(), {}) + self.app = memcache.MemcacheMiddleware(FakeApp(), {}) def test_cache_middleware(self): req = Request.blank('/something', environ={'REQUEST_METHOD': 'GET'}) From 0e881304ae9beefe9c3b0d2126546d8cc3f943e2 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 24 Aug 2010 14:04:44 +0000 Subject: [PATCH 08/15] Changed default port of proxy and auth to be 8080, and cleaned up exits --- bin/swift-account-server | 2 +- bin/swift-auth-server | 4 ++-- bin/swift-container-server | 2 +- bin/swift-object-server | 2 +- bin/swift-proxy-server | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/swift-account-server b/bin/swift-account-server index 8b9a1ec68c..55e2a10857 100755 --- a/bin/swift-account-server +++ b/bin/swift-account-server @@ -20,6 +20,6 @@ from swift.common.wsgi import run_wsgi if __name__ == '__main__': if len(sys.argv) != 2: - print "Usage: %s CONFIG_FILE" % sys.argv[0] + sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) run_wsgi(sys.argv[1], 'account-server', default_port=6002) diff --git a/bin/swift-auth-server b/bin/swift-auth-server index 62e1d8b592..2f2a3eeb3d 100755 --- a/bin/swift-auth-server +++ b/bin/swift-auth-server @@ -20,6 +20,6 @@ from swift.common.wsgi import run_wsgi if __name__ == '__main__': if len(sys.argv) != 2: - print "Usage: %s CONFIG_FILE" % sys.argv[0] - run_wsgi(sys.argv[1], 'auth-server', default_port=11000) + sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) + run_wsgi(sys.argv[1], 'auth-server', default_port=8080) diff --git a/bin/swift-container-server b/bin/swift-container-server index 02201041b1..a15a35cf04 100755 --- a/bin/swift-container-server +++ b/bin/swift-container-server @@ -20,5 +20,5 @@ from swift.common.wsgi import run_wsgi if __name__ == '__main__': if len(sys.argv) != 2: - print "Usage: %s CONFIG_FILE" % sys.argv[0] + sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) run_wsgi(sys.argv[1], 'container-server', default_port=6001) diff --git a/bin/swift-object-server b/bin/swift-object-server index 58b5120bd8..39c9246d6b 100755 --- a/bin/swift-object-server +++ b/bin/swift-object-server @@ -20,5 +20,5 @@ from swift.common.wsgi import run_wsgi if __name__ == '__main__': if len(sys.argv) != 2: - print "Usage: %s CONFIG_FILE" % sys.argv[0] + sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) run_wsgi(sys.argv[1], 'object-server', default_port=6000) diff --git a/bin/swift-proxy-server b/bin/swift-proxy-server index 2ed1b5adf1..7599f87280 100755 --- a/bin/swift-proxy-server +++ b/bin/swift-proxy-server @@ -20,5 +20,5 @@ from swift.common.wsgi import run_wsgi if __name__ == '__main__': if len(sys.argv) != 2: - print "Usage: %s CONFIG_FILE" % sys.argv[0] - run_wsgi(sys.argv[1], 'proxy-server', default_port=80) + sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) + run_wsgi(sys.argv[1], 'proxy-server', default_port=8080) From 459e1e95bbf10ee9413db9c029d2f961364e209c Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 24 Aug 2010 14:08:16 +0000 Subject: [PATCH 09/15] Changed default cluster url to point to new default proxy port of 8080, and set auth default port back to 11000 --- bin/swift-auth-server | 3 +-- etc/auth-server.conf-sample | 2 +- swift/auth/server.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/swift-auth-server b/bin/swift-auth-server index 2f2a3eeb3d..a8f03c6e92 100755 --- a/bin/swift-auth-server +++ b/bin/swift-auth-server @@ -21,5 +21,4 @@ from swift.common.wsgi import run_wsgi if __name__ == '__main__': if len(sys.argv) != 2: sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) - run_wsgi(sys.argv[1], 'auth-server', default_port=8080) - + run_wsgi(sys.argv[1], 'auth-server', default_port=11000) diff --git a/etc/auth-server.conf-sample b/etc/auth-server.conf-sample index ae02fb07da..0f818d4231 100644 --- a/etc/auth-server.conf-sample +++ b/etc/auth-server.conf-sample @@ -16,7 +16,7 @@ use = egg:swift#auth # log_facility = LOG_LOCAL0 # log_level = INFO # reseller_prefix = AUTH -# default_cluster_url = http://127.0.0.1:9000/v1 +# default_cluster_url = http://127.0.0.1:8080/v1 # token_life = 86400 # log_headers = False # node_timeout = 10 diff --git a/swift/auth/server.py b/swift/auth/server.py index 48b40ab50f..5658790c88 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -93,7 +93,7 @@ class AuthController(object): self.logger = get_logger(conf) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.default_cluster_url = \ - conf.get('default_cluster_url', 'http://127.0.0.1:9000/v1') + conf.get('default_cluster_url', 'http://127.0.0.1:8080/v1') self.token_life = int(conf.get('token_life', 86400)) self.log_headers = conf.get('log_headers') == 'True' if ring: From 04a5ccb4b1e243ee163ec28478743ccff4959f39 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 24 Aug 2010 14:10:36 +0000 Subject: [PATCH 10/15] Added vm_test_mode to the sample configs --- etc/account-server.conf-sample | 1 + etc/container-server.conf-sample | 1 + etc/object-server.conf-sample | 1 + 3 files changed, 3 insertions(+) diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index 2c072285f5..d7cbf2d656 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -18,6 +18,7 @@ use = egg:swift#account [account-replicator] # log_name = account-replicator +# vm_test_mode = no # log_facility = LOG_LOCAL0 # log_level = INFO # per_diff = 1000 diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index f069543f96..e17ccd30be 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -20,6 +20,7 @@ use = egg:swift#container [container-replicator] # log_name = container-replicator +# vm_test_mode = no # per_diff = 1000 # concurrency = 8 # run_pause = 30 diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index 61756a6194..a72ef879d7 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -25,6 +25,7 @@ use = egg:swift#object [object-replicator] # log_name = object-replicator +# vm_test_mode = no # daemonize = on # run_pause = 30 # concurrency = 1 From 83e8d76d9f44745b74d76814ecee1b003c1816b0 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 24 Aug 2010 14:23:49 +0000 Subject: [PATCH 11/15] Updated SAIO to be in line with the logging refactor --- doc/source/development_saio.rst | 50 --------------------------------- 1 file changed, 50 deletions(-) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 00e3ae3df1..f14b9ca4de 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -165,7 +165,6 @@ good idea what to do on other environments. #. Create `/etc/swift/auth-server.conf`:: [DEFAULT] - log_name = auth user = [pipeline:main] @@ -178,7 +177,6 @@ good idea what to do on other environments. #. Create `/etc/swift/proxy-server.conf`:: [DEFAULT] - log_name = proxy bind_port = 8080 user = @@ -200,7 +198,6 @@ good idea what to do on other environments. #. Create `/etc/swift/account-server/1.conf`:: [DEFAULT] - log_name = account devices = /srv/1/node mount_check = false bind_port = 6012 @@ -213,19 +210,15 @@ good idea what to do on other environments. use = egg:swift#account [account-replicator] - log_name = account-replicator vm_test_mode = yes [account-auditor] - log_name = account-auditor [account-reaper] - log_name = account-reaper #. Create `/etc/swift/account-server/2.conf`:: [DEFAULT] - log_name = account devices = /srv/2/node mount_check = false bind_port = 6022 @@ -238,19 +231,15 @@ good idea what to do on other environments. use = egg:swift#account [account-replicator] - log_name = account-replicator vm_test_mode = yes [account-auditor] - log_name = account-auditor [account-reaper] - log_name = account-reaper #. Create `/etc/swift/account-server/3.conf`:: [DEFAULT] - log_name = account devices = /srv/3/node mount_check = false bind_port = 6032 @@ -263,19 +252,15 @@ good idea what to do on other environments. use = egg:swift#account [account-replicator] - log_name = account-replicator vm_test_mode = yes [account-auditor] - log_name = account-auditor [account-reaper] - log_name = account-reaper #. Create `/etc/swift/account-server/4.conf`:: [DEFAULT] - log_name = account devices = /srv/4/node mount_check = false bind_port = 6042 @@ -288,19 +273,15 @@ good idea what to do on other environments. use = egg:swift#account [account-replicator] - log_name = account-replicator vm_test_mode = yes [account-auditor] - log_name = account-auditor [account-reaper] - log_name = account-reaper #. Create `/etc/swift/container-server/1.conf`:: [DEFAULT] - log_name = container devices = /srv/1/node mount_check = false bind_port = 6011 @@ -313,19 +294,15 @@ good idea what to do on other environments. use = egg:swift#container [container-replicator] - log_name = container-replicator vm_test_mode = yes [container-updater] - log_name = container-updater [container-auditor] - log_name = container-auditor #. Create `/etc/swift/container-server/2.conf`:: [DEFAULT] - log_name = container devices = /srv/2/node mount_check = false bind_port = 6021 @@ -338,20 +315,16 @@ good idea what to do on other environments. use = egg:swift#container [container-replicator] - log_name = container-replicator vm_test_mode = yes [container-updater] - log_name = container-updater [container-auditor] - log_name = container-auditor #. Create `/etc/swift/container-server/3.conf`:: [DEFAULT] - log_name = container devices = /srv/3/node mount_check = false bind_port = 6031 @@ -364,20 +337,16 @@ good idea what to do on other environments. use = egg:swift#container [container-replicator] - log_name = container-replicator vm_test_mode = yes [container-updater] - log_name = container-updater [container-auditor] - log_name = container-auditor #. Create `/etc/swift/container-server/4.conf`:: [DEFAULT] - log_name = container devices = /srv/4/node mount_check = false bind_port = 6041 @@ -390,20 +359,16 @@ good idea what to do on other environments. use = egg:swift#container [container-replicator] - log_name = container-replicator vm_test_mode = yes [container-updater] - log_name = container-updater [container-auditor] - log_name = container-auditor #. Create `/etc/swift/object-server/1.conf`:: [DEFAULT] - log_name = object devices = /srv/1/node mount_check = false bind_port = 6010 @@ -416,19 +381,15 @@ good idea what to do on other environments. use = egg:swift#object [object-replicator] - log_name = object-replicator vm_test_mode = yes [object-updater] - log_name = object-updater [object-auditor] - log_name = object-auditor #. Create `/etc/swift/object-server/2.conf`:: [DEFAULT] - log_name = object devices = /srv/2/node mount_check = false bind_port = 6020 @@ -441,19 +402,15 @@ good idea what to do on other environments. use = egg:swift#object [object-replicator] - log_name = object-replicator vm_test_mode = yes [object-updater] - log_name = object-updater [object-auditor] - log_name = object-auditor #. Create `/etc/swift/object-server/3.conf`:: [DEFAULT] - log_name = object devices = /srv/3/node mount_check = false bind_port = 6030 @@ -466,19 +423,15 @@ good idea what to do on other environments. use = egg:swift#object [object-replicator] - log_name = object-replicator vm_test_mode = yes [object-updater] - log_name = object-updater [object-auditor] - log_name = object-auditor #. Create `/etc/swift/object-server/4.conf`:: [DEFAULT] - log_name = object devices = /srv/4/node mount_check = false bind_port = 6040 @@ -491,14 +444,11 @@ good idea what to do on other environments. use = egg:swift#object [object-replicator] - log_name = object-replicator vm_test_mode = yes [object-updater] - log_name = object-updater [object-auditor] - log_name = object-auditor #. Create `~/bin/resetswift`:: From 708088dd68301e6ea1ed760e7799d12c23738559 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 24 Aug 2010 14:38:36 +0000 Subject: [PATCH 12/15] Fixed proxy server config for saio --- doc/source/development_saio.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index f14b9ca4de..07d54356d0 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -181,9 +181,9 @@ good idea what to do on other environments. user = [pipeline:main] - pipeline = healthcheck cache auth proxy + pipeline = healthcheck cache auth proxy-server - [app:proxy] + [app:proxy-server] use = egg:swift#proxy [filter:auth] From 459cbaaad606e7a1f5c8baf73eccbf2707182b3e Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 24 Aug 2010 14:40:36 +0000 Subject: [PATCH 13/15] Fixed memcache middleware config for proxy --- doc/source/development_saio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 07d54356d0..1d692bf1a5 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -193,7 +193,7 @@ good idea what to do on other environments. use = egg:swift#healthcheck [filter:cache] - use = egg:swift#cache + use = egg:swift#memcache #. Create `/etc/swift/account-server/1.conf`:: From 2dc8eec4bb494a17d213bc6970dd79f94a7722db Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 24 Aug 2010 14:55:20 +0000 Subject: [PATCH 14/15] Fixed container-updater to correctly load conf --- bin/swift-container-updater | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/swift-container-updater b/bin/swift-container-updater index c582d15577..92b7017faa 100755 --- a/bin/swift-container-updater +++ b/bin/swift-container-updater @@ -28,7 +28,7 @@ if __name__ == '__main__': sys.exit() once = len(sys.argv) > 2 and sys.argv[2] == 'once' - conf = readconf(sys.argv[1], 'container-updater') + conf = utils.readconf(sys.argv[1], 'container-updater') utils.drop_privileges(conf.get('user', 'swift')) try: From 608da06a9be881c7f68b80c229473180fafb9c8b Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Tue, 24 Aug 2010 18:48:00 +0000 Subject: [PATCH 15/15] Cleaned up a couple of doc strings in proxy.server --- swift/proxy/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 5cfb23fbdc..140bc5dbc4 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -47,7 +47,7 @@ def update_headers(response, headers): Helper function to update headers in the response. :param response: webob.Response object - :parm headers: dictionary headers + :param headers: dictionary headers """ if hasattr(headers, 'items'): headers = headers.items() @@ -1264,7 +1264,7 @@ class Application(BaseApplication): return None def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI proxy apps.""" conf = global_conf.copy() conf.update(local_conf) - """paste.deploy app factory for creating WSGI proxy apps.""" return Application(conf)