From 0da9da513101c172d8923469e40f9c15b3518e73 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Thu, 3 Mar 2016 11:14:39 +0000 Subject: [PATCH] Allow fallocate_reserve to be a percentage Add the ability to set the fallocate_reserve value as a percentage. This happens automatically when adding the '%' at the end of the value. Having the ability to set a % of free space rather than a byte value is useful especially when drive sizes are heterogenous. The default for fallocate_reserve has been adjusted to 1%, having the fallocate_reserve set seems sensible for all deploys and percentages are far safer to default than byte values (across drives of any size). Tests added for using fallocate_reserve as a percentage. Duplicate tests for fallocate_reserve have been removed. Docs updated to reflect the fallocate_reserve change. Change-Id: I4aea613a708205c917e81d6b2861396655e73238 --- doc/manpages/account-server.conf.5 | 6 +- doc/manpages/container-server.conf.5 | 6 +- doc/manpages/object-server.conf.5 | 6 +- doc/source/deployment_guide.rst | 48 +++++---- etc/account-server.conf-sample | 7 +- etc/container-server.conf-sample | 7 +- etc/object-server.conf-sample | 7 +- swift/common/daemon.py | 5 +- swift/common/utils.py | 26 ++++- swift/common/wsgi.py | 6 +- test/unit/common/test_utils.py | 153 +++++++++++++++++++++++---- 11 files changed, 215 insertions(+), 62 deletions(-) diff --git a/doc/manpages/account-server.conf.5 b/doc/manpages/account-server.conf.5 index 4a7e8c597e..62ee40f186 100644 --- a/doc/manpages/account-server.conf.5 +++ b/doc/manpages/account-server.conf.5 @@ -121,8 +121,10 @@ The default is false. .IP \fBeventlet_debug\fR Debug mode for eventlet library. The default is false. .IP \fBfallocate_reserve\fR -You can set fallocate_reserve to the number of bytes you'd like fallocate to -reserve, whether there is space for the given file size or not. The default is 0. +You can set fallocate_reserve to the number of bytes or percentage of disk +space you'd like fallocate to reserve, whether there is space for the given +file size or not. Percentage will be used if the value ends with a '%'. +The default is 1%. .RE .PD diff --git a/doc/manpages/container-server.conf.5 b/doc/manpages/container-server.conf.5 index 970fa18f2c..fa7b8a6d55 100644 --- a/doc/manpages/container-server.conf.5 +++ b/doc/manpages/container-server.conf.5 @@ -127,8 +127,10 @@ The default is false. .IP \fBeventlet_debug\fR Debug mode for eventlet library. The default is false. .IP \fBfallocate_reserve\fR -You can set fallocate_reserve to the number of bytes you'd like fallocate to -reserve, whether there is space for the given file size or not. The default is 0. +You can set fallocate_reserve to the number of bytes or percentage of disk +space you'd like fallocate to reserve, whether there is space for the given +file size or not. Percentage will be used if the value ends with a '%'. +The default is 1%. .RE .PD diff --git a/doc/manpages/object-server.conf.5 b/doc/manpages/object-server.conf.5 index 2e58de32fb..2c5b8fb327 100644 --- a/doc/manpages/object-server.conf.5 +++ b/doc/manpages/object-server.conf.5 @@ -126,8 +126,10 @@ The default is empty. .IP \fBeventlet_debug\fR Debug mode for eventlet library. The default is false. .IP \fBfallocate_reserve\fR -You can set fallocate_reserve to the number of bytes you'd like fallocate to -reserve, whether there is space for the given file size or not. The default is 0. +You can set fallocate_reserve to the number of bytes or percentage of disk +space you'd like fallocate to reserve, whether there is space for the given +file size or not. Percentage will be used if the value ends with a '%'. +The default is 1%. .IP \fBnode_timeout\fR Request timeout to external services. The default is 3 seconds. .IP \fBconn_timeout\fR diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 083a298578..a19f2c2bc5 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -489,12 +489,14 @@ log_statsd_sample_rate_factor 1.0 log_statsd_metric_prefix eventlet_debug false If true, turn on debug logging for eventlet -fallocate_reserve 0 You can set fallocate_reserve to the - number of bytes you'd like fallocate to - reserve, whether there is space for the - given file size or not. This is useful for - systems that behave badly when they - completely run out of space; you can +fallocate_reserve 1% You can set fallocate_reserve to the + number of bytes or percentage of disk + space you'd like fallocate to reserve, + whether there is space for the given + file size or not. Percentage will be used + if the value ends with a '%'. This is + useful for systems that behave badly when + they completely run out of space; you can make the services pretend they're out of space early. conn_timeout 0.5 Time to wait while attempting to connect @@ -809,13 +811,16 @@ log_statsd_default_sample_rate 1.0 log_statsd_sample_rate_factor 1.0 log_statsd_metric_prefix eventlet_debug false If true, turn on debug logging for eventlet -fallocate_reserve 0 You can set fallocate_reserve to the number of - bytes you'd like fallocate to reserve, whether - there is space for the given file size or not. - This is useful for systems that behave badly - when they completely run out of space; you can - make the services pretend they're out of space - early. +fallocate_reserve 1% You can set fallocate_reserve to the + number of bytes or percentage of disk + space you'd like fallocate to reserve, + whether there is space for the given + file size or not. Percentage will be used + if the value ends with a '%'. This is + useful for systems that behave badly when + they completely run out of space; you can + make the services pretend they're out of + space early. db_preallocation off If you don't mind the extra disk space usage in overhead, you can turn this on to preallocate disk space with SQLite databases to decrease @@ -1024,13 +1029,16 @@ log_statsd_default_sample_rate 1.0 log_statsd_sample_rate_factor 1.0 log_statsd_metric_prefix eventlet_debug false If true, turn on debug logging for eventlet -fallocate_reserve 0 You can set fallocate_reserve to the number of - bytes you'd like fallocate to reserve, whether - there is space for the given file size or not. - This is useful for systems that behave badly - when they completely run out of space; you can - make the services pretend they're out of space - early. +fallocate_reserve 1% You can set fallocate_reserve to the + number of bytes or percentage of disk + space you'd like fallocate to reserve, + whether there is space for the given + file size or not. Percentage will be used + if the value ends with a '%'. This is + useful for systems that behave badly when + they completely run out of space; you can + make the services pretend they're out of + space early. =============================== ========== ============================================= [account-server] diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index 9fa98c6f20..3471747331 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -47,9 +47,10 @@ bind_port = 6002 # # eventlet_debug = false # -# You can set fallocate_reserve to the number of bytes you'd like fallocate to -# reserve, whether there is space for the given file size or not. -# fallocate_reserve = 0 +# You can set fallocate_reserve to the number of bytes or percentage of disk +# space you'd like fallocate to reserve, whether there is space for the given +# file size or not. Percentage will be used if the value ends with a '%'. +# fallocate_reserve = 1% [pipeline:main] pipeline = healthcheck recon account-server diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 5927f5e230..17e37c5b78 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -53,9 +53,10 @@ bind_port = 6001 # # eventlet_debug = false # -# You can set fallocate_reserve to the number of bytes you'd like fallocate to -# reserve, whether there is space for the given file size or not. -# fallocate_reserve = 0 +# You can set fallocate_reserve to the number of bytes or percentage of disk +# space you'd like fallocate to reserve, whether there is space for the given +# file size or not. Percentage will be used if the value ends with a '%'. +# fallocate_reserve = 1% [pipeline:main] pipeline = healthcheck recon container-server diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index e01193bff2..528e293d4a 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -52,9 +52,10 @@ bind_port = 6000 # # eventlet_debug = false # -# You can set fallocate_reserve to the number of bytes you'd like fallocate to -# reserve, whether there is space for the given file size or not. -# fallocate_reserve = 0 +# You can set fallocate_reserve to the number of bytes or percentage of disk +# space you'd like fallocate to reserve, whether there is space for the given +# file size or not. Percentage will be used if the value ends with a '%'. +# fallocate_reserve = 1% # # Time to wait while attempting to connect to another backend node. # conn_timeout = 0.5 diff --git a/swift/common/daemon.py b/swift/common/daemon.py index 7b2ea93c03..a5d415638f 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -92,9 +92,8 @@ def run_daemon(klass, conf_file, section_name='', once=False, **kwargs): if utils.config_true_value(conf.get('disable_fallocate', 'no')): utils.disable_fallocate() # set utils.FALLOCATE_RESERVE if desired - reserve = int(conf.get('fallocate_reserve', 0)) - if reserve > 0: - utils.FALLOCATE_RESERVE = reserve + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value(conf.get('fallocate_reserve', '1%')) # By default, disable eventlet printing stacktraces eventlet_debug = utils.config_true_value(conf.get('eventlet_debug', 'no')) diff --git a/swift/common/utils.py b/swift/common/utils.py index fb4baa4396..b0376d8bcf 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -97,6 +97,9 @@ _libc_accept = None # If set to non-zero, fallocate routines will fail based on free space # available being at or below this amount, in bytes. FALLOCATE_RESERVE = 0 +# Indicates if FALLOCATE_RESERVE is the percentage of free space (True) or +# the number of bytes (False). +FALLOCATE_IS_PERCENT = False # Used by hash_path to offer a bit more security when generating hashes for # paths. It simply appends this value to all paths; guessing the hash a path @@ -453,6 +456,25 @@ def get_trans_id_time(trans_id): return None +def config_fallocate_value(reserve_value): + """ + Returns fallocate reserve_value as an int or float. + Returns is_percent as a boolean. + Returns a ValueError on invalid fallocate value. + """ + try: + if str(reserve_value[-1:]) == '%': + reserve_value = float(reserve_value[:-1]) + is_percent = True + else: + reserve_value = int(reserve_value) + is_percent = False + except ValueError: + raise ValueError('Error: %s is an invalid value for fallocate' + '_reserve.' % reserve_value) + return reserve_value, is_percent + + class FileLikeIter(object): def __init__(self, iterable): @@ -596,7 +618,9 @@ class FallocateWrapper(object): if FALLOCATE_RESERVE > 0: st = os.fstatvfs(fd) free = st.f_frsize * st.f_bavail - length.value - if free <= FALLOCATE_RESERVE: + if FALLOCATE_IS_PERCENT: + free = (float(free) / float(st.f_frsize * st.f_blocks)) * 100 + if float(free) <= float(FALLOCATE_RESERVE): raise OSError( errno.ENOSPC, 'FALLOCATE_RESERVE fail %s <= %s' % (free, diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 2c169eb2a6..6365afb552 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -897,9 +897,9 @@ def run_wsgi(conf_path, app_section, *args, **kwargs): loadapp(conf_path, global_conf=global_conf) # set utils.FALLOCATE_RESERVE if desired - reserve = int(conf.get('fallocate_reserve', 0)) - if reserve > 0: - utils.FALLOCATE_RESERVE = reserve + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value(conf.get('fallocate_reserve', '1%')) + # redirect errors to logger and close stdio capture_stdio(logger) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 626043de3d..ca92d7f58f 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -2583,6 +2583,7 @@ cluster_dfw1 = http://dfw1.host/v1/ class StatVFS(object): f_frsize = 1024 f_bavail = 1 + f_blocks = 100 def fstatvfs(fd): return StatVFS() @@ -2593,17 +2594,20 @@ cluster_dfw1 = http://dfw1.host/v1/ fallocate = utils.FallocateWrapper(noop=True) utils.os.fstatvfs = fstatvfs # Want 1023 reserved, have 1024 * 1 free, so succeeds - utils.FALLOCATE_RESERVE = 1023 + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1023') StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(0)), 0) # Want 1023 reserved, have 512 * 2 free, so succeeds - utils.FALLOCATE_RESERVE = 1023 + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1023') StatVFS.f_frsize = 512 StatVFS.f_bavail = 2 self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(0)), 0) # Want 1024 reserved, have 1024 * 1 free, so fails - utils.FALLOCATE_RESERVE = 1024 + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1024') StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 exc = None @@ -2615,7 +2619,8 @@ cluster_dfw1 = http://dfw1.host/v1/ '[Errno 28] FALLOCATE_RESERVE fail 1024 <= 1024') self.assertEqual(err.errno, errno.ENOSPC) # Want 1024 reserved, have 512 * 2 free, so fails - utils.FALLOCATE_RESERVE = 1024 + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1024') StatVFS.f_frsize = 512 StatVFS.f_bavail = 2 exc = None @@ -2627,7 +2632,8 @@ cluster_dfw1 = http://dfw1.host/v1/ '[Errno 28] FALLOCATE_RESERVE fail 1024 <= 1024') self.assertEqual(err.errno, errno.ENOSPC) # Want 2048 reserved, have 1024 * 1 free, so fails - utils.FALLOCATE_RESERVE = 2048 + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('2048') StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 exc = None @@ -2639,7 +2645,8 @@ cluster_dfw1 = http://dfw1.host/v1/ '[Errno 28] FALLOCATE_RESERVE fail 1024 <= 2048') self.assertEqual(err.errno, errno.ENOSPC) # Want 2048 reserved, have 512 * 2 free, so fails - utils.FALLOCATE_RESERVE = 2048 + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('2048') StatVFS.f_frsize = 512 StatVFS.f_bavail = 2 exc = None @@ -2652,7 +2659,8 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertEqual(err.errno, errno.ENOSPC) # Want 1023 reserved, have 1024 * 1 free, but file size is 1, so # fails - utils.FALLOCATE_RESERVE = 1023 + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1023') StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 exc = None @@ -2665,28 +2673,95 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertEqual(err.errno, errno.ENOSPC) # Want 1022 reserved, have 1024 * 1 free, and file size is 1, so # succeeds - utils.FALLOCATE_RESERVE = 1022 + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1022') StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(1)), 0) - # Want 1023 reserved, have 1024 * 1 free, and file size is 0, so - # succeeds - utils.FALLOCATE_RESERVE = 1023 - StatVFS.f_frsize = 1024 - StatVFS.f_bavail = 1 - self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(0)), 0) - # Want 1024 reserved, have 1024 * 1 free, and even though - # file size is 0, since we're under the reserve, fails - utils.FALLOCATE_RESERVE = 1024 - StatVFS.f_frsize = 1024 - StatVFS.f_bavail = 1 + # Want 1% reserved, have 100 bytes * 2/100 free, and file size is + # 99, so succeeds + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1%') + StatVFS.f_frsize = 100 + StatVFS.f_bavail = 2 + StatVFS.f_blocks = 100 + self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(99)), 0) + # Want 2% reserved, have 50 bytes * 2/50 free, and file size is 49, + # so succeeds + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('2%') + StatVFS.f_frsize = 50 + StatVFS.f_bavail = 2 + StatVFS.f_blocks = 50 + self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(49)), 0) + # Want 100% reserved, have 100 * 100/100 free, and file size is 0, + # so fails. + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('100%') + StatVFS.f_frsize = 100 + StatVFS.f_bavail = 100 + StatVFS.f_blocks = 100 exc = None try: fallocate(0, 1, 0, ctypes.c_uint64(0)) except OSError as err: exc = err self.assertEqual(str(exc), - '[Errno 28] FALLOCATE_RESERVE fail 1024 <= 1024') + '[Errno 28] FALLOCATE_RESERVE fail 100.0 <= ' + '100.0') + self.assertEqual(err.errno, errno.ENOSPC) + # Want 1% reserved, have 100 * 2/100 free, and file size is 101, + # so fails. + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1%') + StatVFS.f_frsize = 100 + StatVFS.f_bavail = 2 + StatVFS.f_blocks = 100 + exc = None + try: + fallocate(0, 1, 0, ctypes.c_uint64(101)) + except OSError as err: + exc = err + self.assertEqual(str(exc), + '[Errno 28] FALLOCATE_RESERVE fail 0.99 <= 1.0') + self.assertEqual(err.errno, errno.ENOSPC) + # Want 98% reserved, have 100 bytes * 99/100 free, and file size + # is 100, so fails + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('98%') + StatVFS.f_frsize = 100 + StatVFS.f_bavail = 99 + StatVFS.f_blocks = 100 + exc = None + try: + fallocate(0, 1, 0, ctypes.c_uint64(100)) + except OSError as err: + exc = err + self.assertEqual(str(exc), + '[Errno 28] FALLOCATE_RESERVE fail 98.0 <= 98.0') + self.assertEqual(err.errno, errno.ENOSPC) + # Want 2% reserved, have 1000 bytes * 21/1000 free, and file size + # is 999, so succeeds. + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('2%') + StatVFS.f_frsize = 1000 + StatVFS.f_bavail = 21 + StatVFS.f_blocks = 1000 + self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(999)), 0) + # Want 2% resereved, have 1000 bytes * 21/1000 free, and file size + # is 1000, so fails. + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('2%') + StatVFS.f_frsize = 1000 + StatVFS.f_bavail = 21 + StatVFS.f_blocks = 1000 + exc = None + try: + fallocate(0, 1, 0, ctypes.c_uint64(1000)) + except OSError as err: + exc = err + self.assertEqual(str(exc), + '[Errno 28] FALLOCATE_RESERVE fail 2.0 <= 2.0') self.assertEqual(err.errno, errno.ENOSPC) finally: utils.FALLOCATE_RESERVE = orig_FALLOCATE_RESERVE @@ -2767,6 +2842,44 @@ cluster_dfw1 = http://dfw1.host/v1/ ts = utils.get_trans_id_time('tx1df4ff4f55ea45f7b2ec2-almostright') self.assertEqual(ts, None) + def test_config_fallocate_value(self): + fallocate_value, is_percent = utils.config_fallocate_value('10%') + self.assertEqual(fallocate_value, 10) + self.assertTrue(is_percent) + fallocate_value, is_percent = utils.config_fallocate_value('10') + self.assertEqual(fallocate_value, 10) + self.assertFalse(is_percent) + try: + fallocate_value, is_percent = utils.config_fallocate_value('ab%') + except ValueError as err: + exc = err + self.assertEqual(str(exc), 'Error: ab% is an invalid value for ' + 'fallocate_reserve.') + try: + fallocate_value, is_percent = utils.config_fallocate_value('ab') + except ValueError as err: + exc = err + self.assertEqual(str(exc), 'Error: ab is an invalid value for ' + 'fallocate_reserve.') + try: + fallocate_value, is_percent = utils.config_fallocate_value('1%%') + except ValueError as err: + exc = err + self.assertEqual(str(exc), 'Error: 1%% is an invalid value for ' + 'fallocate_reserve.') + try: + fallocate_value, is_percent = utils.config_fallocate_value('10.0') + except ValueError as err: + exc = err + self.assertEqual(str(exc), 'Error: 10.0 is an invalid value for ' + 'fallocate_reserve.') + fallocate_value, is_percent = utils.config_fallocate_value('10.5%') + self.assertEqual(fallocate_value, 10.5) + self.assertTrue(is_percent) + fallocate_value, is_percent = utils.config_fallocate_value('10.000%') + self.assertEqual(fallocate_value, 10.000) + self.assertTrue(is_percent) + def test_tpool_reraise(self): with patch.object(utils.tpool, 'execute', lambda f: f()): self.assertTrue(