From 8ad1403d8f0413b41f585e69c50b59edfc925813 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Mon, 5 Oct 2015 10:51:36 -0700 Subject: [PATCH] Allow oslo.log to work on non-linux platforms Especially needed for all the HyperV work on windows. Depends-On: I1a0a2dacbefa548296a6cfef1663ea9d48253c0c Change-Id: Ifcc854e6eb6f4d0e575d1489f0d00f3339705e0b --- doc/source/api/watchers.rst | 12 ++++ oslo_log/handlers.py | 89 ------------------------- oslo_log/log.py | 3 +- oslo_log/tests/unit/test_log.py | 3 +- oslo_log/watchers.py | 111 ++++++++++++++++++++++++++++++++ requirements.txt | 2 +- 6 files changed, 128 insertions(+), 92 deletions(-) create mode 100644 doc/source/api/watchers.rst create mode 100644 oslo_log/watchers.py diff --git a/doc/source/api/watchers.rst b/doc/source/api/watchers.rst new file mode 100644 index 00000000..367d8c2e --- /dev/null +++ b/doc/source/api/watchers.rst @@ -0,0 +1,12 @@ +================== + oslo_log.watchers +================== + +.. automodule:: oslo_log.watchers + :members: + :undoc-members: + :show-inheritance: + +.. seealso:: + + :doc:`../usage` diff --git a/oslo_log/handlers.py b/oslo_log/handlers.py index e6500640..9ecbcd69 100644 --- a/oslo_log/handlers.py +++ b/oslo_log/handlers.py @@ -10,15 +10,11 @@ # License for the specific language governing permissions and limitations # under the License. -import errno import inspect import logging import logging.config import logging.handlers import os -import pyinotify -import stat -import time try: import syslog except ImportError: @@ -119,88 +115,3 @@ class ColorHandler(logging.StreamHandler): def format(self, record): record.color = self.LEVEL_COLORS[record.levelno] return logging.StreamHandler.format(self, record) - - -class FileKeeper(pyinotify.ProcessEvent): - def my_init(self, watched_handler, watched_file): - self._watched_handler = watched_handler - self._watched_file = watched_file - - def process_default(self, event): - if event.name == self._watched_file: - self._watched_handler.reopen_file() - - -class EventletThreadedNotifier(pyinotify.ThreadedNotifier): - - def loop(self): - """Eventlet friendly ThreadedNotifier - - EventletFriendlyThreadedNotifier contains additional time.sleep() - call insude loop to allow switching to other thread when eventlet - is used. - It can be used with eventlet and native threads as well. - """ - - while not self._stop_event.is_set(): - self.process_events() - time.sleep(0) - ref_time = time.time() - if self.check_events(): - self._sleep(ref_time) - self.read_events() - - -class FastWatchedFileHandler(logging.handlers.WatchedFileHandler, object): - """Frequency of reading events. - - Watching thread sleeps max(0, READ_FREQ - (TIMEOUT / 1000)) seconds. - """ - READ_FREQ = 1 - - """Poll timeout in milliseconds. - - See https://docs.python.org/2/library/select.html#select.poll.poll""" - TIMEOUT = 500 - - def __init__(self, logpath, *args, **kwargs): - self._log_file = os.path.basename(logpath) - self._log_dir = os.path.dirname(logpath) - super(FastWatchedFileHandler, self).__init__(logpath, *args, **kwargs) - self._watch_file() - - def _watch_file(self): - mask = pyinotify.IN_MOVED_FROM | pyinotify.IN_DELETE - watch_manager = pyinotify.WatchManager() - handler = FileKeeper(watched_handler=self, - watched_file=self._log_file) - notifier = EventletThreadedNotifier( - watch_manager, - default_proc_fun=handler, - read_freq=FastWatchedFileHandler.READ_FREQ, - timeout=FastWatchedFileHandler.TIMEOUT) - notifier.daemon = True - watch_manager.add_watch(self._log_dir, mask) - notifier.start() - - def reopen_file(self): - try: - # stat the file by path, checking for existence - sres = os.stat(self.baseFilename) - except OSError as err: - if err.errno == errno.ENOENT: - sres = None - else: - raise - # compare file system stat with that of our stream file handle - if (not sres or - sres[stat.ST_DEV] != self.dev or - sres[stat.ST_INO] != self.ino): - if self.stream is not None: - # we have an open file handle, clean it up - self.stream.flush() - self.stream.close() - self.stream = None - # open a new file handle and get new stat info from that fd - self.stream = self._open() - self._statstream() diff --git a/oslo_log/log.py b/oslo_log/log.py index 9f3949fd..df7cf3ca 100644 --- a/oslo_log/log.py +++ b/oslo_log/log.py @@ -313,7 +313,8 @@ def _setup_logging_from_conf(conf, project, version): logpath = _get_log_file_path(conf) if logpath: if conf.watch_log_file and platform.system() == 'Linux': - file_handler = handlers.FastWatchedFileHandler + from oslo_log import watchers + file_handler = watchers.FastWatchedFileHandler else: file_handler = logging.handlers.WatchedFileHandler diff --git a/oslo_log/tests/unit/test_log.py b/oslo_log/tests/unit/test_log.py index 99c87775..68c6f648 100644 --- a/oslo_log/tests/unit/test_log.py +++ b/oslo_log/tests/unit/test_log.py @@ -684,8 +684,9 @@ class FastWatchedFileHandlerTestCase(BaseTestCase): self._config() logger = log._loggers[None].logger self.assertEqual(1, len(logger.handlers)) + from oslo_log import watchers self.assertIsInstance(logger.handlers[0], - handlers.FastWatchedFileHandler) + watchers.FastWatchedFileHandler) def test_log(self): log_path = self._config() diff --git a/oslo_log/watchers.py b/oslo_log/watchers.py new file mode 100644 index 00000000..d967e214 --- /dev/null +++ b/oslo_log/watchers.py @@ -0,0 +1,111 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import errno +import logging +import logging.config +import logging.handlers +import os +import pyinotify +import stat +import time +try: + import syslog +except ImportError: + syslog = None + +"""Linux specific pyinotify based logging handlers""" + + +class _FileKeeper(pyinotify.ProcessEvent): + def my_init(self, watched_handler, watched_file): + self._watched_handler = watched_handler + self._watched_file = watched_file + + def process_default(self, event): + if event.name == self._watched_file: + self._watched_handler.reopen_file() + + +class _EventletThreadedNotifier(pyinotify.ThreadedNotifier): + + def loop(self): + """Eventlet friendly ThreadedNotifier + + EventletFriendlyThreadedNotifier contains additional time.sleep() + call insude loop to allow switching to other thread when eventlet + is used. + It can be used with eventlet and native threads as well. + """ + + while not self._stop_event.is_set(): + self.process_events() + time.sleep(0) + ref_time = time.time() + if self.check_events(): + self._sleep(ref_time) + self.read_events() + + +class FastWatchedFileHandler(logging.handlers.WatchedFileHandler, object): + """Frequency of reading events. + + Watching thread sleeps max(0, READ_FREQ - (TIMEOUT / 1000)) seconds. + """ + READ_FREQ = 1 + + """Poll timeout in milliseconds. + + See https://docs.python.org/2/library/select.html#select.poll.poll""" + TIMEOUT = 500 + + def __init__(self, logpath, *args, **kwargs): + self._log_file = os.path.basename(logpath) + self._log_dir = os.path.dirname(logpath) + super(FastWatchedFileHandler, self).__init__(logpath, *args, **kwargs) + self._watch_file() + + def _watch_file(self): + mask = pyinotify.IN_MOVED_FROM | pyinotify.IN_DELETE + watch_manager = pyinotify.WatchManager() + handler = _FileKeeper(watched_handler=self, + watched_file=self._log_file) + notifier = _EventletThreadedNotifier( + watch_manager, + default_proc_fun=handler, + read_freq=FastWatchedFileHandler.READ_FREQ, + timeout=FastWatchedFileHandler.TIMEOUT) + notifier.daemon = True + watch_manager.add_watch(self._log_dir, mask) + notifier.start() + + def reopen_file(self): + try: + # stat the file by path, checking for existence + sres = os.stat(self.baseFilename) + except OSError as err: + if err.errno == errno.ENOENT: + sres = None + else: + raise + # compare file system stat with that of our stream file handle + if (not sres or + sres[stat.ST_DEV] != self.dev or + sres[stat.ST_INO] != self.ino): + if self.stream is not None: + # we have an open file handle, clean it up + self.stream.flush() + self.stream.close() + self.stream = None + # open a new file handle and get new stat info from that fd + self.stream = self._open() + self._statstream() diff --git a/requirements.txt b/requirements.txt index 7bf0967b..d20752f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,4 @@ oslo.i18n>=1.5.0 # Apache-2.0 oslo.utils>=2.0.0 # Apache-2.0 oslo.serialization>=1.4.0 # Apache-2.0 debtcollector>=0.3.0 # Apache-2.0 -pyinotify>=0.9.6 # MIT +pyinotify>=0.9.6;sys_platform!='win32' and sys_platform!='darwin' # MIT