From 3b8e20e5985140b30c0ff7d45d9d37ced09320f8 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Sun, 26 May 2013 15:33:04 -0700 Subject: [PATCH] Set an upper bound to log pusher output queue. Logstash's queues are small. During periods of high load this can cause the log pusher to cache all of the logs it is pulling in. Set an upper bound to its output queue size to force it to block when Logstash cannot keep up. This should prevent OOMing and won't impact log throughput as Logstash is blocking anyways. Change-Id: I9ca31e6dbe454e9c4878fd7ba35bc33bc9df7d83 Reviewed-on: https://review.openstack.org/30572 Reviewed-by: James E. Blair Reviewed-by: Khai Do Approved: Jeremy Stanley Reviewed-by: Jeremy Stanley Tested-by: Jenkins --- modules/openstack_project/files/logstash/log-pusher.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/openstack_project/files/logstash/log-pusher.py b/modules/openstack_project/files/logstash/log-pusher.py index a07f0d3f54..a613f90a7f 100644 --- a/modules/openstack_project/files/logstash/log-pusher.py +++ b/modules/openstack_project/files/logstash/log-pusher.py @@ -288,7 +288,10 @@ class Server(object): self.event_queues = [] self.retrievers = [] # TODO(clarkb) support multiple outputs - self.logqueue = queue.Queue() + # Logstash queues are small. Set an upper bound to our queue + # so that this doesn't turn into an unbounded cache prone to + # OOMing. But set it to a reasonable size to provide some caching. + self.logqueue = queue.Queue(16384) self.processor = None def setup_logging(self):