From 26a7a2c82735bc78df9255e067778d435d8e1e67 Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Tue, 5 Jun 2018 09:58:52 +1000
Subject: [PATCH] Use statsd pipeline

With a few volumes, stats tend to go missing as there's a big flood of
individual packets.  Use the statsd pipeline which is meant for this
sort of batch reporting.

Test-case is updated to handle the batched messages which are
delineated by newlines.
---
 afsmon/cmd/main.py   | 20 +++++++++++++-------
 afsmon/tests/base.py | 11 ++++++++---
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/afsmon/cmd/main.py b/afsmon/cmd/main.py
index 6cc502a..d88987c 100644
--- a/afsmon/cmd/main.py
+++ b/afsmon/cmd/main.py
@@ -53,29 +53,35 @@ class AFSMonCmd(object):
                                                  statsd_args['port']))
         self.statsd = statsd.StatsClient(**statsd_args)
 
+        # With a lot of volumes, we can flood out a lot of stats
+        # quickly.  Use a pipeline to batch.
+        pipe = self.statsd.pipeline()
+
         for f in self.fileservers:
             if f.status != afsmon.FileServerStatus.NORMAL:
                 continue
 
             hn = f.hostname.replace('.', '_')
-            self.statsd.gauge('afs.%s.idle_threads' % hn, f.idle_threads)
-            self.statsd.gauge('afs.%s.calls_waiting' % hn, f.calls_waiting)
+            pipe.gauge('afs.%s.idle_threads' % hn, f.idle_threads)
+            pipe.gauge('afs.%s.calls_waiting' % hn, f.calls_waiting)
             for p in f.partitions:
-                self.statsd.gauge(
+                pipe.gauge(
                     'afs.%s.part.%s.used' % (hn, p.partition), p.used)
-                self.statsd.gauge(
+                pipe.gauge(
                     'afs.%s.part.%s.free' % (hn, p.partition), p.free)
-                self.statsd.gauge(
+                pipe.gauge(
                     'afs.%s.part.%s.total' % (hn, p.partition), p.total)
             for v in f.volumes:
                 if v.perms != 'RW':
                     continue
                 vn = v.volume.replace('.', '_')
-                self.statsd.gauge(
+                pipe.gauge(
                     'afs.%s.vol.%s.used' % (hn, vn), v.used)
-                self.statsd.gauge(
+                pipe.gauge(
                     'afs.%s.vol.%s.quota' % (hn, vn), v.quota)
 
+        pipe.send()
+
     def main(self, args=None):
         if args is None:
             args = sys.argv[1:]
diff --git a/afsmon/tests/base.py b/afsmon/tests/base.py
index 7d76df5..5a0d00a 100644
--- a/afsmon/tests/base.py
+++ b/afsmon/tests/base.py
@@ -14,6 +14,7 @@
 # under the License.
 
 import fixtures
+import itertools
 import logging
 import os
 import select
@@ -119,9 +120,13 @@ class TestCase(testtools.TestCase):
         while time.time() < (start + 5):
             # Note our fake statsd just queues up results in a queue.
             # We just keep going through them until we find one that
-            # matches, or fail out.
-            for stat in self.statsd.stats:
-                k, v = stat.decode('utf-8').split(':')
+            # matches, or fail out.  If a statsd pipeline is used, the
+            # elements are separated by newlines, so flatten out all
+            # the stats first.
+            stats = itertools.chain.from_iterable(
+                [s.decode('utf-8').split('\n') for s in self.statsd.stats])
+            for stat in stats:
+                k, v = stat.split(':')
                 if key == k:
                     if kind is None:
                         # key with no qualifiers is found