Add statsd reporter and test
This commit is contained in:
Ian Wienand
parent
8a1eb3901b
commit
719425d9f2
@ -1,2 +1,2 @@
|
||||
[DEFAULT]
|
||||
test_path=./pyafsmon/tests
|
||||
test_path=./afsmon/tests
|
||||
@ -22,6 +22,8 @@ from datetime import datetime
|
||||
from enum import Enum
|
||||
from prettytable import PrettyTable
|
||||
|
||||
logger = logging.getLogger("afsmon")
|
||||
|
||||
#
|
||||
# Fileserver
|
||||
#
|
||||
@ -66,7 +68,7 @@ class FileServerStats:
|
||||
|
||||
def _get_volumes(self):
|
||||
cmd = ["vos", "listvol", "-long", "-server", self.hostname]
|
||||
logging.debug("Running: %s" % cmd)
|
||||
logger.debug("Running: %s" % cmd)
|
||||
output = subprocess.check_output(
|
||||
cmd, stderr=subprocess.STDOUT).decode('ascii')
|
||||
|
||||
@ -104,7 +106,7 @@ class FileServerStats:
|
||||
|
||||
def _get_calls_waiting(self):
|
||||
cmd = ["rxdebug", self.hostname, "7000", "-rxstats", "-noconns"]
|
||||
logging.debug("Running: %s" % cmd)
|
||||
logger.debug("Running: %s" % cmd)
|
||||
output = subprocess.check_output(
|
||||
cmd, stderr=subprocess.STDOUT).decode('ascii')
|
||||
|
||||
@ -118,7 +120,7 @@ class FileServerStats:
|
||||
|
||||
def _get_partition_stats(self):
|
||||
cmd = ["vos", "partinfo", self.hostname, "-noauth"]
|
||||
logging.debug("Running: %s" % cmd)
|
||||
logger.debug("Running: %s" % cmd)
|
||||
output = subprocess.check_output(
|
||||
cmd, stderr=subprocess.STDOUT).decode('ascii')
|
||||
|
||||
@ -137,12 +139,12 @@ class FileServerStats:
|
||||
|
||||
def _get_fs_stats(self):
|
||||
cmd = ["bos", "status", self.hostname, "-long", "-noauth"]
|
||||
logging.debug("Running: %s" % cmd)
|
||||
logger.debug("Running: %s" % cmd)
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
cmd, stderr=subprocess.STDOUT).decode('ascii')
|
||||
except subprocess.CalledProcessError:
|
||||
logging.debug(" ... failed!")
|
||||
logger.debug(" ... failed!")
|
||||
self.status = FileServerStatus.NO_CONNECTION
|
||||
return
|
||||
|
||||
@ -159,7 +161,7 @@ class FileServerStats:
|
||||
elif re.search('disabled, currently shutdown', output):
|
||||
self.status = FileServerStatus.DISABLED
|
||||
else:
|
||||
logging.debug(output)
|
||||
logger.debug(output)
|
||||
self.status = FileServerStatus.UNKNOWN
|
||||
|
||||
def get_stats(self):
|
||||
@ -223,12 +225,12 @@ def get_fs_addresses(cell):
|
||||
'''
|
||||
fs = []
|
||||
cmd = ["vos", "listaddrs", "-noauth", "-cell", cell]
|
||||
logging.debug("Running: %s" % cmd)
|
||||
logger.debug("Running: %s" % cmd)
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
cmd, stderr=subprocess.STDOUT).decode('ascii')
|
||||
except subprocess.CalledProcessError:
|
||||
logging.debug(" ... failed!")
|
||||
logger.debug(" ... failed!")
|
||||
return []
|
||||
|
||||
for line in output.split('\n'):
|
||||
|
||||
@ -14,40 +14,119 @@
|
||||
import argparse
|
||||
import configparser
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import statsd
|
||||
|
||||
import afsmon
|
||||
|
||||
def main(args=None):
|
||||
logger = logging.getLogger("afsmon.main")
|
||||
|
||||
if args is None:
|
||||
args = sys.argv[1:]
|
||||
class AFSMonCmd:
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='An AFS monitoring tool')
|
||||
def cmd_show(self):
|
||||
for fs in self.fileservers:
|
||||
print(fs)
|
||||
return 0
|
||||
|
||||
parser.add_argument("config", help="Path to config file")
|
||||
parser.add_argument("-d", '--debug', action="store_true")
|
||||
def cmd_statsd(self):
|
||||
# note we're just being careful to let the default values fall
|
||||
# through to StatsClient()
|
||||
statsd_args = {}
|
||||
try:
|
||||
try:
|
||||
statsd_args['host'] = self.config.get('statsd', 'host')
|
||||
except configparser.NoOptionError:
|
||||
pass
|
||||
try:
|
||||
statsd_args['port'] = self.config.get('statsd', 'port')
|
||||
except configparser.NoOptionerror:
|
||||
pass
|
||||
except configparser.NoSectionError:
|
||||
pass
|
||||
if os.getenv('STATSD_HOST', None):
|
||||
statsd_args['host'] = os.environ['STATSD_HOST']
|
||||
if os.getenv('STATSD_PORT', None):
|
||||
statsd_args['port'] = os.environ['STATSD_PORT']
|
||||
logger.debug("Sending stats to %s:%s" % (statsd_args['host'],
|
||||
statsd_args['port']))
|
||||
self.statsd = statsd.StatsClient(**statsd_args)
|
||||
|
||||
args = parser.parse_args(args)
|
||||
for f in self.fileservers:
|
||||
if f.status != afsmon.FileServerStatus.NORMAL:
|
||||
continue
|
||||
|
||||
if args.debug:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
logging.debug("Debugging enabled")
|
||||
hn = f.hostname.replace('.', '_')
|
||||
self.statsd.gauge('afs.%s.idle_threads' % hn, f.idle_threads)
|
||||
self.statsd.gauge('afs.%s.calls_waiting'% hn, f.calls_waiting)
|
||||
for p in f.partitions:
|
||||
self.statsd.gauge(
|
||||
'afs.%s.part.%s.used' % (hn, p.partition), p.used)
|
||||
self.statsd.gauge(
|
||||
'afs.%s.part.%s.free' % (hn, p.partition), p.free)
|
||||
self.statsd.gauge(
|
||||
'afs.%s.part.%s.total' % (hn, p.partition), p.total)
|
||||
for v in f.volumes:
|
||||
if v.perms != 'RW':
|
||||
continue
|
||||
vn = v.volume.replace('.', '_')
|
||||
self.statsd.gauge(
|
||||
'afs.%s.vol.%s.used' % (hn, vn), v.used)
|
||||
self.statsd.gauge(
|
||||
'afs.%s.vol.%s.quota' % (hn, vn), v.quota)
|
||||
|
||||
config = configparser.RawConfigParser()
|
||||
config.read(args.config)
|
||||
|
||||
cell = config.get('main', 'cell').strip()
|
||||
def main(self, args=None):
|
||||
if args is None:
|
||||
args = sys.argv[1:]
|
||||
|
||||
fileservers = afsmon.get_fs_addresses(cell)
|
||||
logging.debug("Found fileservers: %s" % ", ".join(fileservers))
|
||||
self.fileservers = []
|
||||
|
||||
for fileserver in fileservers:
|
||||
logging.debug("Finding stats for: %s" % fileserver)
|
||||
parser = argparse.ArgumentParser(
|
||||
description='An AFS monitoring tool')
|
||||
|
||||
fs = afsmon.FileServerStats(fileserver)
|
||||
fs.get_stats()
|
||||
print(fs)
|
||||
parser.add_argument("-c", "--config", action='store',
|
||||
default="/etc/afsmon.cfg",
|
||||
help="Path to config file")
|
||||
parser.add_argument("-d", '--debug', action="store_true")
|
||||
|
||||
sys.exit(0)
|
||||
subparsers = parser.add_subparsers(title='commands',
|
||||
description='valid commands',
|
||||
dest='command')
|
||||
|
||||
cmd_show = subparsers.add_parser('show', help='show table of results')
|
||||
cmd_show.set_defaults(func=self.cmd_show)
|
||||
|
||||
cmd_statsd = subparsers.add_parser('statsd', help='report to statsd')
|
||||
cmd_statsd.set_defaults(func=self.cmd_statsd)
|
||||
|
||||
self.args = parser.parse_args(args)
|
||||
|
||||
if self.args.debug:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
logger.debug("Debugging enabled")
|
||||
|
||||
if not os.path.exists(self.args.config):
|
||||
raise ValueError("Config file %s does not exist" % self.args.config)
|
||||
|
||||
self.config = configparser.RawConfigParser()
|
||||
self.config.read(self.args.config)
|
||||
|
||||
cell = self.config.get('main', 'cell').strip()
|
||||
|
||||
fs_addrs = afsmon.get_fs_addresses(cell)
|
||||
logger.debug("Found fileservers: %s" % ", ".join(fs_addrs))
|
||||
|
||||
for addr in fs_addrs:
|
||||
logger.debug("Finding stats for: %s" % addr)
|
||||
fs = afsmon.FileServerStats(addr)
|
||||
fs.get_stats()
|
||||
self.fileservers.append(fs)
|
||||
|
||||
# run the subcommand
|
||||
return self.args.func()
|
||||
|
||||
|
||||
def main():
|
||||
cmd = AFSMonCmd()
|
||||
return cmd.main()
|
||||
|
||||
@ -15,11 +15,46 @@
|
||||
|
||||
import os
|
||||
|
||||
import logging
|
||||
import fixtures
|
||||
import select
|
||||
import socket
|
||||
import testtools
|
||||
import threading
|
||||
import time
|
||||
|
||||
_TRUE_VALUES = ('True', 'true', '1', 'yes')
|
||||
|
||||
logger = logging.getLogger("afsmon.tests.base")
|
||||
|
||||
class FakeStatsd(threading.Thread):
|
||||
def __init__(self):
|
||||
threading.Thread.__init__(self)
|
||||
self.daemon = True
|
||||
self.sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
|
||||
self.sock.bind(('', 0))
|
||||
self.port = self.sock.getsockname()[1]
|
||||
self.wake_read, self.wake_write = os.pipe()
|
||||
self.stats = []
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
poll = select.poll()
|
||||
poll.register(self.sock, select.POLLIN)
|
||||
poll.register(self.wake_read, select.POLLIN)
|
||||
ret = poll.poll()
|
||||
for (fd, event) in ret:
|
||||
if fd == self.sock.fileno():
|
||||
data = self.sock.recvfrom(1024)
|
||||
if not data:
|
||||
return
|
||||
self.stats.append(data[0])
|
||||
if fd == self.wake_read:
|
||||
return
|
||||
|
||||
def stop(self):
|
||||
os.write(self.wake_write, b'1\n')
|
||||
|
||||
|
||||
class TestCase(testtools.TestCase):
|
||||
"""Test case base class for all unit tests."""
|
||||
@ -47,4 +82,69 @@ class TestCase(testtools.TestCase):
|
||||
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
|
||||
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
|
||||
|
||||
self.log_fixture = self.useFixture(fixtures.FakeLogger())
|
||||
self.log_fixture = self.useFixture(
|
||||
fixtures.FakeLogger(level=logging.DEBUG))
|
||||
|
||||
self.statsd = FakeStatsd()
|
||||
os.environ['STATSD_HOST'] = '127.0.0.1'
|
||||
os.environ['STATSD_PORT'] = str(self.statsd.port)
|
||||
self.statsd.start()
|
||||
|
||||
def shutdown(self):
|
||||
self.statsd.stop()
|
||||
self.statsd.join()
|
||||
|
||||
def assertReportedStat(self, key, value=None, kind=None):
|
||||
"""Check statsd output
|
||||
|
||||
Check statsd return values. A ``value`` should specify a
|
||||
``kind``, however a ``kind`` may be specified without a
|
||||
``value`` for a generic match. Leave both empy to just check
|
||||
for key presence.
|
||||
|
||||
:arg str key: The statsd key
|
||||
:arg str value: The expected value of the metric ``key``
|
||||
:arg str kind: The expected type of the metric ``key`` For example
|
||||
|
||||
- ``c`` counter
|
||||
- ``g`` gauge
|
||||
- ``ms`` timing
|
||||
- ``s`` set
|
||||
"""
|
||||
if value:
|
||||
self.assertNotEqual(kind, None)
|
||||
|
||||
start = time.time()
|
||||
while time.time() < (start + 5):
|
||||
# Note our fake statsd just queues up results in a queue.
|
||||
# We just keep going through them until we find one that
|
||||
# matches, or fail out.
|
||||
for stat in self.statsd.stats:
|
||||
k, v = stat.decode('utf-8').split(':')
|
||||
if key == k:
|
||||
if kind is None:
|
||||
# key with no qualifiers is found
|
||||
return True
|
||||
|
||||
s_value, s_kind = v.split('|')
|
||||
# if no kind match, look for other keys
|
||||
if kind != s_kind:
|
||||
continue
|
||||
|
||||
if value:
|
||||
# special-case value|ms because statsd can turn
|
||||
# timing results into float of indeterminate
|
||||
# length, hence foiling string matching.
|
||||
if kind == 'ms':
|
||||
if float(value) == float(s_value):
|
||||
return True
|
||||
if value == s_value:
|
||||
return True
|
||||
# otherwise keep looking for other matches
|
||||
continue
|
||||
|
||||
# this key matches
|
||||
return True
|
||||
time.sleep(0.1)
|
||||
|
||||
raise Exception("Key %s not found in reported stats" % key)
|
||||
|
||||
@ -9,14 +9,17 @@
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import afsmon
|
||||
import configparser
|
||||
|
||||
from pyafsmon.tests import base
|
||||
from afsmon.tests import base
|
||||
from afsmon.cmd.main import AFSMonCmd
|
||||
|
||||
"""
|
||||
test_pyafsmon
|
||||
test_afsmon
|
||||
----------------------------------
|
||||
|
||||
Tests for `pyafsmon` module.
|
||||
Tests for `afsmon` module.
|
||||
"""
|
||||
|
||||
class TestPyAFSMon(base.TestCase):
|
||||
@ -24,5 +27,45 @@ class TestPyAFSMon(base.TestCase):
|
||||
def setUp(self):
|
||||
super(TestPyAFSMon, self).setUp()
|
||||
|
||||
def test_blank(self):
|
||||
self.assertEqual(0, 0)
|
||||
def test_statsd(self):
|
||||
cmd = AFSMonCmd()
|
||||
cmd.config = configparser.ConfigParser()
|
||||
|
||||
a = afsmon.FileServerStats('afs01.dfw.openstack.org')
|
||||
a.status = afsmon.FileServerStatus.NORMAL
|
||||
a.idle_threads = 250
|
||||
a.calls_waiting = 0
|
||||
a.partitions = [afsmon.Partition('vicepa', 512, 512, 1024, 50.00)]
|
||||
a.volumes = [
|
||||
afsmon.Volume('mirror.foo', 12345678, 'RW', 512, 1024, 50.00),
|
||||
afsmon.Volume('mirror.moo', 87654321, 'RW', 1024, 2048, 50.00),
|
||||
]
|
||||
|
||||
b = afsmon.FileServerStats('afs02.ord.openstack.org')
|
||||
b.status = afsmon.FileServerStatus.NORMAL
|
||||
b.idle_threads = 100
|
||||
b.calls_waiting = 2
|
||||
b.partitions = [afsmon.Partition('vicepa', 512, 512, 1024, 50.00)]
|
||||
b.volumes = []
|
||||
|
||||
cmd.fileservers = [a, b]
|
||||
|
||||
cmd.cmd_statsd()
|
||||
|
||||
self.assertReportedStat(
|
||||
'afs.afs01_dfw_openstack_org.idle_threads', value='250', kind='g')
|
||||
self.assertReportedStat(
|
||||
'afs.afs02_ord_openstack_org.calls_waiting', value='2', kind='g')
|
||||
self.assertReportedStat(
|
||||
'afs.afs01_dfw_openstack_org.part.vicepa.used',
|
||||
value='512', kind='g')
|
||||
self.assertReportedStat(
|
||||
'afs.afs01_dfw_openstack_org.part.vicepa.total',
|
||||
value='1024', kind='g')
|
||||
self.assertReportedStat(
|
||||
'afs.afs01_dfw_openstack_org.vol.mirror_moo.used',
|
||||
value='1024', kind='g')
|
||||
self.assertReportedStat(
|
||||
'afs.afs01_dfw_openstack_org.vol.mirror_moo.quota',
|
||||
value='2048', kind='g')
|
||||
|
||||
|
||||
@ -5,3 +5,4 @@
|
||||
pbr!=2.1.0,>=2.0.0 # Apache-2.0
|
||||
Babel!=2.4.0,>=2.3.4 # BSD
|
||||
PrettyTable<0.8 # BSD
|
||||
statsd>=3.2.1 # MIT
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user