Collectd plugin: Gnocchi status
Initial commit to add a plugin to collectd for obtaining and storing Gnocchi status metrics. This plug obtains the two metrics provided by Gnocchi's REST API for "the number of metrics having measures to be processed" and "the number of measures to process". This also addes an associated graph in the General OpenStack System Performance Dashboard and the Controller specific dashboard. * no longer requires authentication details in the collectd config * removing the redis exec plugin * configurable interval for the python plugin now Change-Id: Idfc5c719e9b0fbaad5db0632a3e5293bf03d7ab6
This commit is contained in:
akrzos
Alex Krzos
parent
6d55da3b43
commit
39cf6ce92e
@ -71,6 +71,17 @@ collectd_controller: true
|
||||
collectd_ceph: true
|
||||
collectd_compute: false
|
||||
|
||||
# Collect plugins configuration:
|
||||
########################
|
||||
# Gnocchi backlog plugin
|
||||
########################
|
||||
# This should only be enabled on a single controller when monitoring Gnocchi for
|
||||
# performance/scale. This plugin does create a token each interval and adds load
|
||||
# to Gnocchi-api each time it polls for measurements and thus your cloud will pay
|
||||
# a monitoring "tax" by enabling this plugin.
|
||||
gnocchi_status_python_plugin: false
|
||||
gnocchi_status_interval: 30
|
||||
|
||||
########################################
|
||||
# Docker related
|
||||
# (use these if deploying graphite/carbon/grafana as containers)
|
||||
|
||||
@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
HOSTNAME="${COLLECTD_HOSTNAME:-`hostname -f`}"
|
||||
INTERVAL="${COLLECTD_INTERVAL:-10}"
|
||||
PORT=6379
|
||||
|
||||
while true
|
||||
do
|
||||
|
||||
info=$((echo info ; sleep 2) |nc -w 1 $HOSTNAME $PORT 2>&1)
|
||||
connected_clients=$(echo "$info" | egrep ^connected_clients| awk -F: '{ print $2 }' | sed 's/
//g')
|
||||
connected_slaves=$(echo "$info" | egrep ^connected_slaves| awk -F: '{ print $2 }' | sed 's/
//g')
|
||||
uptime=$(echo "$info" | egrep ^uptime_in_seconds| awk -F: '{ print $2 }' | sed 's/
//g')
|
||||
used_memory=$(echo "$info" | egrep ^used_memory:| awk -F: '{ print $2 }' | sed 's/
//g')
|
||||
changes_since_last_save=$(echo "$info" | egrep ^rdb_changes_since_last_save| awk -F: '{ print $2 }' | sed 's/
//g')
|
||||
total_commands_processed=$(echo "$info" | egrep ^total_commands_processed| awk -F: '{ print $2 }' | sed 's/
//g')
|
||||
keys=$(echo "$info" | egrep ^db0:keys| awk -F= '{ print $2 }' | awk -F, '{ print $1 }' | sed 's/
//g')
|
||||
|
||||
echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_connections-clients interval=$INTERVAL N:$connected_clients"
|
||||
echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_connections-slaves interval=$INTERVAL N:$connected_slaves"
|
||||
echo "PUTVAL $HOSTNAME/redis-$PORT/uptime interval=$INTERVAL N:$uptime"
|
||||
echo "PUTVAL $HOSTNAME/redis-$PORT/df-memory interval=$INTERVAL N:$used_memory:U"
|
||||
echo "PUTVAL $HOSTNAME/redis-$PORT/files-unsaved_changes interval=$INTERVAL N:$changes_since_last_save"
|
||||
echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_command-total interval=$INTERVAL N:$total_commands_processed"
|
||||
echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_items-db0 interval=$INTERVAL N:$keys"
|
||||
|
||||
sleep "$INTERVAL"
|
||||
done
|
||||
|
||||
@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python
|
||||
"""Collectd python plugin to read gnocchi status on an OpenStack Controller."""
|
||||
from gnocchiclient.v1 import client
|
||||
from keystoneauth1.identity import v2
|
||||
from keystoneauth1 import session
|
||||
import collectd
|
||||
import os
|
||||
import time
|
||||
|
||||
|
||||
def configure(configobj):
|
||||
global INTERVAL
|
||||
|
||||
config = {c.key: c.values for c in configobj.children}
|
||||
INTERVAL = 10
|
||||
if 'interval' in config:
|
||||
INTERVAL = config['interval'][0]
|
||||
collectd.info('gnocchi_status: Interval: {}'.format(INTERVAL))
|
||||
collectd.register_read(read, INTERVAL)
|
||||
|
||||
def read(data=None):
|
||||
starttime = time.time()
|
||||
|
||||
auth = v2.Password(username=os_username,
|
||||
password=os_password,
|
||||
tenant_name=os_tenant,
|
||||
auth_url=os_auth_url)
|
||||
sess = session.Session(auth=auth)
|
||||
|
||||
gnocchi = client.Client(session=sess)
|
||||
status = gnocchi.status.get()
|
||||
|
||||
metric = collectd.Values()
|
||||
metric.plugin = 'gnocchi_status'
|
||||
metric.interval = INTERVAL
|
||||
metric.type = 'gauge'
|
||||
metric.type_instance = 'measures'
|
||||
metric.values = [status['storage']['summary']['measures']]
|
||||
metric.dispatch()
|
||||
|
||||
metric = collectd.Values()
|
||||
metric.plugin = 'gnocchi_status'
|
||||
metric.interval = INTERVAL
|
||||
metric.type = 'gauge'
|
||||
metric.type_instance = 'metrics'
|
||||
metric.values = [status['storage']['summary']['metrics']]
|
||||
metric.dispatch()
|
||||
|
||||
timediff = time.time() - starttime
|
||||
if timediff > INTERVAL:
|
||||
collectd.warning('gnocchi_status: Took: {} > {}'.format(round(timediff, 2),
|
||||
INTERVAL))
|
||||
|
||||
os_username = os.environ.get('OS_USERNAME')
|
||||
os_password = os.environ.get('OS_PASSWORD')
|
||||
os_tenant = os.environ.get('OS_TENANT_NAME')
|
||||
os_auth_url = os.environ.get('OS_AUTH_URL')
|
||||
|
||||
collectd.info('gnocchi_status: Connecting with user={}, password={}, tenant={}, '
|
||||
'auth_url={}'.format(os_username, os_password, os_tenant, os_auth_url))
|
||||
collectd.register_config(configure)
|
||||
@ -1,6 +1,6 @@
|
||||
---
|
||||
#
|
||||
# Install/run collectd for browbeat
|
||||
# Install/run collectd for Browbeat
|
||||
#
|
||||
|
||||
#
|
||||
@ -22,23 +22,45 @@
|
||||
register: mysql_root_password
|
||||
when: "'controller' in group_names"
|
||||
|
||||
- name: Get overcloudrc
|
||||
remote_user: "{{local_remote_user}}"
|
||||
shell: "cat /home/stack/overcloudrc | grep 'OS' | awk '{gsub(/export /,\"Environment=\");print }'"
|
||||
delegate_to: "{{groups['undercloud'][0]}}"
|
||||
register: overcloudrc_file
|
||||
when: "(gnocchi_status_python_plugin == true) and (inventory_hostname == groups['controller'][0])"
|
||||
|
||||
- name: Add environment variables to collectd.service systemd file
|
||||
become: true
|
||||
lineinfile:
|
||||
dest: /usr/lib/systemd/system/collectd.service
|
||||
insertafter: '\[Service\]'
|
||||
line: "{{item}}"
|
||||
with_items: "{{overcloudrc_file.stdout_lines | default(omit)}}"
|
||||
when: "(gnocchi_status_python_plugin == true) and (inventory_hostname == groups['controller'][0])"
|
||||
|
||||
- name: Reload systemd units
|
||||
command: systemctl daemon-reload
|
||||
become: true
|
||||
when: "(gnocchi_status_python_plugin == true) and (inventory_hostname == groups['controller'][0])"
|
||||
|
||||
- name: Configure collectd.conf
|
||||
template:
|
||||
src={{config_type}}.collectd.conf.j2
|
||||
dest=/etc/collectd.conf
|
||||
owner=root
|
||||
group=root
|
||||
mode="0644"
|
||||
src: "{{config_type}}.collectd.conf.j2"
|
||||
dest: /etc/collectd.conf
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0644
|
||||
become: true
|
||||
|
||||
- name: Copy collectd-redis.sh
|
||||
- name: Copy collectd_gnocchi_status.py
|
||||
copy:
|
||||
src=collectd-redis.sh
|
||||
dest=/usr/local/bin/collectd-redis.sh
|
||||
owner=root
|
||||
group=root
|
||||
mode="0755"
|
||||
src: collectd_gnocchi_status.py
|
||||
dest: /usr/local/bin/collectd_gnocchi_status.py
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0755
|
||||
become: true
|
||||
when: gnocchi_status_python_plugin
|
||||
|
||||
#
|
||||
# Configure selinux bits
|
||||
@ -60,11 +82,11 @@
|
||||
#
|
||||
- name: Collectd policy customization
|
||||
copy:
|
||||
src=custom-collectd.pp
|
||||
dest=/root/custom-collectd.pp
|
||||
owner=root
|
||||
group=root
|
||||
mode="0644"
|
||||
src: custom-collectd.pp
|
||||
dest: /root/custom-collectd.pp
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0644
|
||||
become: true
|
||||
|
||||
- name: Check for collectd custom
|
||||
@ -83,5 +105,8 @@
|
||||
# Start collectd service
|
||||
#
|
||||
- name: Setup collectd service
|
||||
service: name=collectd state=restarted enabled=true
|
||||
service:
|
||||
name: collectd
|
||||
state: restarted
|
||||
enabled: true
|
||||
become: true
|
||||
|
||||
@ -89,6 +89,26 @@ PreCacheChain "PreCache"
|
||||
# Exec nobody "/usr/local/bin/collectd-redis.sh"
|
||||
#</Plugin>
|
||||
|
||||
{%if gnocchi_status_python_plugin %}
|
||||
{%if inventory_hostname == groups['controller'][0] %}
|
||||
<LoadPlugin python>
|
||||
Globals true
|
||||
</LoadPlugin>
|
||||
|
||||
<Plugin python>
|
||||
ModulePath "/usr/local/bin/"
|
||||
LogTraces true
|
||||
Interactive false
|
||||
Import "collectd_gnocchi_status"
|
||||
<Module collectd_gnocchi_status>
|
||||
interval {{gnocchi_status_interval}}
|
||||
</Module>
|
||||
</Plugin>
|
||||
{% else %}
|
||||
# Gnocchi status plugin installed and enabled on {{groups['controller'][0]}}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
<Plugin mysql>
|
||||
<Database "overcloud">
|
||||
Host "localhost"
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
{% set vars = {'panel_idx': 0, 'temp_count': 0} %}
|
||||
{% set mariadb_groups = ['undercloud', 'controller', '*'] %}
|
||||
{% set gnocchi_groups = ['controller', '*'] %}
|
||||
{
|
||||
"dashboard": {
|
||||
"annotations": {
|
||||
@ -3734,6 +3735,94 @@
|
||||
"title": "MYSQL INNODB"
|
||||
},
|
||||
{% endif %}
|
||||
{% if item.template_node_type in gnocchi_groups %}
|
||||
{
|
||||
"title": "Gnocchi Backlog",
|
||||
"height": "250px",
|
||||
"editable": true,
|
||||
"collapse": true,
|
||||
"panels": [
|
||||
{
|
||||
"title": "Metrics/Measures Backlog",
|
||||
"error": false,
|
||||
"span": 12,
|
||||
"editable": true,
|
||||
"type": "graph",
|
||||
"isNew": true,
|
||||
"id": 185,
|
||||
"targets": [
|
||||
{
|
||||
"target": "aliasByMetric(aliasSub($Cloud.$Node.gnocchi_status.*, 'gauge-', ''))",
|
||||
"refId": "B",
|
||||
"textEditor": false
|
||||
}
|
||||
],
|
||||
"datasource": null,
|
||||
"renderer": "flot",
|
||||
"yaxes": [
|
||||
{
|
||||
"label": null,
|
||||
"show": true,
|
||||
"logBase": 1,
|
||||
"min": null,
|
||||
"max": null,
|
||||
"format": "short"
|
||||
},
|
||||
{
|
||||
"label": null,
|
||||
"show": true,
|
||||
"logBase": 1,
|
||||
"min": null,
|
||||
"max": null,
|
||||
"format": "short"
|
||||
}
|
||||
],
|
||||
"xaxis": {
|
||||
"show": true
|
||||
},
|
||||
"grid": {
|
||||
"threshold1": null,
|
||||
"threshold2": null,
|
||||
"threshold1Color": "rgba(216, 200, 27, 0.27)",
|
||||
"threshold2Color": "rgba(234, 112, 112, 0.22)"
|
||||
},
|
||||
"lines": true,
|
||||
"fill": 0,
|
||||
"linewidth": 2,
|
||||
"points": false,
|
||||
"pointradius": 5,
|
||||
"bars": false,
|
||||
"stack": false,
|
||||
"percentage": false,
|
||||
"legend": {
|
||||
"show": true,
|
||||
"values": true,
|
||||
"min": true,
|
||||
"max": true,
|
||||
"current": true,
|
||||
"total": false,
|
||||
"avg": true,
|
||||
"alignAsTable": true,
|
||||
"rightSide": true
|
||||
},
|
||||
"nullPointMode": "connected",
|
||||
"steppedLine": false,
|
||||
"tooltip": {
|
||||
"value_type": "cumulative",
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"msResolution": false
|
||||
},
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"aliasColors": {},
|
||||
"seriesOverrides": [],
|
||||
"links": []
|
||||
}
|
||||
],
|
||||
"showTitle": true
|
||||
},
|
||||
{% endif %}
|
||||
{
|
||||
"collapse": true,
|
||||
"editable": true,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user