diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index 7ecb18233..32b4ccb72 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -71,6 +71,17 @@ collectd_controller: true collectd_ceph: true collectd_compute: false +# Collect plugins configuration: +######################## +# Gnocchi backlog plugin +######################## +# This should only be enabled on a single controller when monitoring Gnocchi for +# performance/scale. This plugin does create a token each interval and adds load +# to Gnocchi-api each time it polls for measurements and thus your cloud will pay +# a monitoring "tax" by enabling this plugin. +gnocchi_status_python_plugin: false +gnocchi_status_interval: 30 + ######################################## # Docker related # (use these if deploying graphite/carbon/grafana as containers) diff --git a/ansible/install/roles/collectd-openstack/files/collectd-redis.sh b/ansible/install/roles/collectd-openstack/files/collectd-redis.sh deleted file mode 100644 index e25c2418a..000000000 --- a/ansible/install/roles/collectd-openstack/files/collectd-redis.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -HOSTNAME="${COLLECTD_HOSTNAME:-`hostname -f`}" -INTERVAL="${COLLECTD_INTERVAL:-10}" -PORT=6379 - -while true -do - -info=$((echo info ; sleep 2) |nc -w 1 $HOSTNAME $PORT 2>&1) -connected_clients=$(echo "$info" | egrep ^connected_clients| awk -F: '{ print $2 }' | sed 's/ //g') -connected_slaves=$(echo "$info" | egrep ^connected_slaves| awk -F: '{ print $2 }' | sed 's/ //g') -uptime=$(echo "$info" | egrep ^uptime_in_seconds| awk -F: '{ print $2 }' | sed 's/ //g') -used_memory=$(echo "$info" | egrep ^used_memory:| awk -F: '{ print $2 }' | sed 's/ //g') -changes_since_last_save=$(echo "$info" | egrep ^rdb_changes_since_last_save| awk -F: '{ print $2 }' | sed 's/ //g') -total_commands_processed=$(echo "$info" | egrep ^total_commands_processed| awk -F: '{ print $2 }' | sed 's/ //g') -keys=$(echo "$info" | egrep ^db0:keys| awk -F= '{ print $2 }' | awk -F, '{ print $1 }' | sed 's/ //g') - -echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_connections-clients interval=$INTERVAL N:$connected_clients" -echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_connections-slaves interval=$INTERVAL N:$connected_slaves" -echo "PUTVAL $HOSTNAME/redis-$PORT/uptime interval=$INTERVAL N:$uptime" -echo "PUTVAL $HOSTNAME/redis-$PORT/df-memory interval=$INTERVAL N:$used_memory:U" -echo "PUTVAL $HOSTNAME/redis-$PORT/files-unsaved_changes interval=$INTERVAL N:$changes_since_last_save" -echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_command-total interval=$INTERVAL N:$total_commands_processed" -echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_items-db0 interval=$INTERVAL N:$keys" - -sleep "$INTERVAL" -done - diff --git a/ansible/install/roles/collectd-openstack/files/collectd_gnocchi_status.py b/ansible/install/roles/collectd-openstack/files/collectd_gnocchi_status.py new file mode 100644 index 000000000..ecdb5b41a --- /dev/null +++ b/ansible/install/roles/collectd-openstack/files/collectd_gnocchi_status.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +"""Collectd python plugin to read gnocchi status on an OpenStack Controller.""" +from gnocchiclient.v1 import client +from keystoneauth1.identity import v2 +from keystoneauth1 import session +import collectd +import os +import time + + +def configure(configobj): + global INTERVAL + + config = {c.key: c.values for c in configobj.children} + INTERVAL = 10 + if 'interval' in config: + INTERVAL = config['interval'][0] + collectd.info('gnocchi_status: Interval: {}'.format(INTERVAL)) + collectd.register_read(read, INTERVAL) + +def read(data=None): + starttime = time.time() + + auth = v2.Password(username=os_username, + password=os_password, + tenant_name=os_tenant, + auth_url=os_auth_url) + sess = session.Session(auth=auth) + + gnocchi = client.Client(session=sess) + status = gnocchi.status.get() + + metric = collectd.Values() + metric.plugin = 'gnocchi_status' + metric.interval = INTERVAL + metric.type = 'gauge' + metric.type_instance = 'measures' + metric.values = [status['storage']['summary']['measures']] + metric.dispatch() + + metric = collectd.Values() + metric.plugin = 'gnocchi_status' + metric.interval = INTERVAL + metric.type = 'gauge' + metric.type_instance = 'metrics' + metric.values = [status['storage']['summary']['metrics']] + metric.dispatch() + + timediff = time.time() - starttime + if timediff > INTERVAL: + collectd.warning('gnocchi_status: Took: {} > {}'.format(round(timediff, 2), + INTERVAL)) + +os_username = os.environ.get('OS_USERNAME') +os_password = os.environ.get('OS_PASSWORD') +os_tenant = os.environ.get('OS_TENANT_NAME') +os_auth_url = os.environ.get('OS_AUTH_URL') + +collectd.info('gnocchi_status: Connecting with user={}, password={}, tenant={}, ' + 'auth_url={}'.format(os_username, os_password, os_tenant, os_auth_url)) +collectd.register_config(configure) diff --git a/ansible/install/roles/collectd-openstack/tasks/main.yml b/ansible/install/roles/collectd-openstack/tasks/main.yml index 630795925..21933027f 100644 --- a/ansible/install/roles/collectd-openstack/tasks/main.yml +++ b/ansible/install/roles/collectd-openstack/tasks/main.yml @@ -1,6 +1,6 @@ --- # -# Install/run collectd for browbeat +# Install/run collectd for Browbeat # # @@ -22,23 +22,45 @@ register: mysql_root_password when: "'controller' in group_names" +- name: Get overcloudrc + remote_user: "{{local_remote_user}}" + shell: "cat /home/stack/overcloudrc | grep 'OS' | awk '{gsub(/export /,\"Environment=\");print }'" + delegate_to: "{{groups['undercloud'][0]}}" + register: overcloudrc_file + when: "(gnocchi_status_python_plugin == true) and (inventory_hostname == groups['controller'][0])" + +- name: Add environment variables to collectd.service systemd file + become: true + lineinfile: + dest: /usr/lib/systemd/system/collectd.service + insertafter: '\[Service\]' + line: "{{item}}" + with_items: "{{overcloudrc_file.stdout_lines | default(omit)}}" + when: "(gnocchi_status_python_plugin == true) and (inventory_hostname == groups['controller'][0])" + +- name: Reload systemd units + command: systemctl daemon-reload + become: true + when: "(gnocchi_status_python_plugin == true) and (inventory_hostname == groups['controller'][0])" + - name: Configure collectd.conf template: - src={{config_type}}.collectd.conf.j2 - dest=/etc/collectd.conf - owner=root - group=root - mode="0644" + src: "{{config_type}}.collectd.conf.j2" + dest: /etc/collectd.conf + owner: root + group: root + mode: 0644 become: true -- name: Copy collectd-redis.sh +- name: Copy collectd_gnocchi_status.py copy: - src=collectd-redis.sh - dest=/usr/local/bin/collectd-redis.sh - owner=root - group=root - mode="0755" + src: collectd_gnocchi_status.py + dest: /usr/local/bin/collectd_gnocchi_status.py + owner: root + group: root + mode: 0755 become: true + when: gnocchi_status_python_plugin # # Configure selinux bits @@ -60,11 +82,11 @@ # - name: Collectd policy customization copy: - src=custom-collectd.pp - dest=/root/custom-collectd.pp - owner=root - group=root - mode="0644" + src: custom-collectd.pp + dest: /root/custom-collectd.pp + owner: root + group: root + mode: 0644 become: true - name: Check for collectd custom @@ -83,5 +105,8 @@ # Start collectd service # - name: Setup collectd service - service: name=collectd state=restarted enabled=true + service: + name: collectd + state: restarted + enabled: true become: true diff --git a/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 b/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 index a4fa9dcb0..986990672 100644 --- a/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 +++ b/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 @@ -89,6 +89,26 @@ PreCacheChain "PreCache" # Exec nobody "/usr/local/bin/collectd-redis.sh" # +{%if gnocchi_status_python_plugin %} +{%if inventory_hostname == groups['controller'][0] %} + + Globals true + + + + ModulePath "/usr/local/bin/" + LogTraces true + Interactive false + Import "collectd_gnocchi_status" + + interval {{gnocchi_status_interval}} + + +{% else %} +# Gnocchi status plugin installed and enabled on {{groups['controller'][0]}} +{% endif %} +{% endif %} + Host "localhost" diff --git a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 index b0875ca09..5768214e9 100644 --- a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 @@ -1,5 +1,6 @@ {% set vars = {'panel_idx': 0, 'temp_count': 0} %} {% set mariadb_groups = ['undercloud', 'controller', '*'] %} +{% set gnocchi_groups = ['controller', '*'] %} { "dashboard": { "annotations": { @@ -3734,6 +3735,94 @@ "title": "MYSQL INNODB" }, {% endif %} + {% if item.template_node_type in gnocchi_groups %} + { + "title": "Gnocchi Backlog", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "Metrics/Measures Backlog", + "error": false, + "span": 12, + "editable": true, + "type": "graph", + "isNew": true, + "id": 185, + "targets": [ + { + "target": "aliasByMetric(aliasSub($Cloud.$Node.gnocchi_status.*, 'gauge-', ''))", + "refId": "B", + "textEditor": false + } + ], + "datasource": null, + "renderer": "flot", + "yaxes": [ + { + "label": null, + "show": true, + "logBase": 1, + "min": null, + "max": null, + "format": "short" + }, + { + "label": null, + "show": true, + "logBase": 1, + "min": null, + "max": null, + "format": "short" + } + ], + "xaxis": { + "show": true + }, + "grid": { + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 2, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": true, + "min": true, + "max": true, + "current": true, + "total": false, + "avg": true, + "alignAsTable": true, + "rightSide": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true, + "sort": 0, + "msResolution": false + }, + "timeFrom": null, + "timeShift": null, + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + {% endif %} { "collapse": true, "editable": true,