From a8ef2462d7f3bf79a4d6206df904fb7c20f8d099 Mon Sep 17 00:00:00 2001 From: jkilpatr Date: Thu, 12 Jan 2017 16:28:08 -0500 Subject: [PATCH] Add StatsD support for Ironic StatsD support works, working on dashboards As documented here http://docs.openstack.org/developer/ironic/deploy/metrics.html Ironic allows performance metrics to be dumped via StatsD, this installs StatsD on the undercloud via epel, sets up a service for it, enables it, and updates Ironic to use it. Change-Id: I793d4d3211ecf6113bd4863a0672ea0cb0de9dd3 --- ansible/install/grafana-dashboards.yml | 1 + ansible/install/group_vars/all.yml | 14 + ansible/install/ironic-statsd.yml | 8 + .../files/openstack_ironic_metrics.json | 698 ++++++++++++++++++ .../roles/grafana-dashboards/tasks/main.yml | 8 +- ...enstack_general_system_performance.json.j2 | 639 ++++++++++++++++ .../roles/statsd-install/tasks/main.yml | 29 + .../templates/statsd.service.j2 | 26 + .../templates/statsd_config.js.j2 | 6 + .../roles/statsd-ironic/defaults/main.yml | 3 + .../roles/statsd-ironic/tasks/main.yml | 30 + ansible/install/statsd.yml | 9 + ...metal-virt-undercloud-tripleo-browbeat.yml | 2 + ci-scripts/tripleo/microbrow.sh | 2 + 14 files changed, 1474 insertions(+), 1 deletion(-) create mode 100644 ansible/install/ironic-statsd.yml create mode 100644 ansible/install/roles/grafana-dashboards/files/openstack_ironic_metrics.json create mode 100644 ansible/install/roles/statsd-install/tasks/main.yml create mode 100644 ansible/install/roles/statsd-install/templates/statsd.service.j2 create mode 100644 ansible/install/roles/statsd-install/templates/statsd_config.js.j2 create mode 100644 ansible/install/roles/statsd-ironic/defaults/main.yml create mode 100644 ansible/install/roles/statsd-ironic/tasks/main.yml create mode 100644 ansible/install/statsd.yml diff --git a/ansible/install/grafana-dashboards.yml b/ansible/install/grafana-dashboards.yml index c35e97606..a5634e7f8 100644 --- a/ansible/install/grafana-dashboards.yml +++ b/ansible/install/grafana-dashboards.yml @@ -18,6 +18,7 @@ - cloud_system_performance_comparsion - cloud_gnocchi_status - three_node_performance_food_groups + - openstack_ironic_metrics cloud_specific_dashboards: - cpu - memory diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index 41f2b732a..42cc2132e 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -117,6 +117,12 @@ graphite_prefix: openstack # credential aren't created when you deploy graphite, use manage.py graphite_username: root graphite_password: calvin +# List of cloud names taken by other infrastructure +# attempting to use them should fail. +forbidden_cloud_names: + - "statsd" + - "stats" + - "stats_counts" ######################################## # Grafana Dashboarding Configuration @@ -136,6 +142,14 @@ dashboards_batch: 20 # For use with all-{cpu, memory, disk, network} openstack dashboards, uses the graphite prefix to create dashboards for specific openstack cloud dashboard_cloud_name: "{{graphite_prefix}}" +######################################## +# StatsD Configuration +# Points at configured Graphite instance +######################################## +statsd_host: +statsd_port: 8125 +statsd_enabled: False + ######################################## # Shaker Configuration ######################################## diff --git a/ansible/install/ironic-statsd.yml b/ansible/install/ironic-statsd.yml new file mode 100644 index 000000000..16d640b4a --- /dev/null +++ b/ansible/install/ironic-statsd.yml @@ -0,0 +1,8 @@ +--- +# +# enables Ironic StatsD metrics and points at the StatsD host configured in groupvars/all.yml + +- hosts: undercloud + remote_user: "{{ local_remote_user }}" + roles: + - statsd-ironic diff --git a/ansible/install/roles/grafana-dashboards/files/openstack_ironic_metrics.json b/ansible/install/roles/grafana-dashboards/files/openstack_ironic_metrics.json new file mode 100644 index 000000000..60e4f35a6 --- /dev/null +++ b/ansible/install/roles/grafana-dashboards/files/openstack_ironic_metrics.json @@ -0,0 +1,698 @@ +{ + "dashboard": { + "annotations": { + "list": [] + }, + "description": "Ironic StatsD data, only populated when StatsD is enabled", + "editable": true, + "gnetId": null, + "hideControls": false, + "id": null, + "links": [], + "refresh": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.conductor.manager.do_sync_power_state.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Power State Sync: 90th Percentile Mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.conductor.manager.ConductorManager.change_node_power_state.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Change Node Power State: 90th percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 3, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.conductor.manager.ConductorManager.set_boot_device.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Set boot device: 90th percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 4, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.conductor.manager.ConductorManager.do_node_deploy.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Do Node Deployment: 90th percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 5, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.api.controllers.v1.node.NodesController.detail.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Node Details API response time: 90th percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 6, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.api.controllers.v1.node.NodesController.get_all.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Node List API response time: 90th Percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 7, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.api.controllers.v1.node.NodeStatesController.power.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node Power State API response time: 90th Percentile Mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.api.controllers.v1.node.NodeStatesController.provision.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Ironic Node Provision State API response time: 90th percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Millisecond", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + } + ], + "schemaVersion": 12, + "sharedCrosshair": false, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allFormat": "glob", + "current": { + "text": "", + "value": "" + }, + "datasource": null, + "hide": 0, + "includeAll": false, + "multi": false, + "multiFormat": "glob", + "name": "Cloud", + "options": [], + "query": "*", + "refresh": 1, + "regex": "/^(?!stats[d]*[_counts]*$).*$/", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "OpenStack Ironic Metrics", + "version": 2 + } +} diff --git a/ansible/install/roles/grafana-dashboards/tasks/main.yml b/ansible/install/roles/grafana-dashboards/tasks/main.yml index 750b8f6b9..8e12c483d 100644 --- a/ansible/install/roles/grafana-dashboards/tasks/main.yml +++ b/ansible/install/roles/grafana-dashboards/tasks/main.yml @@ -12,6 +12,12 @@ # with cloud_dashboards=false # +- name: Check Cloud Name + fail: + msg: "The Cloud name {{dashboard_cloud_name}} is reserved for a service, please use a different one" + when: item == dashboard_cloud_name + with_items: "{{forbidden_cloud_names}}" + - name: Generate General Dashboards template: src: "{{role_path}}/templates/{{item.template_name}}_general_system_performance.json.j2" @@ -39,7 +45,7 @@ - name: Remove Existing Cloud Specific CPU/Memory/Disk/Network/Log Dashboards command: "curl -X DELETE -H 'Content-Type: application/json' http://{{grafana_username}}:{{grafana_password}}@{{grafana_host}}:{{grafana_port}}/api/dashboards/db/{{dashboard_cloud_name}}-{{item|replace('_', '-')}}" - when: overwrite_existing and cloud_dashboards + when: "{{overwrite_existing}} and {{cloud_dashboards}}" with_items: "{{cloud_specific_dashboards}}" - name: Upload Static Dashboards to Grafana diff --git a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 index 2b8642629..238582e93 100644 --- a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 @@ -1,6 +1,7 @@ {% set vars = {'panel_idx': 0, 'temp_count': 0} %} {% set mariadb_groups = ['undercloud', 'controller', '*'] %} {% set gnocchi_groups = ['controller', '*'] %} +{% set ironic_groups = ['undercloud'] %} { "dashboard": { "annotations": { @@ -3823,6 +3824,644 @@ "showTitle": true }, {% endif %} + {% if item.template_node_type in ironic_groups %} + { + "collapse": true, + "editable": true, + "height": "250px", + "showTitle": true, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.conductor.manager.do_sync_power_state.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Power State Sync: 90th Percentile Mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.conductor.manager.ConductorManager.change_node_power_state.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Change Node Power State: 90th percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Ironic StatsD power state" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "showTitle": true, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.conductor.manager.ConductorManager.set_boot_device.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Set boot device: 90th percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.conductor.manager.ConductorManager.do_node_deploy.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Do Node Deployment: 90th percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Ironic StatsD boot/deployment info" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "showTitle": true, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.api.controllers.v1.node.NodesController.detail.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Node Details API response time: 90th percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.api.controllers.v1.node.NodesController.get_all.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Node List API response time: 90th Percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Ironic StatsD API node details/list response time" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "showTitle": true, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.api.controllers.v1.node.NodeStatesController.power.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node Power State API response time: 90th Percentile Mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "stats.timers.$Cloud.ironic.api.controllers.v1.node.NodeStatesController.provision.mean_90" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Ironic Node Provision State API response time: 90th percentile mean", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": "Millisecond", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Ironic StatsD API power/provision state" + }, + {% endif %} { "collapse": true, "editable": true, diff --git a/ansible/install/roles/statsd-install/tasks/main.yml b/ansible/install/roles/statsd-install/tasks/main.yml new file mode 100644 index 000000000..a04d25403 --- /dev/null +++ b/ansible/install/roles/statsd-install/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: Install StatsD + yum: name=statsd state=present + become: True + +- name: Create StatsD configuration folder + file: + path: /etc/statsd + state: directory + become: True + +- name: Template configuration + template: + src: statsd_config.js.j2 + dest: /etc/statsd/config.js + become: True + +- name: Template StatsD service file + template: + src: statsd.service.j2 + dest: /etc/systemd/system/statsd.service.j2 + owner: root + group: root + mode: 0644 + become: True + +- name: bounce systemd and setup StatsD to run on startup + shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable statsd && /usr/bin/systemctl restart statsd + become: True diff --git a/ansible/install/roles/statsd-install/templates/statsd.service.j2 b/ansible/install/roles/statsd-install/templates/statsd.service.j2 new file mode 100644 index 000000000..bed0e7038 --- /dev/null +++ b/ansible/install/roles/statsd-install/templates/statsd.service.j2 @@ -0,0 +1,26 @@ +# This is a systemd file to make statsd work well +# To make this work: +# * place this file in /etc/systemd/system and run the commands: +# +# Credit for this template goes to the venerable Kambiz +# +# systemctl daemon-reload +# systemctl enable graphite-web +# systemctl start graphite-web +# +[Unit] +Description=statsd + +[Service] +Type=simple +TimeoutStartSec=5m + +ExecStart=statsd /etc/statsd/conf.js + +ExecReload=statsd /etc/statsd/conf.js + +Restart=always +RestartSec=30 + +[Install] +WantedBy=default.target diff --git a/ansible/install/roles/statsd-install/templates/statsd_config.js.j2 b/ansible/install/roles/statsd-install/templates/statsd_config.js.j2 new file mode 100644 index 000000000..254da4018 --- /dev/null +++ b/ansible/install/roles/statsd-install/templates/statsd_config.js.j2 @@ -0,0 +1,6 @@ +{ + graphitePort: 2003 +, graphiteHost: "{{graphite_host}}" +, port: {{statsd_port}} +, backends: [ "./backends/graphite" ] +} diff --git a/ansible/install/roles/statsd-ironic/defaults/main.yml b/ansible/install/roles/statsd-ironic/defaults/main.yml new file mode 100644 index 000000000..863739b56 --- /dev/null +++ b/ansible/install/roles/statsd-ironic/defaults/main.yml @@ -0,0 +1,3 @@ +statsd_host: localhost +statsd_port: 8125 +statsd_enabled: True diff --git a/ansible/install/roles/statsd-ironic/tasks/main.yml b/ansible/install/roles/statsd-ironic/tasks/main.yml new file mode 100644 index 000000000..483f42060 --- /dev/null +++ b/ansible/install/roles/statsd-ironic/tasks/main.yml @@ -0,0 +1,30 @@ +--- +- name: Configure Ironic to use StatsD for metrics + ini_file: + dest: /etc/ironic/ironic.conf + section: "{{ item.section }}" + option: "{{ item.option }}" + value: "{{ item.value }}" + become: True + with_items: + - { section: 'metrics', option: 'backend', value: 'statsd' } + - { section: 'metrics', option: 'agent_backend', value: 'statsd' } + - { section: 'metrics', option: 'global_prefix', value: '{{graphite_prefix}}' } + - { section: 'metrics', option: 'agent_global_prefix', value: '{{graphite_prefix}}' } + - { section: 'metrics_statsd', option: 'statsd_host', value: "{{ statsd_host }}"} + - { section: 'metrics_statsd', option: 'statsd_port', value: "{{ statsd_port }}"} + - { section: 'metrics_statsd', option: 'agent_statsd_host', value: "{{ statsd_host }}"} + - { section: 'metrics_statsd', option: 'agent_statsd_port', value: "{{ statsd_port }}"} + when: "{{ statsd_enabled }}" + +- name: Restart Ironic services + service: + name: "{{ item }}" + state: restarted + become: True + with_items: + - openstack-ironic-api + - openstack-ironic-conductor + - openstack-ironic-inspector-dnsmasq + - openstack-ironic-inspector + when: "{{ statsd_enabled }}" diff --git a/ansible/install/statsd.yml b/ansible/install/statsd.yml new file mode 100644 index 000000000..f4ee344fe --- /dev/null +++ b/ansible/install/statsd.yml @@ -0,0 +1,9 @@ +--- +# +# Installs a StatsD server on the StatsD host with the default port, does not secure it + +- hosts: statsd + remote_user: root + roles: + - epel + - statsd-install diff --git a/ansible/oooq/baremetal-virt-undercloud-tripleo-browbeat.yml b/ansible/oooq/baremetal-virt-undercloud-tripleo-browbeat.yml index ba8971683..19be22b26 100644 --- a/ansible/oooq/baremetal-virt-undercloud-tripleo-browbeat.yml +++ b/ansible/oooq/baremetal-virt-undercloud-tripleo-browbeat.yml @@ -14,10 +14,12 @@ hosts: undercloud vars: results_in_httpd: false + statsd_host: "{{ graphite_host }}" roles: - browbeat/common - browbeat/browbeat - browbeat/browbeat-network + - browbeat/statsd-ironic - name: Disable sshd dns hosts: overcloud diff --git a/ci-scripts/tripleo/microbrow.sh b/ci-scripts/tripleo/microbrow.sh index 418e46ddb..12912c048 100755 --- a/ci-scripts/tripleo/microbrow.sh +++ b/ci-scripts/tripleo/microbrow.sh @@ -22,6 +22,8 @@ export VARS="elastic_enabled_template=true \ --extra-vars grafana_enabled_template=false \ --extra-vars elastic_host_template=$ELASTIC_HOST \ --extra-vars graphite_host_template=$GRAPH_HOST \ +--extra-vars statsd_host=$GRAPH_HOST \ +--extra-vars statsd_enabled=True \ --extra-vars grafana_host_template=$GRAPH_HOST \ --extra-vars grafana_username_template=$GRAFANA_USER \ --extra-vars grafana_password_template=$GRAFANA_PASS \