diff --git a/ansible/generate_tripleo_hostfile.sh b/ansible/generate_tripleo_hostfile.sh index 434a64741..5fe4c3836 100755 --- a/ansible/generate_tripleo_hostfile.sh +++ b/ansible/generate_tripleo_hostfile.sh @@ -27,6 +27,11 @@ if [ ${#controller_id} -lt 1 ]; then echo "Error: Controller ID is not reporting correctly. Please see check the openstack-heat-api on the undercloud." exit 1 fi +blockstorage_id=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; openstack stack resource show overcloud BlockStorage > >(grep physical_resource_id) 2>/dev/null" | awk '{print $4}') +if [ ${#blockstorage_id} -lt 1 ]; then + echo "Error: BlockStorage ID is not reporting correctly. Please see check the openstack-heat-api on the undercloud." + exit 1 +fi objectstorage_id=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; openstack stack resource show overcloud ObjectStorage > >(grep physical_resource_id) 2>/dev/null" | awk '{print $4}') if [ ${#objectstorage_id} -lt 1 ]; then echo "Error: ObjectStorage ID is not reporting correctly. Please see check the openstack-heat-api on the undercloud." @@ -48,6 +53,10 @@ if [ ${#controller_ids} -lt 1 ]; then echo "Error: Controller IDs is not reporting correctly. Please see check the openstack-heat-api on the undercloud." exit 1 fi +blockstorage_ids=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; openstack stack resource list ${blockstorage_id} > >(grep -i blockstorage) 2>/dev/null" | awk '{print $2}') +if [ ${#blockstorage_ids} -lt 1 ]; then + echo "Info: No BlockStorage resources." +fi objectstorage_ids=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; openstack stack resource list ${objectstorage_id} > >(grep -i objectstorage) 2>/dev/null" | awk '{print $2}') if [ ${#objectstorage_ids} -lt 1 ]; then echo "Info: No ObjectStorage resources." @@ -72,6 +81,15 @@ do controller_uuids+=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; openstack stack resource show ${controller_id} ${controller} > >(grep -oP \"'nova_server_resource': u'([a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+)'\") 2>/dev/null" | awk '{print $2}' | grep -oP [a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+) fi done +blockstorage_uuids=() +for blockstorage in ${blockstorage_ids} +do + if [[ ${version_tripleo} -lt 2 ]] ; then + blockstorage_uuids+=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; heat resource-show ${blockstorage_id} ${blockstorage} | grep -i nova_server_resource" | awk '{print $4}') + else + blockstorage_uuids+=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; openstack stack resource show ${blockstorage_id} ${blockstorage} > >(grep -oP \"'nova_server_resource': u'([a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+)'\") 2>/dev/null" | awk '{print $2}' | grep -oP [a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+) + fi +done objectstorage_uuids=() for objectstorage in ${objectstorage_ids} do @@ -130,7 +148,9 @@ echo " UserKnownHostsFile=/dev/null" | tee -a ${ssh_config_file} compute_hn=() controller_hn=() -ceph_hn=() +blockstorage_hn=() +objectstorage_hn=() +cephstorage_hn=() IFS=$'\n' for line in $nodes; do uuid=$(echo $line | awk '{print $2}') @@ -138,6 +158,8 @@ for line in $nodes; do IP=$(echo $line | awk '{print $12}' | cut -d "=" -f2) if grep -q $uuid <<< {$controller_uuids}; then controller_hn+=("$host") +elif grep -q $uuid <<< {$blockstorage_uuids}; then + blockstorage_hn+=("$host") elif grep -q $uuid <<< {$objectstorage_uuids}; then objectstorage_hn+=("$host") elif grep -q $uuid <<< {$cephstorage_uuids}; then @@ -170,6 +192,13 @@ if [[ ${#controller_hn} -gt 0 ]]; then echo "${ct}" | tee -a ${ansible_inventory_file} done fi +if [[ ${#blockstorage_hn} -gt 0 ]]; then + echo "" | tee -a ${ansible_inventory_file} + echo "[blockstorage]" | tee -a ${ansible_inventory_file} + for blockstorage in ${blockstorage_hn[@]}; do + echo "${blockstorage}" | tee -a ${ansible_inventory_file} + done +fi if [[ ${#objectstorage_hn} -gt 0 ]]; then echo "" | tee -a ${ansible_inventory_file} echo "[objectstorage]" | tee -a ${ansible_inventory_file} diff --git a/ansible/install/collectd-openstack.yml b/ansible/install/collectd-openstack.yml index cb9f62fce..491ab2ec2 100644 --- a/ansible/install/collectd-openstack.yml +++ b/ansible/install/collectd-openstack.yml @@ -8,6 +8,7 @@ # Or use tags: # ansible-playbook -i hosts install/collectd-openstack.yml --tag "undercloud" # ansible-playbook -i hosts install/collectd-openstack.yml --tag "controller" +# ansible-playbook -i hosts install/collectd-openstack.yml --tag "blockstorage" # ansible-playbook -i hosts install/collectd-openstack.yml --tag "objectstorage" # ansible-playbook -i hosts install/collectd-openstack.yml --tag "cephstorage" # ansible-playbook -i hosts install/collectd-openstack.yml --tag "compute" @@ -46,6 +47,22 @@ ignore_errors: true tags: controller +- hosts: blockstorage + remote_user: "{{ host_remote_user }}" + vars: + config_type: blockstorage + roles: + - { role: common, when: collectd_blockstorage } + - { role: epel, when: collectd_blockstorage } + - { role: collectd-openstack, when: collectd_blockstorage } + tasks: + - name: Collectd off if not collectd_blockstorage + service: name=collectd state=stopped enabled=false + become: true + when: not collectd_blockstorage + ignore_errors: true + tags: blockstorage + - hosts: objectstorage remote_user: "{{ host_remote_user }}" vars: diff --git a/ansible/install/grafana-dashboards.yml b/ansible/install/grafana-dashboards.yml index 136aa8f4e..4fbabfbbd 100644 --- a/ansible/install/grafana-dashboards.yml +++ b/ansible/install/grafana-dashboards.yml @@ -29,6 +29,9 @@ - template_name: openstack template_node_type: controller process_list_name: OpenStack-Controller + - template_name: openstack + template_node_type: blockstorage + process_list_name: OpenStack-BlockStorage - template_name: openstack template_node_type: objectstorage process_list_name: OpenStack-ObjectStorage diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index ec4f12e5d..2dd736531 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -68,6 +68,7 @@ collectd_interval: 10 # Run collectd on specific openstack nodes: collectd_undercloud: true collectd_controller: true +collectd_blockstorage: true collectd_objectstorage: true collectd_cephstorage: true collectd_compute: false diff --git a/ansible/install/roles/collectd-openstack/templates/blockstorage.collectd.conf.j2 b/ansible/install/roles/collectd-openstack/templates/blockstorage.collectd.conf.j2 new file mode 100644 index 000000000..01209ca09 --- /dev/null +++ b/ansible/install/roles/collectd-openstack/templates/blockstorage.collectd.conf.j2 @@ -0,0 +1,139 @@ +# Installed by Browbeat Ansible Installer +# Config type: {{config_type}} + +# Interval default is 10s +Interval {{collectd_interval}} + +# Hostname for this machine, if not defined, use gethostname(2) system call +Hostname "{{inventory_hostname}}" + +# Loaded Plugins: +LoadPlugin "logfile" + + LogLevel "info" + File "/var/log/collectd.log" + Timestamp true + + +LoadPlugin write_graphite +LoadPlugin cpu +LoadPlugin conntrack +LoadPlugin df +LoadPlugin disk +LoadPlugin exec +LoadPlugin interface +LoadPlugin irq +LoadPlugin load +LoadPlugin match_regex +LoadPlugin memory +LoadPlugin numa +LoadPlugin processes +LoadPlugin swap +LoadPlugin tail +LoadPlugin turbostat +LoadPlugin unixsock +LoadPlugin uptime + +# Open unix domain socket for collectdctl + + SocketFile "/var/run/collectd-unixsock" + SocketGroup "collectd" + SocketPerms "0770" + DeleteSocket true + + +PreCacheChain "PreCache" + + + + Plugin "^interface$" + PluginInstance "^tap*" + + Target "stop" + + + + Plugin "^interface$" + PluginInstance "^q.*" + + Target "stop" + + Target "return" + + +# Graphite Host Configuration + + + Host "{{graphite_host}}" + Port "2003" + Prefix "{{graphite_prefix}}." + Protocol "tcp" + LogSendErrors true + StoreRates true + AlwaysAppendDS false + EscapeCharacter "_" + + + + + ValuesPercentage true + + + + Disk "/^[hsv]d[a-z]+[0-9]?$/" + IgnoreSelected false + + +# (akrzos) Including the version of OpenStack that the process was verified as running after +# OpenStack Installation with a comment at the end of each Process/ProcessMatch statement. +# A Minus before the version means the process was not found in that version. (Ex -9) + + # Collectd (Browbeat Installed) + ProcessMatch "collectd" "/usr/sbin/collectd" + + # OVS (OpenStack Installed) + ProcessMatch "ovs-vswitchd" "ovs-vswitchd.+openvswitch" # 10 + ProcessMatch "ovsdb-server" "ovsdb-server.+openvswitch" # 10 + + # Cinder (OpenStack Installed) + ProcessMatch "cinder-volume" "python.+cinder-volume" # 10 + + + + ReportBytes true + ValuesPercentage true + + +# Tail plugin configuration + + + + Instance "cinder-volume" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + + +# Include other collectd configuration files +Include "/etc/collectd.d" diff --git a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_cpu.json.j2 b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_cpu.json.j2 index 40e642674..b6e67e164 100644 --- a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_cpu.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_cpu.json.j2 @@ -1,5 +1,5 @@ {% set vars = {'panel_idx': 0, 'initial': 0} %} -{% set dashboard_groups = ['undercloud', 'controller', 'objectstorage', 'cephstorage', 'compute'] %} +{% set dashboard_groups = ['undercloud', 'controller', 'blockstorage', 'objectstorage', 'cephstorage', 'compute'] %} { "dashboard": { "id": null, diff --git a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_disk.json.j2 b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_disk.json.j2 index 3542295dd..75911c204 100644 --- a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_disk.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_disk.json.j2 @@ -1,5 +1,5 @@ {% set vars = {'panel_idx': 0, 'initial1': 0, 'initial2': 0, 'initial3': 0} %} -{% set dashboard_groups = ['undercloud', 'controller', 'objectstorage', 'cephstorage', 'compute'] %} +{% set dashboard_groups = ['undercloud', 'controller', 'blockstorage', 'objectstorage', 'cephstorage', 'compute'] %} { "dashboard": { "id": null, @@ -398,6 +398,39 @@ "refresh": true, "type": "query" }, + { + "allFormat": "glob", + "current": { + "text": "disk-sda", + "value": "disk-sda" + }, + "datasource": null, + "includeAll": false, + "multi": false, + "multiFormat": "glob", + "name": "compute_disk", + "options": [], + "query": "{{dashboard_cloud_name}}.*compute*.disk-*", + "refresh": true, + "type": "query" + }, + { + "allFormat": "glob", + "current": { + "text": "None", + "value": "", + "isNone": true + }, + "datasource": null, + "includeAll": false, + "multi": false, + "multiFormat": "glob", + "name": "cephstorage_disk", + "options": [], + "query": "{{dashboard_cloud_name}}.*cephstorage*.disk-*", + "refresh": true, + "type": "query" + }, { "allFormat": "glob", "current": { @@ -426,25 +459,9 @@ "includeAll": false, "multi": false, "multiFormat": "glob", - "name": "cephstorage_disk", + "name": "blockstorage_disk", "options": [], - "query": "{{dashboard_cloud_name}}.*cephstorage*.disk-*", - "refresh": true, - "type": "query" - }, - { - "allFormat": "glob", - "current": { - "text": "disk-sda", - "value": "disk-sda" - }, - "datasource": null, - "includeAll": false, - "multi": false, - "multiFormat": "glob", - "name": "compute_disk", - "options": [], - "query": "{{dashboard_cloud_name}}.*compute*.disk-*", + "query": "{{dashboard_cloud_name}}.*blockstorage*.disk-*", "refresh": true, "type": "query" } diff --git a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_disks_all.json.j2 b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_disks_all.json.j2 index df10dd616..02e77c712 100644 --- a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_disks_all.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_disks_all.json.j2 @@ -1,5 +1,5 @@ {% set vars = {'panel_idx': 0, 'initial1': 0, 'initial2': 0, 'initial3': 0} %} -{% set dashboard_groups = ['undercloud', 'controller', 'objectstorage', 'cephstorage', 'compute'] %} +{% set dashboard_groups = ['undercloud', 'controller', 'blockstorage', 'objectstorage', 'cephstorage', 'compute'] %} { "dashboard": { "id": null, @@ -401,9 +401,9 @@ "includeAll": true, "multi": true, "multiFormat": "glob", - "name": "objectstorage_disk", + "name": "compute_disk", "options": [], - "query": "{{dashboard_cloud_name}}.*objectstorage*.disk-*", + "query": "{{dashboard_cloud_name}}.*compute*.disk-*", "refresh": true, "type": "query", "regex": "/disk-[a-z]*$/" @@ -435,9 +435,26 @@ "includeAll": true, "multi": true, "multiFormat": "glob", - "name": "compute_disk", + "name": "objectstorage_disk", "options": [], - "query": "{{dashboard_cloud_name}}.*compute*.disk-*", + "query": "{{dashboard_cloud_name}}.*objectstorage*.disk-*", + "refresh": true, + "type": "query", + "regex": "/disk-[a-z]*$/" + }, + { + "allFormat": "glob", + "current": { + "text": "all", + "value": "all" + }, + "datasource": null, + "includeAll": true, + "multi": true, + "multiFormat": "glob", + "name": "blockstorage_disk", + "options": [], + "query": "{{dashboard_cloud_name}}.*blockstorage*.disk-*", "refresh": true, "type": "query", "regex": "/disk-[a-z]*$/" diff --git a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_log.json.j2 b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_log.json.j2 index 5b59d75e2..9e40f9530 100644 --- a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_log.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_log.json.j2 @@ -1,5 +1,5 @@ {% set vars = {'panel_idx': 0, 'initial': 0} %} -{% set dashboard_groups = ['undercloud', 'controller', 'objectstorage', 'cephstorage', 'compute'] %} +{% set dashboard_groups = ['undercloud', 'controller', 'blockstorage', 'objectstorage', 'cephstorage', 'compute'] %} { "dashboard": { "id": null, diff --git a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_memory.json.j2 b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_memory.json.j2 index 5d23caaf1..d0d62aff3 100644 --- a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_memory.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_memory.json.j2 @@ -1,5 +1,5 @@ {% set vars = {'panel_idx': 0, 'initial': 0} %} -{% set dashboard_groups = ['undercloud', 'controller', 'objectstorage', 'cephstorage', 'compute'] %} +{% set dashboard_groups = ['undercloud', 'controller', 'blockstorage', 'objectstorage', 'cephstorage', 'compute'] %} { "dashboard": { "id": null, diff --git a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_network.json.j2 b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_network.json.j2 index 227575122..b07dd1710 100644 --- a/ansible/install/roles/grafana-dashboards/templates/cloud_specific_network.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/cloud_specific_network.json.j2 @@ -1,5 +1,5 @@ {% set vars = {'panel_idx': 0, 'initial': 0, 'initial2': 0} %} -{% set dashboard_groups = ['undercloud', 'controller', 'objectstorage', 'cephstorage', 'compute'] %} +{% set dashboard_groups = ['undercloud', 'controller', 'blockstorage', 'objectstorage', 'cephstorage', 'compute'] %} { "dashboard": { "id": null, @@ -294,17 +294,16 @@ { "allFormat": "glob", "current": { - "text": "None", - "value": "", - "isNone": true + "text": "interface-br-ex", + "value": "interface-br-ex" }, "datasource": null, "includeAll": false, "multi": false, "multiFormat": "glob", - "name": "objectstorage_interface", + "name": "compute_interface", "options": [], - "query": "{{dashboard_cloud_name}}.*objectstorage*.interface-*", + "query": "{{dashboard_cloud_name}}.*compute*.interface-*", "refresh": true, "type": "query" }, @@ -325,6 +324,23 @@ "refresh": true, "type": "query" }, + { + "allFormat": "glob", + "current": { + "text": "None", + "value": "", + "isNone": true + }, + "datasource": null, + "includeAll": false, + "multi": false, + "multiFormat": "glob", + "name": "objectstorage_interface", + "options": [], + "query": "{{dashboard_cloud_name}}.*objectstorage*.interface-*", + "refresh": true, + "type": "query" + }, { "allFormat": "glob", "current": { @@ -335,9 +351,9 @@ "includeAll": false, "multi": false, "multiFormat": "glob", - "name": "compute_interface", + "name": "blockstorage_interface", "options": [], - "query": "{{dashboard_cloud_name}}.*compute*.interface-*", + "query": "{{dashboard_cloud_name}}.*blockstorage*.interface-*", "refresh": true, "type": "query" } diff --git a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 index f44b138c3..5854c50ab 100644 --- a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 @@ -4092,13 +4092,18 @@ "text": "objectstorage", "value": "objectstorage" }, + { + "selected": false, + "text": "blockstorage", + "value": "blockstorage" + }, { "selected": true, "text": "*", "value": "*" } ], - "query": "undercloud,controller,objectstorage,cephstorage,compute,*", + "query": "undercloud,controller,blockstorage,objectstorage,cephstorage,compute,*", "refresh": false, "type": "custom" }, diff --git a/ansible/install/roles/grafana-dashboards/vars/main.yml b/ansible/install/roles/grafana-dashboards/vars/main.yml index fbe16bd89..7eb92af64 100644 --- a/ansible/install/roles/grafana-dashboards/vars/main.yml +++ b/ansible/install/roles/grafana-dashboards/vars/main.yml @@ -242,6 +242,21 @@ per_process_panels: processes: - collectd + # + # This dashboard should only contain OpenStack BlockStorage Node processes + # + OpenStack-BlockStorage: + - name: "Cinder" + processes: + - cinder-volume + - name: "Open vSwitch" + processes: + - ovs-vswitchd + - ovsdb-server + - name: "Collectd" + processes: + - collectd + # # This dashboard should only contain OpenStack ObjectStorage Node processes #