Collectd agent work

Currently looking to integrate Collectd into Browbeat
+ Install epel onto the hosts
+ Added collectd config files
+ Added collectd.conf
+ Play to do the install..
+ Play tested, works
+ fixed missing configs
+ (akrzos) added filter.conf, moved static config files to files folder, cleaned up collectd playbook
+ (kambiz) fixed the gen_hostfile.sh to include the previous fix on heat-admin key.  (regression!)
+ (kambiz) added selinux bits for collectd to run permissive without going full permissive
+ (kambiz) added a policy for running sample exec plugin via collectd as nobody (may need some fine tuning)

Change-Id: Ied873a7f3e91a13a9462df41334268322c7123a2

ansible/README.md

@@ -49,6 +49,14 @@ Install Shaker
 # ansible-playbook -i hosts install/shaker.yml
 ```
 
+Install Collectd Agent
+Prior to installing the agent, please review the install/group_vars/all to ensure the
+correct params are passed
+```
+# ansible-playbook -i hosts install/collectd
+```
+
+
 ## Performance Checks:
 
 Run the check playbook to identify common performance issues:

ansible/install/collectd.yml

@@ -0,0 +1,12 @@
+---
+#
+# Playbook to install connmon on undercloud/overcloud
+#
+- hosts: controller:compute
+  remote_user: heat-admin
+  vars:
+    ansible_become: true
+    undercloud: false
+  roles:
+  - common
+  - collectd

ansible/install/group_vars/all

@@ -6,6 +6,13 @@ connmon_host: 192.0.2.1
 
 dns_server: 8.8.8.8
 
+# collectd params:
+#  epel_repo for collectd packages
+#  graphite_host, where to send the collectd metrics
+epel_repo: https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+graphite_host: 1.1.1.1
+graphite_prefix: openstack
+
 # pbench repos, you must replace the internal to the correct repo to install pbench
 pbench_internal_repo_file_url: http://pbench.example.com/repo/yum.repos.d/pbench.repo
 pbench_repo_file_url: https://copr.fedoraproject.org/coprs/ndokos/pbench/repo/epel-7/ndokos-pbench-epel-7.repo

ansible/install/roles/collectd/files/collectd-redis.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+HOSTNAME="${COLLECTD_HOSTNAME:-`hostname -f`}"
+INTERVAL="${COLLECTD_INTERVAL:-10}"
+PORT=6379
+
+while true
+do
+
+info=$((echo info ; sleep 2) |nc -w 1 $HOSTNAME $PORT 2>&1)
+connected_clients=$(echo "$info" | egrep ^connected_clients| awk -F: '{ print $2 }')
+connected_slaves=$(echo "$info" | egrep ^connected_slaves| awk -F: '{ print $2 }')
+uptime=$(echo "$info" | egrep ^uptime_in_seconds| awk -F: '{ print $2 }')
+used_memory=$(echo "$info" | egrep ^used_memory:| awk -F: '{ print $2 }')
+changes_since_last_save=$(echo "$info" | egrep ^rdb_changes_since_last_save| awk -F: '{ print $2 }')
+total_commands_processed=$(echo "$info" | egrep ^total_commands_processed| awk -F: '{ print $2 }')
+keys=$(echo "$info" | egrep ^db0:keys| awk -F= '{ print $2 }' | awk -F, '{ print $1 }')
+
+echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_connections-clients interval=$INTERVAL N:$connected_clients"
+echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_connections-slaves interval=$INTERVAL N:$connected_slaves"
+echo "PUTVAL $HOSTNAME/redis-$PORT/uptime interval=$INTERVAL N:$uptime"
+echo "PUTVAL $HOSTNAME/redis-$PORT/df-memory interval=$INTERVAL N:$used_memory:U"
+echo "PUTVAL $HOSTNAME/redis-$PORT/files-unsaved_changes interval=$INTERVAL N:$changes_since_last_save"
+echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_command-total interval=$INTERVAL N:$total_commands_processed"
+echo "PUTVAL $HOSTNAME/redis-$PORT/memcached_items-db0 interval=$INTERVAL N:$keys"
+
+sleep "$INTERVAL"
+done

ansible/install/roles/collectd/files/collectd.conf

@@ -0,0 +1,22 @@
+LoadPlugin syslog
+LoadPlugin cpu
+LoadPlugin disk
+LoadPlugin interface
+LoadPlugin load
+LoadPlugin memory
+LoadPlugin numa
+LoadPlugin processes
+LoadPlugin tail
+LoadPlugin "match_regex"
+PreCacheChain "PreCache"
+<Chain "PreCache">
+  <Rule "ignore_tap">
+    <Match "regex">
+      Plugin "^interface$"
+      PluginInstance "^tap*"
+    </Match>
+    Target "stop"
+  </Rule>
+    Target "return"
+</Chain>
+Include "/etc/collectd.d"

ansible/install/roles/collectd/files/disk.conf

@@ -0,0 +1,6 @@
+LoadPlugin disk
+
+<Plugin disk>
+       Disk "/^[hs]d[a-f][0-9]?$/"
+       IgnoreSelected false
+</Plugin>

ansible/install/roles/collectd/files/filter.conf

@@ -0,0 +1,12 @@
+LoadPlugin "match_regex"
+PreCacheChain "PreCache"
+<Chain "PreCache">
+  <Rule "ignore_tap">
+    <Match "regex">
+      Plugin "^interface$"
+      PluginInstance "^tap*"
+    </Match>
+    Target "stop"
+  </Rule>
+    Target "return"
+</Chain>

ansible/install/roles/collectd/files/processes.conf

@@ -0,0 +1,34 @@
+LoadPlugin processes
+<Plugin processes>
+        ProcessMatch "ceilometer-api" "python.+ceilometer-api"
+        ProcessMatch "ceilometer-alarm-notifier" "python.+ceilometer-alarm-notifier"
+        ProcessMatch "ceilometer-alarm-evaluator" "python.+ceilometer-alarm-evaluator"
+        ProcessMatch "ceilometer-agent-notification" "python.+ceilometer-agent-notification"
+        ProcessMatch "ceilometer-agent-central" "python.+ceilometer-agent-central"
+        ProcessMatch "ceilometer-agent-notification" "python.+ceilometer-agent-notification"
+        ProcessMatch "ceilometer-collector" "python.+ceilometer-collector"
+        ProcessMatch "cinder-api" "python.+cinder-api"
+        ProcessMatch "cinder-scheduler" "python.+cinder-scheduler"
+        ProcessMatch "cinder-volume" "python.+cinder-volume"
+        ProcessMatch "glance-api" "python.+glance-api"
+        ProcessMatch "glance-registry" "python.+glance-registry"
+        ProcessMatch "heat-api" "python.+heat-api"
+        ProcessMatch "heat-engine" "python.+heat-engine"
+        ProcessMatch "keystone-all" "python.+keystone-all"
+        ProcessMatch "mysqld" "/usr/libexec/mysqld"
+        ProcessMatch "neutron-server" "python.+neutron-server"
+        ProcessMatch "nova-api" "python.+nova-api"
+        ProcessMatch "nova-cert" "python.+nova-cert"
+        ProcessMatch "nova-scheduler" "python.+nova-scheduler"
+        ProcessMatch "nova-conductor" "python.+nova-conductor"
+        ProcessMatch "nova-compute" "python.+nova-compute"
+        ProcessMatch "rabbitmq" "/usr/lib64/erlang/erts-.+/bin/beam.smp"
+        ProcessMatch "swift-proxy-server" "python.+swift-proxy-server"
+        ProcessMatch "neutron-ns-metadata-proxy" "python.+neutron-ns-metadata-proxy"
+        ProcessMatch "neutron-openvswitch-agent" "python.+neutron-openvswitch-agent"
+        ProcessMatch "neutron-metadata-agent" "python.+neutron-metadata-agent"
+        ProcessMatch "neutron-l3-agent" "python.+neutron-l3-agent"
+        ProcessMatch "neutron-dhcp-agent" "python.+neutron-dhcp-agent"
+        Process "dnsmasq"
+        ProcessMatch "qemu-kvm" "/usr/libexec/qemu-kvm"
+</Plugin>

ansible/install/roles/collectd/files/redis.conf

@@ -0,0 +1,5 @@
+LoadPlugin exec
+
+<Plugin exec>
+        Exec nobody "/usr/local/bin/collectd-redis.sh"
+</Plugin>

ansible/install/roles/collectd/files/tail.conf

@@ -0,0 +1,57 @@
+LoadPlugin "Tail"
+<Plugin "tail">
+ <File "/var/log/nova/nova-scheduler.log">
+  Instance "nova-scheduler"
+  <Match>
+   Regex " ERROR "
+   DSType "CounterInc"
+   Type "counter"
+   Instance "total"
+  </Match>
+ </File>
+ <File "/var/log/nova/nova-api.log">
+  Instance "nova-api"
+  <Match>
+   Regex " ERROR "
+   DSType "CounterInc"
+   Type "counter"
+   Instance "total"
+  </Match>
+ </File>
+ <File "/var/log/nova/nova-conductor.log">
+  Instance "nova-conductor"
+  <Match>
+   Regex " ERROR "
+   DSType "CounterInc"
+   Type "counter"
+   Instance "total"
+  </Match>
+ </File>
+ <File "/var/log/neutron/openvswitch-agent.log">
+  Instance "openvswitch-agent"
+  <Match>
+   Regex " ERROR "
+   DSType "CounterInc"
+   Type "counter"
+   Instance "total"
+  </Match>
+ </File>
+ <File "/var/log/neutron/metadata-agent.log">
+  Instance "metadata-agent"
+  <Match>
+   Regex " ERROR "
+   DSType "CounterInc"
+   Type "counter"
+   Instance "total"
+  </Match>
+ </File>
+ <File "/var/log/neutron/server.log">
+  Instance "neutron-server"
+  <Match>
+   Regex " ERROR "
+   DSType "CounterInc"
+   Type "counter"
+   Instance "total"
+  </Match>
+ </File>
+</Plugin>

ansible/install/roles/collectd/files/tcpconns.conf

@@ -0,0 +1,5 @@
+LoadPlugin tcpconns
+
+<Plugin tcpconns>
+       ListeningPorts true
+</Plugin>

ansible/install/roles/collectd/tasks/main.yml

@@ -0,0 +1,91 @@
+---
+#
+# Install collectd for browbeat
+#
+
+- name: Install epel repo
+  command: rpm -ivh {{ epel_repo }}
+
+- name: Install collectd agent
+  yum: name=collectd state=present
+
+- name: Copy collectd.conf file
+  copy:
+    src=collectd.conf
+    dest=/etc/collectd.conf
+    owner=root
+    group=root
+    mode=0644
+
+- name: Copy collectd config files
+  copy:
+    src: "{{ item }}"
+    dest: /etc/collectd.d/{{ item }}
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - disk.conf
+    - filter.conf
+    - processes.conf
+    - redis.conf
+    - tail.conf
+    - tcpconns.conf
+
+- name: Copy collectd-redis.sh
+  copy:
+    src=collectd-redis.sh
+    dest=/usr/local/bin/collectd-redis.sh
+    owner=root
+    group=root
+    mode=0755
+
+- name: Configure collectd graphite.conf
+  template:
+    src=graphite.conf.j2
+    dest=/etc/collectd.d/graphite.conf
+    owner=root
+    group=root
+    mode=0644
+  with_items:
+    - ip_address: "{{ graphite_host }}"
+      prefix: "{{ graphite_prefix }}"
+
+- name: Setup collectd service
+  service: name=collectd state=restarted enabled=yes
+
+#
+# Configure selinux bits
+#
+
+- name: Check for collectd permissive
+  shell: semodule -l | grep -q permissive_collectd_t
+  register: collectd_permissive
+  ignore_errors: true
+  changed_when: false
+
+- name: Set permissive for collectd
+  shell: semanage permissive -a collectd_t
+  when: collectd_permissive.rc != 0
+
+#
+# additional policy bits may be needed for exec
+#
+- name: collectd policy customization
+  copy:
+    src=custom-collectd.pp
+    dest=/root/custom-collectd.pp
+    owner=root
+    group=root
+    mode=644
+
+- name: Check for collectd custom
+  shell: semodule -l | grep -q custom-collectd
+  register: collectd_custom
+  ignore_errors: true
+  changed_when: false
+
+- name: Set custom policy for collectd
+  shell: semodule -i /root/custom-collectd.pp
+  when: collectd_custom.rc != 0
+

ansible/install/roles/collectd/templates/graphite.conf.j2

@@ -0,0 +1,14 @@
+LoadPlugin write_graphite
+
+<Plugin write_graphite>
+  <Carbon>
+    Host "{{ item.ip_address }}"
+    Port "2003"
+    Prefix "{{ item.prefix }}."
+    Protocol "tcp"
+    LogSendErrors true
+    StoreRates true
+    AlwaysAppendDS false
+    EscapeCharacter "_"
+  </Carbon>
+</Plugin>