Add collectd cookbook

Will associate monitored processes with roles later.

Change-Id: I33a14255a11f52be87c1551a3dd2b4542c11eebc

chef/cookbooks/collectd/README.md

@@ -0,0 +1,92 @@
+# DESCRIPTION #
+
+Configure and install the [collectd](http://collectd.org/) monitoring daemon.
+
+# REQUIREMENTS #
+
+This cookbook has only been tested on Ubuntu 10.04.
+
+To use the `collectd::collectd_web` recipe you need the [apache2](https://github.com/opscode/cookbooks/tree/master/apache2) cookbook.
+
+The [collectd_plugins](https://github.com/coderanger/chef-collectd_plugins) cookbook is not required, but provides many common plugin definitions for easy reuse.
+
+# ATTRIBUTES #
+
+* collectd.basedir - Base folder for collectd output data.
+* collectd.plugin_dir - Base folder to find plugins.
+* collectd.types_db - Array of files to read graph type information from.
+* collectd.interval - Time period in seconds to wait between data reads.
+
+* collectd.collectd_web.path - Location to install collectd_web to. Defaults to /srv/collectd_web.
+* collectd.collectd_web.hostname - Server name to use for collectd_web Apache site.
+
+# USAGE #
+
+Three main recipes are provided:
+
+* collectd - Install a standalone daemon.
+* collectd::client - Install collectd and configure it to send data to a server.
+* collectd::server - Install collectd and configure it to recieve data from clients.
+
+The client recipe will use the search index to automatically locate the server hosts, so no manual configuration is required.
+
+## Defines ##
+
+Several defines are provided to simplfy configuring plugins
+
+### collectd_plugin ###
+
+The `collectd_plugin` define configures and enables standard collect plugins. Example:
+
+```ruby
+collectd_plugin "interface" do
+  options :interface=>"lo", :ignore_selected=>true
+end
+```
+
+The options hash is converted to collectd-style settings automatically. Any symbol key will be converted to camel-case. In the above example :ignore_selected will be output as the
+key "IgnoreSelected". If the key is already a string, this conversion is skipped. If the value is an array, it will be output as a separate line for each element.
+
+### collectd_python_plugin ###
+
+The `collectd_python_plugin` define configures and enables Python plugins using the collectd-python plugin. Example:
+
+```ruby
+collectd_python_plugin "redis" do
+  options :host=>servers, :verbose=>true
+end
+```
+
+Options are interpreted in the same way as with `collectd_plugin`. This define will not deploy the plugin script as well, so be sure to setup a cookbook_file resource
+or other mechanism to handle distribution. Example:
+
+```ruby
+cookbook_file File.join(node[:collectd][:plugin_dir], "redis.py") do
+  owner "root"
+  group "root"
+  mode "644"
+end
+```
+
+## Web frontend ##
+
+The `collectd::collectd_web` recipe will automatically deploy the [collectd_web](https://github.com/httpdss/collectd-web) frontend using Apache. The 
+[apache2](https://github.com/opscode/cookbooks/tree/master/apache2) cookbook is required for this and is *not* included automatically as this is an optional
+component, so be sure to configure the node with the correct recipes.
+
+# LICENSE & AUTHOR #
+
+Author:: Noah Kantrowitz (<noah@coderanger.net>)
+Copyright:: 2010, Atari, Inc
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

chef/cookbooks/collectd/attributes/default.rb

@@ -0,0 +1,46 @@
+#
+# Cookbook Name:: collectd
+# Attributes:: default
+#
+# Copyright 2010, Atari, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+default[:collectd][:base_dir] = "/var/lib/collectd"
+if platform_family?("rhel")
+  default[:collectd][:package_name] = ["collectd"]
+  default[:collectd][:plugin_dir] = "/usr/lib64/collectd"
+  default[:collectd][:config_file] = "/etc/collectd.conf"
+elsif platform_family?("debian")
+  default[:collectd][:package_name] = ["collectd-core"]
+  default[:collectd][:plugin_dir] = "/usr/lib/collectd"
+  default[:collectd][:config_file] = "/etc/collectd/collectd.conf"
+end
+default[:collectd][:types_db] = ["/usr/share/collectd/types.db"]
+default[:collectd][:interval] = 10
+default[:collectd][:read_threads] = 5
+
+default[:collectd][:collectd_web][:path] = "/srv/collectd_web"
+default[:collectd][:collectd_web][:hostname] = "collectd"
+
+default[:collectd][:plugins] = {"cpu"=>{},
+                                "syslog"=>"",
+                                "disk"=>{"Disk"=>"/^[hsv]d[a-f][0-9]?$/", "IgnoreSelected"=>false},
+                                "interface"=>"",
+                                "load"=>"",
+                                "memory"=>"",
+                                "match_regex"=>""
+                               }
+default[:collectd][:server][:host] = "10.145.81.250"
+default[:collectd][:server][:port] = "4242"
+default[:collectd][:server][:protocol] = "tcp"

chef/cookbooks/collectd/definitions/collectd_plugin.rb

@@ -0,0 +1,52 @@
+#
+# Cookbook Name:: collectd
+# Definition:: collectd_plugin
+#
+# Copyright 2010, Atari, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+define :collectd_plugin, :options => {}, :template => nil, :cookbook => nil do
+  template "/etc/collectd/plugins/#{params[:name]}.conf" do
+    owner "root"
+    group "root"
+    mode "644"
+    if params[:template].nil?
+      source "plugin.conf.erb"
+      cookbook params[:cookbook] || "collectd"
+    else
+      source params[:template]
+      cookbook params[:cookbook]
+    end
+    variables :name=>params[:name], :options=>params[:options]
+    notifies :restart, resources(:service => "collectd")
+  end
+end
+
+define :collectd_python_plugin, :options => {}, :mod => nil, :path => nil do
+  begin
+    t = resources(:template => "/etc/collectd/plugins/python.conf")
+  rescue ArgumentError,Chef::Exceptions::ResourceNotFound
+    collectd_plugin "python" do
+      options :paths=>[node[:collectd][:plugin_dir]], :modules=>{}
+      template "python_plugin.conf.erb"
+      cookbook "collectd"
+    end
+    retry
+  end
+  if not params[:path].nil?
+    t.variables[:options][:paths] << params[:path]
+  end
+  t.variables[:options][:modules][params[:mod] || params[:name]] = params[:options]
+end

chef/cookbooks/collectd/files/default/kairosdb_writer.py

@@ -0,0 +1,258 @@
+#  Copyright 2013 Gregory Durham
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import collectd
+import socket
+import re
+from string import maketrans
+from time import time
+from traceback import format_exc
+
+host = None
+port = None
+prefix = None
+types = {}
+postfix = None
+tags = ""
+host_separator = "_"
+metric_separator = "."
+protocol = "tcp"
+
+def kairosdb_parse_types_file(path):
+    global types
+
+    f = open(path, 'r')
+
+    for line in f:
+        fields = line.split()
+        if len(fields) < 2:
+            continue
+
+        type_name = fields[0]
+
+        if type_name[0] == '#':
+            continue
+
+        v = []
+        for ds in fields[1:]:
+            ds = ds.rstrip(',')
+            ds_fields = ds.split(':')
+
+            if len(ds_fields) != 4:
+                collectd.warning('kairosdb_writer: cannot parse data source %s on type %s' % ( ds, type_name ))
+                continue
+
+            v.append(ds_fields)
+
+        types[type_name] = v
+
+    f.close()
+
+def str_to_num(s):
+    """
+    Convert type limits from strings to floats for arithmetic.
+    Will force U[nlimited] values to be 0.
+    """
+
+    try:
+        n = float(s)
+    except ValueError:
+        n = 0
+
+    return n
+
+def sanitize_field(field):
+    """
+    Santize Metric Fields: replace dot and space with metric_separator. Delete
+    parentheses. Convert to lower case if configured to do so.
+    """
+    field = field.strip()
+    trans = maketrans(' .', metric_separator * 2)
+    field = field.translate(trans, '()')
+    if lowercase_metric_names:
+        field = field.lower()
+    return field
+
+def kairosdb_config(c):
+    global host, port, prefix, postfix, host_separator, \
+            metric_separator, lowercase_metric_names, protocol, \
+            tags
+
+    for child in c.children:
+        if child.key == 'KairosDBHost':
+            host = child.values[0]
+        elif child.key == 'KairosDBPort':
+            port = int(child.values[0])
+        elif child.key == 'TypesDB':
+            for v in child.values:
+                kairosdb_parse_types_file(v)
+        elif child.key == 'LowercaseMetricNames':
+            lowercase_metric_names = True
+        elif child.key == 'MetricPrefix':
+            prefix = child.values[0]
+        elif child.key == 'HostPostfix':
+            postfix = child.values[0]
+        elif child.key == 'HostSeparator':
+            host_separator = child.values[0]
+        elif child.key == 'MetricSeparator':
+            metric_separator = child.values[0]
+        elif child.key == 'KairosDBProtocol':
+            protocol = str(child.values[0])
+        elif child.key == 'Tags':
+            for v in child.values:
+                tags += "%s " % (v)
+
+    tags = tags.replace('.', host_separator)
+
+    if not host:
+        raise Exception('KairosDBHost not defined')
+
+    if not port:
+        raise Exception('KairosDBPort not defined')
+
+    collectd.info('Initializing kairosdb_writer client in %s socket mode.'
+                         % protocol.upper() )
+
+def kairosdb_init():
+    import threading
+
+    d = {
+        'host': host,
+        'port': port,
+        'lowercase_metric_names': lowercase_metric_names,
+        'sock': None,
+        'lock': threading.Lock(),
+        'values': { },
+        'last_connect_time': 0
+    }
+
+    kairosdb_connect(d)
+
+    collectd.register_write(kairosdb_write, data=d)
+
+def kairosdb_connect(data):
+    result = False
+
+    if not data['sock'] and protocol.lower() == 'tcp':
+        # only attempt reconnect every 10 seconds if protocol of type TCP
+        now = time()
+        if now - data['last_connect_time'] < 10:
+            return False
+
+        data['last_connect_time'] = now
+        collectd.info('connecting to %s:%s' % ( data['host'], data['port'] ) )
+        try:
+            data['sock'] = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            data['sock'].connect((host, port))
+            result = True
+        except:
+            result = False
+            collectd.warning('error connecting socket: %s' % format_exc())
+    else:
+        # we're either connected, or protocol does not == tcp. we will send 
+        # data via udp/SOCK_DGRAM call.
+        result = True
+
+    return result
+
+def kairosdb_write_data(data, s):
+    result = False
+    data['lock'].acquire()
+
+    try:
+        if protocol.lower() == 'tcp':
+            data['sock'].sendall(s)
+        else:
+            # send message to via UDP to the line receiver .
+            sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            sock.sendto(s, (host, port))
+        result = True
+    except socket.error, e:
+        data['sock'] = None
+        if isinstance(e.args, tuple):
+            collectd.warning('kairosdb_writer: socket error %d' % e[0])
+        else:
+            collectd.warning('kairosdb_writer: socket error')
+    except:
+        collectd.warning('kairosdb_writer: error sending data: %s' % format_exc())
+
+    data['lock'].release()
+    return result
+
+def kairosdb_write(v, data=None):
+    data['lock'].acquire()
+    if not kairosdb_connect(data) and protocol.lower() == 'tcp':
+        data['lock'].release()
+        collectd.warning('kairosdb_writer: no connection to kairosdb server')
+        return
+
+    data['lock'].release()
+
+    if v.type not in types:
+        collectd.warning('kairosdb_writer: do not know how to handle type %s. do you have all your types.db files configured?' % v.type)
+        return
+
+    v_type = types[v.type]
+
+    if len(v_type) != len(v.values):
+        collectd.warning('kairosdb_writer: differing number of values for type %s' % v.type)
+        return
+
+    metric_fields = []
+    if prefix:
+        metric_fields.append(prefix)
+
+    if postfix:
+        metric_fields.append(postfix)
+
+    metric_fields.append(v.plugin)
+    if v.plugin_instance:
+        metric_fields.append(sanitize_field(v.plugin_instance))
+
+    metric_fields.append(v.type)
+    if v.type_instance:
+        metric_fields.append(sanitize_field(v.type_instance))
+
+    time = v.time
+
+    # we update shared recorded values, so lock to prevent race conditions
+    data['lock'].acquire()
+
+    lines = []
+    i = 0
+    for value in v.values:
+        ds_name = v_type[i][0]
+        ds_type = v_type[i][1]
+
+        path_fields = metric_fields[:]
+        path_fields.append(ds_name)
+
+        metric = '.'.join(path_fields)
+
+        new_value = value
+
+        if new_value is not None:
+            line = 'put %s %d %f %s' % ( metric, time, new_value, tags)
+            collectd.debug(line)
+            lines.append(line)
+
+        i += 1
+
+    data['lock'].release()
+
+    lines.append('')
+    kairosdb_write_data(data, '\n'.join(lines))
+
+collectd.register_config(kairosdb_config)
+collectd.register_init(kairosdb_init)

chef/cookbooks/collectd/libraries/default.rb

@@ -0,0 +1,47 @@
+#
+# Cookbook Name:: collectd
+# Library:: default
+#
+# Copyright 2010, Atari, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+def collectd_key(option)
+  return option.to_s.split('_').map{|x| x.capitalize}.join() if option.instance_of?(Symbol)
+  "#{option}"
+end
+
+def collectd_option(option)
+  return option if option.instance_of?(Fixnum) || option == true || option == false
+  "\"#{option}\""
+end
+
+def collectd_settings(options, level=0)
+  indent = '  ' * level
+  output = []
+  options.each_pair do |key, value|
+    if value.is_a? Array
+      value.each do |subvalue|
+        output << "#{indent}#{collectd_key(key)} #{collectd_option(subvalue)}"
+      end
+    elsif value.is_a? Hash
+      value.each_pair do |name, suboptions|
+        output << "#{indent}<#{key} \"#{name}\">\n#{collectd_settings(suboptions, level+1)}\n#{indent}</#{key}>"
+      end
+    else
+      output << "#{indent}#{collectd_key(key)} #{collectd_option(value)}"
+    end
+  end
+  output.join("\n")
+end

chef/cookbooks/collectd/metadata.rb

@@ -0,0 +1,11 @@
+name             "collectd"
+maintainer       "Noan Kantrowitz"
+maintainer_email "noah@coderanger.net"
+license          "Apache 2.0"
+description      "Install and configure the collectd monitoring daemon"
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version          "1.0.3"
+supports         "ubuntu"
+supports         "centos"
+depends          "apt"
+depends          "yum"

chef/cookbooks/collectd/recipes/client.rb

@@ -0,0 +1,65 @@
+#
+# Cookbook Name:: collectd
+# Recipe:: client
+#
+# Copyright 2010, Atari, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+include_recipe "collectd"
+
+#servers = []
+#search(:node, 'recipes:collectd\\:\\:server') do |n|
+#  servers << n['fqdn']
+#end
+
+#if servers.empty?
+#  raise "No servers found. Please configure at least one node with collectd::server."
+#end
+
+#collectd_plugin "network" do
+#  options :server=>servers
+#end
+
+cookbook_file "#{node['collectd']['plugin_dir']}/kairosdb_writer.py" do
+  source "kairosdb_writer.py"
+  owner "root"
+  group "root"
+  mode 00644
+  notifies :restart, "service[collectd]"
+  action :create_if_missing
+end
+
+case node["platform_family"]
+when "rhel"
+  node.override["collectd"]["plugins"]=node["collectd"]["rhel"]["plugins"].to_hash
+when "debian"
+  node.override["collectd"]["plugins"]=node["collectd"]["debian"]["plugins"].to_hash
+end
+
+node["collectd"]["plugins"].each_pair do |plugin_key, options|
+  collectd_plugin plugin_key do
+    options  options
+  end
+end
+
+collectd_python_plugin "kairosdb_writer" do
+  opts  =    {"KairosDBHost"=>node['collectd']['server']['host'],
+              "KairosDBPort"=>node['collectd']['server']['port'],
+              "KairosDBProtocol"=>node['collectd']['server']['protocol'],
+              "LowercaseMetricNames"=>"true",
+              "Tags" => "host=#{node['fqdn']}\" \"role=OSROLE\" \"location=China.Beijing.TsingHua\" \"cluster=#{node['cluster']}",
+              "TypesDB" => node['collectd']['types_db']
+             }
+  options(opts)         
+end

chef/cookbooks/collectd/recipes/collectd_web.rb

@@ -0,0 +1,53 @@
+#
+# Cookbook Name:: collectd
+# Recipe:: collectd_web
+#
+# Copyright 2010, Atari, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "collectd"
+include_recipe "apache2"
+
+%w(libhtml-parser-perl liburi-perl librrds-perl libjson-perl).each do |name|
+  package name
+end
+
+directory node[:collectd][:collectd_web][:path] do
+  owner "root"
+  group "root"
+  mode "755"
+end
+
+bash "install_collectd_web" do
+  user "root"
+  cwd node[:collectd][:collectd_web][:path]
+  not_if do
+    File.exists?(File.join(node[:collectd][:collectd_web][:path], "index.html"))
+  end
+  code <<-EOH
+    wget --no-check-certificate -O collectd-web.tar.gz https://github.com/httpdss/collectd-web/tarball/master
+    tar --strip-components=1 -xzf collectd-web.tar.gz
+    rm collectd-web.tar.gz
+  EOH
+end
+
+template "/etc/apache2/sites-available/collectd_web.conf" do
+  source "collectd_web.conf.erb"
+  owner "root"
+  group "root"
+  mode "644"
+end
+
+apache_site "collectd_web.conf"

chef/cookbooks/collectd/recipes/default.rb

@@ -0,0 +1,112 @@
+#
+# Cookbook Name:: collectd
+# Recipe:: default
+#
+# Copyright 2010, Atari, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+case node["platform_family"]
+when "debian"
+  package "ubuntu-cloud-keyring" do
+    action :install
+  end
+when "rhel"
+  include_recipe "yum::epel"
+  execute "yum-update" do
+    user "root"
+    command "yum -y update"
+    action :run
+  end
+end
+
+node[:collectd][:package_name].each do |pkg|
+  package pkg do
+    action :install
+  end
+end
+
+service "collectd" do
+  supports :restart => true, :status => true
+end
+
+directory "/etc/collectd" do
+  owner "root"
+  group "root"
+  mode "755"
+end
+
+directory "/etc/collectd/plugins" do
+  owner "root"
+  group "root"
+  mode "755"
+end
+
+directory node[:collectd][:base_dir] do
+  owner "root"
+  group "root"
+  mode "755"
+  recursive true
+end
+
+directory node[:collectd][:plugin_dir] do
+  owner "root"
+  group "root"
+  mode "755"
+  recursive true
+end
+
+%w(collection thresholds).each do |file|
+  template "/etc/collectd/#{file}.conf" do
+    source "#{file}.conf.erb"
+    owner "root"
+    group "root"
+    mode "644"
+    notifies :restart, resources(:service => "collectd")
+  end
+end
+
+template node[:collectd][:config_file] do
+  source "collectd.conf.erb"
+  owner "root"
+  group "root"
+  mode "644"
+  notifies :restart, resources(:service => "collectd")
+end
+
+ruby_block "delete_old_plugins" do
+  block do
+    Dir['/etc/collectd/plugins/*.conf'].each do |path|
+      autogen = false
+      File.open(path).each_line do |line|
+        if line.start_with?('#') and line.include?('autogenerated')
+          autogen = true
+          break
+        end
+      end
+      if autogen
+        begin
+          resources(:template => path)
+        rescue ArgumentError, Chef::Exceptions::ResourceNotFound
+          # If the file is autogenerated and has no template it has likely been removed from the run list
+          Chef::Log.info("Deleting old plugin config in #{path}")
+          File.unlink(path)
+        end
+      end
+    end
+  end
+end
+
+service "collectd" do
+  action [:enable, :start]
+end

chef/cookbooks/collectd/recipes/server.rb

@@ -0,0 +1,24 @@
+#
+# Cookbook Name:: collectd
+# Recipe:: server
+#
+# Copyright 2010, Atari, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "collectd"
+
+collectd_plugin "network" do
+  options :listen=>'0.0.0.0'
+end

chef/cookbooks/collectd/templates/default/collectd.conf.erb

@@ -0,0 +1,18 @@
+# Config file for collectd(1).
+#
+# Some plugins need additional configuration and are disabled by default.
+# Please read collectd.conf(5) for details.
+#
+# You should also read /usr/share/doc/collectd/README.Debian.plugins before
+# enabling any more plugins.
+
+Hostname "<%= @node[:fqdn] %>"
+FQDNLookup true
+BaseDir "<%= @node[:collectd][:base_dir] %>"
+PluginDir "<%= @node[:collectd][:plugin_dir] %>"
+TypesDB "<%= @node[:collectd][:types_db].join('", "') %>"
+Interval <%= @node[:collectd][:interval] %>
+ReadThreads <%= @node[:collectd][:read_threads] %>
+
+Include "/etc/collectd/plugins/*.conf"
+Include "/etc/collectd/thresholds.conf"

chef/cookbooks/collectd/templates/default/collectd_web.conf.erb

@@ -0,0 +1,32 @@
+<VirtualHost *:80>
+    ServerName <%= @node[:collectd][:collectd_web][:hostname] %>
+
+    DocumentRoot <%= @node[:collectd][:collectd_web][:path] %>
+    <Directory />
+        Options FollowSymLinks
+        AllowOverride None
+    </Directory>
+    <Directory <%= @node[:collectd][:collectd_web][:path] %>>
+        Options Indexes FollowSymLinks MultiViews
+        AllowOverride None
+        Order allow,deny
+        Allow from all
+    </Directory>
+
+    ScriptAlias /cgi-bin/ <%= @node[:collectd][:collectd_web][:path] %>/cgi-bin/
+    <Directory "<%= @node[:collectd][:collectd_web][:path] %>/cgi-bin">
+        AllowOverride None
+        Options ExecCGI -MultiViews
+        Order allow,deny
+        Allow from all
+    </Directory>
+
+    ErrorLog /var/log/apache2/error.log
+
+    # Possible values include: debug, info, notice, warn, error, crit,
+    # alert, emerg.
+    LogLevel warn
+
+    CustomLog /var/log/apache2/access.log combined
+    ServerSignature On
+</VirtualHost>

chef/cookbooks/collectd/templates/default/collection.conf.erb

@@ -0,0 +1,2 @@
+datadir: "/var/lib/collectd/rrd/"
+libdir: "/usr/lib/collectd/"

chef/cookbooks/collectd/templates/default/plugin.conf.erb

@@ -0,0 +1,15 @@
+# This file autogenerated by Chef
+# Do not edit, changes will be overwritten
+LoadPlugin "<%= @name %>"
+
+<% if not @options.empty? %>
+<Plugin "<%= @name %>">
+  <% @options.each_pair do |key, value|
+    if value.is_a? Array 
+      value.each do |subvalue| %>
+  <%= collectd_key(key) %> <%= collectd_option(subvalue) %>
+  <% end else %>
+  <%= collectd_key(key) %> <%= collectd_option(value) %>
+  <% end end %>
+</Plugin>
+<% end %>

chef/cookbooks/collectd/templates/default/python_plugin.conf.erb

@@ -0,0 +1,20 @@
+# This file autogenerated by Chef
+# Do not edit, changes will be overwritten
+<LoadPlugin python>
+  Globals true
+</LoadPlugin>
+
+<Plugin python>
+  <% @options[:paths].each do |path| %>
+  ModulePath "<%= path %>"
+  <% end %>
+  <% @options[:modules].each_key do |mod| %>
+  Import "<%= mod %>"
+  <% end %>
+
+  <% @options[:modules].each_pair do |mod, config| %>
+  <Module "<%= mod %>">
+<%= collectd_settings(config, 2) %>
+  </Module>
+  <% end %>
+</Plugin>

chef/cookbooks/collectd/templates/default/thresholds.conf.erb

@@ -0,0 +1,37 @@
+# Threshold configuration for collectd(1).
+#
+# See the section "THRESHOLD CONFIGURATION" in collectd.conf(5) for details.
+
+#<Threshold>
+#	<Type "counter">
+#		WarningMin 0.00
+#		WarningMax 1000.00
+#		FailureMin 0
+#		FailureMax 1200.00
+#		Invert false
+#		Persist false
+#		Instance "some_instance"
+#	</Type>
+#
+#	<Plugin "interface">
+#		Instance "eth0"
+#		<Type "if_octets">
+#			DataSource "rx"
+#			FailureMax 10000000
+#		</Type>
+#	</Plugin>
+#
+#	<Host "hostname">
+#		<Type "cpu">
+#			Instance "idle"
+#			FailureMin 10
+#		</Type>
+#
+#		<Plugin "memory">
+#			<Type "memory">
+#				Instance "cached"
+#				WarningMin 100000000
+#			</Type>
+#		</Plugin>
+#	</Host>
+#</Threshold>

chef/roles/os-block-storage-api.rb

@@ -8,6 +8,13 @@ override_attributes(
     "debianloglist" => {
       "cinder-api" => "/var/log/cinder/cinder-api.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-cinder-api"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-block-storage-scheduler.rb

@@ -8,6 +8,13 @@ override_attributes(
     "debianloglist" => {
       "cinder-scheduler" => "/var/log/cinder/cinder-scheduler.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-cinder-scheduler"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-block-storage-worker.rb

@@ -8,6 +8,13 @@ override_attributes(
     "debianloglist" => {
       "cinder-volume" => "/var/log/cinder/cinder-volume.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-cinder-volume", "iscsid", "multipathd"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-compute-api-metadata.rb

@@ -1,5 +1,14 @@
 name "os-compute-api-metadata"
 description "OpenStack compute metadata API service"
+override_attributes(
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-nova-metadata-api"] }
+      }
+    }
+  }
+)
 run_list(
   "role[os-base]",
   "recipe[openstack-compute::api-metadata]"

chef/roles/os-compute-api-os-compute.rb

@@ -8,6 +8,13 @@ override_attributes(
     "debianloglist" => {
       "nova-api" => "/var/log/nova/nova-api.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-nova-api"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-compute-cert.rb

@@ -1,5 +1,14 @@
 name "os-compute-cert"
 description "OpenStack Compute Cert service"
+override_attributes(
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-nova-cert"] }
+      }
+    }
+  }
+)
 run_list(
   "role[os-base]",
   "recipe[openstack-compute::nova-cert]"

chef/roles/os-compute-scheduler.rb

@@ -10,6 +10,13 @@ override_attributes(
       "nova-scheduler" => "/var/log/nova/nova-scheduler.log",
       "nova-conductor" => "/var/log/nova/nova-conductor.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-nova-scheduler", "openstack-nova-conductor"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-compute-vncproxy.rb

@@ -1,5 +1,14 @@
 name "os-compute-vncproxy"
 description "Nova VNC Proxy"
+override_attributes(
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-nova-xvpvncproxy", "openstack-nova-novncproxy"] }
+      }
+    }
+  }
+)
 run_list(
   "role[os-base]",
   "recipe[openstack-compute::vncproxy]"

chef/roles/os-compute-worker.rb

@@ -8,6 +8,13 @@ override_attributes(
     "debianloglist" => {
       "nova-compute" => "/var/log/nova/nova-compute.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-nova-compute"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-dashboard.rb

@@ -1,5 +1,14 @@
 name "os-dashboard"
 description "Horizon server"
+override_attributes(
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["httpd"]}
+      }
+    }
+  }
+)
 run_list(
   "role[os-base]",
 #  "recipe[openstack-dashboard::db]",

chef/roles/os-ha.rb

@@ -1,5 +1,14 @@
 name "os-ha"
 description "Software load banance"
+override_attributes(
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["haproxy", "keepalived"]}
+      }
+    }
+  }
+)
 run_list(
   "recipe[keepalived]",  
   "recipe[haproxy::tcp_lb]"

chef/roles/os-identity.rb

@@ -8,6 +8,13 @@ override_attributes(
     "debianloglist" => {
       "keystone" => "/var/log/keystone/keystone.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-keystone"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-image-api.rb

@@ -8,6 +8,13 @@ override_attributes(
     "debianloglist" => {
       "glance-api" => "/var/log/glance/api.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-glance-api"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-image-registry.rb

@@ -8,6 +8,13 @@ override_attributes(
     "debianloglist" => {
       "glance-registry" => "/var/log/glance/registry.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["openstack-glance-registry"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-network-server.rb

@@ -8,6 +8,13 @@ override_attributes(
     "rhelloglist" => {
       "quantum-server" => "/var/log/quantum/server.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["quantum-server"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-network.rb

@@ -7,11 +7,18 @@ override_attributes(
       "quantum-dhcp" => "/var/log/quantum/dhcp-agent.log",
       "quantum-l3agent" => "/var/log/quantum/l3-agent.log"
     },
-    "rhelloglist" => {
+    "debianloglist" => {
       "quantum-ovsagent" => "/var/log/quantum/openvswitch-agent.log",
       "quantum-dhcp" => "/var/log/quantum/dhcp-agent.log",
       "quantum-l3agent" => "/var/log/quantum/l3-agent.log"
-    },
+    }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["quantum-dhcp-agent", "quantum-l3-agent", "quantum-openvswitch-agent", "quantum-metadata-agent"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-ops-database.rb

@@ -8,6 +8,13 @@ override_attributes(
     "debianloglist" => {
       "mysqld" => "/var/log/mysql.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["mysqld"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/os-ops-messaging.rb

@@ -8,6 +8,13 @@ override_attributes(
     "debianloglist" => {
       "rabbitmq" => "/var/log/rabbitmq/rabbit\@$hostname.log"
     }
+  },
+  "collectd" => {
+    "rhel" => {
+      "plugins" => {
+        "processes" => { "Process" => ["rabbitmq-server"] }
+      }
+    }
   }
 )
 run_list(

chef/roles/test-synclog.rb

@@ -1,5 +1,6 @@
 name "test-synclog"
 description "Sync application related logs for debugging"
 run_list(
-  "recipe[rsyslog::client]"
+  "recipe[rsyslog::client]",
+  "recipe[collectd::client]"
   )