diff --git a/chef/cookbooks/collectd/README.md b/chef/cookbooks/collectd/README.md new file mode 100644 index 0000000..10d7e5d --- /dev/null +++ b/chef/cookbooks/collectd/README.md @@ -0,0 +1,92 @@ +# DESCRIPTION # + +Configure and install the [collectd](http://collectd.org/) monitoring daemon. + +# REQUIREMENTS # + +This cookbook has only been tested on Ubuntu 10.04. + +To use the `collectd::collectd_web` recipe you need the [apache2](https://github.com/opscode/cookbooks/tree/master/apache2) cookbook. + +The [collectd_plugins](https://github.com/coderanger/chef-collectd_plugins) cookbook is not required, but provides many common plugin definitions for easy reuse. + +# ATTRIBUTES # + +* collectd.basedir - Base folder for collectd output data. +* collectd.plugin_dir - Base folder to find plugins. +* collectd.types_db - Array of files to read graph type information from. +* collectd.interval - Time period in seconds to wait between data reads. + +* collectd.collectd_web.path - Location to install collectd_web to. Defaults to /srv/collectd_web. +* collectd.collectd_web.hostname - Server name to use for collectd_web Apache site. + +# USAGE # + +Three main recipes are provided: + +* collectd - Install a standalone daemon. +* collectd::client - Install collectd and configure it to send data to a server. +* collectd::server - Install collectd and configure it to recieve data from clients. + +The client recipe will use the search index to automatically locate the server hosts, so no manual configuration is required. + +## Defines ## + +Several defines are provided to simplfy configuring plugins + +### collectd_plugin ### + +The `collectd_plugin` define configures and enables standard collect plugins. Example: + +```ruby +collectd_plugin "interface" do + options :interface=>"lo", :ignore_selected=>true +end +``` + +The options hash is converted to collectd-style settings automatically. Any symbol key will be converted to camel-case. In the above example :ignore_selected will be output as the +key "IgnoreSelected". If the key is already a string, this conversion is skipped. If the value is an array, it will be output as a separate line for each element. + +### collectd_python_plugin ### + +The `collectd_python_plugin` define configures and enables Python plugins using the collectd-python plugin. Example: + +```ruby +collectd_python_plugin "redis" do + options :host=>servers, :verbose=>true +end +``` + +Options are interpreted in the same way as with `collectd_plugin`. This define will not deploy the plugin script as well, so be sure to setup a cookbook_file resource +or other mechanism to handle distribution. Example: + +```ruby +cookbook_file File.join(node[:collectd][:plugin_dir], "redis.py") do + owner "root" + group "root" + mode "644" +end +``` + +## Web frontend ## + +The `collectd::collectd_web` recipe will automatically deploy the [collectd_web](https://github.com/httpdss/collectd-web) frontend using Apache. The +[apache2](https://github.com/opscode/cookbooks/tree/master/apache2) cookbook is required for this and is *not* included automatically as this is an optional +component, so be sure to configure the node with the correct recipes. + +# LICENSE & AUTHOR # + +Author:: Noah Kantrowitz () +Copyright:: 2010, Atari, Inc + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/chef/cookbooks/collectd/attributes/default.rb b/chef/cookbooks/collectd/attributes/default.rb new file mode 100644 index 0000000..16575d1 --- /dev/null +++ b/chef/cookbooks/collectd/attributes/default.rb @@ -0,0 +1,46 @@ +# +# Cookbook Name:: collectd +# Attributes:: default +# +# Copyright 2010, Atari, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +default[:collectd][:base_dir] = "/var/lib/collectd" +if platform_family?("rhel") + default[:collectd][:package_name] = ["collectd"] + default[:collectd][:plugin_dir] = "/usr/lib64/collectd" + default[:collectd][:config_file] = "/etc/collectd.conf" +elsif platform_family?("debian") + default[:collectd][:package_name] = ["collectd-core"] + default[:collectd][:plugin_dir] = "/usr/lib/collectd" + default[:collectd][:config_file] = "/etc/collectd/collectd.conf" +end +default[:collectd][:types_db] = ["/usr/share/collectd/types.db"] +default[:collectd][:interval] = 10 +default[:collectd][:read_threads] = 5 + +default[:collectd][:collectd_web][:path] = "/srv/collectd_web" +default[:collectd][:collectd_web][:hostname] = "collectd" + +default[:collectd][:plugins] = {"cpu"=>{}, + "syslog"=>"", + "disk"=>{"Disk"=>"/^[hsv]d[a-f][0-9]?$/", "IgnoreSelected"=>false}, + "interface"=>"", + "load"=>"", + "memory"=>"", + "match_regex"=>"" + } +default[:collectd][:server][:host] = "10.145.81.250" +default[:collectd][:server][:port] = "4242" +default[:collectd][:server][:protocol] = "tcp" diff --git a/chef/cookbooks/collectd/definitions/collectd_plugin.rb b/chef/cookbooks/collectd/definitions/collectd_plugin.rb new file mode 100644 index 0000000..73747b6 --- /dev/null +++ b/chef/cookbooks/collectd/definitions/collectd_plugin.rb @@ -0,0 +1,52 @@ +# +# Cookbook Name:: collectd +# Definition:: collectd_plugin +# +# Copyright 2010, Atari, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +define :collectd_plugin, :options => {}, :template => nil, :cookbook => nil do + template "/etc/collectd/plugins/#{params[:name]}.conf" do + owner "root" + group "root" + mode "644" + if params[:template].nil? + source "plugin.conf.erb" + cookbook params[:cookbook] || "collectd" + else + source params[:template] + cookbook params[:cookbook] + end + variables :name=>params[:name], :options=>params[:options] + notifies :restart, resources(:service => "collectd") + end +end + +define :collectd_python_plugin, :options => {}, :mod => nil, :path => nil do + begin + t = resources(:template => "/etc/collectd/plugins/python.conf") + rescue ArgumentError,Chef::Exceptions::ResourceNotFound + collectd_plugin "python" do + options :paths=>[node[:collectd][:plugin_dir]], :modules=>{} + template "python_plugin.conf.erb" + cookbook "collectd" + end + retry + end + if not params[:path].nil? + t.variables[:options][:paths] << params[:path] + end + t.variables[:options][:modules][params[:mod] || params[:name]] = params[:options] +end diff --git a/chef/cookbooks/collectd/files/default/kairosdb_writer.py b/chef/cookbooks/collectd/files/default/kairosdb_writer.py new file mode 100644 index 0000000..3c19d9b --- /dev/null +++ b/chef/cookbooks/collectd/files/default/kairosdb_writer.py @@ -0,0 +1,258 @@ +# Copyright 2013 Gregory Durham +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import collectd +import socket +import re +from string import maketrans +from time import time +from traceback import format_exc + +host = None +port = None +prefix = None +types = {} +postfix = None +tags = "" +host_separator = "_" +metric_separator = "." +protocol = "tcp" + +def kairosdb_parse_types_file(path): + global types + + f = open(path, 'r') + + for line in f: + fields = line.split() + if len(fields) < 2: + continue + + type_name = fields[0] + + if type_name[0] == '#': + continue + + v = [] + for ds in fields[1:]: + ds = ds.rstrip(',') + ds_fields = ds.split(':') + + if len(ds_fields) != 4: + collectd.warning('kairosdb_writer: cannot parse data source %s on type %s' % ( ds, type_name )) + continue + + v.append(ds_fields) + + types[type_name] = v + + f.close() + +def str_to_num(s): + """ + Convert type limits from strings to floats for arithmetic. + Will force U[nlimited] values to be 0. + """ + + try: + n = float(s) + except ValueError: + n = 0 + + return n + +def sanitize_field(field): + """ + Santize Metric Fields: replace dot and space with metric_separator. Delete + parentheses. Convert to lower case if configured to do so. + """ + field = field.strip() + trans = maketrans(' .', metric_separator * 2) + field = field.translate(trans, '()') + if lowercase_metric_names: + field = field.lower() + return field + +def kairosdb_config(c): + global host, port, prefix, postfix, host_separator, \ + metric_separator, lowercase_metric_names, protocol, \ + tags + + for child in c.children: + if child.key == 'KairosDBHost': + host = child.values[0] + elif child.key == 'KairosDBPort': + port = int(child.values[0]) + elif child.key == 'TypesDB': + for v in child.values: + kairosdb_parse_types_file(v) + elif child.key == 'LowercaseMetricNames': + lowercase_metric_names = True + elif child.key == 'MetricPrefix': + prefix = child.values[0] + elif child.key == 'HostPostfix': + postfix = child.values[0] + elif child.key == 'HostSeparator': + host_separator = child.values[0] + elif child.key == 'MetricSeparator': + metric_separator = child.values[0] + elif child.key == 'KairosDBProtocol': + protocol = str(child.values[0]) + elif child.key == 'Tags': + for v in child.values: + tags += "%s " % (v) + + tags = tags.replace('.', host_separator) + + if not host: + raise Exception('KairosDBHost not defined') + + if not port: + raise Exception('KairosDBPort not defined') + + collectd.info('Initializing kairosdb_writer client in %s socket mode.' + % protocol.upper() ) + +def kairosdb_init(): + import threading + + d = { + 'host': host, + 'port': port, + 'lowercase_metric_names': lowercase_metric_names, + 'sock': None, + 'lock': threading.Lock(), + 'values': { }, + 'last_connect_time': 0 + } + + kairosdb_connect(d) + + collectd.register_write(kairosdb_write, data=d) + +def kairosdb_connect(data): + result = False + + if not data['sock'] and protocol.lower() == 'tcp': + # only attempt reconnect every 10 seconds if protocol of type TCP + now = time() + if now - data['last_connect_time'] < 10: + return False + + data['last_connect_time'] = now + collectd.info('connecting to %s:%s' % ( data['host'], data['port'] ) ) + try: + data['sock'] = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + data['sock'].connect((host, port)) + result = True + except: + result = False + collectd.warning('error connecting socket: %s' % format_exc()) + else: + # we're either connected, or protocol does not == tcp. we will send + # data via udp/SOCK_DGRAM call. + result = True + + return result + +def kairosdb_write_data(data, s): + result = False + data['lock'].acquire() + + try: + if protocol.lower() == 'tcp': + data['sock'].sendall(s) + else: + # send message to via UDP to the line receiver . + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.sendto(s, (host, port)) + result = True + except socket.error, e: + data['sock'] = None + if isinstance(e.args, tuple): + collectd.warning('kairosdb_writer: socket error %d' % e[0]) + else: + collectd.warning('kairosdb_writer: socket error') + except: + collectd.warning('kairosdb_writer: error sending data: %s' % format_exc()) + + data['lock'].release() + return result + +def kairosdb_write(v, data=None): + data['lock'].acquire() + if not kairosdb_connect(data) and protocol.lower() == 'tcp': + data['lock'].release() + collectd.warning('kairosdb_writer: no connection to kairosdb server') + return + + data['lock'].release() + + if v.type not in types: + collectd.warning('kairosdb_writer: do not know how to handle type %s. do you have all your types.db files configured?' % v.type) + return + + v_type = types[v.type] + + if len(v_type) != len(v.values): + collectd.warning('kairosdb_writer: differing number of values for type %s' % v.type) + return + + metric_fields = [] + if prefix: + metric_fields.append(prefix) + + if postfix: + metric_fields.append(postfix) + + metric_fields.append(v.plugin) + if v.plugin_instance: + metric_fields.append(sanitize_field(v.plugin_instance)) + + metric_fields.append(v.type) + if v.type_instance: + metric_fields.append(sanitize_field(v.type_instance)) + + time = v.time + + # we update shared recorded values, so lock to prevent race conditions + data['lock'].acquire() + + lines = [] + i = 0 + for value in v.values: + ds_name = v_type[i][0] + ds_type = v_type[i][1] + + path_fields = metric_fields[:] + path_fields.append(ds_name) + + metric = '.'.join(path_fields) + + new_value = value + + if new_value is not None: + line = 'put %s %d %f %s' % ( metric, time, new_value, tags) + collectd.debug(line) + lines.append(line) + + i += 1 + + data['lock'].release() + + lines.append('') + kairosdb_write_data(data, '\n'.join(lines)) + +collectd.register_config(kairosdb_config) +collectd.register_init(kairosdb_init) diff --git a/chef/cookbooks/collectd/libraries/default.rb b/chef/cookbooks/collectd/libraries/default.rb new file mode 100644 index 0000000..1f3e92b --- /dev/null +++ b/chef/cookbooks/collectd/libraries/default.rb @@ -0,0 +1,47 @@ +# +# Cookbook Name:: collectd +# Library:: default +# +# Copyright 2010, Atari, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +def collectd_key(option) + return option.to_s.split('_').map{|x| x.capitalize}.join() if option.instance_of?(Symbol) + "#{option}" +end + +def collectd_option(option) + return option if option.instance_of?(Fixnum) || option == true || option == false + "\"#{option}\"" +end + +def collectd_settings(options, level=0) + indent = ' ' * level + output = [] + options.each_pair do |key, value| + if value.is_a? Array + value.each do |subvalue| + output << "#{indent}#{collectd_key(key)} #{collectd_option(subvalue)}" + end + elsif value.is_a? Hash + value.each_pair do |name, suboptions| + output << "#{indent}<#{key} \"#{name}\">\n#{collectd_settings(suboptions, level+1)}\n#{indent}" + end + else + output << "#{indent}#{collectd_key(key)} #{collectd_option(value)}" + end + end + output.join("\n") +end diff --git a/chef/cookbooks/collectd/metadata.rb b/chef/cookbooks/collectd/metadata.rb new file mode 100644 index 0000000..af1a1ee --- /dev/null +++ b/chef/cookbooks/collectd/metadata.rb @@ -0,0 +1,11 @@ +name "collectd" +maintainer "Noan Kantrowitz" +maintainer_email "noah@coderanger.net" +license "Apache 2.0" +description "Install and configure the collectd monitoring daemon" +long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) +version "1.0.3" +supports "ubuntu" +supports "centos" +depends "apt" +depends "yum" diff --git a/chef/cookbooks/collectd/recipes/client.rb b/chef/cookbooks/collectd/recipes/client.rb new file mode 100644 index 0000000..3b132c3 --- /dev/null +++ b/chef/cookbooks/collectd/recipes/client.rb @@ -0,0 +1,65 @@ +# +# Cookbook Name:: collectd +# Recipe:: client +# +# Copyright 2010, Atari, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +include_recipe "collectd" + +#servers = [] +#search(:node, 'recipes:collectd\\:\\:server') do |n| +# servers << n['fqdn'] +#end + +#if servers.empty? +# raise "No servers found. Please configure at least one node with collectd::server." +#end + +#collectd_plugin "network" do +# options :server=>servers +#end + +cookbook_file "#{node['collectd']['plugin_dir']}/kairosdb_writer.py" do + source "kairosdb_writer.py" + owner "root" + group "root" + mode 00644 + notifies :restart, "service[collectd]" + action :create_if_missing +end + +case node["platform_family"] +when "rhel" + node.override["collectd"]["plugins"]=node["collectd"]["rhel"]["plugins"].to_hash +when "debian" + node.override["collectd"]["plugins"]=node["collectd"]["debian"]["plugins"].to_hash +end + +node["collectd"]["plugins"].each_pair do |plugin_key, options| + collectd_plugin plugin_key do + options options + end +end + +collectd_python_plugin "kairosdb_writer" do + opts = {"KairosDBHost"=>node['collectd']['server']['host'], + "KairosDBPort"=>node['collectd']['server']['port'], + "KairosDBProtocol"=>node['collectd']['server']['protocol'], + "LowercaseMetricNames"=>"true", + "Tags" => "host=#{node['fqdn']}\" \"role=OSROLE\" \"location=China.Beijing.TsingHua\" \"cluster=#{node['cluster']}", + "TypesDB" => node['collectd']['types_db'] + } + options(opts) +end diff --git a/chef/cookbooks/collectd/recipes/collectd_web.rb b/chef/cookbooks/collectd/recipes/collectd_web.rb new file mode 100644 index 0000000..f0fd1a0 --- /dev/null +++ b/chef/cookbooks/collectd/recipes/collectd_web.rb @@ -0,0 +1,53 @@ +# +# Cookbook Name:: collectd +# Recipe:: collectd_web +# +# Copyright 2010, Atari, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "collectd" +include_recipe "apache2" + +%w(libhtml-parser-perl liburi-perl librrds-perl libjson-perl).each do |name| + package name +end + +directory node[:collectd][:collectd_web][:path] do + owner "root" + group "root" + mode "755" +end + +bash "install_collectd_web" do + user "root" + cwd node[:collectd][:collectd_web][:path] + not_if do + File.exists?(File.join(node[:collectd][:collectd_web][:path], "index.html")) + end + code <<-EOH + wget --no-check-certificate -O collectd-web.tar.gz https://github.com/httpdss/collectd-web/tarball/master + tar --strip-components=1 -xzf collectd-web.tar.gz + rm collectd-web.tar.gz + EOH +end + +template "/etc/apache2/sites-available/collectd_web.conf" do + source "collectd_web.conf.erb" + owner "root" + group "root" + mode "644" +end + +apache_site "collectd_web.conf" diff --git a/chef/cookbooks/collectd/recipes/default.rb b/chef/cookbooks/collectd/recipes/default.rb new file mode 100644 index 0000000..7598f8d --- /dev/null +++ b/chef/cookbooks/collectd/recipes/default.rb @@ -0,0 +1,112 @@ +# +# Cookbook Name:: collectd +# Recipe:: default +# +# Copyright 2010, Atari, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +case node["platform_family"] +when "debian" + package "ubuntu-cloud-keyring" do + action :install + end +when "rhel" + include_recipe "yum::epel" + execute "yum-update" do + user "root" + command "yum -y update" + action :run + end +end + +node[:collectd][:package_name].each do |pkg| + package pkg do + action :install + end +end + +service "collectd" do + supports :restart => true, :status => true +end + +directory "/etc/collectd" do + owner "root" + group "root" + mode "755" +end + +directory "/etc/collectd/plugins" do + owner "root" + group "root" + mode "755" +end + +directory node[:collectd][:base_dir] do + owner "root" + group "root" + mode "755" + recursive true +end + +directory node[:collectd][:plugin_dir] do + owner "root" + group "root" + mode "755" + recursive true +end + +%w(collection thresholds).each do |file| + template "/etc/collectd/#{file}.conf" do + source "#{file}.conf.erb" + owner "root" + group "root" + mode "644" + notifies :restart, resources(:service => "collectd") + end +end + +template node[:collectd][:config_file] do + source "collectd.conf.erb" + owner "root" + group "root" + mode "644" + notifies :restart, resources(:service => "collectd") +end + +ruby_block "delete_old_plugins" do + block do + Dir['/etc/collectd/plugins/*.conf'].each do |path| + autogen = false + File.open(path).each_line do |line| + if line.start_with?('#') and line.include?('autogenerated') + autogen = true + break + end + end + if autogen + begin + resources(:template => path) + rescue ArgumentError, Chef::Exceptions::ResourceNotFound + # If the file is autogenerated and has no template it has likely been removed from the run list + Chef::Log.info("Deleting old plugin config in #{path}") + File.unlink(path) + end + end + end + end +end + +service "collectd" do + action [:enable, :start] +end diff --git a/chef/cookbooks/collectd/recipes/server.rb b/chef/cookbooks/collectd/recipes/server.rb new file mode 100644 index 0000000..08b6320 --- /dev/null +++ b/chef/cookbooks/collectd/recipes/server.rb @@ -0,0 +1,24 @@ +# +# Cookbook Name:: collectd +# Recipe:: server +# +# Copyright 2010, Atari, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "collectd" + +collectd_plugin "network" do + options :listen=>'0.0.0.0' +end diff --git a/chef/cookbooks/collectd/templates/default/collectd.conf.erb b/chef/cookbooks/collectd/templates/default/collectd.conf.erb new file mode 100644 index 0000000..b1e3964 --- /dev/null +++ b/chef/cookbooks/collectd/templates/default/collectd.conf.erb @@ -0,0 +1,18 @@ +# Config file for collectd(1). +# +# Some plugins need additional configuration and are disabled by default. +# Please read collectd.conf(5) for details. +# +# You should also read /usr/share/doc/collectd/README.Debian.plugins before +# enabling any more plugins. + +Hostname "<%= @node[:fqdn] %>" +FQDNLookup true +BaseDir "<%= @node[:collectd][:base_dir] %>" +PluginDir "<%= @node[:collectd][:plugin_dir] %>" +TypesDB "<%= @node[:collectd][:types_db].join('", "') %>" +Interval <%= @node[:collectd][:interval] %> +ReadThreads <%= @node[:collectd][:read_threads] %> + +Include "/etc/collectd/plugins/*.conf" +Include "/etc/collectd/thresholds.conf" diff --git a/chef/cookbooks/collectd/templates/default/collectd_web.conf.erb b/chef/cookbooks/collectd/templates/default/collectd_web.conf.erb new file mode 100644 index 0000000..a0757fc --- /dev/null +++ b/chef/cookbooks/collectd/templates/default/collectd_web.conf.erb @@ -0,0 +1,32 @@ + + ServerName <%= @node[:collectd][:collectd_web][:hostname] %> + + DocumentRoot <%= @node[:collectd][:collectd_web][:path] %> + + Options FollowSymLinks + AllowOverride None + + > + Options Indexes FollowSymLinks MultiViews + AllowOverride None + Order allow,deny + Allow from all + + + ScriptAlias /cgi-bin/ <%= @node[:collectd][:collectd_web][:path] %>/cgi-bin/ + /cgi-bin"> + AllowOverride None + Options ExecCGI -MultiViews + Order allow,deny + Allow from all + + + ErrorLog /var/log/apache2/error.log + + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + + CustomLog /var/log/apache2/access.log combined + ServerSignature On + diff --git a/chef/cookbooks/collectd/templates/default/collection.conf.erb b/chef/cookbooks/collectd/templates/default/collection.conf.erb new file mode 100644 index 0000000..a114511 --- /dev/null +++ b/chef/cookbooks/collectd/templates/default/collection.conf.erb @@ -0,0 +1,2 @@ +datadir: "/var/lib/collectd/rrd/" +libdir: "/usr/lib/collectd/" diff --git a/chef/cookbooks/collectd/templates/default/plugin.conf.erb b/chef/cookbooks/collectd/templates/default/plugin.conf.erb new file mode 100644 index 0000000..ac5a824 --- /dev/null +++ b/chef/cookbooks/collectd/templates/default/plugin.conf.erb @@ -0,0 +1,15 @@ +# This file autogenerated by Chef +# Do not edit, changes will be overwritten +LoadPlugin "<%= @name %>" + +<% if not @options.empty? %> +"> + <% @options.each_pair do |key, value| + if value.is_a? Array + value.each do |subvalue| %> + <%= collectd_key(key) %> <%= collectd_option(subvalue) %> + <% end else %> + <%= collectd_key(key) %> <%= collectd_option(value) %> + <% end end %> + +<% end %> diff --git a/chef/cookbooks/collectd/templates/default/python_plugin.conf.erb b/chef/cookbooks/collectd/templates/default/python_plugin.conf.erb new file mode 100644 index 0000000..681d795 --- /dev/null +++ b/chef/cookbooks/collectd/templates/default/python_plugin.conf.erb @@ -0,0 +1,20 @@ +# This file autogenerated by Chef +# Do not edit, changes will be overwritten + + Globals true + + + + <% @options[:paths].each do |path| %> + ModulePath "<%= path %>" + <% end %> + <% @options[:modules].each_key do |mod| %> + Import "<%= mod %>" + <% end %> + + <% @options[:modules].each_pair do |mod, config| %> + "> +<%= collectd_settings(config, 2) %> + + <% end %> + diff --git a/chef/cookbooks/collectd/templates/default/thresholds.conf.erb b/chef/cookbooks/collectd/templates/default/thresholds.conf.erb new file mode 100644 index 0000000..690d103 --- /dev/null +++ b/chef/cookbooks/collectd/templates/default/thresholds.conf.erb @@ -0,0 +1,37 @@ +# Threshold configuration for collectd(1). +# +# See the section "THRESHOLD CONFIGURATION" in collectd.conf(5) for details. + +# +# +# WarningMin 0.00 +# WarningMax 1000.00 +# FailureMin 0 +# FailureMax 1200.00 +# Invert false +# Persist false +# Instance "some_instance" +# +# +# +# Instance "eth0" +# +# DataSource "rx" +# FailureMax 10000000 +# +# +# +# +# +# Instance "idle" +# FailureMin 10 +# +# +# +# +# Instance "cached" +# WarningMin 100000000 +# +# +# +# diff --git a/chef/roles/os-block-storage-api.rb b/chef/roles/os-block-storage-api.rb index 9ab38df..e0ee610 100644 --- a/chef/roles/os-block-storage-api.rb +++ b/chef/roles/os-block-storage-api.rb @@ -8,6 +8,13 @@ override_attributes( "debianloglist" => { "cinder-api" => "/var/log/cinder/cinder-api.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-cinder-api"] } + } + } } ) run_list( diff --git a/chef/roles/os-block-storage-scheduler.rb b/chef/roles/os-block-storage-scheduler.rb index c9f2bb0..5e78e88 100644 --- a/chef/roles/os-block-storage-scheduler.rb +++ b/chef/roles/os-block-storage-scheduler.rb @@ -8,6 +8,13 @@ override_attributes( "debianloglist" => { "cinder-scheduler" => "/var/log/cinder/cinder-scheduler.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-cinder-scheduler"] } + } + } } ) run_list( diff --git a/chef/roles/os-block-storage-worker.rb b/chef/roles/os-block-storage-worker.rb index 2bddf5f..cd6ad23 100644 --- a/chef/roles/os-block-storage-worker.rb +++ b/chef/roles/os-block-storage-worker.rb @@ -8,6 +8,13 @@ override_attributes( "debianloglist" => { "cinder-volume" => "/var/log/cinder/cinder-volume.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-cinder-volume", "iscsid", "multipathd"] } + } + } } ) run_list( diff --git a/chef/roles/os-compute-api-metadata.rb b/chef/roles/os-compute-api-metadata.rb index 59180c5..87a65f1 100644 --- a/chef/roles/os-compute-api-metadata.rb +++ b/chef/roles/os-compute-api-metadata.rb @@ -1,5 +1,14 @@ name "os-compute-api-metadata" description "OpenStack compute metadata API service" +override_attributes( + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-nova-metadata-api"] } + } + } + } +) run_list( "role[os-base]", "recipe[openstack-compute::api-metadata]" diff --git a/chef/roles/os-compute-api-os-compute.rb b/chef/roles/os-compute-api-os-compute.rb index 157525c..c148ab7 100644 --- a/chef/roles/os-compute-api-os-compute.rb +++ b/chef/roles/os-compute-api-os-compute.rb @@ -8,6 +8,13 @@ override_attributes( "debianloglist" => { "nova-api" => "/var/log/nova/nova-api.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-nova-api"] } + } + } } ) run_list( diff --git a/chef/roles/os-compute-cert.rb b/chef/roles/os-compute-cert.rb index ad9922c..5eb6d7b 100644 --- a/chef/roles/os-compute-cert.rb +++ b/chef/roles/os-compute-cert.rb @@ -1,5 +1,14 @@ name "os-compute-cert" description "OpenStack Compute Cert service" +override_attributes( + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-nova-cert"] } + } + } + } +) run_list( "role[os-base]", "recipe[openstack-compute::nova-cert]" diff --git a/chef/roles/os-compute-scheduler.rb b/chef/roles/os-compute-scheduler.rb index dd9db1d..9ffa272 100644 --- a/chef/roles/os-compute-scheduler.rb +++ b/chef/roles/os-compute-scheduler.rb @@ -10,6 +10,13 @@ override_attributes( "nova-scheduler" => "/var/log/nova/nova-scheduler.log", "nova-conductor" => "/var/log/nova/nova-conductor.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-nova-scheduler", "openstack-nova-conductor"] } + } + } } ) run_list( diff --git a/chef/roles/os-compute-vncproxy.rb b/chef/roles/os-compute-vncproxy.rb index fd3eb08..2a1dec5 100644 --- a/chef/roles/os-compute-vncproxy.rb +++ b/chef/roles/os-compute-vncproxy.rb @@ -1,5 +1,14 @@ name "os-compute-vncproxy" description "Nova VNC Proxy" +override_attributes( + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-nova-xvpvncproxy", "openstack-nova-novncproxy"] } + } + } + } +) run_list( "role[os-base]", "recipe[openstack-compute::vncproxy]" diff --git a/chef/roles/os-compute-worker.rb b/chef/roles/os-compute-worker.rb index b5413ad..cdf1f84 100644 --- a/chef/roles/os-compute-worker.rb +++ b/chef/roles/os-compute-worker.rb @@ -8,6 +8,13 @@ override_attributes( "debianloglist" => { "nova-compute" => "/var/log/nova/nova-compute.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-nova-compute"] } + } + } } ) run_list( diff --git a/chef/roles/os-dashboard.rb b/chef/roles/os-dashboard.rb index d33437f..23f64ab 100644 --- a/chef/roles/os-dashboard.rb +++ b/chef/roles/os-dashboard.rb @@ -1,5 +1,14 @@ name "os-dashboard" description "Horizon server" +override_attributes( + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["httpd"]} + } + } + } +) run_list( "role[os-base]", # "recipe[openstack-dashboard::db]", diff --git a/chef/roles/os-ha.rb b/chef/roles/os-ha.rb index ccb383f..a8c8ea4 100644 --- a/chef/roles/os-ha.rb +++ b/chef/roles/os-ha.rb @@ -1,5 +1,14 @@ name "os-ha" description "Software load banance" +override_attributes( + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["haproxy", "keepalived"]} + } + } + } +) run_list( "recipe[keepalived]", "recipe[haproxy::tcp_lb]" diff --git a/chef/roles/os-identity.rb b/chef/roles/os-identity.rb index fa81f4e..7faf10b 100644 --- a/chef/roles/os-identity.rb +++ b/chef/roles/os-identity.rb @@ -8,6 +8,13 @@ override_attributes( "debianloglist" => { "keystone" => "/var/log/keystone/keystone.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-keystone"] } + } + } } ) run_list( diff --git a/chef/roles/os-image-api.rb b/chef/roles/os-image-api.rb index e915306..cb7014f 100644 --- a/chef/roles/os-image-api.rb +++ b/chef/roles/os-image-api.rb @@ -8,6 +8,13 @@ override_attributes( "debianloglist" => { "glance-api" => "/var/log/glance/api.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-glance-api"] } + } + } } ) run_list( diff --git a/chef/roles/os-image-registry.rb b/chef/roles/os-image-registry.rb index df4447e..b79ecdf 100644 --- a/chef/roles/os-image-registry.rb +++ b/chef/roles/os-image-registry.rb @@ -8,6 +8,13 @@ override_attributes( "debianloglist" => { "glance-registry" => "/var/log/glance/registry.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["openstack-glance-registry"] } + } + } } ) run_list( diff --git a/chef/roles/os-network-server.rb b/chef/roles/os-network-server.rb index 50bac00..e2f894c 100644 --- a/chef/roles/os-network-server.rb +++ b/chef/roles/os-network-server.rb @@ -8,6 +8,13 @@ override_attributes( "rhelloglist" => { "quantum-server" => "/var/log/quantum/server.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["quantum-server"] } + } + } } ) run_list( diff --git a/chef/roles/os-network.rb b/chef/roles/os-network.rb index c4486a8..0083344 100644 --- a/chef/roles/os-network.rb +++ b/chef/roles/os-network.rb @@ -7,11 +7,18 @@ override_attributes( "quantum-dhcp" => "/var/log/quantum/dhcp-agent.log", "quantum-l3agent" => "/var/log/quantum/l3-agent.log" }, - "rhelloglist" => { + "debianloglist" => { "quantum-ovsagent" => "/var/log/quantum/openvswitch-agent.log", "quantum-dhcp" => "/var/log/quantum/dhcp-agent.log", "quantum-l3agent" => "/var/log/quantum/l3-agent.log" - }, + } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["quantum-dhcp-agent", "quantum-l3-agent", "quantum-openvswitch-agent", "quantum-metadata-agent"] } + } + } } ) run_list( diff --git a/chef/roles/os-ops-database.rb b/chef/roles/os-ops-database.rb index f5fa5c7..7388caa 100644 --- a/chef/roles/os-ops-database.rb +++ b/chef/roles/os-ops-database.rb @@ -8,6 +8,13 @@ override_attributes( "debianloglist" => { "mysqld" => "/var/log/mysql.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["mysqld"] } + } + } } ) run_list( diff --git a/chef/roles/os-ops-messaging.rb b/chef/roles/os-ops-messaging.rb index b0322ca..0b679c5 100644 --- a/chef/roles/os-ops-messaging.rb +++ b/chef/roles/os-ops-messaging.rb @@ -8,6 +8,13 @@ override_attributes( "debianloglist" => { "rabbitmq" => "/var/log/rabbitmq/rabbit\@$hostname.log" } + }, + "collectd" => { + "rhel" => { + "plugins" => { + "processes" => { "Process" => ["rabbitmq-server"] } + } + } } ) run_list( diff --git a/chef/roles/test-synclog.rb b/chef/roles/test-synclog.rb index 23748ea..45726c7 100644 --- a/chef/roles/test-synclog.rb +++ b/chef/roles/test-synclog.rb @@ -1,5 +1,6 @@ name "test-synclog" description "Sync application related logs for debugging" run_list( - "recipe[rsyslog::client]" + "recipe[rsyslog::client]", + "recipe[collectd::client]" )