c6493a812b
The monitorstack data collection can export data into elasticsearch. A playbook has been added to deploy the data collection probes which will leverage systemd-timers to run the probes on regular intervals. The systemd timers will be deployed per-probe and run within the utility, compute, and memcached hosts. Any place the probes are deployed an isolated user will ensure to fence the probes from the cluster and limit access. OpenStack probes will only be deployed when an openstack-sdk clouds config is found within the system. Change-Id: Ic5cd5fd51a7e0763c0a2db40af4150b8851bc748 Signed-off-by: cloudnull <kevin@cloudnull.com>
119 lines
4.1 KiB
YAML
119 lines
4.1 KiB
YAML
---
|
|
# Copyright 2018, Rackspace US, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
elastic_index_retention_algorithm: default
|
|
|
|
### Elastic curator variables
|
|
## If any of these retention policy option are undefined a dynamic fact will be
|
|
## generated.
|
|
## These options are all in days.
|
|
# elastic_logstash_retention: 1
|
|
# elastic_apm_retention: 1
|
|
# elastic_auditbeat_retention: 1
|
|
# elastic_filebeat_retention: 1
|
|
# elastic_heartbeat_retention: 1
|
|
# elastic_journalbeat_retention: 1
|
|
# elastic_metricbeat_retention: 1
|
|
# elastic_packetbeat_retention: 1
|
|
# elastic_skydive_retention: 1
|
|
|
|
## These options are all in megabytes.
|
|
# elastic_logstash_size: 1024
|
|
# elastic_apm_size: 1024
|
|
# elastic_auditbeat_size: 1024
|
|
# elastic_filebeat_size: 1024
|
|
# elastic_heartbeat_size: 1024
|
|
# elastic_journalbeat_size: 1024
|
|
# elastic_metricbeat_size: 1024
|
|
# elastic_packetbeat_size: 1024
|
|
# elastic_skydive_size: 1024
|
|
|
|
## WHen a static retention policy option is not defined these options will be
|
|
## used for dynamic fact generation.
|
|
##
|
|
## Facts will be generated for the general retention using the total available
|
|
## storage from the ES data nodes, subtracting 25%. Using the weights, each
|
|
## index will be given a percentage of the total available storage. Indexes with
|
|
## higher weights are expected to use more storage. The list of hosts in a given
|
|
## index will be used to determine the number of days data can exist within an
|
|
## index before it's pruned.
|
|
|
|
## Example:
|
|
# es cluster has 4TiB of storage
|
|
# filebeat is deployed to 100 hosts
|
|
# filebeat has a weight of 10
|
|
# metricbeat is deployed to 125 hosts
|
|
# metricbeat has a weight of 2
|
|
#
|
|
# es storage in MiB: 4194304
|
|
# hosts and weighting total: (100 + 125) x (10 + 2) = 2700
|
|
# filebeat pct: (100 x 10) / 2700 = 0.37
|
|
# filebeat storage allowed: 0.37 * 4194304 = 1551892.48 MiB
|
|
# filebeat days allowed: 1551892.48 / (100 * 1024) = 15.1552 Days
|
|
# filebeat result: 15 days of retention or 1.5TiB of storage, whatever comes first
|
|
# metricbeat pct: (125 x 2) / 2700 = 0.09
|
|
# metricbeat storage allowed: 0.09 * 4194304 = 377487.36 MiB
|
|
# metricbeat days allowed: 377487.36 / (125 * 1024) = 2.94912 Days
|
|
# metricbeat result: 2 days of retention or 38GiB of storage, whatever comes first
|
|
|
|
elastic_beat_retention_policy_hosts:
|
|
logstash:
|
|
make_index: true
|
|
weight: 1
|
|
hosts: "{{ groups['elastic-logstash'] | default([]) }}"
|
|
apm:
|
|
make_index: true
|
|
timeFieldName: '@timestamp'
|
|
weight: 1
|
|
hosts: "{{ groups['apm-server'] | default([]) }}"
|
|
auditbeat:
|
|
timeFieldName: '@timestamp'
|
|
weight: 10
|
|
hosts: "{{ groups['hosts'] | default([]) }}"
|
|
filebeat:
|
|
timeFieldName: '@timestamp'
|
|
weight: 10
|
|
hosts: "{{ groups['hosts'] | default([]) }}"
|
|
syslog:
|
|
make_index: true
|
|
weight: 1
|
|
hosts: "{{ groups['hosts'] | default([]) }}"
|
|
heartbeat:
|
|
timeFieldName: '@timestamp'
|
|
weight: 1
|
|
hosts: "{{ groups['kibana'][:3] | default([]) }}"
|
|
journalbeat:
|
|
timeFieldName: '@timestamp'
|
|
weight: 3
|
|
hosts: "{{ groups['hosts'] | default([]) }}"
|
|
metricbeat:
|
|
timeFieldName: '@timestamp'
|
|
weight: 2
|
|
hosts: "{{ groups['all'] | default([]) }}"
|
|
packetbeat:
|
|
timeFieldName: '@timestamp'
|
|
weight: 1
|
|
hosts: "{{ groups['hosts'] | default([]) }}"
|
|
monitorstack:
|
|
timeFieldName: '@timestamp'
|
|
weight: 1
|
|
hosts: "{{ (groups['nova_compute'] | default([])) | union((groups['utility_all'] | default([]))) | union((groups['memcached_all'] | default([]))) }}"
|
|
skydive:
|
|
weight: 1
|
|
hosts: "{{ (((groups['skydive_analyzers'] | default([])) | length) > 0) | ternary((groups['hosts'] | default([])), []) }}"
|
|
|
|
# Refresh the elasticsearch retention policy local facts.
|
|
elastic_retention_refresh: false
|