0d4a4a92c7
The multi-logstash pipeline setup, while amazingly fast, was crashing and causing index errors when under high load for a long period of time. Because of the crashing behavior and the fact that the folks from Elastic describe multi-pipeline queues to be "beta" at this time the logstash pipelines have been converted back into a single pipeline. The memory backed queue options are now limited by a ram disk (tmpfs) which will ensure that a burst within the queue does not cause OOM issues and ensures a highly performant deployment and limiting memory usage at the same time. Memory backed queues will be enabled when the underlying system is using "rotational" media as detected by ansible facts. This will ensure a fast and consistent experience across all deployment types. Pipeline/ml/template/dashboard setup has been added to the beat configurations which will ensure beats are properly configured even when running in an isolated deployment and outside of normal operations where beats are generally configured on the first data node. Change-Id: Ie3c775f98b14f71bcbed05db9cb1c5aa46d9c436 Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
112 lines
3.3 KiB
YAML
112 lines
3.3 KiB
YAML
---
|
|
# Copyright 2018, Rackspace US, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
- name: Gather variables for each operating system
|
|
include_vars: "{{ item }}"
|
|
with_first_found:
|
|
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
|
|
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
|
|
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
|
|
- "{{ ansible_distribution | lower }}.yml"
|
|
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
|
|
- "{{ ansible_os_family | lower }}.yml"
|
|
tags:
|
|
- always
|
|
|
|
- name: Set elasticsearch variables
|
|
include_vars: "vars_{{ ((inventory_hostname in (groups['kibana'] | default([])) and not inventory_hostname in (groups['elastic-logstash']) | default([]))) | ternary('kibana', 'default') }}.yml"
|
|
tags:
|
|
- always
|
|
|
|
- name: Ensure elasticsearch is installed
|
|
package:
|
|
name: "{{ elasticsearch_distro_packages }}"
|
|
state: "{{ elk_package_state | default('present') }}"
|
|
update_cache: "{{ (ansible_pkg_mgr == 'apt') | ternary('yes', omit) }}"
|
|
register: _package_task
|
|
until: _package_task is success
|
|
retries: 3
|
|
delay: 2
|
|
notify:
|
|
- Enable and restart elastic
|
|
tags:
|
|
- package_install
|
|
|
|
- name: Create elasticsearch systemd service config dir
|
|
file:
|
|
path: "/etc/systemd/system/elasticsearch.service.d"
|
|
state: "directory"
|
|
group: "root"
|
|
owner: "root"
|
|
mode: "0755"
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
|
|
- name: Apply systemd options
|
|
template:
|
|
src: "{{ item.src }}"
|
|
dest: "{{ item.dest }}"
|
|
when:
|
|
- ansible_service_mgr == 'systemd'
|
|
with_items:
|
|
- src: "systemd.elasticsearch-overrides.conf.j2"
|
|
dest: "/etc/systemd/system/elasticsearch.service.d/elasticsearch-overrides.conf"
|
|
notify:
|
|
- Enable and restart elastic
|
|
tags:
|
|
- config
|
|
|
|
- name: Drop elasticsearch conf file
|
|
template:
|
|
src: "{{ item.src }}"
|
|
dest: "{{ item.dest }}"
|
|
with_items:
|
|
- src: "elasticsearch.yml.j2"
|
|
dest: "/etc/elasticsearch/elasticsearch.yml"
|
|
- src: "es-log4j2.properties.j2"
|
|
dest: "/etc/elasticsearch/log4j2.properties"
|
|
notify:
|
|
- Enable and restart elastic
|
|
tags:
|
|
- config
|
|
|
|
- name: Ensure elasticsearch ownership
|
|
file:
|
|
path: "/var/lib/elasticsearch/"
|
|
owner: elasticsearch
|
|
group: elasticsearch
|
|
recurse: true
|
|
register: e_perms
|
|
until: e_perms is success
|
|
retries: 3
|
|
delay: 1
|
|
tags:
|
|
- config
|
|
|
|
- name: Ensure elasticsearch tmp dir
|
|
file:
|
|
path: "/var/lib/elasticsearch/tmp"
|
|
state: directory
|
|
owner: "elasticsearch"
|
|
group: "elasticsearch"
|
|
mode: "0750"
|
|
|
|
- include_tasks: "elasticsearch_nfs_setup.yml"
|
|
when:
|
|
- elastic_shared_fs_repos is defined
|
|
- (elastic_shared_fs_repos | json_query(nfs_query)) | length > 0
|
|
|
|
- include_tasks: "elasticsearch_plugins.yml"
|