openstack/openstack-ansible-ops
Code Issues Proposed changes

Convert playbooks into roles

Browse Source 
This change adds the scaffolding required to get multi-distro support
running in the roles. The change breaks up our playbooks converting all
of the tasks into various roles with internal dependencies. While this
will improve execution time, the change is being done to reduce boiler
plate and to allow us to build on the pattern used in OSA to provide
multi-distro capabilities.

A side effect of this change is a major improvement in idempotency. The
playbooks should now be 100% idempotent.

All of the templates have been left in the main playbook directory. This
was done to help ease the transition. In a future PR the template
structure will be moved into the roles where it needs to be.

The main variable files has been left intact. This file will be carved
up into role defaults in a future PR.

Change-Id: I938a10564128ce4078fa12edcf614dcdbd684b25
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
Kevin Carter 2018-08-08 01:11:51 -05:00
parent ca23f9a987
commit 3a0b3d2cde
No known key found for this signature in database
GPG Key ID: 9443251A787B9FB3
113 changed files with 3626 additions and 2011 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
elk_metrics_6x/calculate_index_retention_default.yml
View File

@ -1,24 +0,0 @@
---
- name: Set assumed buffer storage fact
set_fact:
es_assumed_buffer_storage: "{{ ((es_total_available_storage | int) * 0.25) | round | int }}"
- name: Set usable buffer storage fact(s)
set_fact:
es_usable_buffer_storage: "{{ (es_total_available_storage | int) - (es_assumed_buffer_storage | int) }}"
es_expected_storage: "{{ ((elastic_beat_retention_policy_hosts.values() | map('int') | list) | sum) * (elastic_beat_storage_constant | int) }}"
- name: Set buffer storage fact
set_fact:
es_assumed_usable_storage_per_node: "{{ (es_usable_buffer_storage | int) // (data_nodes | length | int) }}"
- name: Set storage the mulitplier
set_fact:
es_storage_multiplier: "{{ ((es_usable_buffer_storage | int) < (es_expected_storage | int)) | ternary(((elastic_beat_storage_constant | int) * 2), elastic_beat_storage_constant | int) }}"
- name: Set retention facts
set_fact: "elastic_{{ item.key }}_retention={{ (es_assumed_usable_storage_per_node | int) // ((item.value | int) * (es_storage_multiplier | int)) }}"
when:
- hostvars[inventory_hostname]["elastic_" + item.key + "_retention"] is undefined
with_dict: "{{ elastic_beat_retention_policy_hosts }}"

131
elk_metrics_6x/common_task_data_node_hosts.yml
View File

@ -1,131 +0,0 @@
---
- name: Refresh minimal facts
setup:
gather_subset: '!all,!any,network,virtual'
# storage node count is equal to the cluster size
- name: Node count fact
set_fact:
storage_node_count: "{{ groups['elastic-logstash'] | length }}"
tags:
- always
# the elasticserch cluster elects one master from all those which are marked as master-eligible
# 1 node cluster can only have one master
# 2 node clusters have 1 master-eligable nodes to avoid split-brain
# 3 node clusters have 3 master-eligable nodes
# >3 node clusters have (nodes // 2) eligable masters rounded up to the next odd number
- name: Master node count fact
set_fact:
master_node_count: |-
{% set masters = 0 %}
{% if (storage_node_count | int) < 3 %}
{% set masters = 1 %}
{% elif (storage_node_count | int) == 3 %}
{% set masters = 3 %}
{% else %}
{% set masters = (storage_node_count | int ) // 2 %}
{% if ((masters | int) % 2 == 0) %}
{% set masters = (masters | int) + 1 %}
{% endif %}
{% endif %}
{{ masters }}
tags:
- always
# assign node roles
# the first 'master_node_count' hosts in groups['elastic-logstash'] become master-eligible nodes
# the first 'master_node_count' and subsequent alternate hosts in groups['elastic-logstash'] becomes data nodes
- name: Data nodes fact
set_fact:
data_nodes: "{{ (groups['elastic-logstash'][:master_node_count | int] + groups['elastic-logstash'][master_node_count | int::2]) }}"
master_nodes: "{{ groups['elastic-logstash'][:master_node_count | int] }}"
coordination_nodes: >-
{{
(groups['kibana'] | map('extract', hostvars, 'ansible_host') | list)
| map('regex_replace', '(.*)' ,'\1:' ~ elastic_port)
| list
}}
zen_nodes: >-
{{
(groups['elastic-logstash'] | union(groups['kibana'])) | map('extract', hostvars, 'ansible_host') | list
}}
elasticserch_interface_speed: |-
{% set default_interface_fact = hostvars[inventory_hostname]['ansible_' + (elastic_data_interface | replace('-', '_'))] %}
{% set speeds = [] %}
{% if default_interface_fact['type'] == 'bridge' %}
{% for interface in default_interface_fact['interfaces'] %}
{% set interface_fact = hostvars[inventory_hostname]['ansible_' + (interface | replace('-', '_'))] %}
{% if 'speed' in interface_fact %}
{% set speed = (interface_fact['speed'] | default(1000)) | string %}
{% if speed == "-1" %}
{% set _ = speeds.append(1000) %}
{% else %}
{% set _ = speeds.append(speed | int) %}
{% endif %}
{% if 'module' in interface_fact %}
{% set _ = speeds.append((interface_fact['speed'] | default(1000)) | int) %}
{% else %}
{% set _ = speeds.append(1000) %}
{% endif %}
{% endif %}
{% endfor %}
{% else %}
{% if ('module' in default_interface_fact) or (default_interface_fact['type'] == 'bond') %}
{% set speed = (default_interface_fact['speed'] | default(1000)) | string %}
{% if speed == "-1" %}
{% set _ = speeds.append(1000) %}
{% else %}
{% set _ = speeds.append(speed | int) %}
{% endif %}
{% else %}
{% set _ = speeds.append(1000) %}
{% endif %}
{% endif %}
{% set interface_speed = ((speeds | min) * 0.20) | int %}
{{ ((interface_speed | int) > 750) | ternary(750, interface_speed) }}
tags:
- always
- name: Set data node details
set_fact:
elasticsearch_data_node_details: >-
{{
(data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port) | list
}}
logstash_data_node_details: >-
{{
(data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ logstash_beat_input_port) | list
}}
# based on the assignment of roles to hosts, set per host booleans
- name: Node enablement
set_fact:
master_node: "{{ (inventory_hostname in master_nodes) | ternary(true, false) }}"
data_node: "{{ (inventory_hostname in data_nodes) | ternary(true, false) }}"
elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) >= 24) | ternary(24, ansible_processor_cores) }}"
tags:
- always
# Set a data node facts. The data nodes, in the case of elasticsearch are also
# ingest nodes.
- name: Set data nodes
set_fact:
elasticsearch_number_of_replicas: "{{ ((data_nodes | length) > 2) | ternary('2', ((data_nodes | length) > 1) | ternary('1', '0')) }}"
elasticsearch_data_hosts: |-
{% set nodes = elasticsearch_data_node_details %}
{% if inventory_hostname in data_nodes %}
{% set _ = nodes.insert(0, '127.0.0.1:' ~ elastic_port) %}
{% endif %}
{% set data_hosts = nodes | shuffle(seed=inventory_hostname) %}
{{ data_hosts }}
logstash_data_hosts: |-
{% set nodes = logstash_data_node_details %}
{% if inventory_hostname in data_nodes %}
{% set _ = nodes.insert(0, '127.0.0.1:' ~ logstash_beat_input_port) %}
{% endif %}
{% set data_hosts = nodes | shuffle(seed=inventory_hostname) %}
{{ data_hosts }}
tags:
- always

62
elk_metrics_6x/common_task_install_go1.10.1.yml
View File

@ -1,62 +0,0 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Check for go
stat:
path: /opt/go1.10.1/go/bin/go
register: go_path
- name: Install go
block:
- name: GET go
get_url:
url: "https://dl.google.com/go/go1.10.1.linux-amd64.tar.gz"
dest: "/opt/go1.10.1.linux-amd64.tar.gz"
checksum: "sha256:72d820dec546752e5a8303b33b009079c15c2390ce76d67cf514991646c6127b"
register: _get_task
until: _get_task is success
retries: 3
delay: 2
tags:
- package_install
- name: Create go directory
file:
path: "/opt/go1.10.1"
state: directory
- name: Unarchive go
unarchive:
src: "/opt/go1.10.1.linux-amd64.tar.gz"
dest: "/opt/go1.10.1"
remote_src: yes
- name: Create go defaults file
copy:
content: |
GOROOT=/opt/go1.10.1/go
GOPATH=/usr/local
PATH=${PATH}:${GOROOT}/bin
dest: /etc/default/go1.10.1
when:
- (elk_package_state | default('present')) == 'present'
- not go_path.stat.exists | bool
- name: Remove go
file:
path: "/opt/go1.10.1"
state: absent
when:
- (elk_package_state | default('present')) == 'absent'

20
elk_metrics_6x/createElasticIndexes.yml
View File

@ -1,4 +1,16 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Create/Setup known indexes in Elasticsearch
hosts: "elastic-logstash[0]"
become: true
@ -8,12 +20,10 @@
environment: "{{ deployment_environment_variables | default({}) }}"
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
roles:
- role: elastic_retention
tasks:
post_tasks:
- name: Create basic indexes
uri:
url: http://127.0.0.1:9200/{{ item.name }}

88
elk_metrics_6x/installAPMserver.yml
View File

@ -1,4 +1,16 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install apm-server
hosts: apm-server
become: true
@ -8,76 +20,10 @@
vars_files:
- vars/variables.yml
environment: "{{ deployment_environment_variables | default({}) }}"
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
tasks:
- include_tasks: common_task_install_elk_repo.yml
- name: Ensure apm-server is installed
apt:
name: "{{ item }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
with_items:
- apm-server
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
notify:
- Enable and restart apm server
tags:
- package_install
- name: exit playbook after uninstall
meta: end_play
when:
- (elk_package_state | default('present')) == 'absent'
post_tasks:
- name: Create apm-server systemd service config dir
file:
path: "/etc/systemd/system/apm-server.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/apm-server.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- { src: "systemd.general-overrides.conf.j2", dest: "apm-server-overrides.conf" }
notify:
- Enable and restart apm server
- name: Drop apm-server conf file
template:
src: templates/apm-server.yml.j2
dest: /etc/apm-server/apm-server.yml
notify:
- Enable and restart apm server
handlers:
- name: Enable and restart apm server
systemd:
name: "apm-server"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
roles:
- role: elastic_apm_server
- role: elastic_rollup
index_name: apm
tags:
- server-install
- import_playbook: setupAPMserver.yml
- apm-server

85
elk_metrics_6x/installAuditbeat.yml
View File

@ -1,4 +1,16 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install Auditbeat
hosts: hosts
become: true
@ -10,75 +22,10 @@
environment: "{{ deployment_environment_variables | default({}) }}"
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
tasks:
- include_tasks: common_task_install_elk_repo.yml
- name: Ensure Auditbeat is installed
apt:
name: "{{ item }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
with_items:
- audispd-plugins
- auditbeat
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
notify:
- Enable and restart auditbeat
tags:
- package_install
- name: exit playbook after uninstall
meta: end_play
when:
- (elk_package_state | default('present')) == 'absent'
post_tasks:
- name: Create auditbeat systemd service config dir
file:
path: "/etc/systemd/system/auditbeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/auditbeat.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- { src: "systemd.general-overrides.conf.j2", dest: "auditbeat-overrides.conf" }
notify:
- Enable and restart auditbeat
- name: Drop auditbeat conf file
template:
src: templates/auditbeat.yml.j2
dest: /etc/auditbeat/auditbeat.yml
notify:
- Enable and restart auditbeat
handlers:
- name: Enable and restart auditbeat
systemd:
name: "auditbeat"
enabled: "true"
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
roles:
- role: elastic_auditbeat
- role: elastic_rollup
index_name: auditbeat
tags:
- beat-install
- import_playbook: setupAuditbeat.yml

126
elk_metrics_6x/installCurator.yml
View File

@ -1,6 +1,4 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -25,128 +23,8 @@
environment: "{{ deployment_environment_variables | default({}) }}"
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
- name: Query es storage
uri:
url: "http://127.0.0.1:9200/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs"
method: GET
register: elk_data
until: elk_data is success
retries: 5
delay: 5
run_once: true
- name: Set available storage fact
set_fact:
es_total_available_storage: "{{ ((elk_data['json']['nodes'].values() | list) | map(attribute='fs.total.total_in_bytes') | list | sum) // 1024 // 1024 }}"
- name: Compute index retention
include_tasks: "calculate_index_retention_{{ elastic_index_retention_algorithm | default('default') }}.yml"
- name: Ensure virtualenv is installed
apt:
name: "{{ item }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
with_items:
- python-virtualenv
- virtualenv
tags:
- package_install
- name: Ensure curator is installed
pip:
name: "elasticsearch-curator<6"
state: "{{ elk_package_state | default('present') }}"
extra_args: --isolated
virtualenv: /opt/elasticsearch-curator
register: _pip_task
until: _pip_task is success
retries: 3
delay: 2
tags:
- package_install
- name: exit playbook after uninstall
meta: end_play
when:
- (elk_package_state | default('present')) == 'absent'
tasks:
- name: create the system group
group:
name: "curator"
state: "present"
system: "yes"
- name: Create the curator system user
user:
name: "curator"
group: "curator"
comment: "curator user"
shell: "/bin/false"
createhome: "yes"
home: "/var/lib/curator"
- name: Create curator data path
file:
path: "{{ item }}"
state: directory
owner: "curator"
group: "curator"
mode: "0755"
recurse: true
with_items:
- "/var/lib/curator"
- "/var/log/curator"
- "/etc/curator"
- name: Drop curator conf file
template:
src: templates/curator.yml.j2
dest: /var/lib/curator/curator.yml
- name: Drop curator action file
template:
src: templates/curator-actions.yml.j2
dest: /var/lib/curator/actions.yml
post_tasks:
- name: Run the systemd service role
include_role:
name: systemd_service
private: true
vars:
systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}"
systemd_service_restart_changed: false
systemd_user_name: curator
systemd_group_name: curator
systemd_services:
- service_name: "curator"
execstarts:
- /opt/elasticsearch-curator/bin/curator
--config /var/lib/curator/curator.yml
/var/lib/curator/actions.yml
timer:
state: "started"
options:
OnBootSec: 30min
OnUnitActiveSec: 24h
Persistent: true
- name: Enable and restart curator.timer
systemd:
name: "curator.timer"
enabled: true
state: restarted
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
roles:
- role: elastic_curator
tags:
- beat-install

242
elk_metrics_6x/installElastic.yml
View File

@ -1,241 +1,27 @@
---
- name: Run serialization detection
hosts: "elastic-logstash:kibana"
gather_facts: true
vars_files:
- vars/variables.yml
tasks:
- include_tasks: common_task_data_node_hosts.yml
- name: Group by stand alone masters
group_by:
key: elastic_masters
parents: elastic-logstash
when:
- inventory_hostname in master_nodes
- name: Group by non stand alone masters
group_by:
key: elastic_non_masters
parents: elastic-logstash
when:
- inventory_hostname in (data_nodes | difference(master_nodes))
- name: Group by coordinators
group_by:
key: elastic_coordinators
parents: elastic-logstash
when:
- inventory_hostname in groups['kibana']
tags:
- always
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install Elastic Search
hosts: "elastic_coordinators:elastic_masters:elastic_non_masters"
serial: "33%"
hosts: "elastic-logstash:kibana"
become: true
vars_files:
- vars/variables.yml
vars:
temp_dir: /var/lib/elasticsearch/tmp
nfs_query: "[?fstype=='nfs' || fstype=='nfs4']"
environment: "{{ deployment_environment_variables | default({}) }}"
tasks:
- name: Set memory fact to half
set_fact:
h_mem: "{{ (ansible_memtotal_mb | int) // 2 }}"
when:
- h_mem is not defined
tags:
- always
- name: Set elasticsearch facts
set_fact:
elastic_heap_size: "{{ ((h_mem | int) > 30720) | ternary(30720, h_mem) }}"
elastic_log_rotate_path: "/var/log/elasticsearch"
tags:
- always
- name: Set kibana elasticsearch facts
block:
- name: Set kibana as elasticsearch coordinators
set_fact:
elasticsearch_node_master: false
elasticsearch_node_data: false
elasticsearch_node_ingest: false
elastic_coordination_node: true
elastic_heap_size: "{{ (elastic_heap_size | int) // 3 }}"
elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 4) | ternary(4, (ansible_processor_cores // 2)) }}"
when:
- inventory_hostname in (groups['kibana'] | default([])) and
not inventory_hostname in (groups['elastic-logstash'] | default([]))
tags:
- always
- name: Configure systcl vm.max_map_count=262144 on container hosts
sysctl:
name: "vm.max_map_count"
value: "262144"
state: "present"
reload: "yes"
delegate_to: "{{ physical_host }}"
tags:
- sysctl
- name: Ensure mount directories exists
file:
path: "/openstack/{{ inventory_hostname }}/elasticsearch"
state: "directory"
delegate_to: "{{ physical_host }}"
- name: elasticsearch datapath bind mount
lxc_container:
name: "{{ inventory_hostname }}"
container_command: |
[[ ! -d "/var/lib/elasticsearch" ]] && mkdir -p "/var/lib/elasticsearch"
container_config:
- "lxc.mount.entry=/openstack/{{ inventory_hostname }}/elasticsearch var/lib/elasticsearch none bind 0 0"
delegate_to: "{{ physical_host }}"
when:
- physical_host != inventory_hostname
- container_tech | default('lxc') == 'lxc'
- name: Ensure Java is installed
apt:
name: openjdk-8-jre
state: "{{ elk_package_state | default('present') }}"
install_recommends: yes
update_cache: yes
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
tags:
- package_install
- include_tasks: common_task_install_elk_repo.yml
- name: Ensure elasticsearch is installed
apt:
name: "{{ item }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: yes
with_items:
- logrotate
- elasticsearch
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
notify:
- Enable and restart elastic
tags:
- package_install
- name: exit playbook after uninstall
meta: end_play
when:
- (elk_package_state | default('present')) == 'absent'
post_tasks:
- name: Create elasticsearch systemd service config dir
file:
path: "/etc/systemd/system/elasticsearch.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/elasticsearch.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- { src: "systemd.elasticsearch-overrides.conf.j2", dest: "elasticsearch-overrides.conf" }
notify:
- Enable and restart elastic
- name: Drop elasticsearch conf file
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
with_items:
- src: templates/elasticsearch.yml.j2
dest: /etc/elasticsearch/elasticsearch.yml
- src: templates/jvm.options.j2
dest: /etc/elasticsearch/jvm.options
- src: templates/es-log4j2.properties.j2
dest: /etc/elasticsearch/log4j2.properties
notify:
- Enable and restart elastic
tags:
- config
- name: Ensure elasticsearch ownership
file:
path: /var/lib/elasticsearch
owner: elasticsearch
group: elasticsearch
recurse: true
register: e_perms
until: e_perms is success
retries: 3
delay: 1
tags:
- config
- name: Ensure logstash tmp dir
file:
path: "/var/lib/elasticsearch/tmp"
state: directory
owner: "elasticsearch"
group: "elasticsearch"
mode: "0750"
- name: Create logrotate config
template:
src: "templates/logrotate.j2"
dest: "/etc/logrotate.d/elasticsearch"
- name: Ensure nfs client is available if necessary
package:
name: nfs-common
state: present
when:
- elastic_shared_fs_repos is defined
- (elastic_shared_fs_repos|json_query(nfs_query)) | length > 0
- name: Ensure backup filesystems are mounted
mount:
fstype: "{{ item.fstype }}"
src: "{{ item.src }}"
opts: "{{ item.opts }}"
path: "{{ item.path }}"
state: "{{ item.state }}"
with_items:
- "{{ elastic_shared_fs_repos }}"
when:
- elastic_shared_fs_repos is defined and elastic_shared_fs_repos | length > 0
handlers:
- name: Enable and restart elastic
systemd:
name: "elasticsearch"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
roles:
- role: elasticsearch
tags:
- server-install

194
elk_metrics_6x/installFilebeat.yml
View File

@ -1,4 +1,16 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install Filebeat
hosts: hosts
become: true
@ -10,184 +22,10 @@
environment: "{{ deployment_environment_variables | default({}) }}"
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
tasks:
- include_tasks: common_task_install_elk_repo.yml
- name: Ensure Filebeat is installed
apt:
name: "{{ item }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
with_items:
- filebeat
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
notify:
- Enable and restart filebeat
tags:
- package_install
- name: exit playbook after uninstall
meta: end_play
when:
- (elk_package_state | default('present')) == 'absent'
post_tasks:
- name: Create filebeat systemd service config dir
file:
path: "/etc/systemd/system/filebeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/filebeat.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- { src: "systemd.general-overrides.conf.j2", dest: "filebeat-overrides.conf" }
notify:
- Enable and restart filebeat
- name: Check for apache
stat:
path: /etc/apache2
register: apache2
- name: Check for auditd
stat:
path: /etc/audit
register: audit
- name: Check for ceph
stat:
path: /var/log/ceph
register: ceph
- name: Check for cinder
stat:
path: /var/log/cinder
register: cinder
- name: Check for glance
stat:
path: /var/log/glance
register: glance
- name: Check for heat
stat:
path: /var/log/heat
register: heat
- name: Check for horizon
stat:
path: /var/log/horizon
register: horizon
- name: Check for httpd
stat:
path: /var/log/httpd
register: httpd
- name: Check for keystone
stat:
path: /var/log/keystone
register: keystone
- name: Check for mysql
stat:
path: /var/lib/mysql
register: mysql
- name: Check for neutron
stat:
path: /var/log/neutron
register: neutron
- name: Check for nginx
stat:
path: /var/log/nginx
register: nginx
- name: Check for nova
stat:
path: /var/log/nova
register: nova
- name: Check for octavia
stat:
path: /var/log/octavia
register: octavia
- name: Check for swift
stat:
path: /var/log/swift
register: swift
- name: Check for rabbitmq
stat:
path: /var/lib/rabbitmq
register: rabbitmq
- name: Check for designate
stat:
path: /var/log/designate
register: designate
- name: Check for osquery
stat:
path: /var/log/osquery/osqueryd.results.log
register: osquery
- name: Set discovery facts
set_fact:
apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}"
nginx_enabled: "{{ nginx.stat.exists | bool }}"
auditd_enabled: "{{ audit.stat.exists | bool }}"
mysql_enabled: "{{ (mysql.stat.exists | bool) or (inventory_hostname in groups['galera_all'] | default([])) }}"
ceph_enabled: "{{ (ceph.stat.exists | bool) or (inventory_hostname in groups['ceph_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*ceph.*') | list | length) > 0) }}"
cinder_enabled: "{{ (cinder.stat.exists | bool) or (inventory_hostname in groups['cinder_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*cinder.*') | list | length) > 0) }}"
glance_enabled: "{{ (glance.stat.exists | bool) or (inventory_hostname in groups['glance_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*glance.*') | list | length) > 0) }}"
heat_enabled: "{{ (heat.stat.exists | bool) or (inventory_hostname in groups['heat_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*heat.*') | list | length) > 0) }}"
horizon_enabled: "{{ (horizon.stat.exists | bool) or (inventory_hostname in groups['horizon_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*horizon.*') | list | length) > 0) }}"
keystone_enabled: "{{ (keystone.stat.exists | bool) or (inventory_hostname in groups['keystone_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*keystone.*') | list | length) > 0) }}"
neutron_enabled: "{{ (neutron.stat.exists | bool) or (inventory_hostname in groups['neutron_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*neutron.*') | list | length) > 0) }}"
nova_enabled: "{{ (nova.stat.exists | bool) or (inventory_hostname in groups['nova_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*nova.*') | list | length) > 0) }}"
octavia_enabled: "{{ (octavia.stat.exists | bool) or (inventory_hostname in groups['octavia_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*octavia.*') | list | length) > 0) }}"
swift_enabled: "{{ (swift.stat.exists | bool) or (inventory_hostname in groups['swift_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*swift.*') | list | length) > 0) }}"
rabbitmq_enabled: "{{ (rabbitmq.stat.exists | bool) or (inventory_hostname in groups['rabbitmq_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*rabbit.*') | list | length) > 0) }}"
designate_enabled: "{{ (designate.stat.exists | bool) or (inventory_hostname in groups['designate_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*designate.*') | list | length) > 0) }}"
osquery_enabled: "{{ osquery.stat.exists | bool }}"
- name: Drop Filebeat conf file
template:
src: templates/filebeat.yml.j2
dest: /etc/filebeat/filebeat.yml
notify:
- Enable and restart filebeat
handlers:
- name: Enable and restart filebeat
systemd:
name: "filebeat"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
roles:
- role: elastic_filebeat
- role: elastic_rollup
index_name: filebeat
tags:
- beat-install
- import_playbook: setupFilebeat.yml

80
elk_metrics_6x/installHeartbeat.yml
View File

@ -1,4 +1,15 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Set heartbeat host deployment group
hosts: kibana
@ -11,7 +22,6 @@
parents: kibana
when:
- inventory_hostname in groups['kibana'][:3]
tags:
- always
@ -26,70 +36,10 @@
environment: "{{ deployment_environment_variables | default({}) }}"
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
tasks:
- include_tasks: common_task_install_elk_repo.yml
- name: Ensure heartbeat is installed
apt:
name: "heartbeat-elastic"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
tags:
- package_install
- name: exit playbook after uninstall
meta: end_play
when:
- (elk_package_state | default('present')) == 'absent'
post_tasks:
- name: Create heartbeat systemd service config dir
file:
path: "/etc/systemd/system/heartbeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/heartbeat.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- { src: "systemd.general-overrides.conf.j2", dest: "heartbeat-overrides.conf" }
notify:
- Enable and restart heartbeat
- name: Drop heartbeat conf file
template:
src: templates/heartbeat.yml.j2
dest: /etc/heartbeat/heartbeat.yml
notify:
- Enable and restart heartbeat
handlers:
- name: Enable and restart heartbeat
systemd:
name: "heartbeat-elastic"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
roles:
- role: elastic_heartbeat
- role: elastic_rollup
index_name: heartbeat
tags:
- beat-install
- import_playbook: setupHeartbeat.yml

145
elk_metrics_6x/installJournalbeat.yml
View File

@ -1,6 +1,4 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -33,7 +31,7 @@
key: journalbeat_deployment_hosts
parents: all_journalbeat_deployments
when:
- physical_host is defined and
- physical_host is undefined or
physical_host == inventory_hostname
tags:
@ -50,143 +48,10 @@
environment: "{{ deployment_environment_variables | default({}) }}"
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
tasks:
- include_tasks: common_task_install_go1.10.1.yml
- name: Check for journal directory
stat:
path: /var/log/journal
register: journal_dir
- name: Ensure libsystemd-dev is installed
apt:
name: "{{ item }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
with_items:
- gcc
- git
- libsystemd-dev
tags:
- package_install
- name: exit playbook after uninstall
meta: end_play
when:
- (elk_package_state | default('present')) == 'absent'
post_tasks:
- name: Create journalbeat systemd service config dir
file:
path: "/etc/systemd/system/journalbeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/journalbeat.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- { src: "systemd.general-overrides.conf.j2", dest: "journalbeat-overrides.conf" }
notify:
- Enable and restart journalbeat
- name: create the system group
group:
name: "journalbeat"
state: "present"
system: "yes"
- name: Create the journalbeat user
user:
name: "journalbeat"
group: "journalbeat"
comment: "journalbeat user"
shell: "/bin/false"
createhome: "yes"
home: "/usr/share/journalbeat"
- name: Create journalbeat data path
file:
path: "{{ item }}"
state: directory
owner: "journalbeat"
group: "journalbeat"
mode: "0755"
with_items:
- "/usr/share/journalbeat"
- "/var/lib/journalbeat"
- "/etc/journalbeat"
- name: Install journalbeat
shell: /opt/go1.10.1/go/bin/go get -v github.com/mheese/journalbeat
args:
creates: /usr/local/bin/journalbeat
environment:
PATH: "{{ ansible_env.PATH }}:/opt/go1.10.1/go/bin"
GOPATH: /usr/local
notify:
- Enable and restart journalbeat
- name: Drop journalbeat conf file
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
with_items:
- src: templates/journalbeat.yml.j2
dest: /etc/journalbeat/journalbeat.yml
notify:
- Enable and restart journalbeat
- name: Run the systemd service role
include_role:
name: systemd_service
private: true
vars:
systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}"
systemd_service_restart_changed: false
systemd_services:
- service_name: "journalbeat"
execstarts:
- /usr/local/bin/journalbeat
-c /etc/journalbeat/journalbeat.yml
-path.home /usr/share/journalbeat
-path.config /etc/journalbeat
-path.data /var/lib/journalbeat
-path.logs /var/log/beats
config_overrides:
Service:
EnvironmentFile: "-/etc/default/go1.10.1"
Unit:
Documentation: https://github.com/mheese/journalbeat/blob/master/README.md
Wants: network-online.target
After: network-online.target
handlers:
- name: Enable and restart journalbeat
systemd:
name: "journalbeat"
enabled: true
state: restarted
daemon_reload: yes
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
roles:
- role: elastic_journalbeat
- role: elastic_rollup
index_name: journalbeat
tags:
- beat-install
- import_playbook: setupJournalbeat.yml

117
elk_metrics_6x/installKibana.yml
View File

@ -1,4 +1,16 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install Kibana
hosts: kibana
become: true
@ -7,109 +19,8 @@
environment: "{{ deployment_environment_variables | default({}) }}"
tasks:
- include_tasks: common_task_install_elk_repo.yml
- name: Ensure Nginx is installed
apt:
name: "{{ item }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: yes
with_items:
- nginx
- apache2-utils
- python-passlib
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
notify:
- Enable and restart nginx
tags:
- package_install
- name: create kibana user to access web interface
htpasswd:
path: "/etc/nginx/htpasswd.users"
name: "{{ kibana_username }}"
password: "{{ kibana_password }}"
owner: root
mode: 0644
- name: Drop Nginx default conf file
template:
src: templates/nginx_default.j2
dest: /etc/nginx/sites-available/default
notify:
- Enable and restart nginx
- name: Ensure kibana is installed
apt:
name: kibana
state: "{{ elk_package_state | default('present') }}"
update_cache: yes
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
notify:
- Enable and restart kibana
tags:
- package_install
- name: exit playbook after uninstall
meta: end_play
when:
- (elk_package_state | default('present')) == 'absent'
post_tasks:
- name: Create kibana systemd service config dir
file:
path: "/etc/systemd/system/kibana.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/kibana.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- { src: "systemd.general-overrides.conf.j2", dest: "kibana-overrides.conf" }
notify:
- Enable and restart kibana
- name: Drop kibana conf file
template:
src: templates/kibana.yml.j2
dest: /etc/kibana/kibana.yml
mode: "0666"
notify:
- Enable and restart kibana
handlers:
- name: Enable and restart kibana
systemd:
name: "kibana"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
- name: Enable and restart nginx
systemd:
name: "nginx"
enabled: true
state: restarted
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
roles:
- role: elastic_kibana
tags:
- server-install

278
elk_metrics_6x/installLogstash.yml
View File

@ -1,276 +1,26 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install Logstash
hosts: elastic-logstash
serial: "50%"
become: true
vars_files:
- vars/variables.yml
vars:
temp_dir: /var/lib/logstash/tmp
logstash_pipelines: "{{lookup('template', 'templates/logstash-pipelines.yml.j2') }}"
environment: "{{ deployment_environment_variables | default({}) }}"
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
tasks:
- name: Set quarter memory fact
set_fact:
q_mem: "{{ (ansible_memtotal_mb | int) // 3 }}"
when:
- q_mem is not defined
tags:
- always
- name: Set processor cores fact
set_fact:
q_storage: "{{ (ansible_processor_cores | int) * (ansible_processor_threads_per_core | int) * 2 }}"
when:
- q_storage is not defined
tags:
- always
- name: Set logstash facts
set_fact:
elastic_heap_size: "{{ ((q_mem | int) > 30720) | ternary(30720, q_mem) }}"
logstash_queue_size: "{{ ((((q_storage | int) >= 2) | ternary(q_storage, 2) | int) * 1024) // ((logstash_pipelines | from_yaml) | length) }}"
elastic_log_rotate_path: "/var/log/logstash"
tags:
- always
- include_tasks: common_task_install_elk_repo.yml
- name: Configure systcl vm.max_map_count=262144 on container hosts
sysctl:
name: "vm.max_map_count"
value: "262144"
state: "present"
reload: "yes"
delegate_to: "{{ physical_host }}"
tags:
- sysctl
- name: Ensure mount directories exists
file:
path: "/openstack/{{ inventory_hostname }}/logstash"
state: "directory"
delegate_to: "{{ physical_host }}"
- name: logstash datapath bind mount
lxc_container:
name: "{{ inventory_hostname }}"
container_command: |
[[ ! -d "/var/lib/logstash" ]] && mkdir -p "/var/lib/logstash"
container_config:
- "lxc.mount.entry=/openstack/{{ inventory_hostname }}/logstash var/lib/logstash none bind 0 0"
delegate_to: "{{ physical_host }}"
when:
- physical_host != inventory_hostname
- container_tech | default('lxc') == 'lxc'
- name: Ensure Java is installed
apt:
name: openjdk-8-jre
state: "{{ elk_package_state | default('present') }}"
install_recommends: yes
update_cache: yes
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
tags:
- package_install
- name: Ensure Logstash is installed
apt:
name: "{{ item }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: yes
with_items:
- logrotate
- logstash
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
notify:
- Enable and restart logstash
tags:
- package_install
- name: exit playbook after uninstall
meta: end_play
when:
- (elk_package_state | default('present')) == 'absent'
post_tasks:
- name: Create logstash systemd service config dir
file:
path: "/etc/systemd/system/logstash.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/logstash.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- { src: "systemd.general-overrides.conf.j2", dest: "logstash-overrides.conf" }
notify:
- Enable and restart logstash
- name: Create patterns directory
file:
name: "/opt/logstash/patterns"
owner: "logstash"
group: "logstash"
state: directory
tags:
- logstash-patterns
- name: Logstash Extra Patterns
template:
src: "{{ item }}"
dest: "/opt/logstash/patterns/{{ item }}"
owner: "logstash"
group: "logstash"
with_items:
- extras
when:
- logstash_deploy_filters
notify:
- Enable and restart logstash
tags:
- logstash-filters
- config
- name: Run kafka output block
block:
- name: Copy kafka keystore into place
copy:
src: "{{ logstash_kafka_ssl_keystore_location }}"
dest: "/var/lib/logstash/{{ logstash_kafka_ssl_keystore_location | basename }}"
when:
- logstash_kafka_ssl_keystore_location is defined
- name: Copy kafka truststore into place
copy:
src: "{{ logstash_kafka_ssl_truststore_location }}"
dest: "/var/lib/logstash/{{ logstash_kafka_ssl_truststore_location | basename }}"
when:
- logstash_kafka_ssl_truststore_location is defined
when:
- logstash_kafka_options is defined
- name: Drop logstash conf file(s)
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
with_items:
- src: templates/jvm.options.j2
dest: /etc/logstash/jvm.options
- src: templates/logstash.yml.j2
dest: /etc/logstash/logstash.yml
- src: templates/logstash-pipelines.yml.j2
dest: /etc/logstash/pipelines.yml
- src: "templates/logrotate.j2"
dest: "/etc/logrotate.d/logstash"
notify:
- Enable and restart logstash
tags:
- config
- name: Ensure logstash ownership
file:
path: /var/lib/logstash
owner: logstash
group: logstash
recurse: true
register: l_perms
until: l_perms is success
retries: 3
delay: 1
- name: Ensure logstash tmp dir
file:
path: "/var/lib/logstash/tmp"
state: directory
owner: "logstash"
group: "logstash"
mode: "0750"
- name: Run arcsight output block
block:
- name: Initialise arcsight local facts
ini_file:
dest: "/etc/ansible/facts.d/logstash.fact"
section: arcsight
option: "initialise"
value: true
- name: refresh local facts
setup:
filter: ansible_local
gather_subset: "!all"
tags:
- nova-config
- name: Setup arcsight smart connector
shell: >-
/usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }}
-M "arcsight.var.input.smartconnector.bootstrap_servers={{ item.host }}:{{ item.port }}"
-M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}"
-M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}"
with_items: "{{ logstash_arcsight_smart_connectors }}"
run_once: true
register: smart_connector
until: smart_connector is success
retries: 5
delay: 5
- name: Setup arcsight event broker
shell: >-
/usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }}
-M "arcsight.var.input.eventbroker.bootstrap_servers={{ item.host }}:{{ item.port }}"
-M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}"
-M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}"
with_items: "{{ logstash_arcsight_event_brokers }}"
run_once: true
register: event_broker
until: event_broker is success
retries: 5
delay: 5
- name: Set arcsight local facts
ini_file:
dest: "/etc/ansible/facts.d/logstash.fact"
section: arcsight
option: "{{ item.host }}"
value: "{{ item.port }}"
with_items: "{{ logstash_arcsight_smart_connectors | union(logstash_arcsight_event_brokers) }}"
when:
- logstash_arcsight_smart_connectors or
logstash_arcsight_event_brokers
handlers:
- name: Enable and restart logstash
systemd:
name: "logstash"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
roles:
- role: elastic_logstash
tags:
- server-install

236
elk_metrics_6x/installMetricbeat.yml
View File

@ -1,4 +1,16 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install Metricsbeat
hosts: all
become: true
@ -10,226 +22,10 @@
vars_files:
- vars/variables.yml
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
tasks:
- include_tasks: common_task_install_elk_repo.yml
- name: Ensure Metricsbeat is installed
apt:
name: metricbeat
state: "{{ elk_package_state | default('present') }}"
update_cache: true
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
notify:
- Enable and restart metricbeat
tags:
- package_install
- name: exit playbook after uninstall
meta: end_play
when:
- (elk_package_state | default('present')) == 'absent'
post_tasks:
- name: Create metricbeat systemd service config dir
file:
path: "/etc/systemd/system/metricbeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/metricbeat.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- { src: "systemd.general-overrides.conf.j2", dest: "metricbeat-overrides.conf" }
notify:
- Enable and restart metricbeat
- name: Check for apache
stat:
path: /etc/apache2/sites-available
register: apache2
- name: Check for ceph
stat:
path: /etc/ceph
register: ceph
# gather ceph stats from localhost
# except when a list of mons is provided
- name: Set ceph stats hosts
set_fact:
ceph_stats_hosts: |-
{% set ceph_stats = [] %}
{% if (ceph_mons is defined) and (ceph_mons | length > 0) %}
{% for mon in ceph_mons %}
{% set _ = ceph_stats.insert(loop.index, (mon + ":5000")) %}
{% endfor %}
{% else %}
{% set ceph_stats = [ ansible_hostname + ":5000" ] %}
{% endif %}
{{ ceph_stats }}
- name: Check for etcd
stat:
path: /etc/etcd
register: etcd
- name: Check for docker
stat:
path: /var/run/docker.sock
register: docker
- name: Check for haproxy
stat:
path: /etc/haproxy
register: haproxy
- name: Check for httpd
stat:
path: /etc/httpd
register: httpd
- name: Check for kvm
stat:
path: /var/run/libvirt/libvirt-sock
register: kvm
- name: Check for memcached
stat:
path: /etc/memcached.conf
register: memcached
- name: Check for mysql
stat:
path: /var/lib/mysql
register: mysql
- name: Check for nginx
stat:
path: /etc/nginx/nginx.conf
register: nginx
- name: Check for rabbitmq
stat:
path: /var/lib/rabbitmq
register: rabbitmq
- name: Check for uwsgi
stat:
path: /etc/uwsgi
register: uwsgi
- name: Check for uwsgi stats sockets
find:
paths: /tmp
file_type: any
patterns: '*uwsgi-stats.sock'
register: uwsgi_find_sockets
- name: Set discovery facts
set_fact:
apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}"
# enable ceph on: cinder volume hosts when we have a list of ceph mons
# otherwise: all hosts which have /etc/ceph
ceph_enabled: |-
{% set ceph_detect = false %}
{% if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups['cinder_volume']) %}
{% set ceph_detect = true %}
{% else %}
{% set ceph_detect = ceph.stat.exists | bool %}
{% endif %}
{{ ceph_detect }}
docker_enabled: "{{ docker.stat.exists | bool }}"
etcd_enabled: "{{ etcd.stat.exists | bool }}"
haproxy_enabled: "{{ haproxy.stat.exists | bool }}"
kvm_enabled: "{{ kvm.stat.exists | bool }}"
memcached_enabled: "{{ memcached.stat.exists | bool }}"
mysql_enabled: "{{ mysql.stat.exists | bool }}"
nginx_enabled: "{{ nginx.stat.exists | bool }}"
rabbitmq_enabled: "{{ rabbitmq.stat.exists | bool }}"
uwsgi_enabled: "{{ uwsgi.stat.exists | bool }}"
uwsgi_sockets: "{{ uwsgi_find_sockets }}"
# Apache 2 stats enablement
- name: Drop apache2 stats site config
template:
src: apache-status.conf.j2
dest: /etc/apache2/sites-available/apache-status.conf
when: apache_enabled
- name: Enable apache2 stats site
file:
src: /etc/apache2/sites-available/apache-status.conf
dest: /etc/apache2/sites-enabled/apache-status.conf
state: link
when: apache_enabled
- name: Ensure apache2 stats mode is enabled
apache2_module:
name: status
state: present
when: apache_enabled
- name: Reload apache2
service:
name: apache2
state: reloaded
when: apache_enabled
# NGINX stats enablement
- name: Drop nginx stats site config
template:
src: nginx-status.conf.j2
dest: /etc/nginx/sites-available/nginx-status.conf
when: nginx_enabled
- name: Enable nginx stats site
file:
src: /etc/nginx/sites-available/nginx-status.conf
dest: /etc/nginx/sites-enabled/nginx-status.conf
state: link
when: nginx_enabled
- name: Reload nginx
service:
name: nginx
state: reloaded
when: nginx_enabled
- name: Drop metricbeat conf file
template:
src: templates/metricbeat.yml.j2
dest: /etc/metricbeat/metricbeat.yml
notify:
- Enable and restart metricbeat
handlers:
- name: Enable and restart metricbeat
systemd:
name: "metricbeat"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
roles:
- role: elastic_metricbeat
- role: elastic_rollup
index_name: metricbeat
tags:
- beat-install
- import_playbook: setupMetricbeat.yml

85
elk_metrics_6x/installPacketbeat.yml
View File

@ -1,4 +1,16 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install Packetbeat
hosts: hosts
become: true
@ -10,75 +22,10 @@
vars_files:
- vars/variables.yml
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
tasks:
- include_tasks: common_task_install_elk_repo.yml
- name: Ensure packetbeat is installed
apt:
name: "{{ item }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
with_items:
- tcpdump
- packetbeat
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
notify:
- Enable and restart packetbeat
tags:
- package_install
- name: exit playbook after uninstall
meta: end_play
when:
- (elk_package_state | default('present')) == 'absent'
post_tasks:
- name: Create packetbeat systemd service config dir
file:
path: "/etc/systemd/system/packetbeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/packetbeat.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- { src: "systemd.general-overrides.conf.j2", dest: "packetbeat-overrides.conf" }
notify:
- Enable and restart packetbeat
- name: Drop packetbeat conf file
template:
src: templates/packetbeat.yml.j2
dest: /etc/packetbeat/packetbeat.yml
notify:
- Enable and restart packetbeat
handlers:
- name: Enable and restart packetbeat
systemd:
name: "packetbeat"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config
roles:
- role: elastic_packetbeat
- role: elastic_rollup
index_name: packetbeat
tags:
- beat-install
- import_playbook: setupPacketbeat.yml

25
elk_metrics_6x/roles/elastic_apm_server/handlers/main.yml Normal file
View File

@ -0,0 +1,25 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Enable and restart apm server
systemd:
name: "apm-server"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config

33
elk_metrics_6x/roles/elastic_apm_server/meta/main.yml Normal file
View File

@ -0,0 +1,33 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x apm-server role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_data_hosts
- role: elastic_repositories

30
elk_metrics_6x/roles/elastic_apm_server/tasks/apm_setup.yml Normal file
View File

@ -0,0 +1,30 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Load templates
shell: >-
apm-server setup
{{ item }}
-E 'apm-server.host=localhost:8200'
-E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-e -v
with_items:
- "--template"
- "--dashboards"
register: templates
until: templates is success
retries: 5
delay: 5
run_once: true

70
elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml Normal file
View File

@ -0,0 +1,70 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Ensure apm-server is installed
package:
name: "{{ apm_server_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
register: _package_task
until: _package_task is success
retries: 3
delay: 2
notify:
- Enable and restart apm server
tags:
- package_install
- name: Create apm-server systemd service config dir
file:
path: "/etc/systemd/system/apm-server.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/apm-server.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- src: "systemd.general-overrides.conf.j2"
dest: "apm-server-overrides.conf"
notify:
- Enable and restart apm server
- name: Drop apm-server conf file
template:
src: templates/apm-server.yml.j2
dest: /etc/apm-server/apm-server.yml
notify:
- Enable and restart apm server
- include_tasks: apm_setup.yml
tags:
- setup

17
elk_metrics_6x/roles/elastic_apm_server/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,17 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apm_server_distro_packages:
- apm-server

25
elk_metrics_6x/roles/elastic_auditbeat/handlers/main.yml Normal file
View File

@ -0,0 +1,25 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Enable and restart auditbeat
systemd:
name: "auditbeat"
enabled: "true"
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config

33
elk_metrics_6x/roles/elastic_auditbeat/meta/main.yml Normal file
View File

@ -0,0 +1,33 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x auditbeat role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_data_hosts
- role: elastic_repositories

32
elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml Normal file
View File

@ -0,0 +1,32 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Load templates
shell: >-
auditbeat setup
{{ item }}
-E 'output.logstash.enabled=false'
-E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-e -v
with_items:
- "--template"
- "--dashboards"
register: templates
until: templates is success
retries: 5
delay: 5
run_once: true
tags:
- setup

68
elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml Normal file
View File

@ -0,0 +1,68 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Ensure beat is installed
package:
name: "{{ auditbeat_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
register: _package_task
until: _package_task is success
retries: 3
delay: 2
notify:
- Enable and restart auditbeat
tags:
- package_install
- name: Create auditbeat systemd service config dir
file:
path: "/etc/systemd/system/auditbeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/auditbeat.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- src: "systemd.general-overrides.conf.j2"
dest: "auditbeat-overrides.conf"
notify:
- Enable and restart auditbeat
- name: Drop auditbeat conf file
template:
src: templates/auditbeat.yml.j2
dest: /etc/auditbeat/auditbeat.yml
notify:
- Enable and restart auditbeat
- include_tasks: auditbeat_setup.yml

18
elk_metrics_6x/roles/elastic_auditbeat/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,18 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
auditbeat_distro_packages:
- audispd-plugins
- auditbeat

24
elk_metrics_6x/roles/elastic_curator/handlers/main.yml Normal file
View File

@ -0,0 +1,24 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Enable and restart curator.timer
systemd:
name: "curator.timer"
enabled: true
state: restarted
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config

32
elk_metrics_6x/roles/elastic_curator/meta/main.yml Normal file
View File

@ -0,0 +1,32 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x curator role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_retention

111
elk_metrics_6x/roles/elastic_curator/tasks/main.yml Normal file
View File

@ -0,0 +1,111 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Ensure virtualenv is installed
package:
name: "{{ curator_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
tags:
- package_install
- name: Ensure curator is installed
pip:
name: "elasticsearch-curator<6"
state: "{{ elk_package_state | default('present') }}"
extra_args: --isolated
virtualenv: /opt/elasticsearch-curator
register: _pip_task
until: _pip_task is success
retries: 3
delay: 2
tags:
- package_install
- name: create the system group
group:
name: "curator"
state: "present"
system: "yes"
- name: Create the curator system user
user:
name: "curator"
group: "curator"
comment: "curator user"
shell: "/bin/false"
createhome: "yes"
home: "/var/lib/curator"
- name: Create curator data path
file:
path: "{{ item }}"
state: directory
owner: "curator"
group: "curator"
mode: "0755"
recurse: true
with_items:
- "/var/lib/curator"
- "/var/log/curator"
- "/etc/curator"
- name: Drop curator conf file
template:
src: templates/curator.yml.j2
dest: /var/lib/curator/curator.yml
notify:
- Enable and restart curator.timer
- name: Drop curator action file
template:
src: templates/curator-actions.yml.j2
dest: /var/lib/curator/actions.yml
notify:
- Enable and restart curator.timer
- name: Run the systemd service role
include_role:
name: systemd_service
private: true
vars:
systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}"
systemd_service_restart_changed: false
systemd_user_name: curator
systemd_group_name: curator
systemd_services:
- service_name: "curator"
execstarts:
- /opt/elasticsearch-curator/bin/curator
--config /var/lib/curator/curator.yml
/var/lib/curator/actions.yml
timer:
state: "started"
options:
OnBootSec: 30min
OnUnitActiveSec: 24h
Persistent: true

18
elk_metrics_6x/roles/elastic_curator/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,18 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
curator_distro_packages:
- python-virtualenv
- virtualenv

31
elk_metrics_6x/roles/elastic_data_hosts/meta/main.yml Normal file
View File

@ -0,0 +1,31 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x data hosts role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies: []

25
elk_metrics_6x/roles/elastic_data_hosts/tasks/main.yml Normal file
View File

@ -0,0 +1,25 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Refresh minimal facts
setup:
gather_subset: '!all,!any,network,virtual'
tags:
- always
- name: Load data node variables
include_vars: "data-node-variables.yml"
tags:
- always

117
elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml Normal file
View File

@ -0,0 +1,117 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# storage node count is equal to the cluster size
storage_node_count: "{{ groups['elastic-logstash'] | length }}"
# the elasticserch cluster elects one master from all those which are marked as master-eligible
# 1 node cluster can only have one master
# 2 node clusters have 1 master-eligable nodes to avoid split-brain
# 3 node clusters have 3 master-eligable nodes
# >3 node clusters have (nodes // 2) eligable masters rounded up to the next odd number
master_node_count: |-
{% set masters = 0 %}
{% if (storage_node_count | int) < 3 %}
{% set masters = 1 %}
{% elif (storage_node_count | int) == 3 %}
{% set masters = 3 %}
{% else %}
{% set masters = (storage_node_count | int ) // 2 %}
{% if ((masters | int) % 2 == 0) %}
{% set masters = (masters | int) + 1 %}
{% endif %}
{% endif %}
{{ masters }}
# Assign node roles
# the first 'master_node_count' hosts in groups['elastic-logstash'] become master-eligible nodes
# the first 'master_node_count' and subsequent alternate hosts in groups['elastic-logstash'] becomes data nodes
data_nodes: "{{ (groups['elastic-logstash'][:master_node_count | int] + groups['elastic-logstash'][master_node_count | int::2]) }}"
master_nodes: "{{ groups['elastic-logstash'][:master_node_count | int] }}"
coordination_nodes: >-
{{
(groups['kibana'] | map('extract', hostvars, 'ansible_host') | list)
| map('regex_replace', '(.*)' ,'\1:' ~ elastic_port)
| list
}}
zen_nodes: >-
{{
(groups['elastic-logstash'] | union(groups['kibana'])) | map('extract', hostvars, 'ansible_host') | list
}}
elasticserch_interface_speed: |-
{% set default_interface_fact = hostvars[inventory_hostname]['ansible_' + (elastic_data_interface | replace('-', '_'))] %}
{% set speeds = [] %}
{% if default_interface_fact['type'] == 'bridge' %}
{% for interface in default_interface_fact['interfaces'] %}
{% set interface_fact = hostvars[inventory_hostname]['ansible_' + (interface | replace('-', '_'))] %}
{% if 'speed' in interface_fact %}
{% set speed = (interface_fact['speed'] | default(1000)) | string %}
{% if speed == "-1" %}
{% set _ = speeds.append(1000) %}
{% else %}
{% set _ = speeds.append(speed | int) %}
{% endif %}
{% if 'module' in interface_fact %}
{% set _ = speeds.append((interface_fact['speed'] | default(1000)) | int) %}
{% else %}
{% set _ = speeds.append(1000) %}
{% endif %}
{% endif %}
{% endfor %}
{% else %}
{% if ('module' in default_interface_fact) or (default_interface_fact['type'] == 'bond') %}
{% set speed = (default_interface_fact['speed'] | default(1000)) | string %}
{% if speed == "-1" %}
{% set _ = speeds.append(1000) %}
{% else %}
{% set _ = speeds.append(speed | int) %}
{% endif %}
{% else %}
{% set _ = speeds.append(1000) %}
{% endif %}
{% endif %}
{% set interface_speed = ((speeds | min) * 0.20) | int %}
{{ ((interface_speed | int) > 750) | ternary(750, interface_speed) }}
elasticsearch_data_node_details: >-
{{
(data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port) | list
}}
logstash_data_node_details: >-
{{
(data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ logstash_beat_input_port) | list
}}
# based on the assignment of roles to hosts, set per host booleans
master_node: "{{ (inventory_hostname in master_nodes) | ternary(true, false) }}"
data_node: "{{ (inventory_hostname in data_nodes) | ternary(true, false) }}"
elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) >= 24) | ternary(24, ansible_processor_cores) }}"
# Set a data node facts. The data nodes, in the case of elasticsearch are also
# ingest nodes.
elasticsearch_number_of_replicas: "{{ ((data_nodes | length) > 2) | ternary('2', ((data_nodes | length) > 1) | ternary('1', '0')) }}"
elasticsearch_data_hosts: |-
{% set nodes = elasticsearch_data_node_details %}
{% if inventory_hostname in data_nodes %}
{% set _ = nodes.insert(0, '127.0.0.1:' ~ elastic_port) %}
{% endif %}
{% set data_hosts = nodes | shuffle(seed=inventory_hostname) %}
{{ data_hosts }}
logstash_data_hosts: |-
{% set nodes = logstash_data_node_details %}
{% if inventory_hostname in data_nodes %}
{% set _ = nodes.insert(0, '127.0.0.1:' ~ logstash_beat_input_port) %}
{% endif %}
{% set data_hosts = nodes | shuffle(seed=inventory_hostname) %}
{{ data_hosts }}

22
elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml Normal file
View File

@ -0,0 +1,22 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## Adds option to set the UID/GID of a given service user.
# service_group_gid: 5000
# service_owner_uid: 5000
# Option to define third memory
q_mem: "{{ (ansible_memtotal_mb | int) // 3 }}"
# Option to define half memory
h_mem: "{{ (ansible_memtotal_mb | int) // 2 }}"

32
elk_metrics_6x/roles/elastic_dependencies/meta/main.yml Normal file
View File

@ -0,0 +1,32 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
allow_duplicates: true
galaxy_info:
author: OpenStack
description: Elastic v6.x dependencies role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies: []

162
elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml Normal file
View File

@ -0,0 +1,162 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Check for service_name var
fail:
msg: >-
The required variable [ service_name ] is undefined.
when:
- service_name is undefined
- name: Check for service_owner var
fail:
msg: >-
The required variable [ service_owner ] is undefined.
when:
- service_owner is undefined
- name: Check for service_group var
fail:
msg: >-
The required variable [ service_group ] is undefined.
when:
- service_group is undefined
- name: Load service variables
include_vars: "vars_{{ service_name }}.yml"
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Set elastic heap defaults
set_fact:
elastic_heap_size_default: "{{ _elastic_heap_size_default }}"
- name: Configure systcl vm.max_map_count=262144 on container hosts
sysctl:
name: "vm.max_map_count"
value: "262144"
state: "present"
reload: "yes"
delegate_to: "{{ physical_host }}"
tags:
- sysctl
- name: Physical host block
block:
- name: Check for directory
stat:
path: "/var/lib/{{ service_name }}"
register: service_dir
- name: Check for existing rollup job
debug:
msg: >-
The service data directory [ /var/lib/{{ service_name }} ] already
exists. To ensure no data is lost, the linked directory path to
[ /openstack/{{ inventory_hostname }}/{{ service_name }} ] will not be
created for this host.
when:
- service_dir.stat.isdir is defined and
service_dir.stat.isdir
- name: Ensure service directories data-path exists
file:
path: "/openstack/{{ inventory_hostname }}/{{ service_name }}"
state: "directory"
owner: "{{ service_owner }}"
group: "{{ service_group }}"
when:
- service_dir.stat.isdir is defined and
not service_dir.stat.isdir
- name: Ensure data link exists
file:
src: "/openstack/{{ inventory_hostname }}/{{ service_name }}"
dest: "/var/lib/{{ service_name }}"
owner: "{{ service_owner }}"
group: "{{ service_group }}"
state: link
when:
- service_dir.stat.isdir is defined and
not service_dir.stat.isdir
when:
- physical_host == inventory_hostname
- name: elasticsearch datapath bind mount
lxc_container:
name: "{{ inventory_hostname }}"
container_command: |
[[ ! -d "/var/lib/{{ service_name }}" ]] && mkdir -p "/var/lib/{{ service_name }}"
container_config:
- "lxc.mount.entry=/openstack/{{ inventory_hostname }}/{{ service_name }} var/lib/{{ service_name }} none bind 0 0"
delegate_to: "{{ physical_host }}"
when:
- physical_host != inventory_hostname
- container_tech | default('lxc') == 'lxc'
- name: Ensure Java is installed
package:
name: "{{ elastic_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
install_recommends: yes
update_cache: yes
register: _package_task
until: _package_task is success
retries: 3
delay: 2
tags:
- package_install
- name: Create the system group
group:
name: "{{ service_group }}"
gid: "{{ service_group_gid | default(omit) }}"
state: "present"
system: "yes"
- name: Create the system user
user:
name: "{{ service_owner }}"
uid: "{{ service_owner_uid | default(omit) }}"
group: "{{ service_group }}"
shell: "/bin/false"
system: "yes"
createhome: "yes"
home: "/var/lib/{{ service_name }}"
- name: Ensure service directories exists
file:
path: "/etc/{{ service_name }}"
state: "directory"
owner: "{{ service_owner }}"
group: "{{ service_group }}"
- name: Drop jvm conf file(s)
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
with_items:
- src: templates/jvm.options.j2
dest: /etc/{{ service_name }}/jvm.options

17
elk_metrics_6x/roles/elastic_dependencies/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,17 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
elastic_distro_packages:
- openjdk-8-jre

15
elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml Normal file
View File

@ -0,0 +1,15 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Set memory fact to half
_elastic_heap_size_default: "{{ ((h_mem | int) > 30720) | ternary(30720, h_mem) }}"

15
elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml Normal file
View File

@ -0,0 +1,15 @@
---
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Set quarter memory fact
_elastic_heap_size_default: "{{ ((q_mem | int) > 30720) | ternary(30720, q_mem) }}"

25
elk_metrics_6x/roles/elastic_filebeat/handlers/main.yml Normal file
View File

@ -0,0 +1,25 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Enable and restart filebeat
systemd:
name: "filebeat"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config

33
elk_metrics_6x/roles/elastic_filebeat/meta/main.yml Normal file
View File

@ -0,0 +1,33 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x filebeat role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_data_hosts
- role: elastic_repositories

32
elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml Normal file
View File

@ -0,0 +1,32 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Load templates
shell: >-
filebeat setup
{{ item }}
-E 'output.logstash.enabled=false'
-E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-e -v
with_items:
- "--template"
- "--dashboards"
register: templates
until: templates is success
retries: 5
delay: 5
run_once: true
tags:
- setup

177
elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml Normal file
View File

@ -0,0 +1,177 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Ensure beat is installed
package:
name: "{{ filebeat_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
register: _package_task
until: _package_task is success
retries: 3
delay: 2
notify:
- Enable and restart filebeat
tags:
- package_install
- name: Create filebeat systemd service config dir
file:
path: "/etc/systemd/system/filebeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/filebeat.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- { src: "systemd.general-overrides.conf.j2", dest: "filebeat-overrides.conf" }
notify:
- Enable and restart filebeat
- name: Check for apache
stat:
path: /etc/apache2
register: apache2
- name: Check for auditd
stat:
path: /etc/audit
register: audit
- name: Check for ceph
stat:
path: /var/log/ceph
register: ceph
- name: Check for cinder
stat:
path: /var/log/cinder
register: cinder
- name: Check for glance
stat:
path: /var/log/glance
register: glance
- name: Check for heat
stat:
path: /var/log/heat
register: heat
- name: Check for horizon
stat:
path: /var/log/horizon
register: horizon
- name: Check for httpd
stat:
path: /var/log/httpd
register: httpd
- name: Check for keystone
stat:
path: /var/log/keystone
register: keystone
- name: Check for mysql
stat:
path: /var/lib/mysql
register: mysql
- name: Check for neutron
stat:
path: /var/log/neutron
register: neutron
- name: Check for nginx
stat:
path: /var/log/nginx
register: nginx
- name: Check for nova
stat:
path: /var/log/nova
register: nova
- name: Check for octavia
stat:
path: /var/log/octavia
register: octavia
- name: Check for swift
stat:
path: /var/log/swift
register: swift
- name: Check for rabbitmq
stat:
path: /var/lib/rabbitmq
register: rabbitmq
- name: Check for designate
stat:
path: /var/log/designate
register: designate
- name: Check for osquery
stat:
path: /var/log/osquery/osqueryd.results.log
register: osquery
- name: Set discovery facts
set_fact:
apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}"
nginx_enabled: "{{ nginx.stat.exists | bool }}"
auditd_enabled: "{{ audit.stat.exists | bool }}"
mysql_enabled: "{{ (mysql.stat.exists | bool) or (inventory_hostname in groups['galera_all'] | default([])) }}"
ceph_enabled: "{{ (ceph.stat.exists | bool) or (inventory_hostname in groups['ceph_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*ceph.*') | list | length) > 0) }}"
cinder_enabled: "{{ (cinder.stat.exists | bool) or (inventory_hostname in groups['cinder_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*cinder.*') | list | length) > 0) }}"
glance_enabled: "{{ (glance.stat.exists | bool) or (inventory_hostname in groups['glance_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*glance.*') | list | length) > 0) }}"
heat_enabled: "{{ (heat.stat.exists | bool) or (inventory_hostname in groups['heat_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*heat.*') | list | length) > 0) }}"
horizon_enabled: "{{ (horizon.stat.exists | bool) or (inventory_hostname in groups['horizon_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*horizon.*') | list | length) > 0) }}"
keystone_enabled: "{{ (keystone.stat.exists | bool) or (inventory_hostname in groups['keystone_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*keystone.*') | list | length) > 0) }}"
neutron_enabled: "{{ (neutron.stat.exists | bool) or (inventory_hostname in groups['neutron_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*neutron.*') | list | length) > 0) }}"
nova_enabled: "{{ (nova.stat.exists | bool) or (inventory_hostname in groups['nova_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*nova.*') | list | length) > 0) }}"
octavia_enabled: "{{ (octavia.stat.exists | bool) or (inventory_hostname in groups['octavia_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*octavia.*') | list | length) > 0) }}"
swift_enabled: "{{ (swift.stat.exists | bool) or (inventory_hostname in groups['swift_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*swift.*') | list | length) > 0) }}"
rabbitmq_enabled: "{{ (rabbitmq.stat.exists | bool) or (inventory_hostname in groups['rabbitmq_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*rabbit.*') | list | length) > 0) }}"
designate_enabled: "{{ (designate.stat.exists | bool) or (inventory_hostname in groups['designate_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*designate.*') | list | length) > 0) }}"
osquery_enabled: "{{ osquery.stat.exists | bool }}"
- name: Drop Filebeat conf file
template:
src: templates/filebeat.yml.j2
dest: /etc/filebeat/filebeat.yml
notify:
- Enable and restart filebeat
- include_tasks: filebeat_setup.yml

17
elk_metrics_6x/roles/elastic_filebeat/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,17 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
filebeat_distro_packages:
- filebeat

25
elk_metrics_6x/roles/elastic_heartbeat/handlers/main.yml Normal file
View File

@ -0,0 +1,25 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Enable and restart heartbeat
systemd:
name: "heartbeat-elastic"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config

33
elk_metrics_6x/roles/elastic_heartbeat/meta/main.yml Normal file
View File

@ -0,0 +1,33 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x heartbeat role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_data_hosts
- role: elastic_repositories

32
elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml Normal file
View File

@ -0,0 +1,32 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Load templates
shell: >-
heartbeat setup
{{ item }}
-E 'output.logstash.enabled=false'
-E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-e -v
with_items:
- "--template"
- "--dashboards"
register: templates
until: templates is success
retries: 5
delay: 5
run_once: true
tags:
- setup

63
elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml Normal file
View File

@ -0,0 +1,63 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Ensure beat is installed
package:
name: "{{ heartbeat_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
register: _package_task
until: _package_task is success
retries: 3
delay: 2
notify:
- Enable and restart heartbeat
tags:
- package_install
- name: Create heartbeat systemd service config dir
file:
path: "/etc/systemd/system/heartbeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Create heartbeat configs
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
mode: "0644"
with_items:
- src: "systemd.general-overrides.conf.j2"
dest: "/etc/systemd/system/heartbeat.service.d/heartbeat-overrides.conf"
- src: templates/heartbeat.yml.j2
dest: /etc/heartbeat/heartbeat.yml
notify:
- Enable and restart heartbeat
- include_tasks: heartbeat_setup.yml

17
elk_metrics_6x/roles/elastic_heartbeat/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,17 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
heartbeat_distro_packages:
- heartbeat-elastic

25
elk_metrics_6x/roles/elastic_journalbeat/handlers/main.yml Normal file
View File

@ -0,0 +1,25 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Enable and restart journalbeat
systemd:
name: "journalbeat"
enabled: true
state: restarted
daemon_reload: yes
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config

33
elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml Normal file
View File

@ -0,0 +1,33 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x journalbeat role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_data_hosts
- role: go_install_1.10

40
elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml Normal file
View File

@ -0,0 +1,40 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Notice
debug:
msg: >-
The setup task for templates and dashboards is broken due to missing
configuration. Once the following issue
[ https://github.com/mheese/journalbeat/issues/136 ] is
resolved, this should be enabled.
# - name: Load templates
# shell: >-
# /usr/local/bin/journalbeat -setup
# {{ item }}
# -E 'output.logstash.enabled=false'
# -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
# -e -v
# with_items:
# - "--template"
# - "--dashboards"
# register: templates
# until: templates is success
# retries: 5
# delay: 5
# run_once: true
# tags:
# - setup

134
elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml Normal file
View File

@ -0,0 +1,134 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Check for journal directory
stat:
path: /var/log/journal
register: journal_dir
- name: exit playbook after uninstall
meta: end_play
when:
- not journal_dir.stat.exists | bool
- name: Ensure beat is installed
package:
name: "{{ journalbeat_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
register: _package_task
until: _package_task is success
retries: 3
delay: 2
notify:
- Enable and restart journalbeat
tags:
- package_install
- name: create the system group
group:
name: "journalbeat"
state: "present"
system: "yes"
- name: Create the journalbeat user
user:
name: "journalbeat"
group: "journalbeat"
comment: "journalbeat user"
shell: "/bin/false"
createhome: "yes"
home: "/usr/share/journalbeat"
- name: Create journalbeat data path
file:
path: "{{ item }}"
state: directory
owner: "journalbeat"
group: "journalbeat"
mode: "0755"
with_items:
- "/usr/share/journalbeat"
- "/var/lib/journalbeat"
- "/etc/journalbeat"
- name: Install journalbeat
shell: /opt/go1.10.1/go/bin/go get -v github.com/mheese/journalbeat
args:
creates: /usr/local/bin/journalbeat
environment:
PATH: "{{ ansible_env.PATH }}:/opt/go1.10.1/go/bin"
GOPATH: /usr/local
notify:
- Enable and restart journalbeat
- name: Create journalbeat systemd service config dir
file:
path: "/etc/systemd/system/journalbeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Drop journalbeat conf files
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
mode: "0644"
with_items:
- src: "systemd.general-overrides.conf.j2"
dest: "/etc/systemd/system/journalbeat.service.d/journalbeat-overrides.conf"
- src: templates/journalbeat.yml.j2
dest: /etc/journalbeat/journalbeat.yml
notify:
- Enable and restart journalbeat
- name: Run the systemd service role
include_role:
name: systemd_service
private: true
vars:
systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}"
systemd_service_restart_changed: false
systemd_services:
- service_name: "journalbeat"
execstarts:
- /usr/local/bin/journalbeat
-c /etc/journalbeat/journalbeat.yml
-path.home /usr/share/journalbeat
-path.config /etc/journalbeat
-path.data /var/lib/journalbeat
-path.logs /var/log/beats
config_overrides:
Service:
EnvironmentFile: "-/etc/default/go1.10.1"
Unit:
Documentation: https://github.com/mheese/journalbeat/blob/master/README.md
Wants: network-online.target
After: network-online.target
- include_tasks: journalbeat_setup.yml

19
elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,19 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
journalbeat_distro_packages:
- gcc
- git
- libsystemd-dev

16
elk_metrics_6x/roles/elastic_kibana/defaults/main.yml Normal file
View File

@ -0,0 +1,16 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
kibana_enable_basic_auth: false

37
elk_metrics_6x/roles/elastic_kibana/handlers/main.yml Normal file
View File

@ -0,0 +1,37 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Enable and restart nginx
systemd:
name: "nginx"
enabled: true
state: restarted
when:
- (elk_package_state | default('present')) != 'absent'
listen: Enable and restart services
tags:
- config
- name: Enable and restart kibana
systemd:
name: "kibana"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
listen: Enable and restart services
tags:
- config

32
elk_metrics_6x/roles/elastic_kibana/meta/main.yml Normal file
View File

@ -0,0 +1,32 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x kibana role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_repositories

84
elk_metrics_6x/roles/elastic_kibana/tasks/main.yml Normal file
View File

@ -0,0 +1,84 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Ensure distro packages are installed
package:
name: "{{ kibana_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: yes
register: _package_task
until: _package_task is success
retries: 3
delay: 2
notify:
- Enable and restart services
tags:
- package_install
- name: create kibana user to access web interface
htpasswd:
path: "/etc/nginx/htpasswd.users"
name: "{{ kibana_username }}"
password: "{{ kibana_password }}"
owner: root
mode: 0644
when:
- kibana_enable_basic_auth
- name: Drop Nginx default conf file
template:
src: templates/nginx_default.j2
dest: /etc/nginx/sites-available/default
notify:
- Enable and restart services
- name: Create kibana systemd service config dir
file:
path: "/etc/systemd/system/kibana.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/kibana.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- src: "systemd.general-overrides.conf.j2"
dest: "kibana-overrides.conf"
notify:
- Enable and restart services
- name: Drop kibana conf file
template:
src: templates/kibana.yml.j2
dest: /etc/kibana/kibana.yml
mode: "0666"
notify:
- Enable and restart services

20
elk_metrics_6x/roles/elastic_kibana/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,20 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
kibana_distro_packages:
- apache2-utils
- kibana
- nginx
- python-passlib

24
elk_metrics_6x/roles/elastic_logstash/defaults/main.yml Normal file
View File

@ -0,0 +1,24 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
temp_dir: /var/lib/logstash/tmp
logstash_pipelines: "{{lookup('template', 'templates/logstash-pipelines.yml.j2') }}"
# Set processor cores fact
q_storage: "{{ (ansible_processor_cores | int) * (ansible_processor_threads_per_core | int) * 2 }}"
# Set logstash facts
logstash_queue_size: "{{ ((((q_storage | int) >= 2) | ternary(q_storage, 2) | int) * 1024) // ((logstash_pipelines | from_yaml) | length) }}"
elastic_log_rotate_path: "/var/log/logstash"

25
elk_metrics_6x/roles/elastic_logstash/handlers/main.yml Normal file
View File

@ -0,0 +1,25 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Enable and restart logstash
systemd:
name: "logstash"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config

37
elk_metrics_6x/roles/elastic_logstash/meta/main.yml Normal file
View File

@ -0,0 +1,37 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x logstash role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_data_hosts
- role: elastic_repositories
- role: elastic_dependencies
service_name: logstash
service_owner: logstash
service_group: logstash

62
elk_metrics_6x/roles/elastic_logstash/tasks/logstash_arcsight.yml Normal file
View File

@ -0,0 +1,62 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Initialise arcsight local facts
ini_file:
dest: "/etc/ansible/facts.d/logstash.fact"
section: arcsight
option: "initialise"
value: true
- name: refresh local facts
setup:
filter: ansible_local
gather_subset: "!all"
tags:
- nova-config
- name: Setup arcsight smart connector
shell: >-
/usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }}
-M "arcsight.var.input.smartconnector.bootstrap_servers={{ item.host }}:{{ item.port }}"
-M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}"
-M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}"
with_items: "{{ logstash_arcsight_smart_connectors }}"
run_once: true
register: smart_connector
until: smart_connector is success
retries: 5
delay: 5
- name: Setup arcsight event broker
shell: >-
/usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }}
-M "arcsight.var.input.eventbroker.bootstrap_servers={{ item.host }}:{{ item.port }}"
-M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}"
-M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}"
with_items: "{{ logstash_arcsight_event_brokers }}"
run_once: true
register: event_broker
until: event_broker is success
retries: 5
delay: 5
- name: Set arcsight local facts
ini_file:
dest: "/etc/ansible/facts.d/logstash.fact"
section: arcsight
option: "{{ item.host }}"
value: "{{ item.port }}"
with_items: "{{ logstash_arcsight_smart_connectors | union(logstash_arcsight_event_brokers) }}"

24
elk_metrics_6x/roles/elastic_logstash/tasks/logstash_kafka_ssl.yml Normal file
View File

@ -0,0 +1,24 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Copy kafka keystore into place
copy:
src: "{{ logstash_kafka_ssl_keystore_location }}"
dest: "/var/lib/logstash/{{ logstash_kafka_ssl_keystore_location | basename }}"
- name: Copy kafka truststore into place
copy:
src: "{{ logstash_kafka_ssl_truststore_location }}"
dest: "/var/lib/logstash/{{ logstash_kafka_ssl_truststore_location | basename }}"

129
elk_metrics_6x/roles/elastic_logstash/tasks/main.yml Normal file
View File

@ -0,0 +1,129 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Ensure Logstash is installed
package:
name: "{{ logstash_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: yes
register: _package_task
until: _package_task is success
retries: 3
delay: 2
notify:
- Enable and restart logstash
tags:
- package_install
- name: Create logstash systemd service config dir
file:
path: "/etc/systemd/system/logstash.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "/etc/systemd/system/logstash.service.d/{{ item.dest }}"
mode: "0644"
with_items:
- src: "systemd.general-overrides.conf.j2"
dest: "logstash-overrides.conf"
notify:
- Enable and restart logstash
- name: Create patterns directory
file:
name: "/opt/logstash/patterns"
owner: "logstash"
group: "logstash"
state: directory
tags:
- logstash-patterns
- name: Logstash Extra Patterns
template:
src: "extras"
dest: "/opt/logstash/patterns/extras"
owner: "logstash"
group: "logstash"
when:
- logstash_deploy_filters
notify:
- Enable and restart logstash
tags:
- logstash-filters
- config
- name: Run kafka ssl deployment
include_tasks: logstash_kafka_ssl.yml
when:
- logstash_kafka_options is defined
- logstash_kafka_ssl_keystore_location is defined
- name: Drop logstash conf file(s)
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
with_items:
- src: templates/logstash.yml.j2
dest: /etc/logstash/logstash.yml
- src: templates/logstash-pipelines.yml.j2
dest: /etc/logstash/pipelines.yml
- src: "templates/logrotate.j2"
dest: "/etc/logrotate.d/logstash"
notify:
- Enable and restart logstash
tags:
- config
- name: Ensure logstash ownership
file:
path: "/var/lib/logstash/"
owner: logstash
group: logstash
recurse: true
register: l_perms
until: l_perms is success
retries: 3
delay: 1
- name: Ensure logstash tmp dir
file:
path: "/var/lib/logstash/tmp"
state: directory
owner: "logstash"
group: "logstash"
mode: "0750"
- name: Deploy arcsight collector
include_tasks: logstash_arcsight.yml
when:
- logstash_arcsight_smart_connectors or
logstash_arcsight_event_brokers

18
elk_metrics_6x/roles/elastic_logstash/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,18 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
logstash_distro_packages:
- logrotate
- logstash

25
elk_metrics_6x/roles/elastic_metricbeat/handlers/main.yml Normal file
View File

@ -0,0 +1,25 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Enable and restart metricbeat
systemd:
name: "metricbeat"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config

33
elk_metrics_6x/roles/elastic_metricbeat/meta/main.yml Normal file
View File

@ -0,0 +1,33 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x metricbeat role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_data_hosts
- role: elastic_repositories

236
elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml Normal file
View File

@ -0,0 +1,236 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Ensure beat is installed
package:
name: "{{ metricbeat_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
register: _package_task
until: _package_task is success
retries: 3
delay: 2
notify:
- Enable and restart metricbeat
tags:
- package_install
- name: Check for apache
stat:
path: /etc/apache2/sites-available
register: apache2
- name: Check for ceph
stat:
path: /etc/ceph
register: ceph
# gather ceph stats from localhost
# except when a list of mons is provided
- name: Set ceph stats hosts
set_fact:
ceph_stats_hosts: |-
{% set ceph_stats = [] %}
{% if (ceph_mons is defined) and (ceph_mons | length > 0) %}
{% for mon in ceph_mons %}
{% set _ = ceph_stats.insert(loop.index, (mon + ":5000")) %}
{% endfor %}
{% else %}
{% set ceph_stats = [ ansible_hostname + ":5000" ] %}
{% endif %}
{{ ceph_stats }}
- name: Check for etcd
stat:
path: /etc/etcd
register: etcd
- name: Check for docker
stat:
path: /var/run/docker.sock
register: docker
- name: Check for haproxy
stat:
path: /etc/haproxy
register: haproxy
- name: Check for httpd
stat:
path: /etc/httpd
register: httpd
- name: Check for kvm
stat:
path: /var/run/libvirt/libvirt-sock
register: kvm
- name: Check for memcached
stat:
path: /etc/memcached.conf
register: memcached
- name: Check for mysql
stat:
path: /var/lib/mysql
register: mysql
- name: Check for nginx
stat:
path: /etc/nginx/nginx.conf
register: nginx
- name: Check for rabbitmq
stat:
path: /var/lib/rabbitmq
register: rabbitmq
- name: Check for uwsgi
stat:
path: /etc/uwsgi
register: uwsgi
- name: Check for uwsgi stats sockets
find:
paths: /tmp
file_type: any
patterns: '*uwsgi-stats.sock'
register: uwsgi_find_sockets
- name: Set discovery facts
set_fact:
apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}"
# enable ceph on: cinder volume hosts when we have a list of ceph mons
# otherwise: all hosts which have /etc/ceph
ceph_enabled: |-
{% set ceph_detect = false %}
{% if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups['cinder_volume']) %}
{% set ceph_detect = true %}
{% else %}
{% set ceph_detect = ceph.stat.exists | bool %}
{% endif %}
{{ ceph_detect }}
docker_enabled: "{{ docker.stat.exists | bool }}"
etcd_enabled: "{{ etcd.stat.exists | bool }}"
haproxy_enabled: "{{ haproxy.stat.exists | bool }}"
kvm_enabled: "{{ kvm.stat.exists | bool }}"
memcached_enabled: "{{ memcached.stat.exists | bool }}"
mysql_enabled: "{{ mysql.stat.exists | bool }}"
nginx_enabled: "{{ nginx.stat.exists | bool }}"
rabbitmq_enabled: "{{ rabbitmq.stat.exists | bool }}"
uwsgi_enabled: "{{ uwsgi.stat.exists | bool }}"
uwsgi_sockets: "{{ uwsgi_find_sockets }}"
# Apache 2 stats enablement
- name: Drop apache2 stats site config
template:
src: apache-status.conf.j2
dest: /etc/apache2/sites-available/apache-status.conf
when: apache_enabled
- name: Enable apache2 stats site
file:
src: /etc/apache2/sites-available/apache-status.conf
dest: /etc/apache2/sites-enabled/apache-status.conf
state: link
when: apache_enabled
- name: Ensure apache2 stats mode is enabled
apache2_module:
name: status
state: present
register: apache_status_mod
when: apache_enabled
- name: Reload apache2
service:
name: apache2
state: reloaded
when:
- apache_enabled
- apache_status_mod is changed
# NGINX stats enablement
- name: Drop nginx stats site config
template:
src: nginx-status.conf.j2
dest: /etc/nginx/sites-available/nginx-status.conf
when: nginx_enabled
- name: Enable nginx stats site
file:
src: /etc/nginx/sites-available/nginx-status.conf
dest: /etc/nginx/sites-enabled/nginx-status.conf
state: link
register: nginx_status
when: nginx_enabled
- name: Reload nginx
service:
name: nginx
state: reloaded
when:
- nginx_enabled
- nginx_status is changed
- name: Create metricbeat systemd service config dir
file:
path: "/etc/systemd/system/metricbeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
when:
- ansible_service_mgr == 'systemd'
- name: Apply systemd options
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
mode: "0644"
when:
- ansible_service_mgr == 'systemd'
with_items:
- src: "systemd.general-overrides.conf.j2"
dest: "/etc/systemd/system/metricbeat.service.d/metricbeat-overrides.conf"
notify:
- Enable and restart metricbeat
- name: Drop metricbeat conf file
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
mode: "0644"
with_items:
- src: templates/metricbeat.yml.j2
dest: /etc/metricbeat/metricbeat.yml
notify:
- Enable and restart metricbeat
- include_tasks: metricbeat_setup.yml

32
elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml Normal file
View File

@ -0,0 +1,32 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Load templates
shell: >-
metricbeat setup
{{ item }}
-E 'output.logstash.enabled=false'
-E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-e -v
with_items:
- "--template"
- "--dashboards"
register: templates
until: templates is success
retries: 5
delay: 5
run_once: true
tags:
- setup

17
elk_metrics_6x/roles/elastic_metricbeat/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,17 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
metricbeat_distro_packages:
- metricbeat

25
elk_metrics_6x/roles/elastic_packetbeat/handlers/main.yml Normal file
View File

@ -0,0 +1,25 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Enable and restart packetbeat
systemd:
name: "packetbeat"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config

33
elk_metrics_6x/roles/elastic_packetbeat/meta/main.yml Normal file
View File

@ -0,0 +1,33 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x packetbeat role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_data_hosts
- role: elastic_repositories

63
elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml Normal file
View File

@ -0,0 +1,63 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Ensure beat is installed
package:
name: "{{ packetbeat_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: true
register: _package_task
until: _package_task is success
retries: 3
delay: 2
notify:
- Enable and restart packetbeat
tags:
- package_install
- name: Create packetbeat systemd service config dir
file:
path: "/etc/systemd/system/packetbeat.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Drop packetbeat conf files
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
mode: "0644"
with_items:
- src: "systemd.general-overrides.conf.j2"
dest: "/etc/systemd/system/packetbeat.service.d/packetbeat-overrides.conf"
- src: templates/packetbeat.yml.j2
dest: /etc/packetbeat/packetbeat.yml
notify:
- Enable and restart packetbeat
- include_tasks: packetbeat_setup.yml

32
elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml Normal file
View File

@ -0,0 +1,32 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Load templates
shell: >-
packetbeat setup
{{ item }}
-E 'output.logstash.enabled=false'
-E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-e -v
with_items:
- "--template"
- "--dashboards"
register: templates
until: templates is success
retries: 5
delay: 5
run_once: true
tags:
- setup

18
elk_metrics_6x/roles/elastic_packetbeat/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,18 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
packetbeat_distro_packages:
- tcpdump
- packetbeat

31
elk_metrics_6x/roles/elastic_repositories/meta/main.yml Normal file
View File

@ -0,0 +1,31 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x repositories role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies: []

12
elk_metrics_6x/common_task_install_elk_repo.yml → elk_metrics_6x/roles/elastic_repositories/tasks/elastic_apt_repos.yml
View File

@ -24,18 +24,6 @@
tags:
- package_install
- name: enable apt-transport-https
apt:
name: apt-transport-https
state: present
update_cache: yes
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
tags:
- package_install
- name: add elk repo to apt sources list
apt_repository:
repo: "{{ elk_repo.repo }}"

40
elk_metrics_6x/roles/elastic_repositories/tasks/main.yml Normal file
View File

@ -0,0 +1,40 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Install repo disto packages
package:
name: "{{ elastic_repo_distro_packages }}"
state: present
update_cache: yes
register: _apt_task
until: _apt_task is success
retries: 3
delay: 2
tags:
- package_install
- include_tasks: "elastic_{{ ansible_pkg_mgr }}_repos.yml"

17
elk_metrics_6x/roles/elastic_repositories/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,17 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
elastic_repo_distro_packages:
- apt-transport-https

16
elk_metrics_6x/roles/elastic_retention/defaults/main.yml Normal file
View File

@ -0,0 +1,16 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
elastic_index_retention_algorithm: default

32
elk_metrics_6x/roles/elastic_retention/meta/main.yml Normal file
View File

@ -0,0 +1,32 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x retention role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_data_hosts

35
elk_metrics_6x/roles/elastic_retention/tasks/main.yml Normal file
View File

@ -0,0 +1,35 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Query es storage
uri:
url: "http://{{ coordination_nodes[0] }}/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs"
method: GET
register: elk_data
until: elk_data is success
retries: 5
delay: 5
run_once: true
- name: Load data node variables
include_vars: "calculate_index_retention_{{ elastic_index_retention_algorithm }}.yml"
tags:
- always
- name: Set retention facts
set_fact: "elastic_{{ item.key }}_retention={{ (es_assumed_usable_storage_per_node | int) // ((item.value | int) * (es_storage_multiplier | int)) }}"
when:
- hostvars[inventory_hostname]["elastic_" + item.key + "_retention"] is undefined
with_dict: "{{ elastic_beat_retention_policy_hosts }}"

30
elk_metrics_6x/roles/elastic_retention/vars/calculate_index_retention_default.yml Normal file
View File

@ -0,0 +1,30 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Set available storage fact
es_total_available_storage: "{{ ((elk_data['json']['nodes'].values() | list) | map(attribute='fs.total.total_in_bytes') | list | sum) // 1024 // 1024 }}"
# Set assumed buffer storage fact
es_assumed_buffer_storage: "{{ ((es_total_available_storage | int) * 0.25) | round | int }}"
# Set usable buffer storage fact(s)
es_usable_buffer_storage: "{{ (es_total_available_storage | int) - (es_assumed_buffer_storage | int) }}"
es_expected_storage: "{{ ((elastic_beat_retention_policy_hosts.values() | map('int') | list) | sum) * (elastic_beat_storage_constant | int) }}"
# Set buffer storage fact
es_assumed_usable_storage_per_node: "{{ (es_usable_buffer_storage | int) // (data_nodes | length | int) }}"
# Set storage the mulitplier
es_storage_multiplier: "{{ ((es_usable_buffer_storage | int) < (es_expected_storage | int)) | ternary(((elastic_beat_storage_constant | int) * 2), elastic_beat_storage_constant | int) }}"

16
elk_metrics_6x/roles/elastic_rollup/defaults/main.yml Normal file
View File

@ -0,0 +1,16 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
elastic_allow_rollup_purge: false

32
elk_metrics_6x/roles/elastic_rollup/meta/main.yml Normal file
View File

@ -0,0 +1,32 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x rollup role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_retention

96
elk_metrics_6x/roles/elastic_rollup/tasks/main.yml Normal file
View File

@ -0,0 +1,96 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Get rollup job
uri:
url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}"
method: GET
register: check_rollup
until: check_rollup is success
retries: 3
delay: 5
run_once: true
- name: Check for existing rollup job
debug:
msg: >-
An existing rollup job was found for {{ index_name }}. In order to
re-create this rollup job the old job will need to be purged. If you're
OK with the old rollup job being purged, add the following option
`elastic_allow_rollup_purge=yes` to the command line and rerun the
playbook.
when:
- check_rollup['json']['jobs'] | length > 0
- not elastic_allow_rollup_purge | bool
- name: Create rollup block
block:
- name: Set retention days fact
set_fact:
days_until_rollup: |-
{% set index_retention = [] %}
{% for item in ansible_play_hosts %}
{% set _ = index_retention.append((hostvars[item]['elastic_' + index_name + '_retention'] | int) // 3) %}
{% endfor %}
{{ index_retention | min }}
run_once: true
- name: Create rollup job
uri:
url: "{{ item.url }}"
method: "{{ item.method }}"
body: "{{ item.index_options | to_json }}"
status_code: "{{ item.status_code }}"
body_format: json
register: elk_indexes
until: elk_indexes is success
retries: 3
delay: 5
when:
- hostvars[inventory_hostname]['elastic_' + index_name + '_retention'] > days_until_rollup
with_items:
- url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}/_stop"
method: POST
status_code: 200,404
index_options: {}
- url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}"
method: DELETE
status_code: 200,404
index_options: {}
- url: "http://{{ coordination_nodes[0] }}/rollup_{{ index_name }}"
method: DELETE
status_code: 200,404
index_options: {}
- url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}"
method: PUT
status_code: 200
index_options:
index_pattern: "{{ index_name }}-*"
rollup_index: "rollup_{{ index_name }}"
cron: "*/30 * * * * ?"
page_size: 1000
groups:
date_histogram:
field: "@timestamp"
interval: "1h"
delay: "{{ days_until_rollup }}d"
- url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}/_start"
method: POST
status_code: 200
index_options: {}
run_once: true
when:
- check_rollup['json']['jobs'] | length < 1 or
elastic_allow_rollup_purge | bool

19
elk_metrics_6x/roles/elasticsearch/defaults/main.yml Normal file
View File

@ -0,0 +1,19 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
elastic_log_rotate_path: "/var/log/elasticsearch"
temp_dir: /var/lib/elasticsearch/tmp
nfs_query: "[?fstype=='nfs' || fstype=='nfs4']"

25
elk_metrics_6x/roles/elasticsearch/handlers/main.yml Normal file
View File

@ -0,0 +1,25 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Enable and restart elastic
systemd:
name: "elasticsearch"
enabled: true
state: restarted
daemon_reload: true
when:
- (elk_package_state | default('present')) != 'absent'
tags:
- config

37
elk_metrics_6x/roles/elasticsearch/meta/main.yml Normal file
View File

@ -0,0 +1,37 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x elasticsearch role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies:
- role: elastic_data_hosts
- role: elastic_repositories
- role: elastic_dependencies
service_name: elasticsearch
service_owner: elasticsearch
service_group: elasticsearch

29
elk_metrics_6x/roles/elasticsearch/tasks/elasticsearch_nfs_setup.yml Normal file
View File

@ -0,0 +1,29 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Ensure nfs client is available if necessary
package:
name: nfs-common
state: present
- name: Ensure backup filesystems are mounted
mount:
fstype: "{{ item.fstype }}"
src: "{{ item.src }}"
opts: "{{ item.opts }}"
path: "{{ item.path }}"
state: "{{ item.state }}"
with_items:
- "{{ elastic_shared_fs_repos }}"

102
elk_metrics_6x/roles/elasticsearch/tasks/main.yml Normal file
View File

@ -0,0 +1,102 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather variables for each operating system
include_vars: "{{ item }}"
with_first_found:
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
- "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
- "{{ ansible_os_family | lower }}.yml"
tags:
- always
- name: Set elasticsearch variables
include_vars: "vars_{{ ((inventory_hostname in (groups['kibana'] | default([])) and not inventory_hostname in (groups['elastic-logstash']) | default([]))) | ternary('kibana', 'default') }}.yml"
tags:
- always
- name: Ensure elasticsearch is installed
package:
name: "{{ elasticsearch_distro_packages }}"
state: "{{ elk_package_state | default('present') }}"
update_cache: yes
register: _package_task
until: _package_task is success
retries: 3
delay: 2
notify:
- Enable and restart elastic
tags:
- package_install
- name: Create elasticsearch systemd service config dir
file:
path: "/etc/systemd/system/elasticsearch.service.d"
state: "directory"
group: "root"
owner: "root"
mode: "0755"
- name: Drop elasticsearch conf file
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
with_items:
- src: templates/elasticsearch.yml.j2
dest: /etc/elasticsearch/elasticsearch.yml
- src: templates/jvm.options.j2
dest: /etc/elasticsearch/jvm.options
- src: templates/es-log4j2.properties.j2
dest: /etc/elasticsearch/log4j2.properties
- src: "systemd.elasticsearch-overrides.conf.j2"
dest: "/etc/systemd/system/elasticsearch.service.d/elasticsearch-overrides.conf"
notify:
- Enable and restart elastic
tags:
- config
- name: Ensure elasticsearch ownership
file:
path: "/var/lib/elasticsearch/"
owner: elasticsearch
group: elasticsearch
recurse: true
register: e_perms
until: e_perms is success
retries: 3
delay: 1
tags:
- config
- name: Ensure elasticsearch tmp dir
file:
path: "/var/lib/elasticsearch/tmp"
state: directory
owner: "elasticsearch"
group: "elasticsearch"
mode: "0750"
- name: Create logrotate config
template:
src: "templates/logrotate.j2"
dest: "/etc/logrotate.d/elasticsearch"
- include_tasks: "elasticsearch_nfs_setup.yml"
when:
- elastic_shared_fs_repos is defined
- (elastic_shared_fs_repos | json_query(nfs_query)) | length > 0

18
elk_metrics_6x/roles/elasticsearch/vars/ubuntu.yml Normal file
View File

@ -0,0 +1,18 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
elasticsearch_distro_packages:
- logrotate
- elasticsearch

17
elk_metrics_6x/roles/elasticsearch/vars/vars_default.yml Normal file
View File

@ -0,0 +1,17 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Set elasticsearch facts
elastic_heap_size: "{{ elastic_heap_size_default }}"

22
elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml Normal file
View File

@ -0,0 +1,22 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Set kibana as elasticsearch coordinators
elasticsearch_node_master: false
elasticsearch_node_data: false
elasticsearch_node_ingest: false
elastic_coordination_node: true
elastic_heap_size: "{{ (elastic_heap_size_default | int) // 3 }}"
elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 4) | ternary(4, (ansible_processor_cores // 2)) }}"

14
elk_metrics_6x/roles/go_install_1.10/handlers/main.yml Normal file
View File

@ -0,0 +1,14 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

31
elk_metrics_6x/roles/go_install_1.10/meta/main.yml Normal file
View File

@ -0,0 +1,31 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v6.x go install role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- xenial
categories:
- cloud
- development
- elasticsearch
- elastic-stack
dependencies: []

45
elk_metrics_6x/roles/go_install_1.10/tasks/go_install.yml Normal file
View File

@ -0,0 +1,45 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: GET go
get_url:
url: "https://dl.google.com/go/go1.10.1.linux-amd64.tar.gz"
dest: "/opt/go1.10.1.linux-amd64.tar.gz"
checksum: "sha256:72d820dec546752e5a8303b33b009079c15c2390ce76d67cf514991646c6127b"
register: _get_task
until: _get_task is success
retries: 3
delay: 2
tags:
- package_install
- name: Create go directory
file:
path: "/opt/go1.10.1"
state: directory
- name: Unarchive go
unarchive:
src: "/opt/go1.10.1.linux-amd64.tar.gz"
dest: "/opt/go1.10.1"
remote_src: yes
- name: Create go defaults file
copy:
content: |
GOROOT=/opt/go1.10.1/go
GOPATH=/usr/local
PATH=${PATH}:${GOROOT}/bin
dest: /etc/default/go1.10.1

23
elk_metrics_6x/roles/go_install_1.10/tasks/main.yml Normal file
View File

@ -0,0 +1,23 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Check for go
stat:
path: /opt/go1.10.1/go/bin/go
register: go_path
- include_tasks: go_install.yml
when:
- not go_path.stat.exists | bool

31
elk_metrics_6x/setupAPMserver.yml
View File

@ -1,31 +0,0 @@
---
- name: Load apm-server Dashboards
hosts: apm-server[0]
gather_facts: true
vars_files:
- vars/variables.yml
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
tasks:
- name: Load templates
shell: >-
apm-server setup
{{ item }}
-E 'apm-server.host=localhost:8200'
-E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-e -v
with_items:
- "--template"
- "--dashboards"
register: templates
until: templates is success
retries: 5
delay: 5
tags:
- beat-setup

31
elk_metrics_6x/setupAuditbeat.yml
View File

@ -1,31 +0,0 @@
---
- name: Load Auditbeat Dashboards
hosts: hosts[0]
gather_facts: true
vars_files:
- vars/variables.yml
pre_tasks:
- include_tasks: common_task_data_node_hosts.yml
tags:
- always
tasks:
- name: Load templates
shell: >-
auditbeat setup
{{ item }}
-E 'output.logstash.enabled=false'
-E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-e -v
with_items:
- "--template"
- "--dashboards"
register: templates
until: templates is success
retries: 5
delay: 5
tags:
- beat-setup

Some files were not shown because too many files have changed in this diff Show More