Add systemd configs and update playbook uniformity

Systemd overrides have been added to the service unit files for all
beats and services. All of the playbooks have been updated to make them
look and feel uniform.

This also sets handlers within the playbooks so that we're improving the
idempotence.

Change-Id: I2dd3183dae4bfddc607cc74f9dfb7af115b80abc
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/installAPMserver.yml

@@ -29,25 +29,53 @@
       until: _apt_task is success
       retries: 3
       delay: 2
+      notify:
+        - Enable and restart apm server
       tags:
         - package_install
 
     - name: exit playbook after uninstall
       meta: end_play
       when:
-        - elk_package_state | default('present') == 'absent'
+        - (elk_package_state | default('present')) == 'absent'
 
   post_tasks:
+    - name: Create apm-server systemd service config dir
+      file:
+        path: "/etc/systemd/system/apm-server.service.d"
+        state: "directory"
+        group: "root"
+        owner: "root"
+        mode: "0755"
+
+    - name: Apply systemd options
+      template:
+        src: "{{ item.src }}"
+        dest: "/etc/systemd/system/apm-server.service.d/{{ item.dest }}"
+        mode: "0644"
+      with_items:
+        - { src: "systemd.general-overrides.conf.j2", dest: "apm-server-overrides.conf" }
+      notify:
+        - Enable and restart apm server
+
     - name: Drop apm-server conf file
       template:
         src: templates/apm-server.yml.j2
         dest: /etc/apm-server/apm-server.yml
+      notify:
+        - Enable and restart apm server
 
-    - name: Enable and restart APM Server
+  handlers:
+    - name: Enable and restart apm server
       systemd:
         name: "apm-server"
         enabled: true
         state: restarted
+        daemon_reload: true
+      when:
+        - (elk_package_state | default('present')) != 'absent'
+      tags:
+        - config
 
   tags:
     - server-install

elk_metrics_6x/installAuditbeat.yml

@@ -30,25 +30,53 @@
       until: _apt_task is success
       retries: 3
       delay: 2
+      notify:
+        - Enable and restart auditbeat
       tags:
         - package_install
 
     - name: exit playbook after uninstall
       meta: end_play
       when:
-        - elk_package_state | default('present') == 'absent'
+        - (elk_package_state | default('present')) == 'absent'
 
   post_tasks:
+    - name: Create auditbeat systemd service config dir
+      file:
+        path: "/etc/systemd/system/auditbeat.service.d"
+        state: "directory"
+        group: "root"
+        owner: "root"
+        mode: "0755"
+
+    - name: Apply systemd options
+      template:
+        src: "{{ item.src }}"
+        dest: "/etc/systemd/system/auditbeat.service.d/{{ item.dest }}"
+        mode: "0644"
+      with_items:
+        - { src: "systemd.general-overrides.conf.j2", dest: "auditbeat-overrides.conf" }
+      notify:
+        - Enable and restart auditbeat
+
     - name: Drop auditbeat conf file
       template:
         src: templates/auditbeat.yml.j2
         dest: /etc/auditbeat/auditbeat.yml
+      notify:
+        - Enable and restart auditbeat
 
+  handlers:
     - name: Enable and restart auditbeat
       systemd:
         name: "auditbeat"
         enabled: "true"
         state: restarted
+        daemon_reload: true
+      when:
+        - (elk_package_state | default('present')) != 'absent'
+      tags:
+        - config
 
   tags:
     - beat-install

elk_metrics_6x/installCurator.yml

@@ -70,7 +70,7 @@
     - name: Ensure virtualenv is installed
       apt:
         name: "{{ item }}"
-        state: "present"
+        state: "{{ elk_package_state | default('present') }}"
         update_cache: true
       with_items:
         - python-virtualenv
@@ -94,7 +94,7 @@
     - name: exit playbook after uninstall
       meta: end_play
       when:
-        - elk_package_state | default('present') == 'absent'
+        - (elk_package_state | default('present')) == 'absent'
 
   tasks:
     - name: create the system group
@@ -141,7 +141,8 @@
         name: systemd_service
         private: true
       vars:
-        systemd_service_enabled: true
+        systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}"
+        systemd_service_restart_changed: false
         systemd_user_name: curator
         systemd_group_name: curator
         systemd_services:
@@ -162,6 +163,10 @@
         name: "curator.timer"
         enabled: true
         state: restarted
+      when:
+        - (elk_package_state | default('present')) != 'absent'
+      tags:
+        - config
 
   tags:
     - beat-install

elk_metrics_6x/installElastic.yml

@@ -99,6 +99,12 @@
       tags:
         - package_install
 
+    - name: exit playbook after uninstall
+      meta: end_play
+      when:
+        - (elk_package_state | default('present')) == 'absent'
+
+  post_tasks:
     - name: Create elasticsearch systemd service config dir
       file:
         path: "/etc/systemd/system/elasticsearch.service.d"
@@ -161,6 +167,8 @@
         enabled: true
         state: restarted
         daemon_reload: true
+      when:
+        - (elk_package_state | default('present')) != 'absent'
       tags:
         - config
 

elk_metrics_6x/installFilebeat.yml

@@ -29,13 +29,34 @@
       until: _apt_task is success
       retries: 3
       delay: 2
+      notify:
+        - Enable and restart filebeat
       tags:
         - package_install
 
     - name: exit playbook after uninstall
       meta: end_play
       when:
-        - elk_package_state | default('present') == 'absent'
+        - (elk_package_state | default('present')) == 'absent'
+
+  post_tasks:
+    - name: Create filebeat systemd service config dir
+      file:
+        path: "/etc/systemd/system/filebeat.service.d"
+        state: "directory"
+        group: "root"
+        owner: "root"
+        mode: "0755"
+
+    - name: Apply systemd options
+      template:
+        src: "{{ item.src }}"
+        dest: "/etc/systemd/system/filebeat.service.d/{{ item.dest }}"
+        mode: "0644"
+      with_items:
+        - { src: "systemd.general-overrides.conf.j2", dest: "filebeat-overrides.conf" }
+      notify:
+        - Enable and restart filebeat
 
     - name: Check for apache
       stat:
@@ -147,18 +168,24 @@
         designate_enabled: "{{ (designate.stat.exists | bool) or (inventory_hostname in groups['designate_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*designate.*') | list | length) > 0) }}"
         osquery_enabled: "{{ osquery.stat.exists | bool }}"
 
-
-  post_tasks:
     - name: Drop Filebeat conf file
       template:
         src: templates/filebeat.yml.j2
         dest: /etc/filebeat/filebeat.yml
+      notify:
+        - Enable and restart filebeat
 
-    - name: Enable and restart Filebeat
+  handlers:
+    - name: Enable and restart filebeat
       systemd:
         name: "filebeat"
         enabled: true
         state: restarted
+        daemon_reload: true
+      when:
+        - (elk_package_state | default('present')) != 'absent'
+      tags:
+        - config
 
   tags:
     - beat-install

elk_metrics_6x/installHeartbeat.yml

@@ -33,19 +33,45 @@
     - name: exit playbook after uninstall
       meta: end_play
       when:
-        - elk_package_state | default('present') == 'absent'
+        - (elk_package_state | default('present')) == 'absent'
 
   post_tasks:
+    - name: Create heartbeat systemd service config dir
+      file:
+        path: "/etc/systemd/system/heartbeat.service.d"
+        state: "directory"
+        group: "root"
+        owner: "root"
+        mode: "0755"
+
+    - name: Apply systemd options
+      template:
+        src: "{{ item.src }}"
+        dest: "/etc/systemd/system/heartbeat.service.d/{{ item.dest }}"
+        mode: "0644"
+      with_items:
+        - { src: "systemd.general-overrides.conf.j2", dest: "heartbeat-overrides.conf" }
+      notify:
+        - Enable and restart heartbeat
+
     - name: Drop heartbeat conf file
       template:
         src: templates/heartbeat.yml.j2
         dest: /etc/heartbeat/heartbeat.yml
+      notify:
+        - Enable and restart heartbeat
 
+  handlers:
     - name: Enable and restart heartbeat
       systemd:
         name: "heartbeat-elastic"
         enabled: true
         state: restarted
+        daemon_reload: true
+      when:
+        - (elk_package_state | default('present')) != 'absent'
+      tags:
+        - config
 
   tags:
     - beat-install

elk_metrics_6x/installJournalbeat.yml

@@ -81,8 +81,26 @@
     - name: exit playbook after uninstall
       meta: end_play
       when:
-        - elk_package_state | default('present') == 'absent'
+        - (elk_package_state | default('present')) == 'absent'
-        - ansible_service_mgr == "systemd"
+
+  post_tasks:
+    - name: Create journalbeat systemd service config dir
+      file:
+        path: "/etc/systemd/system/journalbeat.service.d"
+        state: "directory"
+        group: "root"
+        owner: "root"
+        mode: "0755"
+
+    - name: Apply systemd options
+      template:
+        src: "{{ item.src }}"
+        dest: "/etc/systemd/system/journalbeat.service.d/{{ item.dest }}"
+        mode: "0644"
+      with_items:
+        - { src: "systemd.general-overrides.conf.j2", dest: "journalbeat-overrides.conf" }
+      notify:
+        - Enable and restart journalbeat
 
     - name: create the system group
       group:
@@ -136,6 +154,7 @@
         name: systemd_service
         private: true
       vars:
+        systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}"
         systemd_service_restart_changed: false
         systemd_services:
           - service_name: "journalbeat"
@@ -161,6 +180,10 @@
         enabled: true
         state: restarted
         daemon_reload: yes
+      when:
+        - (elk_package_state | default('present')) != 'absent'
+      tags:
+        - config
 
   tags:
     - beat-install

elk_metrics_6x/installKibana.yml

@@ -23,6 +23,8 @@
       until: _apt_task is success
       retries: 3
       delay: 2
+      notify:
+        - Enable and restart nginx
       tags:
         - package_install
 
@@ -38,12 +40,8 @@
       template:
         src: templates/nginx_default.j2
         dest: /etc/nginx/sites-available/default
-
+      notify:
-    - name: Enable and restart nginx
+        - Enable and restart nginx
-      service:
-        name: "nginx"
-        enabled: true
-        state: restarted
 
     - name: Ensure kibana  is installed
       apt:
@@ -59,6 +57,12 @@
       tags:
         - package_install
 
+    - name: exit playbook after uninstall
+      meta: end_play
+      when:
+        - (elk_package_state | default('present')) == 'absent'
+
+  post_tasks:
     - name: Create kibana systemd service config dir
       file:
         path: "/etc/systemd/system/kibana.service.d"
@@ -73,7 +77,7 @@
         dest: "/etc/systemd/system/kibana.service.d/{{ item.dest }}"
         mode: "0644"
       with_items:
-        - { src: "systemd.kibana-overrides.conf.j2", dest: "kibana-overrides.conf" }
+        - { src: "systemd.general-overrides.conf.j2", dest: "kibana-overrides.conf" }
       notify:
         - Enable and restart kibana
 
@@ -92,6 +96,20 @@
         enabled: true
         state: restarted
         daemon_reload: true
+      when:
+        - (elk_package_state | default('present')) != 'absent'
+      tags:
+        - config
+
+    - name: Enable and restart nginx
+      systemd:
+        name: "nginx"
+        enabled: true
+        state: restarted
+      when:
+        - (elk_package_state | default('present')) != 'absent'
+      tags:
+        - config
 
   tags:
     - server-install

elk_metrics_6x/installLogstash.yml

@@ -86,6 +86,12 @@
       tags:
         - package_install
 
+    - name: exit playbook after uninstall
+      meta: end_play
+      when:
+        - (elk_package_state | default('present')) == 'absent'
+
+  post_tasks:
     - name: Create logstash systemd service config dir
       file:
         path: "/etc/systemd/system/logstash.service.d"
@@ -100,7 +106,7 @@
         dest: "/etc/systemd/system/logstash.service.d/{{ item.dest }}"
         mode: "0644"
       with_items:
-        - { src: "systemd.logstash-overrides.conf.j2", dest: "logstash-overrides.conf" }
+        - { src: "systemd.general-overrides.conf.j2", dest: "logstash-overrides.conf" }
       notify:
         - Enable and restart logstash
 
@@ -233,6 +239,10 @@
         enabled: true
         state: restarted
         daemon_reload: true
+      when:
+        - (elk_package_state | default('present')) != 'absent'
+      tags:
+        - config
 
   tags:
     - server-install

elk_metrics_6x/installMetricbeat.yml

@@ -27,13 +27,34 @@
       until: _apt_task is success
       retries: 3
       delay: 2
+      notify:
+        - Enable and restart metricbeat
       tags:
         - package_install
 
     - name: exit playbook after uninstall
       meta: end_play
       when:
-        - elk_package_state | default('present') == 'absent'
+        - (elk_package_state | default('present')) == 'absent'
+
+  post_tasks:
+    - name: Create metricbeat systemd service config dir
+      file:
+        path: "/etc/systemd/system/metricbeat.service.d"
+        state: "directory"
+        group: "root"
+        owner: "root"
+        mode: "0755"
+
+    - name: Apply systemd options
+      template:
+        src: "{{ item.src }}"
+        dest: "/etc/systemd/system/metricbeat.service.d/{{ item.dest }}"
+        mode: "0644"
+      with_items:
+        - { src: "systemd.general-overrides.conf.j2", dest: "metricbeat-overrides.conf" }
+      notify:
+        - Enable and restart metricbeat
 
     - name: Check for apache
       stat:
@@ -189,17 +210,25 @@
         state: reloaded
       when: nginx_enabled
 
-  post_tasks:
     - name: Drop metricbeat conf file
       template:
         src: templates/metricbeat.yml.j2
         dest: /etc/metricbeat/metricbeat.yml
+      notify:
+        - Enable and restart metricbeat
 
+  handlers:
     - name: Enable and restart metricbeat
       systemd:
         name: "metricbeat"
         enabled: true
         state: restarted
+        daemon_reload: true
+      when:
+        - (elk_package_state | default('present')) != 'absent'
+      tags:
+        - config
+
   tags:
     - beat-install
 

elk_metrics_6x/installPacketbeat.yml

@@ -30,25 +30,53 @@
       until: _apt_task is success
       retries: 3
       delay: 2
+      notify:
+        - Enable and restart packetbeat
       tags:
         - package_install
 
     - name: exit playbook after uninstall
       meta: end_play
       when:
-        - elk_package_state | default('present') == 'absent'
+        - (elk_package_state | default('present')) == 'absent'
 
   post_tasks:
+    - name: Create packetbeat systemd service config dir
+      file:
+        path: "/etc/systemd/system/packetbeat.service.d"
+        state: "directory"
+        group: "root"
+        owner: "root"
+        mode: "0755"
+
+    - name: Apply systemd options
+      template:
+        src: "{{ item.src }}"
+        dest: "/etc/systemd/system/packetbeat.service.d/{{ item.dest }}"
+        mode: "0644"
+      with_items:
+        - { src: "systemd.general-overrides.conf.j2", dest: "packetbeat-overrides.conf" }
+      notify:
+        - Enable and restart packetbeat
+
     - name: Drop packetbeat conf file
       template:
         src: templates/packetbeat.yml.j2
         dest: /etc/packetbeat/packetbeat.yml
+      notify:
+        - Enable and restart packetbeat
 
+  handlers:
     - name: Enable and restart packetbeat
       systemd:
         name: "packetbeat"
         enabled: true
         state: restarted
+        daemon_reload: true
+      when:
+        - (elk_package_state | default('present')) != 'absent'
+      tags:
+        - config
 
   tags:
     - beat-install

elk_metrics_6x/templates/systemd.logstash-overrides.conf.j2

@@ -1,13 +0,0 @@
-[Service]
-# This creates a specific slice to operate from. The accounting options give us
-# the ability to see resource usage through the `systemd-cgtop` command and
-# further isolate this service from the host machine.
-Slice=elastic.slice
-CPUAccounting=true
-BlockIOAccounting=true
-MemoryAccounting=true
-TasksAccounting=true
-
-# Sandbox setup
-PrivateTmp=true
-PrivateDevices={{ ((ansible_os_family | lower) != "redhat") | lower }}