openstack/openstack-ansible-ops
Code Issues Proposed changes

Additional cleanup and simplification

Browse Source 
The role is further being pruned and made more efficient.

Change-Id: I75de7063164959041193526e519682ba80185250
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
Kevin Carter 2018-10-18 21:13:51 -05:00
parent e0e8579785
commit fd2005baf4
No known key found for this signature in database
GPG Key ID: 9443251A787B9FB3
9 changed files with 113 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
osquery/installOSquery.yml
View File

@ -101,6 +101,7 @@
dest: "{{ item }}"
state: "absent"
delegate_to: localhost
run_once: true
with_items:
- "/tmp/{{ kolide_fleet_ssl_cert | basename }}"
- "/tmp/{{ kolide_fleet_ssl_key | basename }}"

4
osquery/roles/fleet/tasks/fleetMigrateDB.yml
View File

@ -17,3 +17,7 @@
command: /usr/local/bin/fleet prepare db --config=/etc/fleet/fleet_config.yml --no-prompt
changed_when: false
run_once: true
register: _fleetctl_db_task
until: _fleetctl_db_task is success
retries: 3
delay: 2

11
osquery/roles/fleet/tasks/fleetRegisterAdmin.yml
View File

@ -16,6 +16,10 @@
- name: set fleetctl default context
command: /usr/local/bin/fleetctl config set --address https://127.0.0.1:{{ kolide_fleet_port }} --tls-skip-verify
changed_when: false
register: _fleetctl_set_task
until: _fleetctl_set_task is success
retries: 3
delay: 2
- name: register admin account
command: /usr/local/bin/fleetctl setup --email {{ kolide_fleet_admin_email }} --password {{ kolide_fleet_admin_password }}
@ -24,7 +28,14 @@
- fleet_register_admin.rc == 0
failed_when:
- fleet_register_admin.rc not in [0, 1]
until: fleet_register_admin is success
retries: 3
delay: 2
- name: login admin account
command: /usr/local/bin/fleetctl login --email {{ kolide_fleet_admin_email }} --password {{ kolide_fleet_admin_password }}
changed_when: false
register: _fleetctl_login_task
until: _fleetctl_login_task is success
retries: 3
delay: 2

108
osquery/roles/osquery/defaults/main.yml
View File

@ -2,8 +2,6 @@
# Enable or disable the installation of the osquery debug packages.
osquery_debug_packages_install: false
osquery_template: 'osquery.conf.j2'
osquery_upload_packs:
- osquery-snapshots-pack
- osquery-monitoring2-pack
@ -18,51 +16,38 @@ osquery_packs:
- "osquery-snapshots-pack"
- "osquery-monitoring2-pack"
osquery_config_plugin: 'filesystem'
osquery_logger_plugin: 'filesystem'
# osquery_flags:
# - '--enroll_secret_path=/etc/osquery/osquery_enroll_secret'
# - '--tls_server_certs=/etc/osquery/kolide.crt'
# - '--tls_hostname=acme.kolide.co'
# - '--host_identifier=hostname'
# - '--enroll_tls_endpoint=/api/v1/osquery/enroll'
# - '--config_plugin=tls'
# - '--config_tls_endpoint=/api/v1/osquery/config'
# - '--config_tls_refresh=10'
# - '--disable_distributed=false'
# - '--distributed_plugin=tls'
# - '--distributed_interval=10'
# - '--distributed_tls_max_attempts=3'
# - '--distributed_tls_read_endpoint=/api/v1/osquery/distributed/read'
# - '--distributed_tls_write_endpoint=/api/v1/osquery/distributed/write'
# - '--logger_plugin=tls'
# - '--logger_tls_endpoint=/api/v1/osquery/log'
# - '--logger_tls_period=10'
osquery_flags: []
## if using zentral, kolide or else
## https://github.com/zentralopensource/zentral/blob/f460b10a95d4ea1e515aea3363f55733465d1d9c/zentral/contrib/osquery/deb_script/template.sh
## https://docs.kolide.co/kolide/current/infrastructure/adding-hosts-to-kolide.html
#osquery_enroll_secret: ""
#osquery_flags:
# - '--enroll_secret_path=/etc/osquery/osquery_enroll_secret'
# - '--tls_server_certs=/etc/osquery/kolide.crt'
# - '--tls_hostname=acme.kolide.co'
# - '--host_identifier=hostname'
# - '--enroll_tls_endpoint=/api/v1/osquery/enroll'
# - '--config_plugin=tls'
# - '--config_tls_endpoint=/api/v1/osquery/config'
# - '--config_tls_refresh=10'
# - '--disable_distributed=false'
# - '--distributed_plugin=tls'
# - '--distributed_interval=10'
# - '--distributed_tls_max_attempts=3'
# - '--distributed_tls_read_endpoint=/api/v1/osquery/distributed/read'
# - '--distributed_tls_write_endpoint=/api/v1/osquery/distributed/write'
# - '--logger_plugin=tls'
# - '--logger_tls_endpoint=/api/v1/osquery/log'
# - '--logger_tls_period=10'
osquery_fim: true
osquery_fim_interval: 900
## Take care if using a lot /tmp. can trigger
## 'Expiring events for subscriber: file_events (overflowed limit 1000)'
## => losing many queries results (fim or not)
osquery_fim_filepaths:
- name: homes_sshdir
list:
- "/root/.ssh/%%"
- "/home/%/.ssh/%%"
- "/home/lib/%/.ssh/%%"
- name: etc
list:
- "/etc/%%"
## Take care if using a lot /tmp. can trigger
## 'Expiring events for subscriber: file_events (overflowed limit 1000)'
## => losing many queries results (fim or not)
# - name: tmp
# list:
# - "/tmp/%%"
- name: bin
list:
- "/bin/%%"
@ -82,20 +67,43 @@ osquery_fim_excludepaths:
list:
- /tmp/too_many_events/
## queries snapshots: 1/week (or 1/month? nok)
## queries snapshots: 1/week
## Max interval 1/w: https://github.com/theopolis/osquery/commit/b76dee8a1fddccb500bc4a058daa1b39083b9dbb
osquery_snapshot_interval: 604800
osquery_snapshot_interval2: 604800
#osquery_snapshot_interval2: 2592000
## making schedule query faster to have logs faster...
osquery_testing: false
osquery_testing_pause: false
osquery_testing_fim_load: false
osquery_testing_fim_load_bigger: 100M
osquery_testing_fim_load_smaller: 1000
## profiling?
osquery_profiling: false
## full config might be too long for travis (max 50min)
#osquery_profiling_conf: /etc/osquery/osquery.conf
osquery_profiling_conf: /usr/share/osquery/packs/incident-response.conf
osquery_options:
options:
## Splay the scheduled interval for queries.
## This is very helpful to prevent system performance impact when scheduling
## large numbers of queries that run a smaller or similar intervals.
schedule_splay_percent: 10
## Clear events from the osquery backing store after a number of seconds.
events_expiry: 3600
## Enable debug or verbose debug output when logging.
verbose: false
## The number of threads for concurrent query schedule execution.
worker_threads: 2
## Enable schedule profiling, this will fill in averages and totals for
## system/user CPU time and memory for every query in the schedule.
## Add a query: "select * from osquery_schedule" to record the performances.
enable_monitor: true
logger_snapshot_event_type: true
## Define a schedule of queries:
schedule:
## This is a simple example query that outputs basic system information.
system_info:
## The exact query to run.
query: "SELECT hostname, cpu_brand, physical_memory FROM system_info;"
## The interval in seconds to run this query, not an exact interval.
interval: 3600
fim:
query: "select * from file_events;"
removed: false
## fim query interval
interval: 900
## Decorators are normal queries that append data to every query.
decorators:
load:
- "SELECT uuid AS host_uuid FROM system_info;"
- "SELECT user AS username FROM logged_in_users ORDER BY time DESC LIMIT 1;"

9
osquery/roles/osquery/handlers/main.yml
View File

@ -1,9 +1,16 @@
---
- name: Enable and stop the service
systemd:
name: osqueryd
enabled: true
state: stopped
listen: restart osquery
- name: Enable and start the service
systemd:
name: osqueryd
enabled: true
state: restarted
state: started
listen: restart osquery
- name: Enable and start the service

2
osquery/roles/osquery/tasks/osquery_configure.yml
View File

@ -26,7 +26,7 @@
- name: Configure osquery
template:
src: "{{ osquery_template }}"
src: "osquery.conf.j2"
dest: /etc/osquery/osquery.conf
mode: '0644'
backup: yes

154
osquery/roles/osquery/templates/osquery.conf.j2
View File

@ -1,132 +1,30 @@
{{ ansible_managed | comment('c')}}
{
// Configure the daemon below:
"options": {
// Select the osquery config plugin.
"config_plugin": "{{ osquery_config_plugin }}",
{# Set the FIM include path items #}
{%- if (osquery_fim_filepaths | length) > 0 %}
{%- set file_path_dict = {} %}
{%- for item in osquery_fim_filepaths %}
{%- set _ = file_path_dict.__setitem__(item.name, item.list) -%}
{%- endfor %}
{%- set _ = osquery_options.__setitem__("file_paths", file_path_dict) -%}
{%- endif -%}
// Select the osquery logging plugin.
"logger_plugin": "{{ osquery_logger_plugin }}",
{# Set the FIM exlude path items #}
{%- if (osquery_fim_excludepaths | length) > 0 %}
{%- set exclude_file_path_dict = {} %}
{%- for item in osquery_fim_excludepaths %}
{%- set _ = exclude_file_path_dict.__setitem__(item.name, item.list) -%}
{%- endfor %}
{%- set _ = osquery_options.__setitem__("exclude_paths", exclude_file_path_dict) -%}
{%- endif -%}
// The log directory stores info, warning, and errors.
// If the daemon uses the 'filesystem' logging retriever then the log_dir
// will also contain the query results.
//"logger_path": "/var/log/osquery",
{# Set the loaded packs #}
{%- if (osquery_packs | length) > 0 %}
{%- set pack_dict = {} %}
{%- for item in osquery_packs %}
{%- set _ = pack_dict.__setitem__(item, "/usr/share/osquery/packs/" + item + ".conf") -%}
{%- endfor %}
{%- set _ = osquery_options.__setitem__("packs", pack_dict) -%}
{%- endif -%}
// Set 'disable_logging' to true to prevent writing any info, warning, error
// logs. If a logging plugin is selected it will still write query results.
//"disable_logging": "false",
// Query differential results are logged as change-events to assist log
// aggregation operations like searching and transactons.
// Set 'log_results_events' to log differentials as transactions.
//"log_result_events": "true",
// Splay the scheduled interval for queries.
// This is very helpful to prevent system performance impact when scheduling
// large numbers of queries that run a smaller or similar intervals.
"schedule_splay_percent": "10",
// Write the pid of the osqueryd process to a pidfile/mutex.
//"pidfile": "/var/osquery/osquery.pidfile",
// Clear events from the osquery backing store after a number of seconds.
"events_expiry": "3600",
// A filesystem path for disk-based backing storage used for events and
// query results differentials. See also 'use_in_memory_database'.
//"database_path": "/var/osquery/osquery.db",
// Comma-delimited list of table names to be disabled.
// This allows osquery to be launched without certain tables.
//"disable_tables": "foo_bar,time",
// Enable debug or verbose debug output when logging.
"verbose": "false",
// The number of threads for concurrent query schedule execution.
"worker_threads": "2",
// Enable schedule profiling, this will fill in averages and totals for
// system/user CPU time and memory for every query in the schedule.
// Add a query: "select * from osquery_schedule" to record the performances.
"enable_monitor": "true",
"logger_snapshot_event_type": "true"
},
// Define a schedule of queries:
"schedule": {
// This is a simple example query that outputs basic system information.
"system_info": {
// The exact query to run.
"query": "SELECT hostname, cpu_brand, physical_memory FROM system_info;",
// The interval in seconds to run this query, not an exact interval.
{% if osquery_testing or osquery_testing_pause %}
"interval": 10
{% else %}
"interval": 3600
{% endif %}
}{% if osquery_fim %},{% endif %}
{% if osquery_fim %}
"fim" : {
// "query": "select target_path, category, time, action from file_events;",
"query": "select * from file_events;",
"removed": false,
"interval": {% if osquery_testing_fim_load %}30{% else %}{{ osquery_fim_interval }}{% endif %}
}
{% endif %}
},
{% if osquery_fim %}
"file_paths": {
{% for item in osquery_fim_filepaths %}
"{{ item.name }}": [
{% for entry in item.list %}
"{{ entry }}"{% if not loop.last %},{% endif %}
{% endfor %}
]{% if not loop.last %},{% endif %}
{% endfor %}
},
"exclude_paths": {
{% for item in osquery_fim_excludepaths %}
"{{ item.name }}": [
{% for entry in item.list %}
"{{ entry }}"{% if not loop.last %},{% endif %}
{% endfor %}
]{% if not loop.last %},{% endif %}
{% endfor %}
},
{% endif %}
// Decorators are normal queries that append data to every query.
"decorators": {
"load": [
"SELECT uuid AS host_uuid FROM system_info;",
"SELECT user AS username FROM logged_in_users ORDER BY time DESC LIMIT 1;"
]
},
// Add default osquery packs or install your own.
//
// There are several 'default' packs installed with 'make install' or via
// packages and/or Homebrew.
//
// Linux: /usr/share/osquery/packs
// OS X: /var/osquery/packs
// Homebrew: /usr/local/share/osquery/packs
// make install: {PREFIX}/share/osquery/packs
//
"packs": {
{% for item in osquery_packs %}
"{{ item }}": "/usr/share/osquery/packs/{{ item }}.conf"{% if not loop.last %},{% endif %}
{% endfor %}
}
}
{# Render the osquery configuration file as json #}
{{ osquery_options | to_nice_json(indent=4) }}

5
osquery/roles/osquery/templates/osquery.flags.j2
View File

@ -1,6 +1,3 @@
--pidfile=/var/run/osqueryd.pid
{% for flag in osquery_flags %}
{{ flag }}
{% endfor %}
--logger_path=/var/log/osquery
--logger_plugin=filesystem
{% endfor %}

13
osquery/vars/variables.yml
View File

@ -1,19 +1,10 @@
---
kolide_fleet_enable: true
kolide_fleet_ssl_cert: /etc/ssl/certs/fleet.cert
kolide_fleet_ssl_key: /etc/ssl/private/fleet.key
kolide_fleet_ssl_ca_cert: /etc/ssl/certs/fleet-ca.pem
kolide_fleet_ssl_ca_key: /etc/ssl/private/fleet-ca.key
kolide_fleet_ssl_pem: /etc/ssl/private/fleet.pem
# Osquery vars
osquery_enroll_secret_file: /etc/osquery/osquery_enroll_secret
osquery_debug_packages_install: false
osquery_config_plugin: 'filesystem'
osquery_logger_plugin: 'filesystem'
osquery_flags:
- "--tls_server_certs={{ kolide_fleet_ssl_cert }}"
- "--tls_hostname={{ hostvars[groups['kolide-fleet_all'][0]]['ansible_host'] }}:443"
@ -28,7 +19,9 @@ osquery_flags:
- "--distributed_tls_max_attempts=3"
- "--distributed_tls_read_endpoint=/api/v1/osquery/distributed/read"
- "--distributed_tls_write_endpoint=/api/v1/osquery/distributed/write"
- "--logger_plugin=tls"
- "--logger_path=/var/log/osquery"
- "--logger_plugin=filesystem,tls"
- "--logger_tls_endpoint=/api/v1/osquery/log"
- "--logger_tls_period=10"
- "--enroll_secret_path={{ osquery_enroll_secret_file }}"
- "--pidfile=/var/run/osqueryd.pid"