From 5ddbde3310676ce082822eebd944458d7d661b5c Mon Sep 17 00:00:00 2001 From: Victor Palma Date: Mon, 16 Jul 2018 15:50:40 -0500 Subject: [PATCH] adding kolide fleet * adds kolide fleet * integrates osquery to kolide fleet server Change-Id: I646364c44bb99d4397bb35068600c49b7bfd62c2 --- osquery/ansible-role-requirements.yml | 29 ++++ osquery/assets/place-holder.svg | 1 + osquery/conf.d/fleet.yml | 6 + osquery/env.d/fleet.yml | 37 +++++ osquery/fleetConfig.yml | 26 ++++ osquery/fleetGetEnrollmentToken.yml | 22 +++ osquery/fleetMigrateDB.yml | 18 +++ osquery/fleetRegisterAdmin.yml | 32 +++++ osquery/fleetRequirements.yml | 40 ++++++ osquery/fleetSSL.yml | 27 ++++ osquery/fleetSSLdistribute.yml | 30 ++++ osquery/fleetSSLkeyCreate.yml | 37 +++++ osquery/fleetSSLselfSigned.yml | 18 +++ osquery/fleetSSLstore.yml | 29 ++++ osquery/fleetSSLuserProvided.yml | 47 +++++++ osquery/fleetServerInstall.yml | 50 +++++++ osquery/fleetService.yml | 31 +++++ osquery/fleetStartService.yml | 45 ++++++ osquery/haproxy.example | 9 ++ osquery/installKolideFleet.yml | 42 ++++++ osquery/installMariaDB.yml | 29 ++++ osquery/installOsquery.yml | 35 ++++- osquery/inventory.example.yml | 31 ++++- osquery/readme.rst | 185 ++++++++++++++++++++++--- osquery/site.yml | 18 +++ osquery/templates/fleet_config.yml.j2 | 15 ++ osquery/templates/fleet_initd.yml.j2 | 70 ++++++++++ osquery/templates/fleet_service.yml.j2 | 17 +++ osquery/templates/redis_initd.yml.j2 | 69 +++++++++ osquery/templates/redis_service.yml.j2 | 18 +++ osquery/vars/variables.yml | 61 ++++++++ 31 files changed, 1099 insertions(+), 25 deletions(-) create mode 100644 osquery/ansible-role-requirements.yml create mode 100644 osquery/assets/place-holder.svg create mode 100644 osquery/conf.d/fleet.yml create mode 100644 osquery/env.d/fleet.yml create mode 100644 osquery/fleetConfig.yml create mode 100644 osquery/fleetGetEnrollmentToken.yml create mode 100644 osquery/fleetMigrateDB.yml create mode 100644 osquery/fleetRegisterAdmin.yml create mode 100644 osquery/fleetRequirements.yml create mode 100644 osquery/fleetSSL.yml create mode 100644 osquery/fleetSSLdistribute.yml create mode 100644 osquery/fleetSSLkeyCreate.yml create mode 100644 osquery/fleetSSLselfSigned.yml create mode 100644 osquery/fleetSSLstore.yml create mode 100644 osquery/fleetSSLuserProvided.yml create mode 100644 osquery/fleetServerInstall.yml create mode 100644 osquery/fleetService.yml create mode 100644 osquery/fleetStartService.yml create mode 100644 osquery/haproxy.example create mode 100644 osquery/installKolideFleet.yml create mode 100644 osquery/installMariaDB.yml create mode 100644 osquery/site.yml create mode 100644 osquery/templates/fleet_config.yml.j2 create mode 100644 osquery/templates/fleet_initd.yml.j2 create mode 100644 osquery/templates/fleet_service.yml.j2 create mode 100644 osquery/templates/redis_initd.yml.j2 create mode 100644 osquery/templates/redis_service.yml.j2 create mode 100644 osquery/vars/variables.yml diff --git a/osquery/ansible-role-requirements.yml b/osquery/ansible-role-requirements.yml new file mode 100644 index 00000000..8df1329c --- /dev/null +++ b/osquery/ansible-role-requirements.yml @@ -0,0 +1,29 @@ +--- +#- name: systemd_service +# scm: git +# src: https://git.openstack.org/openstack/ansible-role-systemd_service +# version: master +- name: config_template + scm: git + src: https://git.openstack.org/openstack/ansible-config_template + version: master +- name: osquery + scm: git + src: https://github.com/juju4/ansible-osquery + version: master +- name: redis + scm: git + src: https://github.com/geerlingguy/ansible-role-redis + version: master +- name: mariadb + scm: git + src: https://github.com/lechuckroh/ansible-role-mariadb + version: master +- name: galera_client + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-galera_client + version: master +- name: galera_server + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-galera_server + version: master diff --git a/osquery/assets/place-holder.svg b/osquery/assets/place-holder.svg new file mode 100644 index 00000000..2cd12daf --- /dev/null +++ b/osquery/assets/place-holder.svg @@ -0,0 +1 @@ +placeholder \ No newline at end of file diff --git a/osquery/conf.d/fleet.yml b/osquery/conf.d/fleet.yml new file mode 100644 index 00000000..dbdf47d3 --- /dev/null +++ b/osquery/conf.d/fleet.yml @@ -0,0 +1,6 @@ +fleet_hosts: + logging1: + ip: 172.22.8.27 +mariadb_hosts: + logging1: + ip: 172.22.8.27 diff --git a/osquery/env.d/fleet.yml b/osquery/env.d/fleet.yml new file mode 100644 index 00000000..8da385f8 --- /dev/null +++ b/osquery/env.d/fleet.yml @@ -0,0 +1,37 @@ +--- +component_skel: + fleet: + belongs_to: + - fleet_all + mariadb: + belongs_to: + - fleet_all + osquery: + belogs_to: + - all + +container_skel: + mariadb_container: + belongs_to: + - mariadb_containers + contains: + - mariadb + fleet_container: + belongs_to: + - fleet_containers + contains: + - fleet + +physical_skel: + mariadb_container: + belongs_to: + - all_containers + mariadb_hosts: + belongs_to: + - hosts + fleet_containers: + belongs_to: + - all_containers + fleet_hosts: + belongs_to: + - hosts diff --git a/osquery/fleetConfig.yml b/osquery/fleetConfig.yml new file mode 100644 index 00000000..76b71191 --- /dev/null +++ b/osquery/fleetConfig.yml @@ -0,0 +1,26 @@ +--- +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: create fleet dir + file: + path: /etc/fleet + state: directory + +- name: Drop fleet conf file + template: + src: templates/fleet_config.yml.j2 + dest: /etc/fleet/fleet_config.yml + tags: + - fleet_config diff --git a/osquery/fleetGetEnrollmentToken.yml b/osquery/fleetGetEnrollmentToken.yml new file mode 100644 index 00000000..03efb6c8 --- /dev/null +++ b/osquery/fleetGetEnrollmentToken.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: retrieve Enrollment Token + command: /usr/local/bin/fleetctl get enroll-secret + register: _enrollment_token + +- name: Set kolide fleet enrollment token fact + set_fact: + kolide_fleet_enroll_secret: "{{ _enrollment_token.stdout }}" diff --git a/osquery/fleetMigrateDB.yml b/osquery/fleetMigrateDB.yml new file mode 100644 index 00000000..b669a15f --- /dev/null +++ b/osquery/fleetMigrateDB.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +- name: Migrate the fleet database + command: /usr/local/bin/fleet prepare db --config=/etc/fleet/fleet_config.yml --no-prompt diff --git a/osquery/fleetRegisterAdmin.yml b/osquery/fleetRegisterAdmin.yml new file mode 100644 index 00000000..b8a22181 --- /dev/null +++ b/osquery/fleetRegisterAdmin.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: check to see if we have alredy registered fleetctl admin + stat: + path: ~/.fleet/config + register: fleet_config_set + +- name: set fleetctl default context + command: fleetctl config set --address https://localhost:{{ kolide_fleet_port }} --tls-skip-verify + when: fleet_config_set.stat.exists == false + +- name: register admin account + command: /usr/local/bin/fleetctl setup --email {{ kolide_fleet_admin_email }} --password {{ kolide_fleet_admin_password }} + register: fleet_register_admin + ignore_errors: true # ignore errors as we might have already set this it will be caought later + +- name: login admin account + command: /usr/local/bin/fleetctl login --email {{ kolide_fleet_admin_email }} --password {{ kolide_fleet_admin_password }} + when: fleet_register_admin['stderr'] == "Kolide Fleet has already been setup" diff --git a/osquery/fleetRequirements.yml b/osquery/fleetRequirements.yml new file mode 100644 index 00000000..9eb0f038 --- /dev/null +++ b/osquery/fleetRequirements.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#- name: Install prerequisites +# apt: name= {{item}} update_cache=yes +# with_items: +# - apt-transport-https +# - ca-certificates +# - curl +# - software-properties-common +- name: Run the systemd service role + include_role: + name: redis + private: true +- name: create fleet dir + file: + path: /etc/fleet/ssl + state: directory + +- name: Debug fleet_config + debug: + var: hostvars[groups['mariadb'][0]]['ansible_host'] + verbosity: 2 + +- name: Drop fleet conf file + template: + src: templates/fleet_config.yml.j2 + dest: /etc/fleet/fleet_config.yml diff --git a/osquery/fleetSSL.yml b/osquery/fleetSSL.yml new file mode 100644 index 00000000..42a1c43e --- /dev/null +++ b/osquery/fleetSSL.yml @@ -0,0 +1,27 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Ensure the private ssl directory exists + file: + dest: "/etc/ssl/private" + state: "directory" + tags: + - fleet-ssl + +- include_tasks: fleetSSLselfSigned.yml + when: + - kolide_fleet_user_ssl_cert is not defined or kolide_fleet_user_ssl_key is not defined + +- include_tasks: fleetSSLuserProvided.yml diff --git a/osquery/fleetSSLdistribute.yml b/osquery/fleetSSLdistribute.yml new file mode 100644 index 00000000..ce5ae8bc --- /dev/null +++ b/osquery/fleetSSLdistribute.yml @@ -0,0 +1,30 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Distribute self signed ssl key + copy: + dest: "{{ kolide_fleet_ssl_key }}" + content: "{{ hostvars[groups['fleet'][0]]['kolide_fleet_ssl_key_fact'] | b64decode }}" + mode: "0640" + tags: + - fleet-ssl + +- name: Distribute self signed ssl cert + copy: + dest: "{{ kolide_fleet_ssl_cert }}" + content: "{{ hostvars[groups['fleet'][0]]['kolide_fleet_ssl_cert_fact'] | b64decode }}" + mode: "0640" + tags: + - fleet-ssl diff --git a/osquery/fleetSSLkeyCreate.yml b/osquery/fleetSSLkeyCreate.yml new file mode 100644 index 00000000..302e37b6 --- /dev/null +++ b/osquery/fleetSSLkeyCreate.yml @@ -0,0 +1,37 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Remove self signed certs and keys for regen + file: + dest: "{{ item }}" + state: "absent" + with_items: + - "{{ kolide_fleet_ssl_pem }}" + - "{{ kolide_fleet_ssl_key }}" + - "{{ kolide_fleet_ssl_cert }}" + tags: + - fleet-ssl + +- name: Create self-signed ssl cert + command: > + openssl req -new -nodes -sha256 -x509 -subj + "{{ kolide_fleet_ssl_self_signed_subject }}" + -days 3650 + -keyout {{ kolide_fleet_ssl_key }} + -out {{ kolide_fleet_ssl_cert }} + -extensions v3_ca + creates={{ kolide_fleet_ssl_cert }} + tags: + - kolide_fleet-ssl diff --git a/osquery/fleetSSLselfSigned.yml b/osquery/fleetSSLselfSigned.yml new file mode 100644 index 00000000..21a68439 --- /dev/null +++ b/osquery/fleetSSLselfSigned.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include_tasks: fleetSSLkeyCreate.yml +- include_tasks: fleetSSLstore.yml +- include_tasks: fleetSSLdistribute.yml diff --git a/osquery/fleetSSLstore.yml b/osquery/fleetSSLstore.yml new file mode 100644 index 00000000..0dde5771 --- /dev/null +++ b/osquery/fleetSSLstore.yml @@ -0,0 +1,29 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Store ssl cert + slurp: + src: "{{ kolide_fleet_ssl_cert }}" + register: _kolide_fleet_ssl_cert + +- name: Store ssl key + slurp: + src: "{{ kolide_fleet_ssl_key }}" + register: _kolide_fleet_ssl_key + +- name: Register a fact for the cert and key + set_fact: + kolide_fleet_ssl_cert_fact: "{{ _kolide_fleet_ssl_cert.content }}" + kolide_fleet_ssl_key_fact: "{{ _kolide_fleet_ssl_key.content }}" diff --git a/osquery/fleetSSLuserProvided.yml b/osquery/fleetSSLuserProvided.yml new file mode 100644 index 00000000..5f8350ec --- /dev/null +++ b/osquery/fleetSSLuserProvided.yml @@ -0,0 +1,47 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Drop user provided ssl cert + copy: + src: "{{ kolide_fleet_user_ssl_cert }}" + dest: "{{ kolide_fleet_ssl_cert }}" + owner: "root" + group: "root" + mode: "0644" + when: kolide_fleet_user_ssl_cert is defined + tags: + - fleet-ssl + +- name: Drop user provided ssl key + copy: + src: "{{ kolide_fleet_user_ssl_key }}" + dest: "{{ kolide_fleet_ssl_key }}" + owner: "root" + group: "root" + mode: "0640" + when: kolide_fleet_user_ssl_key is defined + tags: + - fleet-ssl + +- name: Drop user provided ssl CA cert + copy: + src: "{{ kolide_fleet_user_ssl_ca_cert }}" + dest: "{{ kolide_fleet_ssl_ca_cert }}" + owner: "root" + group: "root" + mode: "0644" + when: kolide_fleet_user_ssl_ca_cert is defined + tags: + - fleet-ssl diff --git a/osquery/fleetServerInstall.yml b/osquery/fleetServerInstall.yml new file mode 100644 index 00000000..c61cedc2 --- /dev/null +++ b/osquery/fleetServerInstall.yml @@ -0,0 +1,50 @@ +--- +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install prerequisites + apt: name={{ item }} update_cache=yes + with_items: + - unzip + - ca-certificates + - apt-transport-https + - software-properties-common + - curl +- name: Run the systemd service role + include_role: + name: redis + private: true + +- name: create fleet dir + file: + path: '/tmp/fleet_{{ kolide_fleet_version }}' + state: directory + +- name: Unarchive Fleet binaries + unarchive: + src: '{{ kolide_fleet_url }}/{{ kolide_fleet_version }}/fleet_{{ kolide_fleet_version }}.zip' + dest: '/tmp/fleet_{{ kolide_fleet_version}}/' + remote_src: yes + +- name: Copy unarchived binaries + copy: + src: '/tmp/fleet_{{ kolide_fleet_version }}/linux/{{ item }}' + dest: '/usr/local/bin/' + mode: '0755' + owner: 'root' + group: 'root' + remote_src: yes + with_items: + - 'fleet' + - 'fleetctl' diff --git a/osquery/fleetService.yml b/osquery/fleetService.yml new file mode 100644 index 00000000..ea143369 --- /dev/null +++ b/osquery/fleetService.yml @@ -0,0 +1,31 @@ +--- +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Run the systemd service role + include_role: + name: systemd_service + private: true + vars: + systemd_service_restart_changed: false + systemd_services: + - service_name: "kolide-fleet" + execstarts: + - /usr/local/bin/fleet serve --config=/etc/fleet/fleet_config.yml + config_overrides: + Unit: + Wants: network-online.target + Requires: redis-server.service + tags: + - server-install diff --git a/osquery/fleetStartService.yml b/osquery/fleetStartService.yml new file mode 100644 index 00000000..133f1fb3 --- /dev/null +++ b/osquery/fleetStartService.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +- name: Start redis fleet kolide service - initd + service: + name: redis + enabled: yes + state: restarted + when: + - ansible_service_mgr == "init" + tags: + - config +- name: Start fleet kolide service - initd + service: + name: redis + enabled: yes + state: restarted + when: + - ansible_service_mgr == "init" + tags: + - config + +- name: Enable and restart kolide - systemd + systemd: + name: "kolide-fleet" + enabled: true + state: restarted + when: + - ansible_service_mgr == "systemd" + tags: + - config diff --git a/osquery/haproxy.example b/osquery/haproxy.example new file mode 100644 index 00000000..3020119a --- /dev/null +++ b/osquery/haproxy.example @@ -0,0 +1,9 @@ +haproxy_extra_services: + - service: + haproxy_service_name: kolide-fleet + haproxy_ssl: False + haproxy_backend_nodes: "{{ groups['fleet'] | default([]) }}" # Fleet nodes + haproxy_port: 8443 # This is set using the "kolide_fleet_port" variable + haproxy_check_port: 443 # This is set using the "kolide_fleet_port" variable + haproxy_backend_port: 443 # This is set using the "kolide_fleet_port" variable + haproxy_balance_type: tcp diff --git a/osquery/installKolideFleet.yml b/osquery/installKolideFleet.yml new file mode 100644 index 00000000..9f77fe34 --- /dev/null +++ b/osquery/installKolideFleet.yml @@ -0,0 +1,42 @@ +--- +- name: Install Kolide Fleet + hosts: "fleet" + become: true + + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + gather_facts: "{{ osa_gather_facts | default(True) }}" + + tasks: + # install SSL certs + - include_tasks: fleetSSL.yml + + # install software requirements + - include_tasks: fleetRequirements.yml + + # install kolide fleet server + - include_tasks: fleetServerInstall.yml + + # drop the configuration + - include_tasks: fleetConfig.yml + + # add files for systemd + - include_tasks: fleetService.yml + when: + - ansible_service_mgr == "systemd" + + # migrate the database + - include_tasks: fleetMigrateDB.yml + run_once: true + + # start fleet via systemd + - include_tasks: fleetStartService.yml + + # configure kolide fleet & set admin account + - include_tasks: fleetRegisterAdmin.yml + + # retrieve and set enrollment token + - include_tasks: fleetGetEnrollmentToken.yml diff --git a/osquery/installMariaDB.yml b/osquery/installMariaDB.yml new file mode 100644 index 00000000..3351182a --- /dev/null +++ b/osquery/installMariaDB.yml @@ -0,0 +1,29 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install MariaDB server + hosts: mariadb + become: true + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + gather_facts: "{{ osa_gather_facts | default(True) }}" + serial: 1 + user: root + + roles: + - role: "mariadb" diff --git a/osquery/installOsquery.yml b/osquery/installOsquery.yml index ef0ae3a1..51acc307 100644 --- a/osquery/installOsquery.yml +++ b/osquery/installOsquery.yml @@ -1,5 +1,5 @@ --- -# Copyright 2016, Rackspace US, Inc. +# Copyright 2018, Rackspace US, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +14,37 @@ # limitations under the License. - name: Install osquery - hosts: osquery_hosts + hosts: hosts become: true + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + gather_facts: "{{ osa_gather_facts | default(True) }}" + + pre_tasks: + - name: create osquery dir + file: + path: /etc/osquery/ssl + state: directory + + - name: Debug fleet_config + debug: + var: hostvars[groups['fleet'][0]]['ansible_host'] + verbosity: 2 + + - name: Distribute self signed ssl cert + copy: + dest: "{{ kolide_fleet_ssl_cert }}" + content: "{{ hostvars[groups['fleet'][0]]['kolide_fleet_ssl_cert_fact'] | b64decode }}" + mode: "0640" + + - name: write enroll secret + copy: + dest: "{{ osquery_enroll_secret_dir }}" + content: "{{ hostvars[groups['fleet'][0]]['kolide_fleet_enroll_secret'] }}" + mode: "0640" roles: - - osquery + - role: "osquery" diff --git a/osquery/inventory.example.yml b/osquery/inventory.example.yml index 7b3006e7..f087661e 100644 --- a/osquery/inventory.example.yml +++ b/osquery/inventory.example.yml @@ -1,5 +1,32 @@ --- -# This is the location where osquery(s) will live -osquery_hosts: + +################################## ALL HOSTS ################################## +all: + hosts: + # Local host + localhost: + ansible_connection: local + +################################## REQUIRED ################################### + logging01: + ansible_host: 172.16.27.100 + ansible_user: root + + vars: {} + + +################################### GROUPS #################################### + +# The hosts group is used to target physical host machines. Enter all physical +# host machines here. +hosts: + hosts: + logging01: + +# This is the location where fleet(s) will live +fleet: + hosts: + logging01: +osquery: hosts: all: diff --git a/osquery/readme.rst b/osquery/readme.rst index 6da9d8df..14dd21a5 100644 --- a/osquery/readme.rst +++ b/osquery/readme.rst @@ -1,42 +1,187 @@ -Install OSQuery -############### +Install OSQuery and Kolide fleet +################################ :tags: openstack, ansible +Table of Contents +================= + + * [About this repository](#about-this-repository) + * [OpenStack-Ansible Integration](#openstack-ansible-integration) + * [TODO](#todo) + + About this repository --------------------- This set of playbooks will deploy osquery. If this is being deployed as part of an OpenStack all of the inventory needs will be provided for. -There multiple ways to aggregate the data. At this point this repo does not provide -one of said methods. It is currently intended to be utilized with the `elk_metrics_6x`. - -It is the intention that at a later point to the ability to configure osquery to report -to a centralized place like (kolide/fleet)[https://github.com/kolide/fleet], (zentral)[https://github.com/zentralopensource/zentral], -etc. **These playbooks require Ansible 2.4+.** -Deployment Process ------------------- +Highlevel overview of Osquery & Kolide Fleet infrastructure these playbooks will +build and operate against. -Clone the osa ops repo +.. image:: assets/place-holder.svg + :scale: 50 % + :alt: Osquery & Kolide Fleet Architecture Diagram + :align: center + +OpenStack-Ansible Integration +----------------------------- + +These playbooks can be used as standalone inventory or as an integrated part of +an OpenStack-Ansible deployment. For a simple example of standalone inventory, +see ``inventory.example.yml``. + +Setup | system configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Clone the osquery-osa repo .. code-block:: bash cd /opt git clone https://github.com/openstack/openstack-ansible-ops -Clone the osquery role - -.. code-block:: bash - - cd /opt - git clone https://github.com/devx/ansible-osquery.git /etc/ansible/roles/osquery - -install osquery +Copy the env.d file into place .. code-block:: bash cd /opt/openstack-ansible-ops/osquery - openstack-ansible installOsquery.yml + cp env.d/fleet.yml /etc/openstack_deploy/env.d/ + +Copy the conf.d file into place + +.. code-block:: bash + + cp conf.d/fleet.yml /etc/openstack_deploy/conf.d/ + +In **fleet.yml**, list your logging hosts under fleet-logstash_hosts to create +the kolide fleet cluster in multiple containers and one logging host under +`fleet_hosts` to create the fleet container + +.. code-block:: bash + + vi /etc/openstack_deploy/conf.d/fleet.yml + +Create the containers + +.. code-block:: bash + + cd /opt/openstack-ansible/playbooks + openstack-ansible lxc-containers-create.yml -e 'container_group=fleet' + + +Update the `/etc/hosts` file + +.. code-block:: bash + + cd /opt/openstack-ansible/playbooks + openstack-ansible openstack-hosts-setup.yml -e 'container_group=fleet' + + + +Create an haproxy entry for kolide-fleet service 8443 + +.. code-block:: bash + + cd /opt/openstack-ansible-ops/osquery + cat haproxy.example >> /etc/openstack_deploy/user_variables.yml + + cd /opt/openstack-ansible/playbooks/ + openstack-ansible haproxy-install.yml --tags=haproxy-service-config + + +Deploying | Installing with embedded Ansible +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If this is being executed on a system that already has Ansible installed but is +incompatible with these playbooks the script `bootstrap-embedded-ansible.sh` can +be sourced to grab an embedded version of Ansible prior to executing the +playbooks. + +.. code-block:: bash + + source bootstrap-embedded-ansible.sh + + +Deploying | Manually resolving the dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This playbook has external role dependencies. If Ansible is not installed with +the `bootstrap-ansible.sh` script these dependencies can be resolved with the +``ansible-galaxy`` command and the ``ansible-role-requirements.yml`` file. + +* Example galaxy execution + +.. code-block:: bash + + ansible-galaxy install -r ansible-role-requirements.yml + + +In the even that some of the modules are alread installed execute the following + +.. code-block:: bash + + ansible-galaxy install -r ansible-role-requirements.yml --ignore-errors + + +Once the dependencies are set make sure to set the action plugin path to the +location of the config_template action directory. This can be done using the +environment variable `ANSIBLE_ACTION_PLUGINS` or through the use of an +`ansible.cfg` file. + + +Deploying | The environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Install master/data Fleet nodes on the elastic-logstash containers, +deploy logstash, deploy Kibana, and then deploy all of the service beats. + +.. code-block:: bashG + + cd /opt/openstack-ansible-ops/osquery + ansible-playbook site.yml $USER_VARS + + +* The `openstack-ansible` command can be used if the version of ansible on the + system is greater than **2.5**. This will automatically pick up the necessary + group_vars for hosts in an OSA deployment. + +* If required add ``-e@/opt/openstack-ansible/inventory/group_vars/all/all.yml`` + to import sufficient OSA group variables to define the OpenStack release. + Journalbeat will then deploy onto all hosts/containers for releases prior to + Rocky, and hosts only for Rocky onwards. If the variable ``openstack_release`` + is undefined the default behaviour is to deploy Journalbeat to hosts only. + +* Alternatively if using the embedded ansible, create a symlink to include all + of the OSA group_vars. These are not available by default with the embedded + ansible and can be symlinked into the ops repo. + +.. code-block:: bash + + ln -s /opt/openstack-ansible/inventory/group_vars /opt/openstack-ansible-ops/osquery/group_vars + + +The individual playbooks found within this repository can be independently run +at anytime. + +Architecture | Data flow +^^^^^^^^^^^^^^^^^^^^^^^^ + +This diagram outlines the data flow from within an Elastic-Stack deployment. + +.. image:: assets/place-holder.svg + :scale: 50 % + :alt: Kolide & Osquery Data Flow Diagram + :align: center + +TODO +---- +The following is a list of open items. + - [ ] Test Redhat familly Operating Systems + - [ ] missing mariadb cluster (should all work needs additional vars) + - [ ] use haproxy instead of the kolide fleet server ip + - [ ] add/update tags + - [ ] add testing diff --git a/osquery/site.yml b/osquery/site.yml new file mode 100644 index 00000000..b92fa751 --- /dev/null +++ b/osquery/site.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- import_playbook: installMariaDB.yml +- import_playbook: installKolideFleet.yml +- import_playbook: installOsquery.yml diff --git a/osquery/templates/fleet_config.yml.j2 b/osquery/templates/fleet_config.yml.j2 new file mode 100644 index 00000000..d39100dc --- /dev/null +++ b/osquery/templates/fleet_config.yml.j2 @@ -0,0 +1,15 @@ +mysql: + address: {{ hostvars[groups['mariadb'][0]]['ansible_host'] }}:3306 + database: {{ kolide_fleet_db_name }} + username: {{ kolide_fleet_db_user }} + password: {{ kolide_fleet_db_password }} +redis: + address: localhost:6379 +auth: + jwt_key: {{ kolide_fleet_jwt_key }} +server: + address: {{ kolide_fleet_address }} + cert: {{ kolide_fleet_ssl_cert }} + key: {{ kolide_fleet_ssl_key }} +logging: + json: true diff --git a/osquery/templates/fleet_initd.yml.j2 b/osquery/templates/fleet_initd.yml.j2 new file mode 100644 index 00000000..562824f2 --- /dev/null +++ b/osquery/templates/fleet_initd.yml.j2 @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +############### +# SysV Init Information +# description: docker daemon. +### BEGIN INIT INFO +# Provides: me +# Required-Start: +# Required-Stop: +# Default-Start: 2 3 5 +# Default-Stop: 0 1 6 +# Short-Description: manage docker daemon +# Description: docker daemon +### END INIT INFO + + +HOME=path/to/code +EXEC=docker +NAME="fleet_service" + +LABEL={{ kolide_fleet_version }} +PORT=8412 + +case "$1" in + start) + RUNNING=`docker inspect -f '{{.State.Running}}' ${NAME}` + + if [[ ${RUNNING} == "true" ]]; then + # + # No need to start it + # + echo "Process is running" + exit 1; + if + echo "Starting container: '$NAME'..." + docker run --restart=always -v /etc/fleet:/etc/fleet --name=fleet_service kolide/fleet:{{ kolide_fleet_version }} -p 8412:8412 --config=/etc/fleet/fleet_config.yml + ;; + stop) + echo "Stopping $NAME..." + + docker stop -t 2 ${NAME} &> /dev/null + + if [[ 0 != $? ]]; then + echo "Could not stop container" + exit 4; + fi + ;; + status) + RUNNING=`docker inspect -f '{{.State.Running}}' ${NAME}` + + if [[ "true" == ${RUNNING} ]]; then + echo "$NAME is running" + else + echo "$NAME is not running" + fi + ;; + inspect) + docker inspect ${NAME} + ;; + logs) + docker logs ${NAME} + ;; + restart) + $0 stop + $0 start + ;; + *) + echo "Please use start, stop, restart, inspect or status as first argument" + ;; +esac diff --git a/osquery/templates/fleet_service.yml.j2 b/osquery/templates/fleet_service.yml.j2 new file mode 100644 index 00000000..1c94f6c9 --- /dev/null +++ b/osquery/templates/fleet_service.yml.j2 @@ -0,0 +1,17 @@ +[Unit] +Requires=redis.service +After=redis.service + +[Service] +TimeoutStartSec=0 +ExecStartPre=-/usr/bin/docker pull kolide/fleet:{{ kolide_fleet_version }} +ExecStartPre=-/usr/bin/docker kill fleet_service +ExecStartPre=-/usr/bin/docker rm fleet_service +ExecStart=/usr/bin/docker run -v /etc/fleet:/etc/fleet --name fleet_service kolide/fleet:{{ kolide_fleet_version }} --config=/etc/fleet/fleet_config.yml + +ExecStop=-/usr/bin/docker stop fleet_fleet + +ExecReload=/usr/bin/docker restart fleet_service + +[Install] +WantedBy=multi-user.target diff --git a/osquery/templates/redis_initd.yml.j2 b/osquery/templates/redis_initd.yml.j2 new file mode 100644 index 00000000..47279c75 --- /dev/null +++ b/osquery/templates/redis_initd.yml.j2 @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +############### +# SysV Init Information +# description: docker daemon. +### BEGIN INIT INFO +# Provides: me +# Required-Start: +# Required-Stop: +# Default-Start: 2 3 5 +# Default-Stop: 0 1 6 +# Short-Description: manage docker daemon +# Description: docker daemon +### END INIT INFO + + +HOME=path/to/code +EXEC=docker +NAME="redis_service" + +LABEL={{ kolide_fleet_version }} + +case "$1" in + start) + RUNNING=`docker inspect -f '{{.State.Running}}' ${NAME}` + + if [[ ${RUNNING} == "true" ]]; then + # + # No need to start it + # + echo "Process is running" + exit 1; + if + echo "Starting container: '$NAME'..." + docker run -d --restart=always --name=redis_service redis:4-alpine + ;; + stop) + echo "Stopping $NAME..." + + docker stop -t 2 ${NAME} &> /dev/null + + if [[ 0 != $? ]]; then + echo "Could not stop container" + exit 4; + fi + ;; + status) + RUNNING=`docker inspect -f '{{.State.Running}}' ${NAME}` + + if [[ "true" == ${RUNNING} ]]; then + echo "$NAME is running" + else + echo "$NAME is not running" + fi + ;; + inspect) + docker inspect ${NAME} + ;; + logs) + docker logs ${NAME} + ;; + restart) + $0 stop + $0 start + ;; + *) + echo "Please use start, stop, restart, inspect or status as first argument" + ;; +esac diff --git a/osquery/templates/redis_service.yml.j2 b/osquery/templates/redis_service.yml.j2 new file mode 100644 index 00000000..154de212 --- /dev/null +++ b/osquery/templates/redis_service.yml.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=Redis Container +After=docker.service +Requires=docker.service + +[Service] +Type=simple +restart=always +TimeoutStartSec=120 +ExecStartPre=-/usr/bin/docker pull redis:4-alpine +ExecStartPre=-/usr/bin/docker kill redis_service +ExecStartPre=-/usr/bin/docker rm redis_service +ExecStart=/usr/bin/docker run --name redis_service redis:alpine + +ExecStop=/usr/bin/docker stop redis_service + +[Install] +WantedBy=multi-user.target diff --git a/osquery/vars/variables.yml b/osquery/vars/variables.yml new file mode 100644 index 00000000..c6eaa1c2 --- /dev/null +++ b/osquery/vars/variables.yml @@ -0,0 +1,61 @@ +kolide_fleet_enable: true +kolide_fleet_cluster: false + +# Kolide Fleet vars +kolide_fleet_db_name: fleet +kolide_fleet_db_user: fleet +kolide_fleet_db_password: fleetSecrete + +kolide_fleet_port: "443" +kolide_fleet_address: "0.0.0.0:{{ kolide_fleet_port }}" +kolide_fleet_version: "2.0.0-rc3" +kolide_fleet_url: "https://github.com/kolide/fleet/releases/download" + +kolide_fleet_admin_email: admin@openstack.org +kolide_fleet_admin_password: AdminSecrete + +kolide_fleet_ssl_cert: /etc/ssl/certs/fleet.cert +kolide_fleet_ssl_key: /etc/ssl/private/fleet.key +kolide_fleet_ssl_pem: /etc/ssl/private/fleet.pem +kolide_fleet_ssl_ca_cert: /etc/ssl/certs/fleet-ca.pem +kolide_fleet_ssl_self_signed_subject: "/C=US/ST=Texas/L=San Antonio/O=IT/CN={{ external_lb_vip_address }}/subjectAltName=IP.1={{ external_lb_vip_address }}/subjectAltName=IP.2={{ ansible_host }}}/subjectAltName=IP.3=localhost" + +kolide_fleet_ssl_protocol: "{{ ssl_protocol | default('ALL -SSLv2 -SSLv3') }}" +kolide_fleet_ssl_cipher_suite: "{{ ssl_cipher_suite | default('ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS') }}" + +#kolide_fleet_osquery_enroll_secret: "{{ kolide_fleet_enroll_secret }}" + +# Osquery vars +osquery_enroll_secret_dir: /etc/osquery/osquery_enroll_secret + +osquery_flags: + - "--tls_server_certs={{ kolide_fleet_ssl_cert }}" + - "--tls_hostname={{ hostvars[groups['fleet'][0]]['ansible_host'] }}:{{ kolide_fleet_port }}" + - "--host_identifier=hostname" + - "--enroll_tls_endpoint=/api/v1/osquery/enroll" + - "--config_plugin=tls" + - "--config_tls_endpoint=/api/v1/osquery/config" + - "--config_tls_refresh=10" + - "--disable_distributed=false" + - "--distributed_plugin=tls" + - "--distributed_interval=10" + - "--distributed_tls_max_attempts=3" + - "--distributed_tls_read_endpoint=/api/v1/osquery/distributed/read" + - "--distributed_tls_write_endpoint=/api/v1/osquery/distributed/write" + - "--logger_plugin=tls" + - "--logger_tls_endpoint=/api/v1/osquery/log" + - "--logger_tls_period=10" + - "--enroll_secret_path={{ osquery_enroll_secret_dir }}" + +# MariaDB/Gallera Variables +mariadb_root_password: fleetSecrete +mariadb_bind_address: "0.0.0.0" +mariadb_root_remote: 1 +mariadb_databases: + - name: "{{ kolide_fleet_db_name }}" + +mariadb_users: + - name: "{{ kolide_fleet_db_user }}" + password: "{{ kolide_fleet_db_password }}" + priv: " {{ kolide_fleet_db_name }}.*:ALL" + host: "%"