From fbf309baee6cf95f5c194c7ab432448c201edbec Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Thu, 27 Feb 2020 12:23:21 -0500 Subject: [PATCH] Add common logging with filebeat This commit 1. Provides a playbook to install the filebeat agent on all undercloud/overcloud nodes 2. Provides another playbook that adds the browbeat uuid to the filebeat config file and starts filebeat during browbeat run 3. Corresponding changes in browbeat.py and browbeat/tools.py to run the playbook to insert custom browbeat uuid in the filebeat configuration. Change-Id: Idd2efaf931f4ff581db715a04adef738f81d281c --- ansible/common_logging/browbeat_logging.yml | 21 ++++++ ansible/common_logging/install_logging.yml | 30 ++++++++ .../roles/browbeat_logging/handlers/main.yml | 13 ++++ .../roles/browbeat_logging/tasks/main.yml | 15 ++++ .../roles/filebeat_setup/tasks/main.yml | 27 +++++++ .../filebeat_setup/templates/filebeat.yml.j2 | 44 +++++++++++ .../roles/filebeat_setup/vars/16.yml | 75 +++++++++++++++++++ ansible/install/group_vars/all.yml | 15 +++- browbeat-complete.yaml | 3 + browbeat-config.yaml | 3 + browbeat.py | 19 ++++- browbeat/schema/browbeat.yml | 10 +++ browbeat/tools.py | 9 +++ conf/browbeat-glance-complete.yaml | 3 + conf/browbeat-keystone-complete.yaml | 3 + conf/browbeat-keystone-minimal.yaml | 3 + conf/browbeat-workloads.yml | 3 + conf/ceilometer-minimal.yaml | 3 + conf/ceilometer-persist-response-timing.yaml | 3 + conf/gnocchi-minimal.yaml | 3 + conf/quickstart.yml | 3 + ...nova-neutron-gnocchi-10k-1000-per-30m.yaml | 3 + ...y-nova-neutron-gnocchi-1k-100-per-30m.yaml | 3 + ...y-nova-neutron-gnocchi-1k-500-per-30m.yaml | 3 + ...y-nova-neutron-gnocchi-5k-500-per-30m.yaml | 3 + ...va-persist-instances-with-network-fip.yaml | 3 + ...ist-instances-with-network-volume-fip.yaml | 3 + ...persist-instances-with-network-volume.yaml | 3 + ...y-nova-persist-instances-with-network.yaml | 3 + ...ry-nova-persist-instances-with-volume.yaml | 3 + conf/telemetry-nova-persist-instances.yaml | 3 + doc/source/installation.rst | 25 +++++-- doc/source/usage.rst | 18 +++++ tests/data/invalid_browbeat.yml | 1 + tests/data/invalid_browbeat_workload.yml | 1 + tests/data/valid_browbeat.yml | 3 + 36 files changed, 374 insertions(+), 12 deletions(-) create mode 100644 ansible/common_logging/browbeat_logging.yml create mode 100644 ansible/common_logging/install_logging.yml create mode 100644 ansible/common_logging/roles/browbeat_logging/handlers/main.yml create mode 100644 ansible/common_logging/roles/browbeat_logging/tasks/main.yml create mode 100644 ansible/common_logging/roles/filebeat_setup/tasks/main.yml create mode 100644 ansible/common_logging/roles/filebeat_setup/templates/filebeat.yml.j2 create mode 100644 ansible/common_logging/roles/filebeat_setup/vars/16.yml diff --git a/ansible/common_logging/browbeat_logging.yml b/ansible/common_logging/browbeat_logging.yml new file mode 100644 index 000000000..6b881ed91 --- /dev/null +++ b/ansible/common_logging/browbeat_logging.yml @@ -0,0 +1,21 @@ +- hosts: undercloud + remote_user: "{{ local_remote_user }}" + vars_files: + - ../install/group_vars/all.yml + roles: + - { role: browbeat_logging } + +- hosts: controller + remote_user: "{{ host_remote_user }}" + vars_files: + - ../install/group_vars/all.yml + roles: + - { role: browbeat_logging } + +- hosts: compute + remote_user: "{{ host_remote_user }}" + vars_files: + - ../install/group_vars/all.yml + roles: + - { role: browbeat_logging } + diff --git a/ansible/common_logging/install_logging.yml b/ansible/common_logging/install_logging.yml new file mode 100644 index 000000000..5f6664c8a --- /dev/null +++ b/ansible/common_logging/install_logging.yml @@ -0,0 +1,30 @@ +- hosts: undercloud + remote_user: "{{ local_remote_user }}" + vars_files: + - ../install/group_vars/all.yml + vars: + config_type: undercloud + roles: + - { role: osp_version } + - { role: filebeat_setup } + +- hosts: controller + remote_user: "{{ host_remote_user }}" + vars_files: + - ../install/group_vars/all.yml + vars: + config_type: controller + roles: + - { role: osp_version } + - { role: filebeat_setup } + +- hosts: compute + remote_user: "{{ host_remote_user }}" + vars_files: + - ../install/group_vars/all.yml + vars: + config_type: compute + roles: + - { role: osp_version } + - { role: filebeat_setup } + diff --git a/ansible/common_logging/roles/browbeat_logging/handlers/main.yml b/ansible/common_logging/roles/browbeat_logging/handlers/main.yml new file mode 100644 index 000000000..1b294ce92 --- /dev/null +++ b/ansible/common_logging/roles/browbeat_logging/handlers/main.yml @@ -0,0 +1,13 @@ +- name: restart filebeat + service: + name: filebeat + state: restarted + become: true + +- name: stop filebeat + service: + name: filebeat + state: stopped + become: true + when: not logging_status|bool + diff --git a/ansible/common_logging/roles/browbeat_logging/tasks/main.yml b/ansible/common_logging/roles/browbeat_logging/tasks/main.yml new file mode 100644 index 000000000..c1c18cd55 --- /dev/null +++ b/ansible/common_logging/roles/browbeat_logging/tasks/main.yml @@ -0,0 +1,15 @@ +- name: check if filebeat config is present + stat: + path: /etc/filebeat/filebeat.yml + register: filebeat_config + +- name: insert browbeat uuid + lineinfile: + path: /etc/filebeat/filebeat.yml + regexp: '^\s+browbeat_uuid' + line: ' browbeat_uuid: "{{browbeat_uuid}}"' + become: true + when: filebeat_config.stat.exists + notify: + - restart filebeat + - stop filebeat diff --git a/ansible/common_logging/roles/filebeat_setup/tasks/main.yml b/ansible/common_logging/roles/filebeat_setup/tasks/main.yml new file mode 100644 index 000000000..0451a12c0 --- /dev/null +++ b/ansible/common_logging/roles/filebeat_setup/tasks/main.yml @@ -0,0 +1,27 @@ +- name: Install filebeat RPM + yum: + name: "{{ filebeat_url }}" + state: present + become: true + +- name: include vars + include_vars: + file: vars/{{rhosp_major}}.yml + +- name: Template the filebeat configuration file + template: + src: filebeat.yml.j2 + dest: /etc/filebeat/filebeat.yml + owner: root + group: root + mode: 0644 + become: true + +- name: Start and enable filebeat + service: + name: filebeat + state: stopped + enabled: no + become: true + when: start_filebeat is defined and start_filebeat|bool + diff --git a/ansible/common_logging/roles/filebeat_setup/templates/filebeat.yml.j2 b/ansible/common_logging/roles/filebeat_setup/templates/filebeat.yml.j2 new file mode 100644 index 000000000..b6db5fe2c --- /dev/null +++ b/ansible/common_logging/roles/filebeat_setup/templates/filebeat.yml.j2 @@ -0,0 +1,44 @@ +#=========================== Filebeat inputs ============================= + +filebeat.inputs: + +- type: log + + enabled: true + + # Paths that should be crawled and fetched. Glob based paths. + paths: +{% for log in filebeat_logs[config_type] %} + - {{ log }} +{% endfor %} + + + fields: + cloud_name: {{ cloud_prefix }} + browbeat_uuid: "" + +setup.ilm.enabled: false +setup.template.name: "browbeat-logs" +setup.template.pattern: "browbeat-logs-*" +#============================= Filebeat modules =============================== + +filebeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s + +#==================== Elasticsearch template setting ========================== + +setup.template.settings: + index.number_of_shards: 1 + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["{{ es_ip }}:{{ es_local_port }}"] + index: "browbeat-logs-{{ cloud_prefix }}" diff --git a/ansible/common_logging/roles/filebeat_setup/vars/16.yml b/ansible/common_logging/roles/filebeat_setup/vars/16.yml new file mode 100644 index 000000000..b584d99a9 --- /dev/null +++ b/ansible/common_logging/roles/filebeat_setup/vars/16.yml @@ -0,0 +1,75 @@ +--- +filebeat_logs: + undercloud: + - /var/log/containers/nova/nova-api.log + - /var/log/containers/nova/nova-compute.log + - /var/log/containers/nova/nova-conductor.log + - /var/log/containers/nova/nova-scheduler.log + - /var/log/containers/glance/api.log + - /var/log/containers/heat/heat-engine.log + - /var/log/containers/heat/heat_api.log + - /var/log/containers/ironic/ironic-conductor.log + - /var/log/containers/ironic-inspector/ironic-inspector.log + - /var/log/containers/keystone/keystone.log + - /var/log/containers/mistral/api.log + - /var/log/containers/mistral/engine.log + - /var/log/containers/mistral/event-engine.log + - /var/log/containers/mistral/executor.log + - /var/log/containers/mysql/mariadb.log + - /var/log/containers/neutron/dhcp-agent.log + - /var/log/containers/neutron/ironic-neutron-agent.log + - /var/log/containers/neutron/l3-agent.log + - /var/log/containers/neutron/openvswitch-agent.log + - /var/log/containers/neutron/server.log + - /var/log/containers/placement/placement.log + - /var/log/containers/zaqar/zaqar-server.log + - /var/log/containers/httpd/heat-api/heat_api_wsgi_error.log + - /var/log/containers/httpd/ironic-api/ironic_wsgi_error.log + - /var/log/containers/httpd/ironic-pxe/ipxe_vhost_error.log + - /var/log/containers/httpd/keystone/keystone_wsgi_error.log + - /var/log/containers/httpd/nova-api/nova_api_wsgi_error.log + - /var/log/containers/httpd/placement/placement_wsgi_error.log + - /var/log/containers/httpd/zaqar/zaqar_wsgi_error.log + controller: + - /var/log/containers/cinder/cinder-api.log + - /var/log/containers/cinder/cinder-manage.log + - /var/log/containers/cinder/cinder-rowsflush.log + - /var/log/containers/cinder/cinder-scheduler.log + - /var/log/containers/cinder/cinder-volume.log + - /var/log/containers/glance/api.log + - /var/log/containers/heat/heat_api_cfn.log + - /var/log/containers/heat/heat_api.log + - /var/log/containers/heat/heat-engine.log + - /var/log/containers/keystone/keystone.log + - /var/log/containers/mysql/mysqld.log + - /var/log/containers/neutron/server.log + - /var/log/containers/neutron/l3-agent.log + - /var/log/containers/neutron/dhcp-agent.log + - /var/log/containers/neutron/metadata-agent.log + - /var/log/containers/neutron/openvswitch-agent.log + - /var/log/containers/nova/nova-api.log + - /var/log/containers/nova/nova-conductor.log + - /var/log/containers/nova/nova-scheduler.log + - /var/log/containers/nova/nova-metadata-api.log + - /var/log/containers/nova/nova-novncproxy.log + - /var/log/containers/openvswitch/ovn-controller.log + - /var/log/containers/openvswitch/ovn-northd.log + - /var/log/containers/openvswitch/ovsdb-server-nb.log + - /var/log/containers/openvswitch/ovsdb-server-sb.log + - /var/log/containers/placement/placement.log + - /var/log/containers/rabbitmq/rabbit@{{ inventory_hostname }}.log + - /var/log/containers/httpd/heat-api/heat_api_wsgi_error.log + - /var/log/containers/httpd/ironic-api/ironic_wsgi_error.log + - /var/log/containers/httpd/ironic-pxe/ipxe_vhost_error.log + - /var/log/containers/httpd/keystone/keystone_wsgi_error.log + - /var/log/containers/httpd/nova-api/nova_api_wsgi_error.log + - /var/log/containers/httpd/placement/placement_wsgi_error.log + - /var/log/containers/httpd/cinder-api/cinder_wsgi_error.log + - /var/log/containers/httpd/heat-api-cfn/heat_api_cfn_wsgi_error.log + - /var/log/containers/httpd/nova-metadata/nova_metadata_wsgi_error.log + - /var/log/containers/swift/swift.log + compute: + - /var/log/containers/libvirt/libvirtd.log + - /var/log/containers/neutron/ovn-metadata-agent.log + - /var/log/containers/nova/nova-compute.log + - /var/log/containers/openvswitch/ovn-controller.log diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index 9d70febf6..90558fb4e 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -45,6 +45,9 @@ shaker_venv: "{{browbeat_path}}/.shaker-venv" # Shaker version to Install shaker_version: 1.3.2 +# Custom name to use for the cloud +cloud_prefix: openstack + # Configuration items to adjust browbeat results served through httpd browbeat_results_port: 9001 browbeat_results_in_httpd: true @@ -324,7 +327,7 @@ graphite_host: graphite_port: 80 carbon_cache_port: 2003 # Graphite prefix / Cloud name used both with graphite and grafana dashboards -graphite_prefix: +graphite_prefix: "{{cloud_prefix}}" # Graphite username and password for login on the dashboard # credential aren't created when you deploy graphite, use manage.py graphite_username: root @@ -404,7 +407,7 @@ rsyslog_elasticsearch_server: "{{es_ip}}" rsyslog_elasticsearch_port: "{{es_local_port}}" rsyslog_aggregator_server: "{{es_ip}}" rsyslog_aggregator_port: "7894" -rsyslog_cloud_name: "{{graphite_prefix}}" +rsyslog_cloud_name: "{{cloud_prefix}}" disk_backed_rsyslog: false rsyslog_forwarding: true # If true up to 2gb of messages will be logged @@ -415,10 +418,16 @@ rsyslog_forwarding: true # due to utilizing elasticsearch connectors, general # usage may want to disable this option due to security reasons # in which case you should set this to false -es_ip: + +## Elasticsearch options ## +es_ip: 1.1.1.1 es_local_port: 9200 es_listen_external: true elastic5: false + +## Filebeat options ## +filebeat_url: https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.6.0-x86_64.rpm + ### kibana options ### # change this to affect nginx-wrapped htpasswd authentication kibana_user: admin diff --git a/browbeat-complete.yaml b/browbeat-complete.yaml index 740064344..3a2a43e90 100644 --- a/browbeat-complete.yaml +++ b/browbeat-complete.yaml @@ -6,6 +6,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: true @@ -27,6 +28,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/browbeat-config.yaml b/browbeat-config.yaml index fcc2a41bd..a4244ac03 100644 --- a/browbeat-config.yaml +++ b/browbeat-config.yaml @@ -10,6 +10,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -31,6 +32,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/browbeat.py b/browbeat.py index ae1ac4348..522364b71 100755 --- a/browbeat.py +++ b/browbeat.py @@ -161,9 +161,10 @@ def main(): "If you meant 'all' use: './browbeat.py all' or './browbeat.py'") exit(1) + browbeat_uuid = browbeat.elastic.browbeat_uuid result_dir_ts = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S") _logger.info("Browbeat test suite kicked off") - _logger.info("Browbeat UUID: {}".format(browbeat.elastic.browbeat_uuid)) + _logger.info("Browbeat UUID: {}".format(browbeat_uuid)) if _config['elasticsearch']['enabled']: _logger.info("Checking for Metadata") metadata_exists = tools.check_metadata() @@ -176,6 +177,11 @@ def main(): _logger.info("Regathering Metadata") tools.gather_metadata() + if _config['filebeat']['enabled']: + _logger.info("Enabling filebeat for log collection" + " with browbeat_uuid {}".format(browbeat_uuid)) + tools.common_logging(browbeat_uuid, logging_status=True) + _logger.info("Running workload(s): {}".format(','.join(_cli_args.workloads))) # Iteration rerun_type pushes rerun logic down to the workload itself. This allows the workload @@ -194,15 +200,20 @@ def main(): if base.WorkloadBase.failure > 0: _logger.info( - "Browbeat finished with test failures, UUID: {}".format(browbeat.elastic.browbeat_uuid)) + "Browbeat finished with test failures, UUID: {}".format(browbeat_uuid)) sys.exit(1) if base.WorkloadBase.index_failures > 0: _logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}" - .format(browbeat.elastic.browbeat_uuid)) + .format(browbeat_uuid)) sys.exit(2) - _logger.info("Browbeat finished successfully, UUID: {}".format(browbeat.elastic.browbeat_uuid)) + if _config['filebeat']['enabled']: + _logger.info("Disabling filebeat for log collection" + " for browbeat_uuid {}".format(browbeat_uuid)) + tools.common_logging("", logging_status=False) + + _logger.info("Browbeat finished successfully, UUID: {}".format(browbeat_uuid)) sys.exit(0) diff --git a/browbeat/schema/browbeat.yml b/browbeat/schema/browbeat.yml index c39fcbc63..e05bae4e2 100644 --- a/browbeat/schema/browbeat.yml +++ b/browbeat/schema/browbeat.yml @@ -29,6 +29,9 @@ mapping: metadata_playbook: type: str required: True + logging_playbook: + type: str + required: True ssh_config: type: str required: True @@ -78,6 +81,13 @@ mapping: port: type: int required: True + filebeat: + required: True + type: map + mapping: + enabled: + type: bool + required: True rally: required: True type: map diff --git a/browbeat/tools.py b/browbeat/tools.py index cd1692af1..7698a3605 100644 --- a/browbeat/tools.py +++ b/browbeat/tools.py @@ -92,6 +92,15 @@ class Tools(object): self.logger.info("Metadata about cloud has been gathered") return True + def common_logging(self, browbeat_uuid, logging_status): + os.putenv("ANSIBLE_SSH_ARGS", " -F {}".format(self.config['ansible']['ssh_config'])) + ansible_cmd = \ + 'ansible-playbook -i {} {} -e "browbeat_uuid={} logging_status={}"' \ + .format(self.config['ansible']['hosts'], + self.config['ansible']['logging_playbook'], + browbeat_uuid, logging_status) + self.run_cmd(ansible_cmd) + def post_process(self, cli): workloads = {} workloads['shaker'] = re.compile("shaker") diff --git a/conf/browbeat-glance-complete.yaml b/conf/browbeat-glance-complete.yaml index 7387fc333..8f4ea5c12 100644 --- a/conf/browbeat-glance-complete.yaml +++ b/conf/browbeat-glance-complete.yaml @@ -6,6 +6,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -27,6 +28,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/browbeat-keystone-complete.yaml b/conf/browbeat-keystone-complete.yaml index 83325914a..39fe57ba1 100644 --- a/conf/browbeat-keystone-complete.yaml +++ b/conf/browbeat-keystone-complete.yaml @@ -7,6 +7,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -28,6 +29,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/browbeat-keystone-minimal.yaml b/conf/browbeat-keystone-minimal.yaml index 4c45c9aed..50e98c86d 100644 --- a/conf/browbeat-keystone-minimal.yaml +++ b/conf/browbeat-keystone-minimal.yaml @@ -7,6 +7,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -28,6 +29,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/browbeat-workloads.yml b/conf/browbeat-workloads.yml index 48d2b60f9..f7235a91b 100644 --- a/conf/browbeat-workloads.yml +++ b/conf/browbeat-workloads.yml @@ -6,6 +6,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -27,6 +28,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/ceilometer-minimal.yaml b/conf/ceilometer-minimal.yaml index eadbfefe5..9179e385f 100644 --- a/conf/ceilometer-minimal.yaml +++ b/conf/ceilometer-minimal.yaml @@ -6,6 +6,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -27,6 +28,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/ceilometer-persist-response-timing.yaml b/conf/ceilometer-persist-response-timing.yaml index 2908ea823..04a0a426c 100644 --- a/conf/ceilometer-persist-response-timing.yaml +++ b/conf/ceilometer-persist-response-timing.yaml @@ -9,6 +9,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -30,6 +31,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/gnocchi-minimal.yaml b/conf/gnocchi-minimal.yaml index 6ae6600f6..16269ba8d 100644 --- a/conf/gnocchi-minimal.yaml +++ b/conf/gnocchi-minimal.yaml @@ -7,6 +7,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -28,6 +29,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/quickstart.yml b/conf/quickstart.yml index 32733175c..c7bf2ba3f 100644 --- a/conf/quickstart.yml +++ b/conf/quickstart.yml @@ -6,6 +6,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -27,6 +28,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml index 5d11f862d..d66ef7f53 100644 --- a/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml @@ -13,6 +13,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -34,6 +35,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml index 8dcbd60d5..1ea12c935 100644 --- a/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml @@ -13,6 +13,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -34,6 +35,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml index e3a57f1bf..334f9e338 100644 --- a/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml @@ -13,6 +13,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -34,6 +35,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml index 02efe1dd6..242675f81 100644 --- a/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml @@ -13,6 +13,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -34,6 +35,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 5 sleep_after: 5 diff --git a/conf/telemetry-nova-persist-instances-with-network-fip.yaml b/conf/telemetry-nova-persist-instances-with-network-fip.yaml index ae46c0b32..aae5deb26 100644 --- a/conf/telemetry-nova-persist-instances-with-network-fip.yaml +++ b/conf/telemetry-nova-persist-instances-with-network-fip.yaml @@ -10,6 +10,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -31,6 +32,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 0 sleep_after: 0 diff --git a/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml b/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml index 0b7b20de4..7e0132e34 100644 --- a/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml +++ b/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml @@ -10,6 +10,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -31,6 +32,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 0 sleep_after: 0 diff --git a/conf/telemetry-nova-persist-instances-with-network-volume.yaml b/conf/telemetry-nova-persist-instances-with-network-volume.yaml index 0eab8ab3d..ee1604038 100644 --- a/conf/telemetry-nova-persist-instances-with-network-volume.yaml +++ b/conf/telemetry-nova-persist-instances-with-network-volume.yaml @@ -9,6 +9,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -30,6 +31,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 0 sleep_after: 0 diff --git a/conf/telemetry-nova-persist-instances-with-network.yaml b/conf/telemetry-nova-persist-instances-with-network.yaml index b3a190c2f..9a0260601 100644 --- a/conf/telemetry-nova-persist-instances-with-network.yaml +++ b/conf/telemetry-nova-persist-instances-with-network.yaml @@ -11,6 +11,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -32,6 +33,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 0 sleep_after: 0 diff --git a/conf/telemetry-nova-persist-instances-with-volume.yaml b/conf/telemetry-nova-persist-instances-with-volume.yaml index 62b3e7f18..9b101433c 100644 --- a/conf/telemetry-nova-persist-instances-with-volume.yaml +++ b/conf/telemetry-nova-persist-instances-with-volume.yaml @@ -9,6 +9,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -30,6 +31,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 0 sleep_after: 0 diff --git a/conf/telemetry-nova-persist-instances.yaml b/conf/telemetry-nova-persist-instances.yaml index 9f1ea6395..8d7c399e9 100644 --- a/conf/telemetry-nova-persist-instances.yaml +++ b/conf/telemetry-nova-persist-instances.yaml @@ -10,6 +10,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/stockpile.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -31,6 +32,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: false rally: sleep_before: 0 sleep_after: 0 diff --git a/doc/source/installation.rst b/doc/source/installation.rst index cc4b81fe2..1fcedf11e 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -89,8 +89,8 @@ can also work with Queens release but it is not recommended. [stack@ospd ansible]$ ansible-playbook -i hosts install/collectd.yml -(Optional) Install Rsyslogd logging with aggregation -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +(Optional) Install Rsyslogd logging with aggregation (not maintained) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ First configure the values rsyslog values and elasticsearch parameters in `ansible/install/group_vars/all.yml`. If you have a large number of hosts @@ -125,6 +125,22 @@ http://docs.grafana.org/http_api/auth/#create-api-token [stack@ospd ansible]$ ansible-playbook -i hosts install/browbeat.yml # if not run before. [stack@ospd ansible]$ ansible-playbook -i hosts install/grafana-dashboards.yml +(Optional) Install Browbeat Common Logging through filebeat +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Browbeat can be used to setup common logging on your OpenStack Cluster using Filebeat on the +client side and Elasticsearch on the server side. Set the `cloud_prefix` and `es_ip` in +`install/group_vars/all.yml` before running the playbook to setup common logging for your cloud. + +:: + + [stack@ospd ansible]$ # update the vars + [stack@ospd ansible]$ vi install/group_vars/all.yml + [stack@ospd ansible]$ # install filebeat + [stack@ospd ansible]$ ansible-playbook -i hosts common_logging/install_logging.yml + [stack@ospd ansible]$ # install and start filebeat + [stack@ospd ansible]$ ansible-playbook -i hosts common_logging/install_logging.yml -e "start_filebeat=True" + Not mantained (Pre-Pike): Run Overcloud checks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -604,9 +620,8 @@ Graphite dashboard included and it is recommended to install collectd on your monitoring host such that you can see if you hit resource issues with your monitoring host. - -Install Kibana Visualizations -------------------------------- +(Optional) Install Kibana Visualizations +---------------------------------------- 1. Update install/group_vars/all.yml (es_ip) to identify your ELK host. 2. Install Kibana Visualizations via Ansible playbook diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 1c6f2c24d..01b5c4f15 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -87,6 +87,24 @@ using some simple searches such as: shaker_uuid: 97092334-34e8-446c-87d6-6a0f361b9aa8 AND record.concurrency: 1 AND result.result_type: bandwidth shaker_uuid: c918a263-3b0b-409b-8cf8-22dfaeeaf33e AND record.concurrency:1 AND record.test:Bi-Directional +Correlating test run with logs +------------------------------ + +If filebeat is enabled in the browbeat configuration file and filebeat was previously installed by running: + +:: + + $ ansible-playbook -i hosts common_logging/install_logging.yml + +as explained in the installation documentation, then + +By enabling filebeat logging within the browbeat configuration file, a playbook `ansible/common_logging/browbeat_logging.yml` +is run which appends browbeat_uuid to log messages and starts filebeat pre-browbeat workload run so that log messages have +browbeat uuid appended and clears the uuid from the configuration file and stops filebeat from sending more logs post-browbeat +workload run + + + Interpreting Browbeat Results ----------------------------- diff --git a/tests/data/invalid_browbeat.yml b/tests/data/invalid_browbeat.yml index 5d3c2479f..b2678b0b3 100644 --- a/tests/data/invalid_browbeat.yml +++ b/tests/data/invalid_browbeat.yml @@ -7,6 +7,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/site.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false diff --git a/tests/data/invalid_browbeat_workload.yml b/tests/data/invalid_browbeat_workload.yml index ca2f2a3f0..784dd9abc 100644 --- a/tests/data/invalid_browbeat_workload.yml +++ b/tests/data/invalid_browbeat_workload.yml @@ -6,6 +6,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/site.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false diff --git a/tests/data/valid_browbeat.yml b/tests/data/valid_browbeat.yml index b125f9384..8c4990cd7 100644 --- a/tests/data/valid_browbeat.yml +++ b/tests/data/valid_browbeat.yml @@ -6,6 +6,7 @@ browbeat: ansible: hosts: ansible/hosts metadata_playbook: ansible/gather/site.yml + logging_playbook: ansible/common_logging/browbeat_logging.yml ssh_config: ansible/ssh-config elasticsearch: enabled: false @@ -27,6 +28,8 @@ grafana: port: 3000 dashboards: - openstack-general-system-performance +filebeat: + enabled: true rally: sleep_before: 0 sleep_after: 0