From 9b2a238241cbadfc8e4dec09b63ed27c0e3e2f5a Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Wed, 15 Mar 2023 11:30:42 +0000 Subject: [PATCH] Add OpenSearch This change is backported from stable/zed, where it replaced Elasticsearch - in stable/yoga it's going to be added alongside Elasticsearch for users that will be migrating to Rocky Linux 9 - which does not support elasticsearch. Co-authored-by: Doug Szumski Co-authored-by: Kyle Dean Change-Id: Iab10ce7ea5d5f21a40b1f99b28e3290b7e9ce895 (cherry picked from commit e1ec02eddf9bb33fd9e34c1e4c79b7791f6fd22c) --- README.rst | 4 +- ansible/group_vars/all.yml | 36 ++- ansible/inventory/all-in-one | 8 + ansible/inventory/multinode | 8 + ansible/opensearch-migration.yml | 108 ++++++++ ansible/roles/cloudkitty/defaults/main.yml | 2 +- ansible/roles/common/defaults/main.yml | 9 + ansible/roles/common/tasks/config.yml | 6 + .../templates/conf/output/00-local.conf.j2 | 31 +++ .../templates/conf/output/01-es.conf.j2 | 2 +- .../conf/output/03-opensearch.conf.j2 | 33 +++ .../cron-logrotate-opensearch.conf.j2 | 3 + ansible/roles/destroy/tasks/cleanup_host.yml | 1 + ansible/roles/freezer/defaults/main.yml | 4 +- ansible/roles/grafana/defaults/main.yml | 12 + ansible/roles/loadbalancer/tasks/precheck.yml | 26 ++ ansible/roles/opensearch/defaults/main.yml | 132 +++++++++ ansible/roles/opensearch/handlers/main.yml | 33 +++ .../opensearch/tasks/check-containers.yml | 18 ++ ansible/roles/opensearch/tasks/check.yml | 1 + .../roles/opensearch/tasks/config-host.yml | 17 ++ ansible/roles/opensearch/tasks/config.yml | 63 +++++ .../opensearch/tasks/config_validate.yml | 1 + ansible/roles/opensearch/tasks/copy-certs.yml | 6 + .../opensearch/tasks/deploy-containers.yml | 2 + ansible/roles/opensearch/tasks/deploy.yml | 12 + .../roles/opensearch/tasks/loadbalancer.yml | 7 + ansible/roles/opensearch/tasks/main.yml | 2 + ansible/roles/opensearch/tasks/precheck.yml | 25 ++ ansible/roles/opensearch/tasks/pull.yml | 3 + .../roles/opensearch/tasks/reconfigure.yml | 2 + ansible/roles/opensearch/tasks/register.yml | 7 + ansible/roles/opensearch/tasks/stop.yml | 6 + ansible/roles/opensearch/tasks/upgrade.yml | 48 ++++ .../templates/opensearch-dashboards.json.j2 | 23 ++ .../opensearch/templates/opensearch.json.j2 | 23 ++ .../opensearch/templates/opensearch.yml.j2 | 21 ++ .../templates/opensearch_dashboards.yml.j2 | 12 + ansible/roles/opensearch/vars/main.yml | 2 + .../prometheus-elasticsearch-exporter.json.j2 | 2 +- ansible/roles/telegraf/defaults/main.yml | 1 + .../roles/telegraf/templates/telegraf.conf.j2 | 5 + ansible/site.yml | 18 ++ ...> central-logging-guide-elasticsearch.rst} | 10 +- .../central-logging-guide-opensearch.rst | 252 ++++++++++++++++++ .../logging-and-monitoring/index.rst | 3 +- .../logging-and-monitoring/monasca-guide.rst | 4 + etc/kolla/globals.yml | 3 + etc/kolla/passwords.yml | 5 + .../add-opensearch-53ef174195acce45.yaml | 10 + tests/run.yml | 17 +- tests/setup_gate.sh | 4 +- tests/templates/globals-default.j2 | 7 +- tests/templates/inventory.j2 | 12 +- tests/test-prometheus-efk.sh | 14 +- tests/test-prometheus-opensearch.sh | 189 +++++++++++++ tests/test-venus.sh | 24 +- tools/cleanup-host | 5 + tools/kolla-ansible | 6 + zuul.d/base.yaml | 12 +- zuul.d/jobs.yaml | 42 +++ zuul.d/project.yaml | 5 + 62 files changed, 1374 insertions(+), 35 deletions(-) create mode 100644 ansible/opensearch-migration.yml create mode 100644 ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 create mode 100644 ansible/roles/common/templates/cron-logrotate-opensearch.conf.j2 create mode 100644 ansible/roles/opensearch/defaults/main.yml create mode 100644 ansible/roles/opensearch/handlers/main.yml create mode 100644 ansible/roles/opensearch/tasks/check-containers.yml create mode 100644 ansible/roles/opensearch/tasks/check.yml create mode 100644 ansible/roles/opensearch/tasks/config-host.yml create mode 100644 ansible/roles/opensearch/tasks/config.yml create mode 100644 ansible/roles/opensearch/tasks/config_validate.yml create mode 100644 ansible/roles/opensearch/tasks/copy-certs.yml create mode 100644 ansible/roles/opensearch/tasks/deploy-containers.yml create mode 100644 ansible/roles/opensearch/tasks/deploy.yml create mode 100644 ansible/roles/opensearch/tasks/loadbalancer.yml create mode 100644 ansible/roles/opensearch/tasks/main.yml create mode 100644 ansible/roles/opensearch/tasks/precheck.yml create mode 100644 ansible/roles/opensearch/tasks/pull.yml create mode 100644 ansible/roles/opensearch/tasks/reconfigure.yml create mode 100644 ansible/roles/opensearch/tasks/register.yml create mode 100644 ansible/roles/opensearch/tasks/stop.yml create mode 100644 ansible/roles/opensearch/tasks/upgrade.yml create mode 100644 ansible/roles/opensearch/templates/opensearch-dashboards.json.j2 create mode 100644 ansible/roles/opensearch/templates/opensearch.json.j2 create mode 100644 ansible/roles/opensearch/templates/opensearch.yml.j2 create mode 100644 ansible/roles/opensearch/templates/opensearch_dashboards.yml.j2 create mode 100644 ansible/roles/opensearch/vars/main.yml rename doc/source/reference/logging-and-monitoring/{central-logging-guide.rst => central-logging-guide-elasticsearch.rst} (96%) create mode 100644 doc/source/reference/logging-and-monitoring/central-logging-guide-opensearch.rst create mode 100644 releasenotes/notes/add-opensearch-53ef174195acce45.yaml mode change 100755 => 100644 tests/test-prometheus-efk.sh create mode 100755 tests/test-prometheus-opensearch.sh diff --git a/README.rst b/README.rst index fdd1e1293e..5fdd42cb6a 100644 --- a/README.rst +++ b/README.rst @@ -84,8 +84,8 @@ Kolla Ansible deploys containers for the following infrastructure components: `InfluxDB `__, `Prometheus `__, and `Grafana `__ for performance monitoring. -- `Elasticsearch `__ and - `Kibana `__ to search, analyze, +- `OpenSearch `__ and + `OpenSearch Dashboards `__ to search, analyze, and visualize log messages. - `Etcd `__ a distributed reliable key-value store. - `Fluentd `__ as an open source data collector diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 0c5ce4bb02..e98427b006 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -193,14 +193,22 @@ default_extra_volumes: [] # Arbitrary unique number from 0..255 keepalived_virtual_router_id: "51" - -####################### +###################### # Elasticsearch Options ####################### elasticsearch_datadir_volume: "elasticsearch" elasticsearch_internal_endpoint: "{{ internal_protocol }}://{{ elasticsearch_address | put_address_in_context('url') }}:{{ elasticsearch_port }}" +####################### +## Opensearch Options +######################## +opensearch_datadir_volume: "opensearch" + +opensearch_internal_endpoint: "{{ internal_protocol }}://{{ opensearch_address | put_address_in_context('url') }}:{{ opensearch_port }}" +opensearch_dashboards_user: "opensearch" +opensearch_log_index_prefix: "{{ kibana_log_prefix if kibana_log_prefix is defined else 'flog' }}" + ################### # Messaging options ################### @@ -436,6 +444,13 @@ octavia_api_port: "9876" octavia_api_listen_port: "{{ octavia_api_port }}" octavia_health_manager_port: "5555" +# NOTE: If an external ElasticSearch cluster port is specified, +# we default to using that port in services with ElasticSearch +# endpoints. This is for backwards compatibility. +opensearch_port: "{{ elasticsearch_port | default('9200') }}" +opensearch_dashboards_port: "5601" +opensearch_dashboards_port_external: "{{ opensearch_dashboards_port }}" + ovn_nb_db_port: "6641" ovn_sb_db_port: "6642" ovn_nb_connection: "{% for host in groups['ovn-nb-db'] %}tcp:{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ ovn_nb_db_port }}{% if not loop.last %},{% endif %}{% endfor %}" @@ -788,15 +803,23 @@ skip_stop_containers: [] #################### elasticsearch_address: "{{ kolla_internal_fqdn }}" -enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool or (enable_cloudkitty | bool and cloudkitty_storage_backend == 'elasticsearch') else 'no' }}" +enable_elasticsearch: "{{ 'yes' if not enable_opensearch | bool and (enable_central_logging | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool or (enable_cloudkitty | bool and cloudkitty_storage_backend == 'elasticsearch')) else 'no' }}" # If using Curator an actions file will need to be defined. Please see # the documentation. enable_elasticsearch_curator: "no" -enable_kibana: "{{ 'yes' if enable_central_logging | bool or enable_monasca | bool else 'no' }}" +enable_kibana: "{{ 'yes' if (enable_central_logging | bool and not enable_opensearch | bool) or enable_monasca | bool else 'no' }}" enable_kibana_external: "{{ enable_kibana | bool }}" +# NOTE: If an external ElasticSearch cluster address is configured, all +# services with ElasticSearch endpoints should be configured to log +# to the external cluster by default. This is for backwards compatibility. +opensearch_address: "{{ elasticsearch_address if elasticsearch_address is defined else kolla_internal_fqdn }}" +enable_opensearch: "{{ kolla_base_distro == 'rocky' and enable_central_logging | bool }}" +enable_opensearch_dashboards: "{{ enable_opensearch | bool }}" +enable_opensearch_dashboards_external: "{{ enable_opensearch_dashboards | bool }}" + #################### # Redis options #################### @@ -809,7 +832,8 @@ redis_connection_string_extras: "&db=0&socket_timeout=60&retry_on_timeout=yes" # valid values: ["elasticsearch", "redis"] osprofiler_backend: "elasticsearch" elasticsearch_connection_string: "elasticsearch://{{ elasticsearch_address | put_address_in_context('url') }}:{{ elasticsearch_port }}" -osprofiler_backend_connection_string: "{{ redis_connection_string if osprofiler_backend == 'redis' else elasticsearch_connection_string }}" +opensearch_connection_string: "elasticsearch://{{ opensearch_address | put_address_in_context('url') }}:{{ opensearch_port }}" +osprofiler_backend_connection_string: "{{ redis_connection_string if osprofiler_backend == 'redis' else opensearch_connection_string if (enable_opensearch | bool) else elasticsearch_connection_string }}" #################### # RabbitMQ options @@ -1129,7 +1153,7 @@ enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}" enable_prometheus_alertmanager_external: "{{ enable_prometheus_alertmanager | bool }}" enable_prometheus_ceph_mgr_exporter: "no" enable_prometheus_openstack_exporter: "{{ enable_prometheus | bool }}" -enable_prometheus_elasticsearch_exporter: "{{ enable_prometheus | bool and enable_elasticsearch | bool }}" +enable_prometheus_elasticsearch_exporter: "{{ enable_prometheus | bool and (enable_elasticsearch | bool or enable_opensearch | bool) }}" enable_prometheus_blackbox_exporter: "{{ enable_prometheus | bool }}" enable_prometheus_rabbitmq_exporter: "{{ enable_prometheus | bool and enable_rabbitmq | bool }}" enable_prometheus_libvirt_exporter: "{{ enable_prometheus | bool and enable_nova | bool and nova_compute_virt_type in ['kvm', 'qemu'] }}" diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 2d6cdd47fb..58548ac124 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -240,6 +240,13 @@ common [elasticsearch-curator:children] elasticsearch +[opensearch:children] +control + +# Opensearch dashboards +[opensearch-dashboards:children] +opensearch + # Glance [glance-api:children] glance @@ -693,6 +700,7 @@ monitoring [prometheus-elasticsearch-exporter:children] elasticsearch +opensearch [prometheus-blackbox-exporter:children] monitoring diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index a62cf37676..3be7e28617 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -258,6 +258,13 @@ common [elasticsearch-curator:children] elasticsearch +[opensearch:children] +control + +# Opensearch dashboards +[opensearch-dashboards:children] +opensearch + # Glance [glance-api:children] glance @@ -711,6 +718,7 @@ monitoring [prometheus-elasticsearch-exporter:children] elasticsearch +opensearch [prometheus-blackbox-exporter:children] monitoring diff --git a/ansible/opensearch-migration.yml b/ansible/opensearch-migration.yml new file mode 100644 index 0000000000..ab56782f16 --- /dev/null +++ b/ansible/opensearch-migration.yml @@ -0,0 +1,108 @@ +--- +- import_playbook: gather-facts.yml + +- name: Prechecks + gather_facts: false + hosts: localhost + serial: '{{ kolla_serial|default("0") }}' + tasks: + - assert: + that: enable_opensearch | bool + fail_msg: "OpenSearch is not enabled" + - assert: + that: not (enable_monasca | bool) + fail_msg: "Monasca is enabled, it's not supported by OpenSearch" + - assert: + that: not (enable_skydive | bool) + fail_msg: "Skydive is enabled, it's not supported by OpenSearch" + - assert: + that: not (enable_elasticsearch | bool) + fail_msg: "ElasticSearch is enabled, please disable it" + +- name: Stop and remove Elasticsearch services containers + gather_facts: false + hosts: elasticsearch + serial: '{{ kolla_serial|default("0") }}' + tasks: + - name: Stop and remove ElasticSearch + become: true + kolla_docker: + action: "stop_and_remove_container" + name: "elasticsearch" + when: + - inventory_hostname in groups['elasticsearch'] + + - name: Stop and remove ElasticSearch Curator + become: true + kolla_docker: + action: "stop_and_remove_container" + name: "elasticsearch_curator" + when: + - inventory_hostname in groups['elasticsearch-curator'] + + - name: Stop and remove Kibana + become: true + kolla_docker: + action: "stop_and_remove_container" + name: "kibana" + when: + - inventory_hostname in groups['kibana'] + +- name: Remove Elasticsearch services loadbalancer config + gather_facts: false + hosts: loadbalancer + serial: '{{ kolla_serial|default("0") }}' + tasks: + - name: Delete ElasticSearch load-balancer config + file: + path: "{{ node_config_directory }}/haproxy/services.d/elasticsearch.cfg" + state: "absent" + become: true + when: + - inventory_hostname in groups['loadbalancer'] + + - name: Delete Kibana load-balancer config + file: + path: "{{ node_config_directory }}/haproxy/services.d/kibana.cfg" + state: "absent" + become: true + when: + - inventory_hostname in groups['loadbalancer'] + +- name: Migrate Elasticsearch data + gather_facts: false + hosts: opensearch + tasks: + - name: Create OpenSearch Docker volume + become: true + command: "docker volume create opensearch" + + - name: Migrate ElasticSearch data to OpenSearch + become: true + command: "mv /var/lib/docker/volumes/elasticsearch/_data/nodes /var/lib/docker/volumes/opensearch/_data/" + +- name: Deploy OpenSearch and OpenSearch Dashboards + gather_facts: false + hosts: + - opensearch + serial: '{{ kolla_serial|default("0") }}' + roles: + - opensearch + +- name: Run loadbalancer role + gather_facts: false + hosts: + - loadbalancer + roles: + - { role: loadbalancer } + tasks: + - include_role: + name: opensearch + tasks_from: loadbalancer + +- name: Run grafana role to register OpenSearch datasource + gather_facts: false + hosts: grafana + serial: '{{ kolla_serial|default("0") }}' + roles: + - { role: grafana, when: enable_grafana | bool } diff --git a/ansible/roles/cloudkitty/defaults/main.yml b/ansible/roles/cloudkitty/defaults/main.yml index 3d4e0da1b9..f11a196590 100644 --- a/ansible/roles/cloudkitty/defaults/main.yml +++ b/ansible/roles/cloudkitty/defaults/main.yml @@ -146,7 +146,7 @@ cloudkitty_influxdb_name: "cloudkitty" cloudkitty_elasticsearch_index_name: "cloudkitty" # Set the elasticsearch host URL. -cloudkitty_elasticsearch_url: "{{ internal_protocol }}://{{ elasticsearch_address }}:{{ elasticsearch_port }}" +cloudkitty_elasticsearch_url: "{{ internal_protocol }}://{{ opensearch_address }}:{{ opensearch_port }}" # Path of the CA certificate to trust for HTTPS connections. # cloudkitty_elasticsearch_cafile: "{{ openstack_cacert }}" diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml index 4879d81111..406d27eab7 100644 --- a/ansible/roles/common/defaults/main.yml +++ b/ansible/roles/common/defaults/main.yml @@ -45,6 +45,15 @@ fluentd_elasticsearch_ssl_verify: "true" fluentd_elasticsearch_cacert: "{{ openstack_cacert }}" fluentd_elasticsearch_request_timeout: "60s" +fluentd_opensearch_path: "" +fluentd_opensearch_scheme: "{{ internal_protocol }}" +fluentd_opensearch_user: "" +fluentd_opensearch_password: "" +fluentd_opensearch_ssl_version: "TLSv1_2" +fluentd_opensearch_ssl_verify: "true" +fluentd_opensearch_cacert: "{{ openstack_cacert }}" +fluentd_opensearch_request_timeout: "60s" + #################### # Docker #################### diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index 31be824d10..a4e6f5a5f6 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -80,6 +80,9 @@ {{ ( enable_elasticsearch | bool or ( elasticsearch_address != kolla_internal_fqdn )) and ( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }} + log_direct_to_opensearch: >- + {{ enable_opensearch | bool or + ( opensearch_address != kolla_internal_fqdn ) }} # Inputs fluentd_input_files: "{{ default_input_files_enabled | customise_fluentd(customised_input_files) }}" default_input_files_enabled: "{{ default_input_files | selectattr('enabled') | map(attribute='name') | list }}" @@ -130,6 +133,8 @@ enabled: "{{ log_direct_to_elasticsearch }}" - name: "conf/output/02-monasca.conf.j2" enabled: "{{ enable_monasca | bool and monasca_ingest_control_plane_logs | bool }}" + - name: "conf/output/03-opensearch.conf.j2" + enabled: "{{ log_direct_to_opensearch }}" customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | list }}" template: src: "td-agent.conf.j2" @@ -190,6 +195,7 @@ - { name: "nova", enabled: "{{ enable_nova | bool }}" } - { name: "nova-libvirt", enabled: "{{ enable_nova | bool and enable_nova_libvirt_container | bool }}" } - { name: "octavia", enabled: "{{ enable_octavia | bool }}" } + - { name: "opensearch", enabled: "{{ enable_opensearch | bool or enable_opensearch_dashboards | bool }}" } - { name: "openvswitch", enabled: "{{ enable_openvswitch | bool }}" } - { name: "outward-rabbitmq", enabled: "{{ enable_outward_rabbitmq | bool }}" } - { name: "placement", enabled: "{{ enable_placement | bool }}" } diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2 index 05dbf73834..4654d4cd30 100644 --- a/ansible/roles/common/templates/conf/output/00-local.conf.j2 +++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2 @@ -64,6 +64,37 @@ chunk_limit_size 8m +{% elif log_direct_to_opensearch %} + + @type opensearch + host {{ opensearch_address }} + port {{ opensearch_port }} + scheme {{ fluentd_opensearch_scheme }} +{% if fluentd_opensearch_path != '' %} + path {{ fluentd_opensearch_path }} +{% endif %} +{% if fluentd_opensearch_scheme == 'https' %} + ssl_version {{ fluentd_opensearch_ssl_version }} + ssl_verify {{ fluentd_opensearch_ssl_verify }} +{% if fluentd_opensearch_cacert | length > 0 %} + ca_file {{ fluentd_opensearch_cacert }} +{% endif %} +{% endif %} +{% if fluentd_opensearch_user != '' and fluentd_opensearch_password != ''%} + user {{ fluentd_opensearch_user }} + password {{ fluentd_opensearch_password }} +{% endif %} + logstash_format true + logstash_prefix {{ opensearch_log_index_prefix }} + reconnect_on_error true + request_timeout {{ fluentd_opensearch_request_timeout }} + suppress_type_name true + + @type file + path /var/lib/fluentd/data/opensearch.buffer/{{ item.facility }}.* + flush_interval 15s + + {% endif %} {% endfor %} diff --git a/ansible/roles/common/templates/conf/output/01-es.conf.j2 b/ansible/roles/common/templates/conf/output/01-es.conf.j2 index 3f73971f3d..0b3ebaa107 100644 --- a/ansible/roles/common/templates/conf/output/01-es.conf.j2 +++ b/ansible/roles/common/templates/conf/output/01-es.conf.j2 @@ -3,7 +3,7 @@ @type elasticsearch host {{ elasticsearch_address }} - port {{ elasticsearch_port }} + port {{ elasticsearch_port | default('9200') }} scheme {{ fluentd_elasticsearch_scheme }} {% if fluentd_elasticsearch_path != '' %} path {{ fluentd_elasticsearch_path }} diff --git a/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 b/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 new file mode 100644 index 0000000000..e40b3f98cb --- /dev/null +++ b/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 @@ -0,0 +1,33 @@ + + @type copy + + @type opensearch + host {{ opensearch_address }} + port {{ opensearch_port }} + scheme {{ fluentd_opensearch_scheme }} +{% if fluentd_opensearch_path != '' %} + path {{ fluentd_opensearch_path }} +{% endif %} +{% if fluentd_opensearch_scheme == 'https' %} + ssl_version {{ fluentd_opensearch_ssl_version }} + ssl_verify {{ fluentd_opensearch_ssl_verify }} +{% if fluentd_opensearch_cacert | length > 0 %} + ca_file {{ fluentd_opensearch_cacert }} +{% endif %} +{% endif %} +{% if fluentd_opensearch_user != '' and fluentd_opensearch_password != ''%} + user {{ fluentd_opensearch_user }} + password {{ fluentd_opensearch_password }} +{% endif %} + logstash_format true + logstash_prefix {{ opensearch_log_index_prefix }} + reconnect_on_error true + request_timeout {{ fluentd_opensearch_request_timeout }} + suppress_type_name true + + @type file + path /var/lib/fluentd/data/opensearch.buffer/openstack.* + flush_interval 15s + + + diff --git a/ansible/roles/common/templates/cron-logrotate-opensearch.conf.j2 b/ansible/roles/common/templates/cron-logrotate-opensearch.conf.j2 new file mode 100644 index 0000000000..4f02e14a74 --- /dev/null +++ b/ansible/roles/common/templates/cron-logrotate-opensearch.conf.j2 @@ -0,0 +1,3 @@ +"/var/log/kolla/opensearch/*.log" +{ +} diff --git a/ansible/roles/destroy/tasks/cleanup_host.yml b/ansible/roles/destroy/tasks/cleanup_host.yml index 99ccfcc99f..7d382747fd 100644 --- a/ansible/roles/destroy/tasks/cleanup_host.yml +++ b/ansible/roles/destroy/tasks/cleanup_host.yml @@ -14,6 +14,7 @@ kolla_internal_vip_address: "{{ kolla_internal_vip_address }}" kolla_external_vip_address: "{{ kolla_external_vip_address }}" kolla_dev_repos_directory: "{{ kolla_dev_repos_directory }}" + opensearch_datadir_volume: "{{ opensearch_datadir_volume }}" destroy_include_dev: "{{ destroy_include_dev }}" - block: diff --git a/ansible/roles/freezer/defaults/main.yml b/ansible/roles/freezer/defaults/main.yml index b08d907750..b4e338e093 100644 --- a/ansible/roles/freezer/defaults/main.yml +++ b/ansible/roles/freezer/defaults/main.yml @@ -35,8 +35,8 @@ freezer_database_user: "{% if use_preconfigured_databases | bool and use_common_ freezer_database_address: "{{ database_address | put_address_in_context('url') }}:{{ database_port }}" freezer_elasticsearch_replicas: "1" freezer_es_protocol: "{{ internal_protocol }}" -freezer_es_address: "{{ elasticsearch_address }}" -freezer_es_port: "{{ elasticsearch_port }}" +freezer_es_address: "{{ opensearch_address if enable_opensearch | bool else elasticsearch_address }}" +freezer_es_port: "{{ opensearch_port if enable_opensearch | bool else elasticsearch_port }}" #################### # Docker diff --git a/ansible/roles/grafana/defaults/main.yml b/ansible/roles/grafana/defaults/main.yml index c17ab6d64a..bb87a15aa9 100644 --- a/ansible/roles/grafana/defaults/main.yml +++ b/ansible/roles/grafana/defaults/main.yml @@ -51,6 +51,18 @@ grafana_data_sources: jsonData: esVersion: 5 timeField: "@timestamp" + opensearch: + enabled: "{{ enable_opensearch | bool }}" + data: + name: "opensearch" + type: "grafana-opensearch-datasource" + access: "proxy" + url: "{{ opensearch_internal_endpoint }}" + jsonData: + flavor: "elasticsearch" + database: "[flog-]YYYY.MM.DD" + version: "7.0.0" + timeField: "@timestamp" monasca: enabled: "{{ enable_monasca | bool }}" data: diff --git a/ansible/roles/loadbalancer/tasks/precheck.yml b/ansible/roles/loadbalancer/tasks/precheck.yml index 131b5aeccd..1bd4a03670 100644 --- a/ansible/roles/loadbalancer/tasks/precheck.yml +++ b/ansible/roles/loadbalancer/tasks/precheck.yml @@ -704,6 +704,32 @@ - haproxy_stat.find('octavia_api') == -1 - haproxy_vip_prechecks +- name: Checking free port for OpenSearch HAProxy + wait_for: + host: "{{ kolla_internal_vip_address }}" + port: "{{ opensearch_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - enable_opensearch | bool + - inventory_hostname in groups['loadbalancer'] + - haproxy_stat.find('opensearch') == -1 + - haproxy_vip_prechecks + +- name: Checking free port for OpenSearch Dashboards HAProxy + wait_for: + host: "{{ kolla_internal_vip_address }}" + port: "{{ opensearch_dashboards_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - enable_opensearch_dashboards | bool + - inventory_hostname in groups['loadbalancer'] + - haproxy_stat.find('opensearch_dashboards') == -1 + - haproxy_vip_prechecks + - name: Checking free port for RabbitMQ Management HAProxy wait_for: host: "{{ kolla_internal_vip_address }}" diff --git a/ansible/roles/opensearch/defaults/main.yml b/ansible/roles/opensearch/defaults/main.yml new file mode 100644 index 0000000000..f8719ba22d --- /dev/null +++ b/ansible/roles/opensearch/defaults/main.yml @@ -0,0 +1,132 @@ +--- +opensearch_services: + opensearch: + container_name: opensearch + group: opensearch + enabled: true + image: "{{ opensearch_image_full }}" + environment: + OPENSEARCH_JAVA_OPTS: "{{ opensearch_java_opts }}" + volumes: "{{ opensearch_default_volumes + opensearch_extra_volumes }}" + dimensions: "{{ opensearch_dimensions }}" + healthcheck: "{{ opensearch_healthcheck }}" + haproxy: + opensearch: + enabled: "{{ enable_opensearch }}" + mode: "http" + external: false + port: "{{ opensearch_port }}" + frontend_http_extra: + - "option dontlog-normal" + opensearch-dashboards: + container_name: opensearch_dashboards + group: opensearch-dashboards + enabled: "{{ enable_opensearch_dashboards }}" + environment: + OPENSEARCH_DASHBOARDS_SECURITY_PLUGIN: "False" + image: "{{ opensearch_dashboards_image_full }}" + volumes: "{{ opensearch_dashboards_default_volumes + opensearch_dashboards_extra_volumes }}" + dimensions: "{{ opensearch_dashboards_dimensions }}" + healthcheck: "{{ opensearch_dashboards_healthcheck }}" + haproxy: + opensearch-dashboards: + enabled: "{{ enable_opensearch_dashboards }}" + mode: "http" + external: false + port: "{{ opensearch_dashboards_port }}" + auth_user: "{{ opensearch_dashboards_user }}" + auth_pass: "{{ opensearch_dashboards_password }}" + opensearch_dashboards_external: + enabled: "{{ enable_opensearch_dashboards_external | bool }}" + mode: "http" + external: true + port: "{{ opensearch_dashboards_port_external }}" + auth_user: "{{ opensearch_dashboards_user }}" + auth_pass: "{{ opensearch_dashboards_password }}" + + +#################### +# Opensearch +#################### + +# Register Opensearch internal endpoint in the Keystone service catalogue +opensearch_enable_keystone_registration: False + +opensearch_cluster_name: "kolla_logging" +opensearch_heap_size: "1g" +opensearch_java_opts: "{% if opensearch_heap_size %}-Xms{{ opensearch_heap_size }} -Xmx{{ opensearch_heap_size }}{% endif %} -Dlog4j2.formatMsgNoLookups=true" + +#################### +# Keystone +#################### +opensearch_openstack_auth: "{{ openstack_auth }}" + +opensearch_ks_services: + - name: "opensearch" + type: "log-storage" + description: "Opensearch" + endpoints: + - {'interface': 'internal', 'url': '{{ opensearch_internal_endpoint }}'} + +####################### +# OpenSearch Dashboards +####################### +opensearch_dashboards_default_app_id: "discover" +opensearch_dashboards_opensearch_request_timeout: 300000 +opensearch_dashboards_opensearch_shard_timeout: 0 +opensearch_dashboards_opensearch_ssl_verify: true + +#################### +# Docker +#################### +opensearch_install_type: "{{ kolla_install_type }}" +opensearch_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ opensearch_install_type }}-opensearch" +opensearch_tag: "{{ openstack_tag }}" +opensearch_image_full: "{{ opensearch_image }}:{{ opensearch_tag }}" + +opensearch_dashboards_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ opensearch_install_type }}-opensearch-dashboards" +opensearch_dashboards_tag: "{{ openstack_tag }}" +opensearch_dashboards_image_full: "{{ opensearch_dashboards_image }}:{{ opensearch_dashboards_tag }}" + +opensearch_dimensions: "{{ default_container_dimensions }}" +opensearch_dashboards_dimensions: "{{ default_container_dimensions }}" + +opensearch_enable_healthchecks: "{{ enable_container_healthchecks }}" +opensearch_healthcheck_interval: "{{ default_container_healthcheck_interval }}" +opensearch_healthcheck_retries: "{{ default_container_healthcheck_retries }}" +opensearch_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}" +opensearch_healthcheck_test: ["CMD-SHELL", "healthcheck_curl http://{{ api_interface_address | put_address_in_context('url') }}:{{ opensearch_port }}"] +opensearch_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}" +opensearch_healthcheck: + interval: "{{ opensearch_healthcheck_interval }}" + retries: "{{ opensearch_healthcheck_retries }}" + start_period: "{{ opensearch_healthcheck_start_period }}" + test: "{% if opensearch_enable_healthchecks | bool %}{{ opensearch_healthcheck_test }}{% else %}NONE{% endif %}" + timeout: "{{ opensearch_healthcheck_timeout }}" +opensearch_dashboards_enable_healthchecks: "{{ enable_container_healthchecks }}" +opensearch_dashboards_healthcheck_interval: "{{ default_container_healthcheck_interval }}" +opensearch_dashboards_healthcheck_retries: "{{ default_container_healthcheck_retries }}" +opensearch_dashboards_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}" +opensearch_dashboards_healthcheck_test: ["CMD-SHELL", "healthcheck_curl http://{{ api_interface_address | put_address_in_context('url') }}:{{ opensearch_dashboards_port }}"] +opensearch_dashboards_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}" +opensearch_dashboards_healthcheck: + interval: "{{ opensearch_dashboards_healthcheck_interval }}" + retries: "{{ opensearch_dashboards_healthcheck_retries }}" + start_period: "{{ opensearch_dashboards_healthcheck_start_period }}" + test: "{% if opensearch_dashboards_enable_healthchecks | bool %}{{ opensearch_dashboards_healthcheck_test }}{% else %}NONE{% endif %}" + timeout: "{{ opensearch_dashboards_healthcheck_timeout }}" + +opensearch_default_volumes: + - "{{ node_config_directory }}/opensearch/:{{ container_config_directory }}/" + - "/etc/localtime:/etc/localtime:ro" + - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}" + - "{{ opensearch_datadir_volume }}:/var/lib/opensearch/data" + - "kolla_logs:/var/log/kolla/" +opensearch_dashboards_default_volumes: + - "{{ node_config_directory }}/opensearch-dashboards/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}" + - "kolla_logs:/var/log/kolla/" + +opensearch_extra_volumes: "{{ default_extra_volumes }}" +opensearch_dashboards_extra_volumes: "{{ default_extra_volumes }}" diff --git a/ansible/roles/opensearch/handlers/main.yml b/ansible/roles/opensearch/handlers/main.yml new file mode 100644 index 0000000000..3c147ad893 --- /dev/null +++ b/ansible/roles/opensearch/handlers/main.yml @@ -0,0 +1,33 @@ +--- +- name: Restart opensearch container + vars: + service_name: "opensearch" + service: "{{ opensearch_services[service_name] }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + environment: "{{ service.environment }}" + volumes: "{{ service.volumes }}" + dimensions: "{{ service.dimensions }}" + healthcheck: "{{ service.healthcheck | default(omit) }}" + when: + - kolla_action != "config" + +- name: Restart opensearch-dashboards container + vars: + service_name: "opensearch-dashboards" + service: "{{ opensearch_services[service_name] }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + dimensions: "{{ service.dimensions }}" + environment: "{{ service.environment | default(omit) }}" + image: "{{ service.image }}" + name: "{{ service.container_name }}" + volumes: "{{ service.volumes }}" + when: + - kolla_action != "config" diff --git a/ansible/roles/opensearch/tasks/check-containers.yml b/ansible/roles/opensearch/tasks/check-containers.yml new file mode 100644 index 0000000000..5efb102e97 --- /dev/null +++ b/ansible/roles/opensearch/tasks/check-containers.yml @@ -0,0 +1,18 @@ +--- +- name: Check opensearch containers + become: true + kolla_docker: + action: "compare_container" + common_options: "{{ docker_common_options }}" + dimensions: "{{ item.value.dimensions }}" + environment: "{{ item.value.environment | default(omit) }}" + healthcheck: "{{ item.value.healthcheck | default(omit) }}" + name: "{{ item.value.container_name }}" + image: "{{ item.value.image }}" + volumes: "{{ item.value.volumes }}" + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ opensearch_services }}" + notify: + - "Restart {{ item.key }} container" diff --git a/ansible/roles/opensearch/tasks/check.yml b/ansible/roles/opensearch/tasks/check.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/opensearch/tasks/check.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/opensearch/tasks/config-host.yml b/ansible/roles/opensearch/tasks/config-host.yml new file mode 100644 index 0000000000..8e092b487b --- /dev/null +++ b/ansible/roles/opensearch/tasks/config-host.yml @@ -0,0 +1,17 @@ +--- +- name: Setting sysctl values + become: true + vars: + should_set: "{{ item.value != 'KOLLA_UNSET' }}" + sysctl: + name: "{{ item.name }}" + state: "{{ should_set | ternary('present', 'absent') }}" + value: "{{ should_set | ternary(item.value, omit) }}" + sysctl_set: "{{ should_set }}" + sysctl_file: "{{ kolla_sysctl_conf_path }}" + with_items: + - { name: "vm.max_map_count", value: 262144} + when: + - set_sysctl | bool + - item.value != 'KOLLA_SKIP' + - inventory_hostname in groups['opensearch'] diff --git a/ansible/roles/opensearch/tasks/config.yml b/ansible/roles/opensearch/tasks/config.yml new file mode 100644 index 0000000000..afaecb1d75 --- /dev/null +++ b/ansible/roles/opensearch/tasks/config.yml @@ -0,0 +1,63 @@ +--- +- name: Ensuring config directories exist + file: + path: "{{ node_config_directory }}/{{ item.key }}" + state: "directory" + owner: "{{ config_owner_user }}" + group: "{{ config_owner_group }}" + mode: "0770" + become: true + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ opensearch_services }}" + +- include_tasks: copy-certs.yml + when: + - kolla_copy_ca_into_containers | bool + +- name: Copying over config.json files for services + template: + src: "{{ item.key }}.json.j2" + dest: "{{ node_config_directory }}/{{ item.key }}/config.json" + mode: "0660" + become: true + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ opensearch_services }}" + notify: + - Restart {{ item.key }} container + +- name: Copying over opensearch service config file + merge_yaml: + sources: + - "{{ role_path }}/templates/opensearch.yml.j2" + - "{{ node_custom_config }}/opensearch.yml" + - "{{ node_custom_config }}/opensearch/opensearch.yml" + - "{{ node_custom_config }}/opensearch/{{ inventory_hostname }}/opensearch.yml" + dest: "{{ node_config_directory }}/opensearch/opensearch.yml" + mode: "0660" + become: true + when: + - inventory_hostname in groups['opensearch'] + - opensearch_services['opensearch'].enabled | bool + notify: + - Restart opensearch container + +- name: Copying over opensearch-dashboards config file + vars: + opensearch_dashboards: "{{ opensearch_services['opensearch-dashboards'] }}" + merge_yaml: + sources: + - "{{ role_path }}/templates/opensearch_dashboards.yml.j2" + - "{{ node_custom_config }}/opensearch/opensearch_dashboards.yml" + - "{{ node_custom_config }}/opensearch/{{ inventory_hostname }}/opensearch_dashboards.yml" + dest: "{{ node_config_directory }}/opensearch-dashboards/opensearch_dashboards.yml" + mode: "0660" + become: true + when: + - inventory_hostname in groups['opensearch-dashboards'] + - opensearch_dashboards.enabled | bool + notify: + - Restart opensearch-dashboards container diff --git a/ansible/roles/opensearch/tasks/config_validate.yml b/ansible/roles/opensearch/tasks/config_validate.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/opensearch/tasks/config_validate.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/opensearch/tasks/copy-certs.yml b/ansible/roles/opensearch/tasks/copy-certs.yml new file mode 100644 index 0000000000..554ac38618 --- /dev/null +++ b/ansible/roles/opensearch/tasks/copy-certs.yml @@ -0,0 +1,6 @@ +--- +- name: "Copy certificates and keys for {{ project_name }}" + import_role: + role: service-cert-copy + vars: + project_services: "{{ opensearch_services }}" diff --git a/ansible/roles/opensearch/tasks/deploy-containers.yml b/ansible/roles/opensearch/tasks/deploy-containers.yml new file mode 100644 index 0000000000..eb24ab5c7a --- /dev/null +++ b/ansible/roles/opensearch/tasks/deploy-containers.yml @@ -0,0 +1,2 @@ +--- +- import_tasks: check-containers.yml diff --git a/ansible/roles/opensearch/tasks/deploy.yml b/ansible/roles/opensearch/tasks/deploy.yml new file mode 100644 index 0000000000..ee17effc62 --- /dev/null +++ b/ansible/roles/opensearch/tasks/deploy.yml @@ -0,0 +1,12 @@ +--- +- import_tasks: config-host.yml + +- import_tasks: config.yml + +- import_tasks: check-containers.yml + +- include_tasks: register.yml + when: opensearch_enable_keystone_registration | bool + +- name: Flush handlers + meta: flush_handlers diff --git a/ansible/roles/opensearch/tasks/loadbalancer.yml b/ansible/roles/opensearch/tasks/loadbalancer.yml new file mode 100644 index 0000000000..8a671d6759 --- /dev/null +++ b/ansible/roles/opensearch/tasks/loadbalancer.yml @@ -0,0 +1,7 @@ +--- +- name: "Configure loadbalancer for {{ project_name }}" + import_role: + name: haproxy-config + vars: + project_services: "{{ opensearch_services }}" + tags: always diff --git a/ansible/roles/opensearch/tasks/main.yml b/ansible/roles/opensearch/tasks/main.yml new file mode 100644 index 0000000000..bc5d1e6257 --- /dev/null +++ b/ansible/roles/opensearch/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include_tasks: "{{ kolla_action }}.yml" diff --git a/ansible/roles/opensearch/tasks/precheck.yml b/ansible/roles/opensearch/tasks/precheck.yml new file mode 100644 index 0000000000..1fc36740a3 --- /dev/null +++ b/ansible/roles/opensearch/tasks/precheck.yml @@ -0,0 +1,25 @@ +--- +- import_role: + name: service-precheck + vars: + service_precheck_services: "{{ opensearch_services }}" + service_name: "{{ project_name }}" + +- name: Get container facts + become: true + kolla_container_facts: + name: + - opensearch + check_mode: false + register: container_facts + +- name: Checking free port for Opensearch + wait_for: + host: "{{ api_interface_address }}" + port: "{{ opensearch_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - container_facts['opensearch'] is not defined + - inventory_hostname in groups['opensearch'] diff --git a/ansible/roles/opensearch/tasks/pull.yml b/ansible/roles/opensearch/tasks/pull.yml new file mode 100644 index 0000000000..53f9c5fda1 --- /dev/null +++ b/ansible/roles/opensearch/tasks/pull.yml @@ -0,0 +1,3 @@ +--- +- import_role: + role: service-images-pull diff --git a/ansible/roles/opensearch/tasks/reconfigure.yml b/ansible/roles/opensearch/tasks/reconfigure.yml new file mode 100644 index 0000000000..5b10a7e111 --- /dev/null +++ b/ansible/roles/opensearch/tasks/reconfigure.yml @@ -0,0 +1,2 @@ +--- +- import_tasks: deploy.yml diff --git a/ansible/roles/opensearch/tasks/register.yml b/ansible/roles/opensearch/tasks/register.yml new file mode 100644 index 0000000000..1fcb8be39b --- /dev/null +++ b/ansible/roles/opensearch/tasks/register.yml @@ -0,0 +1,7 @@ +--- +- import_role: + name: service-ks-register + vars: + service_ks_register_auth: "{{ opensearch_openstack_auth }}" + service_ks_register_services: "{{ opensearch_ks_services }}" + tags: always diff --git a/ansible/roles/opensearch/tasks/stop.yml b/ansible/roles/opensearch/tasks/stop.yml new file mode 100644 index 0000000000..8a9d328909 --- /dev/null +++ b/ansible/roles/opensearch/tasks/stop.yml @@ -0,0 +1,6 @@ +--- +- import_role: + name: service-stop + vars: + project_services: "{{ opensearch_services }}" + service_name: "{{ project_name }}" diff --git a/ansible/roles/opensearch/tasks/upgrade.yml b/ansible/roles/opensearch/tasks/upgrade.yml new file mode 100644 index 0000000000..2891b64e08 --- /dev/null +++ b/ansible/roles/opensearch/tasks/upgrade.yml @@ -0,0 +1,48 @@ +--- +- name: Disable shard allocation + become: true + vars: + opensearch_shard_body: {"transient": {"cluster.routing.allocation.enable": "none"}} + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_cluster/settings" + method: PUT + status_code: 200 + return_content: yes + body: "{{ opensearch_shard_body | to_json }}" # noqa jinja[invalid] + body_format: json + delegate_to: "{{ groups['opensearch'][0] }}" + run_once: true + +- name: Perform a flush + become: true + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_flush" + method: POST + status_code: 200 + return_content: yes + body_format: json + delegate_to: "{{ groups['opensearch'][0] }}" + run_once: true + retries: 10 + delay: 5 + register: result + until: ('status' in result) and result.status == 200 + +- import_tasks: config-host.yml + +- import_tasks: config.yml + +- import_tasks: check-containers.yml + +- include_tasks: register.yml + when: + - opensearch_enable_keystone_registration | bool + +- name: Flush handlers + meta: flush_handlers diff --git a/ansible/roles/opensearch/templates/opensearch-dashboards.json.j2 b/ansible/roles/opensearch/templates/opensearch-dashboards.json.j2 new file mode 100644 index 0000000000..08cddf33f3 --- /dev/null +++ b/ansible/roles/opensearch/templates/opensearch-dashboards.json.j2 @@ -0,0 +1,23 @@ +{ + "command": "/usr/share/opensearch-dashboards/bin/opensearch-dashboards --config /etc/opensearch/opensearch_dashboards.yml", + "config_files": [ + { + "source": "{{ container_config_directory }}/opensearch_dashboards.yml", + "dest": "/etc/opensearch/opensearch_dashboards.yml", + "owner": "opensearch", + "perm": "0640" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/opensearch", + "owner": "opensearch:opensearch", + "recurse": true + }, + { + "path": "/usr/share/opensearch/dashboards/optimize/bundles", + "owner": "opensearch:opensearch", + "recurse": true + } + ] +} diff --git a/ansible/roles/opensearch/templates/opensearch.json.j2 b/ansible/roles/opensearch/templates/opensearch.json.j2 new file mode 100644 index 0000000000..2228f29403 --- /dev/null +++ b/ansible/roles/opensearch/templates/opensearch.json.j2 @@ -0,0 +1,23 @@ +{ + "command": "/usr/share/opensearch/bin/opensearch", + "config_files": [ + { + "source": "{{ container_config_directory }}/opensearch.yml", + "dest": "/usr/share/opensearch/config/opensearch.yml", + "owner": "opensearch", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/lib/opensearch", + "owner": "opensearch:opensearch", + "recurse": true + }, + { + "path": "/var/log/kolla/opensearch", + "owner": "opensearch:opensearch", + "recurse": true + } + ] +} diff --git a/ansible/roles/opensearch/templates/opensearch.yml.j2 b/ansible/roles/opensearch/templates/opensearch.yml.j2 new file mode 100644 index 0000000000..ed4c632190 --- /dev/null +++ b/ansible/roles/opensearch/templates/opensearch.yml.j2 @@ -0,0 +1,21 @@ +{% set num_nodes = groups['opensearch'] | length %} +{% set recover_after_nodes = (num_nodes * 2 / 3) | round(0, 'floor') | int if num_nodes > 1 else 1 %} +plugins.security.disabled: "true" + +node.name: "{{ 'api' | kolla_address | put_address_in_context('url') }}" +network.host: "{{ 'api' | kolla_address | put_address_in_context('url') }}" + +cluster.name: "{{ opensearch_cluster_name }}" +cluster.initial_master_nodes: [{% for host in groups['opensearch'] %}"{{ 'api' | kolla_address(host) }}"{% if not loop.last %},{% endif %}{% endfor %}] +node.master: true +node.data: true +discovery.seed_hosts: [{% for host in groups['opensearch'] %}"{{ 'api' | kolla_address(host) | put_address_in_context('url') }}"{% if not loop.last %},{% endif %}{% endfor %}] + +http.port: {{ opensearch_port }} +gateway.expected_nodes: {{ num_nodes }} +gateway.recover_after_time: "5m" +gateway.recover_after_nodes: {{ recover_after_nodes }} +path.data: "/var/lib/opensearch/data" +path.logs: "/var/log/kolla/opensearch" +indices.fielddata.cache.size: 40% +action.auto_create_index: "true" diff --git a/ansible/roles/opensearch/templates/opensearch_dashboards.yml.j2 b/ansible/roles/opensearch/templates/opensearch_dashboards.yml.j2 new file mode 100644 index 0000000000..b9c20aa40e --- /dev/null +++ b/ansible/roles/opensearch/templates/opensearch_dashboards.yml.j2 @@ -0,0 +1,12 @@ +opensearchDashboards.defaultAppId: "{{ opensearch_dashboards_default_app_id }}" +logging.dest: /var/log/kolla/opensearch/opensearch-dashboards.log +server.port: {{ opensearch_dashboards_port }} +server.host: "{{ api_interface_address }}" +opensearch.hosts: "{{ opensearch_internal_endpoint }}" +opensearch.requestTimeout: {{ opensearch_dashboards_opensearch_request_timeout }} +opensearch.shardTimeout: {{ opensearch_dashboards_opensearch_shard_timeout }} +opensearch.ssl.verificationMode: "{{ 'full' if opensearch_dashboards_opensearch_ssl_verify | bool else 'none' }}" +data.search.usageTelemetry.enabled: false +{% if openstack_cacert | length > 0 %} +opensearch.ssl.certificateAuthorities: {{ openstack_cacert }} +{% endif %} diff --git a/ansible/roles/opensearch/vars/main.yml b/ansible/roles/opensearch/vars/main.yml new file mode 100644 index 0000000000..b3253f7773 --- /dev/null +++ b/ansible/roles/opensearch/vars/main.yml @@ -0,0 +1,2 @@ +--- +project_name: "opensearch" diff --git a/ansible/roles/prometheus/templates/prometheus-elasticsearch-exporter.json.j2 b/ansible/roles/prometheus/templates/prometheus-elasticsearch-exporter.json.j2 index fe881babb7..979303b116 100644 --- a/ansible/roles/prometheus/templates/prometheus-elasticsearch-exporter.json.j2 +++ b/ansible/roles/prometheus/templates/prometheus-elasticsearch-exporter.json.j2 @@ -1,5 +1,5 @@ { - "command": "/opt/elasticsearch_exporter/elasticsearch_exporter --es.uri http://{{ api_interface_address | put_address_in_context('url') }}:{{ elasticsearch_port }} --web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_elasticsearch_exporter_port }}{% if prometheus_elasticsearch_exporter_cmdline_extras %} {{ prometheus_elasticsearch_exporter_cmdline_extras }}{% endif %}", + "command": "/opt/elasticsearch_exporter/elasticsearch_exporter --es.uri http://{{ api_interface_address | put_address_in_context('url') }}:{{ opensearch_port if enable_opensearch | bool else elasticsearch_port }} --web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_elasticsearch_exporter_port }}{% if prometheus_elasticsearch_exporter_cmdline_extras %} {{ prometheus_elasticsearch_exporter_cmdline_extras }}{% endif %}", "config_files": [], "permissions": [ { diff --git a/ansible/roles/telegraf/defaults/main.yml b/ansible/roles/telegraf/defaults/main.yml index 3f97183b84..58ee32e252 100644 --- a/ansible/roles/telegraf/defaults/main.yml +++ b/ansible/roles/telegraf/defaults/main.yml @@ -37,4 +37,5 @@ haproxy_proto: "http" influxdb_proto: "http" rabbitmq_proto: "http" mariadb_proto: "tcp" +opensearch_proto: "http" outward_rabbitmq_proto: "http" diff --git a/ansible/roles/telegraf/templates/telegraf.conf.j2 b/ansible/roles/telegraf/templates/telegraf.conf.j2 index 0e7a0bb251..ec72e5a2c4 100644 --- a/ansible/roles/telegraf/templates/telegraf.conf.j2 +++ b/ansible/roles/telegraf/templates/telegraf.conf.j2 @@ -60,6 +60,11 @@ servers = ["{{ elasticsearch_proto }}://{{ api_interface_address | put_address_in_context('url') }}:{{ elasticsearch_port }}"] local = true cluster_health = true +{% elif inventory_hostname in groups['opensearch'] and enable_opensearch | bool %} +[[inputs.elasticsearch]] + servers = ["{{ opensearch_proto }}://{{ api_interface_address | put_address_in_context('url') }}:{{ opensearch_port }}"] + local = true + cluster_health = true {% endif %} {% if inventory_hostname in groups['rabbitmq'] and enable_rabbitmq | bool %} [[inputs.rabbitmq]] diff --git a/ansible/site.yml b/ansible/site.yml index dae66ea41d..6bbc7d5e23 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -57,6 +57,8 @@ - enable_neutron_{{ enable_neutron | bool }} - enable_nova_{{ enable_nova | bool }} - enable_octavia_{{ enable_octavia | bool }} + - enable_opensearch_{{ enable_opensearch | bool }} + - enable_opensearch_dashboards_{{ enable_opensearch_dashboards | bool }} - enable_openvswitch_{{ enable_openvswitch | bool }}_enable_ovs_dpdk_{{ enable_ovs_dpdk | bool }} - enable_outward_rabbitmq_{{ enable_outward_rabbitmq | bool }} - enable_ovn_{{ enable_ovn | bool }} @@ -283,6 +285,11 @@ tasks_from: loadbalancer tags: octavia when: enable_octavia | bool + - include_role: + name: opensearch + tasks_from: loadbalancer + tags: opensearch + when: enable_opensearch | bool - include_role: name: prometheus tasks_from: loadbalancer @@ -555,6 +562,17 @@ tags: elasticsearch, when: enable_elasticsearch | bool } +- name: Apply role opensearch + gather_facts: false + hosts: + - opensearch + - '&enable_opensearch_True' + serial: '{{ kolla_serial|default("0") }}' + roles: + - { role: opensearch, + tags: opensearch, + when: enable_opensearch | bool } + - name: Apply role kibana gather_facts: false hosts: diff --git a/doc/source/reference/logging-and-monitoring/central-logging-guide.rst b/doc/source/reference/logging-and-monitoring/central-logging-guide-elasticsearch.rst similarity index 96% rename from doc/source/reference/logging-and-monitoring/central-logging-guide.rst rename to doc/source/reference/logging-and-monitoring/central-logging-guide-elasticsearch.rst index f2a41ab76b..ffb2d0b7bd 100644 --- a/doc/source/reference/logging-and-monitoring/central-logging-guide.rst +++ b/doc/source/reference/logging-and-monitoring/central-logging-guide-elasticsearch.rst @@ -1,4 +1,4 @@ -.. _central-logging-guide: +.. _central-logging-guide-elasticsearch: =============== Central Logging @@ -8,6 +8,14 @@ An OpenStack deployment generates vast amounts of log data. In order to successfully monitor this and use it to diagnose problems, the standard "ssh and grep" solution quickly becomes unmanageable. +.. note:: + + OpenSearch support has been backported together with Rocky Linux 9 + support - new deployments using that distribution version will + deploy OpenSearch by default. + For migration from Elasticsearch to OpenSearch and general + OpenSearch docs see :ref:`here ` + Preparation and deployment ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/reference/logging-and-monitoring/central-logging-guide-opensearch.rst b/doc/source/reference/logging-and-monitoring/central-logging-guide-opensearch.rst new file mode 100644 index 0000000000..f7abd4b99c --- /dev/null +++ b/doc/source/reference/logging-and-monitoring/central-logging-guide-opensearch.rst @@ -0,0 +1,252 @@ +.. _central-logging-guide-opensearch: + +============================ +Central Logging - OpenSearch +============================ + +An OpenStack deployment generates vast amounts of log data. In order to +successfully monitor this and use it to diagnose problems, the standard "ssh +and grep" solution quickly becomes unmanageable. + +Preparation and deployment +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Modify the configuration file ``/etc/kolla/globals.yml`` and change +the following: + +.. code-block:: yaml + + enable_central_logging: "yes" + enable_opensearch: "yes" + +Migration +~~~~~~~~~ + +In order to perform Elasticsearch to OpenSearch migration - modify +``/etc/kolla/globals.yml`` and change the following: + +.. code-block:: yaml + + enable_elasticsearch: "no" + enable_opensearch: "yes" + +The migration itself is performed by running following command: + +.. code-block:: console + + kolla-ansible opensearch-migration + +OpenSearch +~~~~~~~~~~ + +Kolla deploys OpenSearch to store, organize and make logs easily accessible. + +By default OpenSearch is deployed on port ``9200``. + +.. note:: + + OpenSearch stores a lot of logs, so if you are running centralized logging, + remember to give ``/var/lib/docker`` adequate space. + + Alternatively it is possible to use a local directory instead of the volume + ``opensearch`` to store the data of OpenSearch. The path can be set via + the variable ``opensearch_datadir_volume``. + +OpenSearch Dashboards +~~~~~~~~~~~~~~~~~~~~~ + +Kolla deploys OpenSearch dashboards to allow operators to +search and visualise logs in a centralised manner. + +After a successful deployment, OpenSearch Dashboards can be accessed using a +browser on ``:5601`` or +``:5601``. + +The default username is ``opensearch``, the password can be located under +```` in ``/etc/kolla/passwords.yml``. + +If you want to prevent OpenSearch Dashboards being exposed on the external +VIP, you can set ``enable_opensearch_dashboards_external`` to ``false`` in +``/etc/kolla/globals.yml``. + +First Login +----------- + +When OpenSearch Dashboards is opened for the first time, it requires creating +a default index pattern. To view, analyse and search logs, at least one +index pattern has to be created. To match indices stored in OpenSearch, +we suggest using the following configuration: + +#. Index pattern - flog-* +#. Time Filter field name - @timestamp +#. Expand index pattern when searching [DEPRECATED] - not checked +#. Use event times to create index names [DEPRECATED] - not checked + +After setting parameters, one can create an index with the *Create* button. + +Search logs - Discover tab +-------------------------- + +Operators can create and store searches based on various fields from logs, for +example, "show all logs marked with ERROR on nova-compute". + +To do this, click the ``Discover`` tab. Fields from the logs can be filtered by +hovering over entries from the left hand side, and clicking ``add`` or +``remove``. Add the following fields: + +* Hostname +* Payload +* severity_label +* programname + +This yields an easy to read list of all log events from each node in the +deployment within the last 15 minutes. A "tail like" functionality can be +achieved by clicking the clock icon in the top right hand corner of the screen, +and selecting ``Auto-refresh``. + +Logs can also be filtered down further. To use the above example, type +``programname:nova-compute`` in the search bar. Click the drop-down arrow from +one of the results, then the small magnifying glass icon from beside the +programname field. This should now show a list of all events from nova-compute +services across the cluster. + +The current search can also be saved by clicking the ``Save Search`` icon +available from the menu on the right hand side. + +Example: using OpenSearch Dashboards to diagnose a common failure +----------------------------------------------------------------- + +The following example demonstrates how OpenSearch can be used to diagnose a +common OpenStack problem, where an instance fails to launch with the error +'No valid host was found'. + +First, re-run the server creation with ``--debug``: + +.. code-block:: console + + openstack --debug server create --image cirros --flavor m1.tiny \ + --key-name mykey --nic net-id=00af016f-dffe-4e3c-a9b8-ec52ccd8ea65 \ + demo1 + +In this output, look for the key ``X-Compute-Request-Id``. This is a unique +identifier that can be used to track the request through the system. An +example ID looks like this: + +.. code-block:: console + + X-Compute-Request-Id: req-c076b50a-6a22-48bf-8810-b9f41176a6d5 + +Taking the value of ``X-Compute-Request-Id``, enter the value into the +OpenSearch Dashboards search bar, minus the leading ``req-``. Assuming some +basic filters have been added as shown in the previous section, OpenSearch +Dashboards should now show the path this request made through the +OpenStack deployment, starting at a ``nova-api`` on a control node, +through the ``nova-scheduler``, ``nova-conductor``, and finally +``nova-compute``. Inspecting the ``Payload`` of the entries marked ``ERROR`` +should quickly lead to the source of the problem. + +While some knowledge is still required of how Nova works in this instance, it +can still be seen how OpenSearch Dashboards helps in tracing this data, +particularly in a large scale deployment scenario. + +Visualize data - Visualize tab +------------------------------ + +In the visualization tab a wide range of charts is available. If any +visualization has not been saved yet, after choosing this tab *Create a new +visualization* panel is opened. If a visualization has already been saved, +after choosing this tab, lately modified visualization is opened. In this +case, one can create a new visualization by choosing *add visualization* +option in the menu on the right. In order to create new visualization, one +of the available options has to be chosen (pie chart, area chart). Each +visualization can be created from a saved or a new search. After choosing +any kind of search, a design panel is opened. In this panel, a chart can be +generated and previewed. In the menu on the left, metrics for a chart can +be chosen. The chart can be generated by pressing a green arrow on the top +of the left-side menu. + +.. note:: + + After creating a visualization, it can be saved by choosing *save + visualization* option in the menu on the right. If it is not saved, it + will be lost after leaving a page or creating another visualization. + +Organize visualizations and searches - Dashboard tab +---------------------------------------------------- + +In the Dashboard tab all of saved visualizations and searches can be +organized in one Dashboard. To add visualization or search, one can choose +*add visualization* option in the menu on the right and then choose an item +from all saved ones. The order and size of elements can be changed directly +in this place by moving them or resizing. The color of charts can also be +changed by checking a colorful dots on the legend near each visualization. + +.. note:: + + After creating a dashboard, it can be saved by choosing *save dashboard* + option in the menu on the right. If it is not saved, it will be lost after + leaving a page or creating another dashboard. + +If a Dashboard has already been saved, it can be opened by choosing *open +dashboard* option in the menu on the right. + +Exporting and importing created items - Settings tab +---------------------------------------------------- + +Once visualizations, searches or dashboards are created, they can be exported +to a JSON format by choosing Settings tab and then Objects tab. Each of the +item can be exported separately by selecting it in the menu. All of the items +can also be exported at once by choosing *export everything* option. +In the same tab (Settings - Objects) one can also import saved items by +choosing *import* option. + +Custom log rules +~~~~~~~~~~~~~~~~ + +Kolla Ansible automatically deploys Fluentd for forwarding OpenStack logs +from across the control plane to a central logging repository. The Fluentd +configuration is split into four parts: Input, forwarding, filtering and +formatting. The following can be customised: + +Custom log filtering +-------------------- + +In some scenarios it may be useful to apply custom filters to logs before +forwarding them. This may be useful to add additional tags to the messages +or to modify the tags to conform to a log format that differs from the one +defined by kolla-ansible. + +Configuration of custom fluentd filters is possible by placing filter +configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on the +control host. + +Custom log formatting +--------------------- + +In some scenarios it may be useful to perform custom formatting of logs before +forwarding them. For example, the JSON formatter plugin can be used to convert +an event to JSON. + +Configuration of custom fluentd formatting is possible by placing filter +configuration files in ``/etc/kolla/config/fluentd/format/*.conf`` on the +control host. + +Custom log forwarding +--------------------- + +In some scenarios it may be useful to forward logs to a logging service other +than elasticsearch. This can be done by configuring custom fluentd outputs. + +Configuration of custom fluentd outputs is possible by placing output +configuration files in ``/etc/kolla/config/fluentd/output/*.conf`` on the +control host. + +Custom log inputs +----------------- + +In some scenarios it may be useful to input logs from other services, e.g. +network equipment. This can be done by configuring custom fluentd inputs. + +Configuration of custom fluentd inputs is possible by placing input +configuration files in ``/etc/kolla/config/fluentd/input/*.conf`` on the +control host. diff --git a/doc/source/reference/logging-and-monitoring/index.rst b/doc/source/reference/logging-and-monitoring/index.rst index 7c164728e5..4515c7d531 100644 --- a/doc/source/reference/logging-and-monitoring/index.rst +++ b/doc/source/reference/logging-and-monitoring/index.rst @@ -8,7 +8,8 @@ logging and monitoring services available in kolla. .. toctree:: :maxdepth: 1 - central-logging-guide + central-logging-guide-elasticsearch + central-logging-guide-opensearch grafana-guide influxdb-guide kafka-guide diff --git a/doc/source/reference/logging-and-monitoring/monasca-guide.rst b/doc/source/reference/logging-and-monitoring/monasca-guide.rst index 2c04f7bdb0..67b91bd08a 100644 --- a/doc/source/reference/logging-and-monitoring/monasca-guide.rst +++ b/doc/source/reference/logging-and-monitoring/monasca-guide.rst @@ -4,6 +4,10 @@ Monasca - Monitoring service ============================ +.. note:: + + Monasca does not support OpenSearch. + Overview ~~~~~~~~ diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 9a950deb7d..2822664f47 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -382,6 +382,9 @@ #enable_nova_ssh: "yes" #enable_octavia: "no" #enable_octavia_driver_agent: "{{ enable_octavia | bool and neutron_plugin_agent == 'ovn' }}" +#enable_opensearch: "{{ kolla_base_distro == 'rocky' and enable_central_logging | bool }}" +#enable_opensearch_dashboards: "{{ enable_opensearch | bool }}" +#enable_opensearch_dashboards_external: "{{ enable_opensearch_dashboards | bool }}" #enable_openvswitch: "{{ enable_neutron | bool and neutron_plugin_agent != 'linuxbridge' }}" #enable_ovn: "{{ enable_neutron | bool and neutron_plugin_agent == 'ovn' }}" #enable_ovs_dpdk: "no" diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml index a7db8b9e30..b79a10afbd 100644 --- a/etc/kolla/passwords.yml +++ b/etc/kolla/passwords.yml @@ -262,3 +262,8 @@ ceph_rgw_keystone_password: # libvirt options ################## libvirt_sasl_password: + +############ +# OpenSearch +############ +opensearch_dashboards_password: diff --git a/releasenotes/notes/add-opensearch-53ef174195acce45.yaml b/releasenotes/notes/add-opensearch-53ef174195acce45.yaml new file mode 100644 index 0000000000..db9d2c73c7 --- /dev/null +++ b/releasenotes/notes/add-opensearch-53ef174195acce45.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Adds support for deploying OpenSearch and OpenSearch dashboards. These + services directly replace ElasticSearch and Kibana which are now + end-of-life. Support for sending logs to a remote ElasticSearch (or + OpenSearch) cluster is maintained. + - | + Adds support for migrating from Elasticsearch to OpenSearch by + running ``kolla-ansible opensearch-migration`` command. diff --git a/tests/run.yml b/tests/run.yml index 6f47397fc7..14ca0df361 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -10,7 +10,7 @@ - name: set facts for commonly used variables vars: # NOTE(yoctozepto): needed here to use in other facts too - openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb', 'prometheus-efk', 'monasca', 'venus'] }}" + openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb', 'prometheus-efk', 'prometheus-opensearch', 'monasca', 'venus'] }}" set_fact: kolla_inventory_path: "/etc/kolla/inventory" logs_dir: "/tmp/logs" @@ -549,8 +549,22 @@ cmd: test-prometheus-efk.sh executable: /bin/bash chdir: "{{ kolla_ansible_src_dir }}" + environment: + KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}" + OPENSEARCH_MIGRATION: "{{ opensearch_migration | default(False) }}" when: scenario == "prometheus-efk" + - name: Run test-prometheus-opensearch.sh script + script: + cmd: test-prometheus-opensearch.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + environment: + TLS_ENABLED: "{{ tls_enabled }}" + when: + - not is_upgrade + - scenario == "prometheus-opensearch" + - name: Run test-venus.sh script script: cmd: test-venus.sh @@ -725,6 +739,7 @@ cmd: tests/test-swift.sh chdir: "{{ kolla_ansible_src_dir }}" when: scenario == 'swift' + when: is_upgrade # Bifrost testing. diff --git a/tests/setup_gate.sh b/tests/setup_gate.sh index 88ae7869f1..827067c946 100755 --- a/tests/setup_gate.sh +++ b/tests/setup_gate.sh @@ -102,8 +102,8 @@ EOF GATE_IMAGES="^cron,^fluentd,^haproxy,^keepalived,^kolla-toolbox,^mariadb" fi - if [[ $SCENARIO == "prometheus-efk" ]]; then - GATE_IMAGES="^cron,^elasticsearch,^fluentd,^grafana,^haproxy,^keepalived,^kibana,^kolla-toolbox,^mariadb,^memcached,^prometheus,^rabbitmq" + if [[ $SCENARIO == "prometheus-opensearch" ]]; then + GATE_IMAGES="^cron,^fluentd,^grafana,^haproxy,^keepalived,^kolla-toolbox,^mariadb,^memcached,^opensearch,^prometheus,^rabbitmq" fi if [[ $SCENARIO == "monasca" ]]; then diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index 77b4c27c81..1c64105470 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -162,8 +162,11 @@ octavia_provider_drivers: "ovn:OVN provider" octavia_provider_agents: "ovn" {% endif %} -{% if scenario == "prometheus-efk" %} +{% if scenario in ["prometheus-efk", "prometheus-opensearch"] %} enable_central_logging: "yes" +{% if scenario == "prometheus-opensearch" %} +enable_opensearch: "yes" +{% endif %} enable_grafana: "yes" enable_prometheus: "yes" enable_prometheus_openstack_exporter: "no" @@ -195,7 +198,7 @@ octavia_network_type: "tenant" {% endif %} {% if scenario == "venus" %} -enable_elasticsearch: "yes" +enable_opensearch: "yes" enable_keystone: "yes" enable_venus: "yes" {% endif %} diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2 index 8a4d5198d4..8fe86989da 100644 --- a/tests/templates/inventory.j2 +++ b/tests/templates/inventory.j2 @@ -310,10 +310,16 @@ common [kolla-toolbox:children] common -# Elasticsearch Curator [elasticsearch-curator:children] elasticsearch +[opensearch:children] +control + +# Opensearch Dashboards +[opensearch-dashboards:children] +opensearch + # Glance [glance-api:children] glance @@ -763,7 +769,11 @@ monitoring monitoring [prometheus-elasticsearch-exporter:children] +{% if is_upgrade %} elasticsearch +{% else %} +opensearch +{% endif %} [prometheus-blackbox-exporter:children] monitoring diff --git a/tests/test-prometheus-efk.sh b/tests/test-prometheus-efk.sh old mode 100755 new mode 100644 index 86406223ad..ac577c5709 --- a/tests/test-prometheus-efk.sh +++ b/tests/test-prometheus-efk.sh @@ -162,13 +162,25 @@ function test_prometheus { echo "SUCCESS: Prometheus" } +function test_opensearch_migration { + echo "MIGRATION: Migrating to Opensearch" + echo "enable_opensearch: true" >> /etc/kolla/globals.yml + RAW_INVENTORY=/etc/kolla/inventory + source ${KOLLA_ANSIBLE_VENV_PATH}/bin/activate + kolla-ansible -i ${RAW_INVENTORY} -vvv opensearch-migration + echo "SUCESS: Migrated to Opensearch" + tests/test-prometheus-opensearch.sh +} + function test_prometheus_efk_logged { . /etc/kolla/admin-openrc.sh - test_kibana test_elasticsearch test_grafana test_prometheus + if [[ "$OPENSEARCH_MIGRATION" == "True" ]]; then + test_opensearch_migration + fi } function test_prometheus_efk { diff --git a/tests/test-prometheus-opensearch.sh b/tests/test-prometheus-opensearch.sh new file mode 100755 index 0000000000..9ce8a76636 --- /dev/null +++ b/tests/test-prometheus-opensearch.sh @@ -0,0 +1,189 @@ +#!/bin/bash + +set -o xtrace +set -o errexit +set -o pipefail + +# Enable unbuffered output +export PYTHONUNBUFFERED=1 + +function check_opensearch_dashboards { + # Perform and validate a basic status page check + OPENSEARCH_DASHBOARDS_URL=${OS_AUTH_URL%:*}:5601/api/status + output_path=$1 + opensearch_dashboards_password=$(awk '$1 == "opensearch_dashboards_password:" { print $2 }' /etc/kolla/passwords.yml) + args=( + --include + --location + --fail + --user + opensearch:$opensearch_dashboards_password + ) + if [[ "$TLS_ENABLED" = "True" ]]; then + args+=(--cacert $OS_CACERT) + fi + if ! curl "${args[@]}" $OPENSEARCH_DASHBOARDS_URL > $output_path; then + return 1 + fi + if ! grep 'Looking good' $output_path >/dev/null; then + return 1 + fi +} + +function check_opensearch { + # Verify that we see a healthy index created due to Fluentd forwarding logs + OPENSEARCH_URL=${OS_AUTH_URL%:*}:9200/_cluster/health + output_path=$1 + args=( + --include + --location + --fail + ) + if [[ "$TLS_ENABLED" = "True" ]]; then + args+=(--cacert $OS_CACERT) + fi + if ! curl "${args[@]}" $OPENSEARCH_URL > $output_path; then + return 1 + fi + # NOTE(mgoddard): Status may be yellow because no indices have been + # created. + if ! grep -E '"status":"(green|yellow)"' $output_path >/dev/null; then + return 1 + fi +} + +function check_grafana { + # Query grafana, and check that the returned page looks like a grafana page. + GRAFANA_URL=${OS_AUTH_URL%:*}:3000 + output_path=$1 + grafana_password=$(awk '$1 == "grafana_admin_password:" { print $2 }' /etc/kolla/passwords.yml) + args=( + --include + --location + --fail + --user + admin:$grafana_password + ) + if [[ "$TLS_ENABLED" = "True" ]]; then + args+=(--cacert $OS_CACERT) + fi + if ! curl "${args[@]}" $GRAFANA_URL > $output_path; then + return 1 + fi + if ! grep 'Grafana' $output_path >/dev/null; then + return 1 + fi +} + +function check_prometheus { + # Query prometheus graph, and check that the returned page looks like a + # prometheus page. + PROMETHEUS_URL=${OS_AUTH_URL%:*}:9091/graph + output_path=$1 + args=( + --include + --location + --fail + ) + if [[ "$TLS_ENABLED" = "True" ]]; then + args+=(--cacert $OS_CACERT) + fi + if ! curl "${args[@]}" $PROMETHEUS_URL > $output_path; then + return 1 + fi + if ! grep 'Prometheus' $output_path >/dev/null; then + return 1 + fi +} + +function test_opensearch_dashboards { + echo "TESTING: OpenSearch Dashboards" + output_path=$(mktemp) + attempt=1 + while ! check_opensearch_dashboards $output_path; do + echo "OpenSearch Dashboards not accessible yet" + attempt=$((attempt+1)) + if [[ $attempt -eq 12 ]]; then + echo "FAILED: OpenSearch Dashboards did not become accessible. Response:" + cat $output_path + return 1 + fi + sleep 10 + done + echo "SUCCESS: OpenSearch Dashboards" +} + +function test_opensearch { + echo "TESTING: OpenSearch" + output_path=$(mktemp) + attempt=1 + while ! check_opensearch $output_path; do + echo "OpenSearch not accessible yet" + attempt=$((attempt+1)) + if [[ $attempt -eq 12 ]]; then + echo "FAILED: OpenSearch did not become accessible. Response:" + cat $output_path + return 1 + fi + sleep 10 + done + echo "SUCCESS: OpenSearch" +} + +function test_grafana { + echo "TESTING: Grafana" + output_path=$(mktemp) + attempt=1 + while ! check_grafana $output_path; do + echo "Grafana not accessible yet" + attempt=$((attempt+1)) + if [[ $attempt -eq 12 ]]; then + echo "FAILED: Grafana did not become accessible. Response:" + cat $output_path + return 1 + fi + sleep 10 + done + echo "SUCCESS: Grafana" +} + +function test_prometheus { + # TODO(mgoddard): Query metrics. + echo "TESTING: Prometheus" + output_path=$(mktemp) + attempt=1 + while ! check_prometheus $output_path; do + echo "Prometheus not accessible yet" + attempt=$((attempt+1)) + if [[ $attempt -eq 12 ]]; then + echo "FAILED: Prometheus did not become accessible. Response:" + cat $output_path + return 1 + fi + sleep 10 + done + echo "SUCCESS: Prometheus" +} + +function test_prometheus_opensearch_logged { + . /etc/kolla/admin-openrc.sh + + test_opensearch_dashboards + test_opensearch + test_grafana + test_prometheus +} + +function test_prometheus_opensearch { + echo "Testing prometheus and OpenSearch" + test_prometheus_opensearch_logged > /tmp/logs/ansible/test-prometheus-opensearch 2>&1 + result=$? + if [[ $result != 0 ]]; then + echo "Testing prometheus and OpenSearch failed. See ansible/test-prometheus-opensearch for details" + else + echo "Successfully tested prometheus and OpenSearch. See ansible/test-prometheus-opensearch for details" + fi + return $result +} + +test_prometheus_opensearch diff --git a/tests/test-venus.sh b/tests/test-venus.sh index 4bb4a4c8fb..0039d67749 100755 --- a/tests/test-venus.sh +++ b/tests/test-venus.sh @@ -7,17 +7,17 @@ set -o pipefail # Enable unbuffered output export PYTHONUNBUFFERED=1 -# TODO(yoctozepto): Avoid duplicating this from prometheus-efk -function check_elasticsearch { +# TODO(yoctozepto): Avoid duplicating this from prometheus-opensearch +function check_opensearch { # Verify that we see a healthy index created due to Fluentd forwarding logs - local es_url=${OS_AUTH_URL%:*}:9200/_cluster/health + local opensearch_url=${OS_AUTH_URL%:*}:9200/_cluster/health output_path=$1 args=( --include --location --fail ) - if ! curl "${args[@]}" $es_url > $output_path; then + if ! curl "${args[@]}" $opensearch_url > $output_path; then return 1 fi # NOTE(mgoddard): Status may be yellow because no indices have been @@ -38,21 +38,21 @@ function check_venus { fi } -function test_elasticsearch { - echo "TESTING: Elasticsearch" +function test_opensearch { + echo "TESTING: OpenSearch" output_path=$(mktemp) attempt=1 - while ! check_elasticsearch $output_path; do - echo "Elasticsearch not accessible yet" + while ! check_opensearch $output_path; do + echo "OpenSearch not accessible yet" attempt=$((attempt+1)) if [[ $attempt -eq 12 ]]; then - echo "FAILED: Elasticsearch did not become accessible. Response:" + echo "FAILED: OpenSearch did not become accessible. Response:" cat $output_path return 1 fi sleep 10 done - echo "SUCCESS: Elasticsearch" + echo "SUCCESS: OpenSearch" } function test_venus { @@ -75,12 +75,12 @@ function test_venus { function test_venus_scenario_logged { . /etc/kolla/admin-openrc.sh - test_elasticsearch + test_opensearch test_venus } function test_venus_scenario { - echo "Testing Venus and EFK" + echo "Testing Venus and OpenSearch" test_venus_scenario_logged > /tmp/logs/ansible/test-venus-scenario 2>&1 result=$? if [[ $result != 0 ]]; then diff --git a/tools/cleanup-host b/tools/cleanup-host index 4a42b0a4e4..e6271bf8c6 100755 --- a/tools/cleanup-host +++ b/tools/cleanup-host @@ -73,6 +73,11 @@ if [[ "$kafka_datadir_volume" != "kafka" && -d "$kafka_datadir_volume" ]]; then rm -rfv $kafka_datadir_volume fi +if [[ "$opensearch_datadir_volume" != "opensearch" && -d "$opensearch_datadir_volume" ]]; then + echo "Removing opensearch volume if it is customzied" + rm -rfv $opensearch_datadir_volume +fi + FOLDER_PATH="/etc/kolla" if [[ -e "$FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.sh" ]]; then diff --git a/tools/kolla-ansible b/tools/kolla-ansible index 2f80ff67a6..2c93460344 100755 --- a/tools/kolla-ansible +++ b/tools/kolla-ansible @@ -246,6 +246,7 @@ upgrade-bifrost genconfig prune-images nova-libvirt-cleanup +opensearch-migration EOF } @@ -553,6 +554,11 @@ EOF ACTION="Cleanup disabled nova_libvirt containers" PLAYBOOK="${BASEDIR}/ansible/nova-libvirt-cleanup.yml" ;; +(opensearch-migration) + ACTION="Migrate to OpenSearch" + PLAYBOOK="${BASEDIR}/ansible/opensearch-migration.yml" + EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=deploy" + ;; (bash-completion) bash_completion exit 0 diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index b2c144fe32..89e36cfbe3 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -234,12 +234,22 @@ vars: scenario: prometheus-efk +- job: + name: kolla-ansible-prometheus-opensearch-base + parent: kolla-ansible-base + voting: false + files: + - ^ansible/roles/(common|opensearch|grafana|prometheus)/ + - ^tests/test-prometheus-opensearch.sh + vars: + scenario: prometheus-opensearch + - job: name: kolla-ansible-venus-base parent: kolla-ansible-base voting: false files: - - ^ansible/roles/(common|elasticsearch|venus)/ + - ^ansible/roles/(common|elasticsearch|opensearch|venus)/ - ^tests/test-venus.sh vars: scenario: venus diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 71ad426de6..2b384e5538 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -573,6 +573,31 @@ base_distro: centos install_type: source +- job: + name: kolla-ansible-centos8s-source-prometheus-opensearch + parent: kolla-ansible-prometheus-opensearch-base + nodeset: kolla-ansible-centos8s + vars: + base_distro: centos + install_type: source + +- job: + name: kolla-ansible-centos8s-source-prometheus-opensearch-migration + parent: kolla-ansible-prometheus-efk-base + nodeset: kolla-ansible-centos8s + vars: + base_distro: centos + install_type: source + opensearch_migration: true + +- job: + name: kolla-ansible-rocky9-source-prometheus-opensearch + parent: kolla-ansible-prometheus-opensearch-base + nodeset: kolla-ansible-rocky9 + vars: + base_distro: rocky + install_type: source + - job: name: kolla-ansible-ubuntu-source-prometheus-efk-focal parent: kolla-ansible-prometheus-efk-base @@ -581,6 +606,23 @@ base_distro: ubuntu install_type: source +- job: + name: kolla-ansible-ubuntu-source-prometheus-opensearch-focal + parent: kolla-ansible-prometheus-opensearch-base + nodeset: kolla-ansible-focal + vars: + base_distro: ubuntu + install_type: source + +- job: + name: kolla-ansible-ubuntu-source-prometheus-opensearch-migration-focal + parent: kolla-ansible-prometheus-efk-base + nodeset: kolla-ansible-focal + vars: + base_distro: ubuntu + install_type: source + opensearch_migration: true + - job: name: kolla-ansible-rocky9-source-venus parent: kolla-ansible-venus-base diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 5a7ee45efd..8c018584c9 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -68,6 +68,11 @@ - kolla-ansible-ubuntu-source-ovn-focal - kolla-ansible-centos8s-source-prometheus-efk - kolla-ansible-ubuntu-source-prometheus-efk-focal + - kolla-ansible-centos8s-source-prometheus-opensearch + - kolla-ansible-centos8s-source-prometheus-opensearch-migration + - kolla-ansible-ubuntu-source-prometheus-opensearch + - kolla-ansible-ubuntu-source-prometheus-opensearch-migration-focal + - kolla-ansible-rocky9-source-prometheus-opensearch - kolla-ansible-centos8s-source-cephadm - kolla-ansible-rocky9-source-cephadm - kolla-ansible-ubuntu-source-cephadm-focal