Merge "Support deploying Monasca Log Persister"
This commit is contained in:
Zuul
Gerrit Code Review
commit
97dcee4c98
@ -449,6 +449,9 @@ monasca
|
||||
[monasca-log-transformer:children]
|
||||
monasca
|
||||
|
||||
[monasca-log-persister:children]
|
||||
monasca
|
||||
|
||||
# Ironic
|
||||
[ironic-api:children]
|
||||
ironic
|
||||
|
||||
@ -458,6 +458,9 @@ monasca
|
||||
[monasca-log-transformer:children]
|
||||
monasca
|
||||
|
||||
[monasca-log-persister:children]
|
||||
monasca
|
||||
|
||||
# Ironic
|
||||
[ironic-api:children]
|
||||
ironic
|
||||
|
||||
@ -27,6 +27,15 @@ monasca_services:
|
||||
- "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro"
|
||||
- "/etc/localtime:/etc/localtime:ro"
|
||||
- "kolla_logs:/var/log/kolla"
|
||||
monasca-log-persister:
|
||||
container_name: monasca_log_persister
|
||||
group: monasca-log-persister
|
||||
enabled: true
|
||||
image: "{{ monasca_logstash_image_full }}"
|
||||
volumes:
|
||||
- "{{ node_config_directory }}/monasca-log-persister/:{{ container_config_directory }}/:ro"
|
||||
- "/etc/localtime:/etc/localtime:ro"
|
||||
- "kolla_logs:/var/log/kolla"
|
||||
|
||||
####################
|
||||
# Databases
|
||||
@ -45,6 +54,7 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}"
|
||||
monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}"
|
||||
monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
|
||||
monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
|
||||
monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}"
|
||||
|
||||
monasca_metrics_topic: "metrics"
|
||||
monasca_raw_logs_topic: "logs"
|
||||
|
||||
@ -62,3 +62,24 @@
|
||||
- config_json.changed | bool
|
||||
or monasca_log_transformer_confs.changed | bool
|
||||
or monasca_log_transformer_container.changed | bool
|
||||
|
||||
- name: Restart monasca-log-persister container
|
||||
vars:
|
||||
service_name: "monasca-log-persister"
|
||||
service: "{{ monasca_services[service_name] }}"
|
||||
config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
|
||||
monasca_log_persister_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
|
||||
kolla_docker:
|
||||
action: "recreate_or_restart_container"
|
||||
common_options: "{{ docker_common_options }}"
|
||||
name: "{{ service.container_name }}"
|
||||
image: "{{ service.image }}"
|
||||
volumes: "{{ service.volumes }}"
|
||||
when:
|
||||
- action != "config"
|
||||
- inventory_hostname in groups[service.group]
|
||||
- service.enabled | bool
|
||||
- config_json.changed | bool
|
||||
or monasca_log_persister_confs.changed | bool
|
||||
or monasca_log_persister_elasticsearch_template.changed | bool
|
||||
or monasca_log_persister_container.changed | bool
|
||||
|
||||
@ -118,6 +118,45 @@
|
||||
notify:
|
||||
- Restart monasca-log-transformer container
|
||||
|
||||
- name: Copying over monasca-log-persister config
|
||||
vars:
|
||||
service: "{{ monasca_services['monasca-log-persister'] }}"
|
||||
template:
|
||||
src: "{{ item }}"
|
||||
dest: "{{ node_config_directory }}/monasca-log-persister/log-persister.conf"
|
||||
mode: "0660"
|
||||
become: true
|
||||
register: monasca_log_persister_confs
|
||||
with_first_found:
|
||||
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-persister.conf"
|
||||
- "{{ node_custom_config }}/monasca/log-persister.conf"
|
||||
- "{{ role_path }}/templates/monasca-log-persister/log-persister.conf.j2"
|
||||
when:
|
||||
- inventory_hostname in groups[service['group']]
|
||||
- service.enabled | bool
|
||||
notify:
|
||||
- Restart monasca-log-persister container
|
||||
|
||||
- name: Copying over monasca-log-persister elasticsearch template
|
||||
vars:
|
||||
service: "{{ monasca_services['monasca-log-persister'] }}"
|
||||
template:
|
||||
src: "{{ item }}"
|
||||
dest: "{{ node_config_directory }}/monasca-log-persister/elasticsearch-template.json"
|
||||
mode: "0660"
|
||||
become: true
|
||||
register: monasca_log_persister_elasticsearch_template
|
||||
with_first_found:
|
||||
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/elasticsearch-template.json"
|
||||
- "{{ node_custom_config }}/monasca/elasticsearch-template.json"
|
||||
- "{{ role_path }}/templates/monasca-log-persister/elasticsearch-template.json"
|
||||
when:
|
||||
- inventory_hostname in groups[service['group']]
|
||||
- service.enabled | bool
|
||||
notify:
|
||||
- Restart monasca-log-persister container
|
||||
|
||||
|
||||
- name: Check monasca containers
|
||||
become: true
|
||||
kolla_docker:
|
||||
|
||||
@ -6,7 +6,8 @@
|
||||
- include: config.yml
|
||||
when: inventory_hostname in groups['monasca-api'] or
|
||||
inventory_hostname in groups['monasca-log-api'] or
|
||||
inventory_hostname in groups['monasca-log-transformer']
|
||||
inventory_hostname in groups['monasca-log-transformer'] or
|
||||
inventory_hostname in groups['monasca-log-persister']
|
||||
|
||||
- include: bootstrap.yml
|
||||
when: inventory_hostname in groups['monasca-api']
|
||||
@ -17,4 +18,5 @@
|
||||
- include: check.yml
|
||||
when: inventory_hostname in groups['monasca-api'] or
|
||||
inventory_hostname in groups['monasca-log-api'] or
|
||||
inventory_hostname in groups['monasca-log-transformer']
|
||||
inventory_hostname in groups['monasca-log-transformer'] or
|
||||
inventory_hostname in groups['monasca-log-persister']
|
||||
|
||||
@ -0,0 +1,56 @@
|
||||
{
|
||||
"aliases": {},
|
||||
"mappings": {
|
||||
"log": {
|
||||
"_all": {
|
||||
"enabled": true,
|
||||
"omit_norms": true
|
||||
},
|
||||
"dynamic_templates": [
|
||||
{
|
||||
"message_field": {
|
||||
"mapping": {
|
||||
"fielddata": {
|
||||
"format": "disabled"
|
||||
},
|
||||
"index": "analyzed",
|
||||
"omit_norms": true,
|
||||
"type": "string"
|
||||
},
|
||||
"match": "message",
|
||||
"match_mapping_type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"other_fields": {
|
||||
"mapping": {
|
||||
"index": "not_analyzed",
|
||||
"type": "string"
|
||||
},
|
||||
"match": "*",
|
||||
"match_mapping_type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"@version": {
|
||||
"index": "not_analyzed",
|
||||
"type": "string"
|
||||
},
|
||||
"creation_time": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"order": 0,
|
||||
"settings": {
|
||||
"index": {
|
||||
"refresh_interval": "5s"
|
||||
}
|
||||
},
|
||||
"template": "monasca-*"
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
# Persist transformed logs to Elasticsearch
|
||||
|
||||
input {
|
||||
kafka {
|
||||
zk_connect => "{{ monasca_zookeeper_servers }}"
|
||||
topic_id => "{{ monasca_transformed_logs_topic }}"
|
||||
group_id => "transformer-logstash-consumer"
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
elasticsearch {
|
||||
index => "monasca-%{[meta][tenantId]}-%{+YYYY.MM.dd}"
|
||||
hosts => [{{ monasca_elasticsearch_servers }}]
|
||||
document_type => "log"
|
||||
template_name => "monasca"
|
||||
template => "/etc/logstash/elasticsearch-template.json"
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
{
|
||||
"command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/monasca/monasca-log-persister.log -f /etc/logstash/conf.d/log-persister.conf",
|
||||
"config_files": [
|
||||
{
|
||||
"source": "{{ container_config_directory }}/log-persister.conf",
|
||||
"dest": "/etc/logstash/conf.d/log-persister.conf",
|
||||
"owner": "logstash",
|
||||
"perm": "0600"
|
||||
},
|
||||
{
|
||||
"source": "{{ container_config_directory }}/elasticsearch-template.json",
|
||||
"dest": "/etc/logstash/elasticsearch-template.json",
|
||||
"owner": "logstash",
|
||||
"perm": "0600"
|
||||
}
|
||||
],
|
||||
"permissions": [
|
||||
{
|
||||
"path": "/var/log/kolla/monasca",
|
||||
"owner": "logstash:kolla",
|
||||
"recurse": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,6 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
Add support for deploying the Monasca Log Persister. The Log
|
||||
Persister is responsible for reading logs from the Kafka processed
|
||||
logs topic and writing them to Elasticsearch.
|
||||
Loading…
Reference in New Issue
Block a user