From ce964e4f8da4d31210673b7c8d2484af7e23d36e Mon Sep 17 00:00:00 2001 From: zhangjianweibj Date: Wed, 9 Oct 2019 15:48:42 +0800 Subject: [PATCH] Support multiple kafka consumers We ues two monasca-persister components and two influxdb nodes.one for collecting metrics and another for backup db.so kafka consumer should hava different groupid option. Change-Id: Iae25b34ea282fcc3e7cf01147fd3e9093ee9a8a0 --- docker/Dockerfile | 2 ++ docker/README.rst | 2 ++ docker/monasca-persister.conf.j2 | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 101b6bab..868727d5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,9 +23,11 @@ ENV \ ZOOKEEPER_URI=zookeeper:2181 \ KAFKA_URI=kafka:9092 \ KAFKA_ALARM_HISTORY_BATCH_SIZE=1000 \ + KAFKA_ALARM_HISTORY_GROUP_ID=1_events \ KAFKA_ALARM_HISTORY_PROCESSORS=1 \ KAFKA_ALARM_HISTORY_WAIT_TIME=15 \ KAFKA_METRICS_BATCH_SIZE=1000 \ + KAFKA_METRICS_GROUP_ID=1_metrics \ KAFKA_METRICS_PROCESSORS=1 \ KAFKA_METRICS_WAIT_TIME=15 \ KAFKA_WAIT_FOR_TOPICS=alarm-state-transitions,metrics \ diff --git a/docker/README.rst b/docker/README.rst index 286cdcd7..5e383bc5 100644 --- a/docker/README.rst +++ b/docker/README.rst @@ -29,10 +29,12 @@ VERBOSE true If true, enable info logging ZOOKEEPER_URI zookeeper:2181 The host and port for zookeeper KAFKA_URI kafka:9092 The host and port for kafka KAFKA_ALARM_HISTORY_BATCH_SIZE 1000 Kafka consumer takes messages in a batch +KAFKA_ALARM_HISTORY_GROUP_ID 1_events Kafka Group from which persister get alarm history KAFKA_ALARM_HISTORY_PROCESSORS 1 Number of processes for alarm history topic KAFKA_ALARM_HISTORY_WAIT_TIME 15 Seconds to wait if the batch size is not reached KAFKA_EVENTS_ENABLE false Enable events persister KAFKA_METRICS_BATCH_SIZE 1000 Kafka consumer takes messages in a batch +KAFKA_METRICS_GROUP_ID 1_metrics Kafka Group from which persister get metrics KAFKA_METRICS_PROCESSORS 1 Number of processes for metrics topic KAFKA_METRICS_WAIT_TIME 15 Seconds to wait if the batch size is not reached DATABASE_BACKEND influxdb Select for backend database diff --git a/docker/monasca-persister.conf.j2 b/docker/monasca-persister.conf.j2 index de3d5168..762983f6 100644 --- a/docker/monasca-persister.conf.j2 +++ b/docker/monasca-persister.conf.j2 @@ -33,7 +33,7 @@ partition_interval_recheck_seconds = 15 [kafka_alarm_history] # Comma separated list of Kafka broker host:port. uri = {{ KAFKA_URI }} -group_id = 1_alarm-state-transitions +group_id = {{ KAFKA_ALARM_HISTORY_GROUP_ID }} topic = alarm-state-transitions consumer_id = 1 client_id = 1 @@ -59,7 +59,7 @@ batch_size = 1 [kafka_metrics] # Comma separated list of Kafka broker host:port uri = {{ KAFKA_URI }} -group_id = 1_metrics +group_id = {{ KAFKA_METRICS_GROUP_ID }} topic = metrics consumer_id = 1 client_id = 1