From 6cd74330869af6b000366bcb4ffdce7dd3d3c1b1 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 28 Jun 2022 15:12:05 +1000 Subject: [PATCH] graphite: fix xFilesFactor When we migrated this to ansible I missed that we didn't bring across the storage-aggregation.conf file. This has had the unfortunate effect of regressing the xFilesFactor set for every newly created graphite stat since the migration. This setting is a percentage (0-1 float) of how much of a "bucket" needs to be non-null to keep the value when rolling up changes. We want this to be zero due to the sporadic nature of data (see the original change I5f416e798e7abedfde776c9571b6fc8cea5f3a33). This only affected newly created statistics, as graphite doesn't modify this setting once it creates the whisper file. This probably helped us overlook this for so long, as longer-existing stats were operating correctly, but newer were dropping data when zoomed out. Restore this setting, and double-check it in testinfra for the future. For simplicity and to get this back to the prior state I will manually update the on-disk .wsp files to this when this change applies. Change-Id: I57873403c4ca9783b1851ba83bfba038f4b90715 --- playbooks/roles/graphite/tasks/main.yaml | 19 +++++++++--- .../graphite/templates/docker-compose.yaml.j2 | 9 +++--- .../templates/storage-aggregation.conf.j2 | 29 +++++++++++++++++++ testinfra/test_graphite.py | 15 ++++++++++ zuul.d/system-config-run.yaml | 1 + 5 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 playbooks/roles/graphite/templates/storage-aggregation.conf.j2 diff --git a/playbooks/roles/graphite/tasks/main.yaml b/playbooks/roles/graphite/tasks/main.yaml index e78aea1ec0..c8cc41e58a 100644 --- a/playbooks/roles/graphite/tasks/main.yaml +++ b/playbooks/roles/graphite/tasks/main.yaml @@ -1,7 +1,11 @@ -- name: Ensure docker-compose directory exists +- name: Ensure docker-compose directories exists file: state: directory - path: /etc/graphite-docker + path: '{{ item }}' + loop: + - /etc/graphite-docker + - /etc/graphite-docker/graphite + - /etc/graphite-docker/graphite/conf - name: Write settings file template: @@ -11,7 +15,12 @@ - name: Write storage config file template: src: storage-schemas.conf.j2 - dest: /etc/graphite-docker/storage-schemas.conf + dest: /etc/graphite-docker/graphite/conf/storage-schemas.conf + +- name: Write storage aggregation config file + template: + src: storage-aggregation.conf.j2 + dest: /etc/graphite-docker/graphite/conf/storage-aggregation.conf - name: Write nginx override config template: @@ -47,11 +56,13 @@ shell: cmd: docker image prune -f -# This is handy to have on the host for checking stat ingestion +# net is handy to have on the host for checking stat ingestion +# whisper utils are handy for inspecting .wsp files for debugging - name: Install netcat package: name: - netcat + - python3-whisper state: present # Removes files not updated for ~9 months, and clears out empty directories diff --git a/playbooks/roles/graphite/templates/docker-compose.yaml.j2 b/playbooks/roles/graphite/templates/docker-compose.yaml.j2 index 75e107e83c..34fdb9258b 100644 --- a/playbooks/roles/graphite/templates/docker-compose.yaml.j2 +++ b/playbooks/roles/graphite/templates/docker-compose.yaml.j2 @@ -8,9 +8,10 @@ services: image: docker.io/graphiteapp/graphite-statsd network_mode: host volumes: - - /etc/graphite-docker/graphite-statsd.conf:/etc/nginx/sites-enabled/graphite-statsd.conf - - /etc/graphite-docker/statsd.js:/opt/statsd/config/udp.js - - /etc/graphite-docker/storage-schemas.conf:/opt/graphite/conf/storage-schemas.conf - - /etc/letsencrypt-certs:/etc/letsencrypt-certs + - /etc/graphite-docker/graphite-statsd.conf:/etc/nginx/sites-enabled/graphite-statsd.conf:ro + - /etc/graphite-docker/statsd.js:/opt/statsd/config/udp.js:ro + - /etc/graphite-docker/graphite/conf/storage-schemas.conf:/opt/graphite/conf/storage-schemas.conf:ro + - /etc/graphite-docker/graphite/conf/storage-aggregation.conf:/opt/graphite/conf/storage-aggregation.conf:ro + - /etc/letsencrypt-certs:/etc/letsencrypt-certs:ro - /opt/graphite/storage:/opt/graphite/storage - /var/log/graphite:/var/log/ diff --git a/playbooks/roles/graphite/templates/storage-aggregation.conf.j2 b/playbooks/roles/graphite/templates/storage-aggregation.conf.j2 new file mode 100644 index 0000000000..bf60170a1f --- /dev/null +++ b/playbooks/roles/graphite/templates/storage-aggregation.conf.j2 @@ -0,0 +1,29 @@ +[min] +pattern = \.lower$ +xFilesFactor = 0 +aggregationMethod = min + +[max] +pattern = \.upper(_\d+)?$ +xFilesFactor = 0 +aggregationMethod = max + +[sum] +pattern = \.sum$ +xFilesFactor = 0 +aggregationMethod = sum + +[count] +pattern = \.count$ +xFilesFactor = 0 +aggregationMethod = sum + +[count_legacy] +pattern = ^stats_counts.* +xFilesFactor = 0 +aggregationMethod = sum + +[default_average] +pattern = .* +xFilesFactor = 0 +aggregationMethod = average diff --git a/testinfra/test_graphite.py b/testinfra/test_graphite.py index 351187943b..fb0a18ba4c 100644 --- a/testinfra/test_graphite.py +++ b/testinfra/test_graphite.py @@ -56,3 +56,18 @@ def test_graphite_data(host): found_value = True assert found_value + +def test_graphite_wsp(host): + # seed some timer data + cmd = ('timeout 20 bash -c ' + '\'while true; do echo -n "example:$((RANDOM % 1000))|ms" ' + '| nc -6 -w 1 -u localhost 8125; done\'') + host.run(cmd) + + wsp_file = '/opt/graphite/storage/whisper/stats/timers/example/mean.wsp' + + wsp = host.file(wsp_file) + assert wsp.exists + + cmd = host.run('whisper-info %s' % wsp_file) + assert 'xFilesFactor: 0.0' in cmd.stdout diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 80863fbc6a..e739cb057f 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -692,6 +692,7 @@ graphite02.opendev.org: host_copy_output: '/var/log/graphite': logs + '/etc/graphite-docker': logs files: - playbooks/bootstrap-bridge.yaml - playbooks/letsencrypt.yaml