Merge "Reduce container metrics cardinality"

ansible/roles/prometheus/defaults/main.yml

@@ -257,3 +257,5 @@ prometheus_openstack_exporter_disabled_dns: "{{ '--disable-service.dns' if not e
 prometheus_openstack_exporter_disabled_object: "{{ '--disable-service.object-store' if not enable_swift | bool else '' }}"
 prometheus_openstack_exporter_disabled_lb: "{{ '--disable-service.load-balancer --disable-metric=neutron-loadbalancers --disable-metric=neutron-loadbalancers_not_active' if not enable_octavia | bool else '' }}"
 prometheus_openstack_exporter_disabled_items: "{{ [prometheus_openstack_exporter_disabled_volume, prometheus_openstack_exporter_disabled_dns, prometheus_openstack_exporter_disabled_object, prometheus_openstack_exporter_disabled_lb|trim]|join(' ')|trim }}"
+
+prometheus_cadvisor_cmdline_extras: "--docker_only --store_container_labels=false --disable_metrics=percpu,referenced_memory,cpu_topology,resctrl,udp,advtcp,sched,hugetlb,memory_numa,tcp,process"

ansible/roles/prometheus/templates/prometheus-cadvisor.json.j2

@@ -1,5 +1,5 @@
 {
-    "command": "/opt/cadvisor --port={{ prometheus_cadvisor_port }} --log_dir=/var/log/kolla/prometheus",
+    "command": "/opt/cadvisor --port={{ prometheus_cadvisor_port }} --log_dir=/var/log/kolla/prometheus {{ prometheus_cadvisor_cmdline_extras }}",
     "config_files": [],
     "permissions": [
         {

etc/kolla/globals.yml

@@ -703,6 +703,10 @@
 # List of extra parameters passed to prometheus. You can add as many to the list.
 #prometheus_cmdline_extras:
 
+# List of extra parameters passed to cAdvisor. By default system cgroups
+# and container labels are not exposed to reduce time series cardinality.
+#prometheus_cadvisor_cmdline_extras: "--docker_only --store_container_labels=false --disable_metrics=percpu,referenced_memory,cpu_topology,resctrl,udp,advtcp,sched,hugetlb,memory_numa,tcp,process"
+
 # Example of setting endpoints for prometheus ceph mgr exporter.
 # You should add all ceph mgr's in your external ceph deployment.
 #prometheus_ceph_mgr_exporter_endpoints:

releasenotes/notes/reduce-cadvisor-cardinality-1213854b9fe0c828.yaml

@@ -0,0 +1,19 @@
+---
+features:
+  - |
+    Adds support for passing extra runtime options to cAdvisor via
+    ``prometheus_cadvisor_cmdline_extras`` new variable. By default
+    system cgroups' metrics are disabled, plus container labels
+    don't get exposed to Prometheus. Expensive metrics that usually
+    should not be exported are also enforced to be disabled - consult
+    `<https://github.com/google/cadvisor/blob/master/docs/runtime_options.md#metrics>`_
+    for a list. These defaults create savings in resources usage by both
+    cAdvisor and Prometheus.
+upgrade:
+  - |
+    cAdvisor has now reduced number of Prometheus metrics and labels
+    exported by default. This means that corresponding timeseries will
+    no longer be created. If existing setup relies on these, eg. for
+    visualisation or alerting, they could be explicitly enabled prior to
+    upgrading with the ``prometheus_cadvisor_cmdline_extras`` new variable.
+    Reference for the possible options: `<https://github.com/google/cadvisor/blob/master/docs/runtime_options.md#metrics>`_.