Add cAdvisor for Prometheus monitoring

cAdvisor (Container Advisor) provides metrics on resource usage and
performance characteristics of running containers.  This change
deploys a cadvisor container and configures prometheus to scrape
data from it.

Change-Id: I55dd4fee954f9be68efda397746861ddaaa0a565
Partially-Implements: blueprint prometheus

ansible/group_vars/all.yml

@@ -273,6 +273,8 @@ prometheus_node_exporter_port: "9100"
 prometheus_mysqld_exporter_port: "9104"
 prometheus_haproxy_exporter_port: "9101"
 prometheus_memcached_exporter_port: "9150"
+# Default cadvisor port of 8080 already in use
+prometheus_cadvisor_port: "18080"
 
 qdrouterd_port: "31459"
 
@@ -803,3 +805,4 @@ enable_prometheus_haproxy_exporter: "{{ enable_haproxy | bool }}"
 enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}"
 enable_prometheus_node_exporter: "yes"
 enable_prometheus_memcached_exporter: "{{ enable_memcached | bool }}"
+enable_prometheus_cadvisor: "yes"

ansible/inventory/all-in-one

@@ -671,3 +671,10 @@ haproxy
 
 [prometheus-memcached-exporter:children]
 memcached
+
+[prometheus-cadvisor:children]
+monitoring
+control
+compute
+network
+storage

ansible/inventory/multinode

@@ -690,3 +690,10 @@ haproxy
 
 [prometheus-memcached-exporter:children]
 memcached
+
+[prometheus-cadvisor:children]
+monitoring
+control
+compute
+network
+storage

ansible/roles/prometheus/defaults/main.yml

@@ -52,6 +52,20 @@ prometheus_services:
       - "{{ node_config_directory }}/prometheus-memcached-exporter/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
+  prometheus-cadvisor:
+    container_name: "prometheus_cadvisor"
+    group: "prometheus-cadvisor"
+    enabled: "{{ enable_prometheus_cadvisor | bool }}"
+    image: "{{ prometheus_cadvisor_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/prometheus-cadvisor/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+      - "/:/rootfs:ro"
+      - "/var/run:/var/run:rw"
+      - "/sys:/sys:ro"
+      - "/varlib/docker/:/var/lib/docker:ro"
+      - "/dev/disk/:/dev/disk:ro"
 
 ####################
 # Database
@@ -83,3 +97,7 @@ prometheus_node_exporter_image_full: "{{ prometheus_node_exporter_image }}:{{ pr
 prometheus_memcached_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-prometheus-memcached-exporter"
 prometheus_memcached_exporter_tag: "{{ prometheus_tag }}"
 prometheus_memcached_exporter_image_full: "{{ prometheus_memcached_exporter_image }}:{{ prometheus_memcached_exporter_tag }}"
+
+prometheus_cadvisor_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-prometheus-cadvisor"
+prometheus_cadvisor_tag: "{{ openstack_release }}"
+prometheus_cadvisor_image_full: "{{ prometheus_cadvisor_image }}:{{ prometheus_cadvisor_tag }}"

ansible/roles/prometheus/handlers/main.yml

@@ -96,3 +96,22 @@
     - service.enabled | bool
     - config_json.changed | bool
       or prometheus_container.changed | bool
+
+- name: Restart prometheus-cadvisor container
+  vars:
+    service_name: "prometheus-cadvisor"
+    service: "{{ prometheus_services[service_name] }}"
+    config_json: "{{ prometheus_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    prometheus_container: "{{ check_prometheus_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or prometheus_container.changed | bool

ansible/roles/prometheus/tasks/precheck.yml

@@ -6,6 +6,7 @@
       - prometheus_node_exporter
       - prometheus_haproxy_exporter
       - prometheus_mysqld_exporter
+      - prometheus_cadvisor
   register: container_facts
 
 - name: Checking free port for Prometheus server
@@ -66,3 +67,15 @@
     - container_facts['prometheus_memcached_exporter'] is not defined
     - inventory_hostname in groups['prometheus-memcached-exporter']
     - enable_prometheus_memcached_exporter | bool
+
+- name: Checking free port for Prometheus cAdvisor
+  wait_for:
+    host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+    port: "{{ prometheus_cadvisor_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - container_facts['prometheus_cadvisor'] is not defined
+    - inventory_hostname in groups['prometheus-cadvisor']
+    - enable_prometheus_cadvisor | bool

ansible/roles/prometheus/templates/prometheus-cadvisor.json.j2

@@ -0,0 +1,11 @@
+{
+    "command": "/opt/cadvisor --port={{ prometheus_cadvisor_port }} --log_dir=/var/log/kolla/prometheus",
+    "config_files": [],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/prometheus",
+            "owner": "prometheus:prometheus",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -48,3 +48,12 @@ scrape_configs:
         - '{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}'
 {% endfor %}
 {% endif %}
+
+{% if enable_prometheus_cadvisor | bool %}
+  - job_name: cadvisor
+    static_configs:
+      - targets:
+{% for host in groups["prometheus-cadvisor"] %}
+        - '{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}'
+{% endfor %}
+{% endif %}

ansible/site.yml

@@ -164,6 +164,7 @@
     - prometheus-node-exporter
     - prometheus-mysqld-exporter
     - prometheus-haproxy-exporter
+    - prometheus-cadvisor
   serial: '{{ kolla_serial|default("0") }}'
   roles:
     - { role: prometheus,

etc/kolla/globals.yml

@@ -452,3 +452,4 @@ tempest_floating_network_name:
 #enable_prometheus_haproxy_exporter: "{{ enable_haproxy | bool }}"
 #enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}"
 #enable_prometheus_node_exporter: "yes"
+#enable_prometheus_cadvisor: "yes"

tests/templates/inventory.j2

@@ -640,3 +640,10 @@ mariadb
 
 [prometheus-haproxy-exporter:children]
 haproxy
+
+[prometheus-cadvisor:children]
+monitoring
+control
+compute
+network
+storage