Merge "add support to index ocp metrics to ES"

ansible/install/collectd-rhoso.yaml

@@ -6,3 +6,8 @@
   roles:
     - { role: collectd-rhoso }
   environment: "{{proxy_env}}"
+
+- name: trigger the cronjob to index data from OCP
+  import_playbook: toggle-indexing-cron-job.yml
+  vars:
+    cron_state: "present"

ansible/install/group_vars/all.yml

@@ -15,6 +15,8 @@ host_remote_user: heat-admin
 is_rhoso_deployment: true
 python_interpreter: /usr/bin/python3.6
 kubeconfig_path: /home/kni/clusterconfigs/auth/kubeconfig
+kube_burner_path: https://github.com/cloud-bulldozer/kube-burner/releases/download/v1.7.12/kube-burner-V1.7.12-linux-x86_64.tar.gz
+ocp_metrics_query: roles/index-ocp-data/templates/metrics.yaml
 
 # OpenStack Installer
 # Tripleo is the only installer supported currently

ansible/install/index-ocp-data.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: yes
+  roles:
+    - index-ocp-data

ansible/install/roles/index-ocp-data/tasks/check_oc.yml

@@ -0,0 +1,22 @@
+---
+- name: Check if oc is installed
+  shell: "which oc"
+  register: oc_location
+  ignore_errors: true
+
+- name: Fail if oc is not installed
+  fail:
+    msg: "oc is not installed"
+  when: oc_location.rc != 0
+
+- name: Check if logged into Kubernetes cluster
+  shell: "oc version"
+  register: oc_version
+  ignore_errors: true
+  environment:
+    KUBECONFIG: "{{ kubeconfig_path }}"
+
+- name: Fail if not logged in
+  fail:
+    msg: "Not logged in to Kubernetes cluster"
+  when: oc_version.rc != 0

ansible/install/roles/index-ocp-data/tasks/main.yml

@@ -0,0 +1,62 @@
+---
+- name: prepare kube-burner args
+  set_fact:
+    uuid: "{{ lookup('pipe', 'uuidgen') }}"
+    es_index: "OSP-metrics-from-OCP-{{ ansible_date_time.year }}.{{ ansible_date_time.month }}"
+    end_time: "{{ ansible_date_time.epoch | int }}"
+    job_name: "{{ cloud_prefix }}"
+    es_server: "http://{{ es_ip }}:{{ es_local_port }}"
+    metrics: "{{ ocp_metrics_query }}"
+
+- name: calculate start time from end time
+  set_fact:
+    start_time: "{{ end_time | int - 1800 }}"
+  when: start_time is undefined
+
+- name: Check if all variables are defined
+  assert:
+    that:
+      - item in vars
+  loop: "{{ required_vars }}"
+  loop_control:
+    label: "{{ item }}"
+  register: var_check_result
+  ignore_errors: true
+
+- name: Extract missing variables
+  set_fact:
+    missing_vars: "{{ var_check_result.results | selectattr('failed', 'equalto', true) | map(attribute='item') | list }}"
+
+- name: Fail if any variable is not defined
+  fail:
+    msg: "Variable '{{ missing_vars | join(', ') }}' is not defined. Aborting playbook execution."
+  when: missing_vars | length > 0
+
+- name: check oc is installed and accessable
+  include_tasks: check_oc.yml
+
+- name: Download kube-burner
+  ansible.builtin.get_url:
+    url: "{{ kube_burner_path }}"
+    dest: /tmp/kube-burner.tar.gz
+
+- name: Extract the kube-burner
+  ansible.builtin.unarchive:
+    src: /tmp/kube-burner.tar.gz
+    dest: /tmp
+    mode: '0774'
+
+- block:
+    - name: retrieve the prometheus url
+      shell: oc get routes -n  openshift-monitoring prometheus-k8s -o=jsonpath='{.spec.host}'
+      register: prometheus_url
+
+    - name: create token to access prometheus
+      shell: oc create token prometheus-k8s -n openshift-monitoring
+      register: token
+  environment:
+    KUBECONFIG: "{{ kubeconfig_path }}"
+
+- name: index data from premetheus to elastic
+  shell: |
+          /tmp/kube-burner index --es-server {{ es_server }} --es-index {{ es_index }} --uuid={{ uuid}} --job-name {{ job_name }} --token={{ token.stdout}} -m={{ metrics }} --start={{ start_time }} --end={{ end_time }} --log-level debug -u https://{{ prometheus_url.stdout }}

ansible/install/roles/index-ocp-data/templates/metrics.yaml

@@ -0,0 +1,41 @@
+# Containers & pod metrics
+#
+- query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!~"POD|",namespace=~"openstack"}[2m]) * 100) by (container, pod, namespace, node)) > 0
+  metricName: containerCPU
+
+- query: sum(container_memory_rss{name!="",container!~"POD|",namespace=~"openstack"}) by (container, pod, namespace, node)
+  metricName: containerMemory
+
+- query: sum(irate(container_network_receive_packets_total{cluster="",namespace=~"openstack", pod!=""}[2m])) by (pod, namespace, node, interface)
+  metricName: containerRecvPackets
+
+- query: sum(irate(container_network_transmit_packets_total{cluster="",namespace=~"openstack", pod!=""}[2m])) by (pod, namespace, node, interface)
+  metricName: containerTranPackets
+
+- query: cluster_version{type="completed"}
+  metricName: clusterVersion
+  instant: true
+
+- query: sum by (cluster_version)(etcd_cluster_version)
+  metricName: etcdVersion
+  instant: true
+
+- query: count(kube_secret_info{namespace='openstack'})
+  metricName: ospSecretCount
+  instant: true
+
+- query: count(kube_deployment_labels{namespace='openstack'})
+  metricName: ospDeploymentCount
+  instant: true
+
+- query: count(kube_configmap_info{namespace='openstack'})
+  metricName: ospConfigmapCount
+  instant: true
+
+- query: count(kube_service_info{namespace='openstack'})
+  metricName: ospServiceCount
+  instant: true
+
+- query: count(kube_statefulset_labels{namespace='openstack'})
+  metricName: ospStatefulsetCount
+  instant: true

ansible/install/roles/index-ocp-data/vars/main.yml

@@ -0,0 +1,8 @@
+required_vars:
+  - es_server
+  - es_index
+  - uuid
+  - job_name
+  - metrics
+  - start_time
+  - end_time

ansible/install/start-collectd-rhoso.yml

@@ -18,3 +18,7 @@
           when: is_statefulset_exist.rc == 0
       environment:
         KUBECONFIG: "{{ kubeconfig_path }}"
+- name: trigger the cronjob to index data from OCP
+  import_playbook: toggle-indexing-cron-job.yml
+  vars:
+    cron_state: "present"

ansible/install/stop-collectd-rhoso.yml

@@ -6,3 +6,8 @@
       environment:
         KUBECONFIG: "{{ kubeconfig_path }}"
       ignore_errors: yes
+
+- name: stop the cronjob to stop index data from OCP
+  import_playbook: toggle-indexing-cron-job.yml
+  vars:
+    cron_state: "absent"

ansible/install/toggle-indexing-cron-job.yml

@@ -0,0 +1,35 @@
+---
+- hosts: localhost
+  vars:
+    cron_state: "present"
+  tasks:
+    - block:
+        - name: Enure the log file exist
+          stat:
+            path: "/tmp/ocp_index_cron.log"
+          register: log_file
+
+        - name: find the age of file(last modification)
+          set_fact:
+            file_age: "{{ ((ansible_date_time.epoch | int) - (log_file.stat.mtime | int)) / 60 }}"
+          when: log_file.stat.exists
+
+        - name: run the ansible task with the start time
+          include_role:
+            name: index-ocp-data
+          vars:
+            start_time: "{{ log_file.stat.mtime | int }}"
+          when: log_file.stat.exists and (file_age | int < 30)
+
+        - name: run the ansible task without start time
+          include_role:
+            name: index-ocp-data
+          when: not log_file.stat.exists or (file_age | int >= 30)
+      when: cron_state == "absent"
+
+    - name: toggle the indexing OCP data cron job
+      cron:
+        name: "Index ocp data every 30 mins"
+        minute: "*/30"
+        job: "PATH=/usr/local/bin:/usr/bin:/bin && /usr/bin/ansible-playbook {{ browbeat_path }}/ansible/install/index-ocp-data.yml > /tmp/ocp_index_cron.log 2>&1"
+        state: "{{ cron_state }}"