From 7ed568cf1535a082997155557719a04a68dcc29d Mon Sep 17 00:00:00 2001 From: Masco Date: Thu, 16 May 2024 15:29:25 +0530 Subject: [PATCH] add support to index ocp metrics to ES after running browbeat tests, the newly added ansible script is invoked and the ocp metrics from prometheus are fetched and indexed in given ES. the indexing is leveraging kube-burner Change-Id: Ic193ee023c44ecc7f750a2cfbb77327c79baed82 --- ansible/install/collectd-rhoso.yaml | 5 ++ ansible/install/group_vars/all.yml | 2 + ansible/install/index-ocp-data.yml | 5 ++ .../roles/index-ocp-data/tasks/check_oc.yml | 22 +++++++ .../roles/index-ocp-data/tasks/main.yml | 62 +++++++++++++++++++ .../index-ocp-data/templates/metrics.yaml | 41 ++++++++++++ .../roles/index-ocp-data/vars/main.yml | 8 +++ ansible/install/start-collectd-rhoso.yml | 4 ++ ansible/install/stop-collectd-rhoso.yml | 5 ++ ansible/install/toggle-indexing-cron-job.yml | 35 +++++++++++ 10 files changed, 189 insertions(+) create mode 100644 ansible/install/index-ocp-data.yml create mode 100644 ansible/install/roles/index-ocp-data/tasks/check_oc.yml create mode 100644 ansible/install/roles/index-ocp-data/tasks/main.yml create mode 100644 ansible/install/roles/index-ocp-data/templates/metrics.yaml create mode 100644 ansible/install/roles/index-ocp-data/vars/main.yml create mode 100644 ansible/install/toggle-indexing-cron-job.yml diff --git a/ansible/install/collectd-rhoso.yaml b/ansible/install/collectd-rhoso.yaml index 4d8ed0be8..da61451bb 100644 --- a/ansible/install/collectd-rhoso.yaml +++ b/ansible/install/collectd-rhoso.yaml @@ -6,3 +6,8 @@ roles: - { role: collectd-rhoso } environment: "{{proxy_env}}" + +- name: trigger the cronjob to index data from OCP + import_playbook: toggle-indexing-cron-job.yml + vars: + cron_state: "present" diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index 0c3251458..3d2fd95fb 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -15,6 +15,8 @@ host_remote_user: heat-admin is_rhoso_deployment: true python_interpreter: /usr/bin/python3.6 kubeconfig_path: /home/kni/clusterconfigs/auth/kubeconfig +kube_burner_path: https://github.com/cloud-bulldozer/kube-burner/releases/download/v1.7.12/kube-burner-V1.7.12-linux-x86_64.tar.gz +ocp_metrics_query: roles/index-ocp-data/templates/metrics.yaml # OpenStack Installer # Tripleo is the only installer supported currently diff --git a/ansible/install/index-ocp-data.yml b/ansible/install/index-ocp-data.yml new file mode 100644 index 000000000..ce6487e33 --- /dev/null +++ b/ansible/install/index-ocp-data.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + gather_facts: yes + roles: + - index-ocp-data diff --git a/ansible/install/roles/index-ocp-data/tasks/check_oc.yml b/ansible/install/roles/index-ocp-data/tasks/check_oc.yml new file mode 100644 index 000000000..903ea9238 --- /dev/null +++ b/ansible/install/roles/index-ocp-data/tasks/check_oc.yml @@ -0,0 +1,22 @@ +--- +- name: Check if oc is installed + shell: "which oc" + register: oc_location + ignore_errors: true + +- name: Fail if oc is not installed + fail: + msg: "oc is not installed" + when: oc_location.rc != 0 + +- name: Check if logged into Kubernetes cluster + shell: "oc version" + register: oc_version + ignore_errors: true + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + +- name: Fail if not logged in + fail: + msg: "Not logged in to Kubernetes cluster" + when: oc_version.rc != 0 diff --git a/ansible/install/roles/index-ocp-data/tasks/main.yml b/ansible/install/roles/index-ocp-data/tasks/main.yml new file mode 100644 index 000000000..f7d0db720 --- /dev/null +++ b/ansible/install/roles/index-ocp-data/tasks/main.yml @@ -0,0 +1,62 @@ +--- +- name: prepare kube-burner args + set_fact: + uuid: "{{ lookup('pipe', 'uuidgen') }}" + es_index: "OSP-metrics-from-OCP-{{ ansible_date_time.year }}.{{ ansible_date_time.month }}" + end_time: "{{ ansible_date_time.epoch | int }}" + job_name: "{{ cloud_prefix }}" + es_server: "http://{{ es_ip }}:{{ es_local_port }}" + metrics: "{{ ocp_metrics_query }}" + +- name: calculate start time from end time + set_fact: + start_time: "{{ end_time | int - 1800 }}" + when: start_time is undefined + +- name: Check if all variables are defined + assert: + that: + - item in vars + loop: "{{ required_vars }}" + loop_control: + label: "{{ item }}" + register: var_check_result + ignore_errors: true + +- name: Extract missing variables + set_fact: + missing_vars: "{{ var_check_result.results | selectattr('failed', 'equalto', true) | map(attribute='item') | list }}" + +- name: Fail if any variable is not defined + fail: + msg: "Variable '{{ missing_vars | join(', ') }}' is not defined. Aborting playbook execution." + when: missing_vars | length > 0 + +- name: check oc is installed and accessable + include_tasks: check_oc.yml + +- name: Download kube-burner + ansible.builtin.get_url: + url: "{{ kube_burner_path }}" + dest: /tmp/kube-burner.tar.gz + +- name: Extract the kube-burner + ansible.builtin.unarchive: + src: /tmp/kube-burner.tar.gz + dest: /tmp + mode: '0774' + +- block: + - name: retrieve the prometheus url + shell: oc get routes -n openshift-monitoring prometheus-k8s -o=jsonpath='{.spec.host}' + register: prometheus_url + + - name: create token to access prometheus + shell: oc create token prometheus-k8s -n openshift-monitoring + register: token + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + +- name: index data from premetheus to elastic + shell: | + /tmp/kube-burner index --es-server {{ es_server }} --es-index {{ es_index }} --uuid={{ uuid}} --job-name {{ job_name }} --token={{ token.stdout}} -m={{ metrics }} --start={{ start_time }} --end={{ end_time }} --log-level debug -u https://{{ prometheus_url.stdout }} diff --git a/ansible/install/roles/index-ocp-data/templates/metrics.yaml b/ansible/install/roles/index-ocp-data/templates/metrics.yaml new file mode 100644 index 000000000..74e5f3c11 --- /dev/null +++ b/ansible/install/roles/index-ocp-data/templates/metrics.yaml @@ -0,0 +1,41 @@ +# Containers & pod metrics +# +- query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!~"POD|",namespace=~"openstack"}[2m]) * 100) by (container, pod, namespace, node)) > 0 + metricName: containerCPU + +- query: sum(container_memory_rss{name!="",container!~"POD|",namespace=~"openstack"}) by (container, pod, namespace, node) + metricName: containerMemory + +- query: sum(irate(container_network_receive_packets_total{cluster="",namespace=~"openstack", pod!=""}[2m])) by (pod, namespace, node, interface) + metricName: containerRecvPackets + +- query: sum(irate(container_network_transmit_packets_total{cluster="",namespace=~"openstack", pod!=""}[2m])) by (pod, namespace, node, interface) + metricName: containerTranPackets + +- query: cluster_version{type="completed"} + metricName: clusterVersion + instant: true + +- query: sum by (cluster_version)(etcd_cluster_version) + metricName: etcdVersion + instant: true + +- query: count(kube_secret_info{namespace='openstack'}) + metricName: ospSecretCount + instant: true + +- query: count(kube_deployment_labels{namespace='openstack'}) + metricName: ospDeploymentCount + instant: true + +- query: count(kube_configmap_info{namespace='openstack'}) + metricName: ospConfigmapCount + instant: true + +- query: count(kube_service_info{namespace='openstack'}) + metricName: ospServiceCount + instant: true + +- query: count(kube_statefulset_labels{namespace='openstack'}) + metricName: ospStatefulsetCount + instant: true diff --git a/ansible/install/roles/index-ocp-data/vars/main.yml b/ansible/install/roles/index-ocp-data/vars/main.yml new file mode 100644 index 000000000..542ef68ac --- /dev/null +++ b/ansible/install/roles/index-ocp-data/vars/main.yml @@ -0,0 +1,8 @@ +required_vars: + - es_server + - es_index + - uuid + - job_name + - metrics + - start_time + - end_time diff --git a/ansible/install/start-collectd-rhoso.yml b/ansible/install/start-collectd-rhoso.yml index 87b667a82..fc9975a56 100644 --- a/ansible/install/start-collectd-rhoso.yml +++ b/ansible/install/start-collectd-rhoso.yml @@ -18,3 +18,7 @@ when: is_statefulset_exist.rc == 0 environment: KUBECONFIG: "{{ kubeconfig_path }}" +- name: trigger the cronjob to index data from OCP + import_playbook: toggle-indexing-cron-job.yml + vars: + cron_state: "present" diff --git a/ansible/install/stop-collectd-rhoso.yml b/ansible/install/stop-collectd-rhoso.yml index 5a12ab6fe..07fc31f19 100644 --- a/ansible/install/stop-collectd-rhoso.yml +++ b/ansible/install/stop-collectd-rhoso.yml @@ -6,3 +6,8 @@ environment: KUBECONFIG: "{{ kubeconfig_path }}" ignore_errors: yes + +- name: stop the cronjob to stop index data from OCP + import_playbook: toggle-indexing-cron-job.yml + vars: + cron_state: "absent" diff --git a/ansible/install/toggle-indexing-cron-job.yml b/ansible/install/toggle-indexing-cron-job.yml new file mode 100644 index 000000000..15cdd2b11 --- /dev/null +++ b/ansible/install/toggle-indexing-cron-job.yml @@ -0,0 +1,35 @@ +--- +- hosts: localhost + vars: + cron_state: "present" + tasks: + - block: + - name: Enure the log file exist + stat: + path: "/tmp/ocp_index_cron.log" + register: log_file + + - name: find the age of file(last modification) + set_fact: + file_age: "{{ ((ansible_date_time.epoch | int) - (log_file.stat.mtime | int)) / 60 }}" + when: log_file.stat.exists + + - name: run the ansible task with the start time + include_role: + name: index-ocp-data + vars: + start_time: "{{ log_file.stat.mtime | int }}" + when: log_file.stat.exists and (file_age | int < 30) + + - name: run the ansible task without start time + include_role: + name: index-ocp-data + when: not log_file.stat.exists or (file_age | int >= 30) + when: cron_state == "absent" + + - name: toggle the indexing OCP data cron job + cron: + name: "Index ocp data every 30 mins" + minute: "*/30" + job: "PATH=/usr/local/bin:/usr/bin:/bin && /usr/bin/ansible-playbook {{ browbeat_path }}/ansible/install/index-ocp-data.yml > /tmp/ocp_index_cron.log 2>&1" + state: "{{ cron_state }}"