From ea0d95bf7f27dad44733c97ba26e0ac1ce4e73bd Mon Sep 17 00:00:00 2001
From: inspurericzhang <zhanglf01@inspur.com>
Date: Tue, 16 Jan 2024 10:58:50 +0000
Subject: [PATCH] delete es index over maxsize

Change-Id: If9b3d712cfb5e50b28c5e74cbfa0e2d5272d3d81
---
 venus/task/core/delete_es_index_task.py | 59 +++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/venus/task/core/delete_es_index_task.py b/venus/task/core/delete_es_index_task.py
index 0a809bc..58c35da 100644
--- a/venus/task/core/delete_es_index_task.py
+++ b/venus/task/core/delete_es_index_task.py
@@ -73,9 +73,68 @@ class DeleteESIndexTask(object):
         except Exception as e:
             LOG.error(_LE("delete es index, catch exception:%s"), str(e))
 
+    def parse_index_size(self, size_str):
+        size_f = 0.0
+        if "kb" in size_str:
+            size_f = float(size_str.replace("kb", "").strip())
+            size_f = size_f * 1024
+        elif "mb" in size_str:
+            size_f = float(size_str.replace("mb", "").strip())
+            size_f = size_f * 1024 * 1024
+        elif "gb" in size_str:
+            size_f = float(size_str.replace("gb", "").strip())
+            size_f = size_f * 1024 * 1024 * 1024
+        elif "tb" in size_str:
+            size_f = float(size_str.replace("tb", "").strip())
+            size_f = size_f * 1024 * 1024 * 1024 * 1024
+        else:
+            pass
+
+        return size_f
+
+    def delete_es_oversize_index(self):
+        log_max = self.config_api.get_config("log_max_gb")
+        if log_max is None:
+            LOG.error(_LE("the config of log_max_gb do not exist"))
+            return
+
+        LOG.info(_LI("es indexes(log) max(GB): %s"), log_max)
+        log_max_int = float(log_max) * 1024 * 1024 * 1024
+        now_log_total = 0
+        today = time.strftime('%Y-%m-%d')
+        try:
+            indexes_array = self.search_lib.get_all_index()
+            for index in indexes_array:
+                size_str = index["store.size"].lower()
+                size_f = self.parse_index_size(size_str)
+                now_log_total = now_log_total + size_f
+
+            while now_log_total > log_max_int:
+                max_diff_days = -1
+                todo_delete_index = None
+                for index in indexes_array:
+                    index_name = index["index"]
+                    index_d = index_name.split('-')[1]
+                    size_str = index["store.size"].lower()
+                    size_f = self.parse_index_size(size_str)
+                    dt_today = datetime.datetime.strptime(today, "%Y-%m-%d")
+                    dt_index = datetime.datetime.strptime(index_d, '%Y.%m.%d')
+                    dt_diff = dt_today - dt_index
+                    if dt_diff.days > max_diff_days:
+                        max_diff_days = dt_diff.days
+                        todo_delete_index = index_name
+
+                if todo_delete_index:
+                    LOG.info(_LI("deleted index %s"), index_name)
+                    self.delete_index(todo_delete_index)
+                    now_log_total = now_log_total - size_f
+        except Exception as e:
+            LOG.error(_LE("delete es index, catch exception:%s"), str(e))
+
     def start_task(self):
         try:
             self.delete_es_outdated_index()
+            self.delete_es_oversize_index()
             LOG.info(_LI("delete es index task done"))
         except Exception as e:
             LOG.error(_LE("delete es index task, catch exception:%s"), str(e))