openstack/venus
Code Issues Proposed changes

delete es index over maxsize

Browse Source 
Change-Id: If9b3d712cfb5e50b28c5e74cbfa0e2d5272d3d81
This commit is contained in:
inspurericzhang 2024-01-16 10:58:50 +00:00
parent 4d673fb1fd
commit ea0d95bf7f
1 changed files with 59 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
venus/task/core/delete_es_index_task.py
View File

@ -73,9 +73,68 @@ class DeleteESIndexTask(object):
except Exception as e: except Exception as e:
LOG.error(_LE("delete es index, catch exception:%s"), str(e)) LOG.error(_LE("delete es index, catch exception:%s"), str(e))
def parse_index_size(self, size_str):
size_f = 0.0
if "kb" in size_str:
size_f = float(size_str.replace("kb", "").strip())
size_f = size_f * 1024
elif "mb" in size_str:
size_f = float(size_str.replace("mb", "").strip())
size_f = size_f * 1024 * 1024
elif "gb" in size_str:
size_f = float(size_str.replace("gb", "").strip())
size_f = size_f * 1024 * 1024 * 1024
elif "tb" in size_str:
size_f = float(size_str.replace("tb", "").strip())
size_f = size_f * 1024 * 1024 * 1024 * 1024
else:
pass
return size_f
def delete_es_oversize_index(self):
log_max = self.config_api.get_config("log_max_gb")
if log_max is None:
LOG.error(_LE("the config of log_max_gb do not exist"))
return
LOG.info(_LI("es indexes(log) max(GB): %s"), log_max)
log_max_int = float(log_max) * 1024 * 1024 * 1024
now_log_total = 0
today = time.strftime('%Y-%m-%d')
try:
indexes_array = self.search_lib.get_all_index()
for index in indexes_array:
size_str = index["store.size"].lower()
size_f = self.parse_index_size(size_str)
now_log_total = now_log_total + size_f
while now_log_total > log_max_int:
max_diff_days = -1
todo_delete_index = None
for index in indexes_array:
index_name = index["index"]
index_d = index_name.split('-')[1]
size_str = index["store.size"].lower()
size_f = self.parse_index_size(size_str)
dt_today = datetime.datetime.strptime(today, "%Y-%m-%d")
dt_index = datetime.datetime.strptime(index_d, '%Y.%m.%d')
dt_diff = dt_today - dt_index
if dt_diff.days > max_diff_days:
max_diff_days = dt_diff.days
todo_delete_index = index_name
if todo_delete_index:
LOG.info(_LI("deleted index %s"), index_name)
self.delete_index(todo_delete_index)
now_log_total = now_log_total - size_f
except Exception as e:
LOG.error(_LE("delete es index, catch exception:%s"), str(e))
def start_task(self): def start_task(self):
try: try:
self.delete_es_outdated_index() self.delete_es_outdated_index()
self.delete_es_oversize_index()
LOG.info(_LI("delete es index task done")) LOG.info(_LI("delete es index task done"))
except Exception as e: except Exception as e:
LOG.error(_LE("delete es index task, catch exception:%s"), str(e)) LOG.error(_LE("delete es index task, catch exception:%s"), str(e))