openstack/devstack
Code Issues Proposed changes

Reduce memory consumption in Cinder services

Browse Source 
This patch reduces memory usage on the Cinder Volume and Backup services
by tuning glibc.

The specific tuning consist on disabling the per thread arenas and
disabling dynamic thresholds.

The Cinder Backup service suffers from high water mark memory usage and
uses excessive memory.  As an example just after 10 restore operations
the service uses almost 1GB of RAM and does not ever free it afterwards.
With this patch the memory consumption of the service is reduced down to
almost 130MB.  If we add a revert from Cinder (Change-Id
I43a20c8687f12bc52b014611cc6977c4c3ca212c) it goes down to 100MB during
my tests.

This glibc tuning is not applied to all Python services because I
haven't done proper testings on them and at first glance they don't seem
to have such great improvements.

Related-bug: #1908805
Change-Id: Ic9030d01468b3189350f83b04a8d1d346c489d3c
This commit is contained in:
Gorka Eguileor 2022-06-08 10:19:50 +02:00
parent 0ae279b54a
commit d5af514ac9
2 changed files with 25 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
functions-common
View File

@ -1564,6 +1564,7 @@ function write_user_unit_file {
local command="$2"
local group=$3
local user=$4
local env_vars="$5"
local extra=""
if [[ -n "$group" ]]; then
extra="Group=$group"
@ -1577,6 +1578,9 @@ function write_user_unit_file {
iniset -sudo $unitfile "Service" "KillMode" "process"
iniset -sudo $unitfile "Service" "TimeoutStopSec" "300"
iniset -sudo $unitfile "Service" "ExecReload" "$KILL_PATH -HUP \$MAINPID"
if [[ -n "$env_vars" ]] ; then
iniset -sudo $unitfile "Service" "Environment" "$env_vars"
fi
if [[ -n "$group" ]]; then
iniset -sudo $unitfile "Service" "Group" "$group"
fi
@ -1591,6 +1595,7 @@ function write_uwsgi_user_unit_file {
local command="$2"
local group=$3
local user=$4
local env_vars="$5"
local unitfile="$SYSTEMD_DIR/$service"
mkdir -p $SYSTEMD_DIR
@ -1605,6 +1610,9 @@ function write_uwsgi_user_unit_file {
iniset -sudo $unitfile "Service" "NotifyAccess" "all"
iniset -sudo $unitfile "Service" "RestartForceExitStatus" "100"
if [[ -n "$env_vars" ]] ; then
iniset -sudo $unitfile "Service" "Environment" "$env_vars"
fi
if [[ -n "$group" ]]; then
iniset -sudo $unitfile "Service" "Group" "$group"
fi
@ -1652,10 +1660,14 @@ function _run_under_systemd {
local systemd_service="devstack@$service.service"
local group=$3
local user=${4:-$STACK_USER}
if [[ -z "$user" ]]; then
user=$STACK_USER
fi
local env_vars="$5"
if [[ "$command" =~ "uwsgi" ]] ; then
write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user"
write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
else
write_user_unit_file $systemd_service "$cmd" "$group" "$user"
write_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
fi
$SYSTEMCTL enable $systemd_service
@ -1676,18 +1688,20 @@ function is_running {
# If the command includes shell metachatacters (;<>*) it must be run using a shell
# If an optional group is provided sg will be used to run the
# command as that group.
# run_process service "command-line" [group] [user]
# run_process service "command-line" [group] [user] [env_vars]
# env_vars must be a space separated list of variable assigments, ie: "A=1 B=2"
function run_process {
local service=$1
local command="$2"
local group=$3
local user=$4
local env_vars="$5"
local name=$service
time_start "run_process"
if is_service_enabled $service; then
_run_under_systemd "$name" "$command" "$group" "$user"
_run_under_systemd "$name" "$command" "$group" "$user" "$env_vars"
fi
time_stop "run_process"
}

9
lib/cinder
View File

@ -552,8 +552,13 @@ function start_cinder {
fi
run_process c-sch "$CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF"
run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF"
run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF"
# Tune glibc for Python Services using single malloc arena for all threads
# and disabling dynamic thresholds to reduce memory usage when using native
# threads directly or via eventlet.tpool
# https://www.gnu.org/software/libc/manual/html_node/Memory-Allocation-Tunables.html
malloc_tuning="MALLOC_ARENA_MAX=1 MALLOC_MMAP_THRESHOLD_=131072 MALLOC_TRIM_THRESHOLD_=262144"
run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF" "" "" "$malloc_tuning"
run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF" "" "" "$malloc_tuning"
# NOTE(jdg): For cinder, startup order matters. To ensure that repor_capabilities is received
# by the scheduler start the cinder-volume service last (or restart it) after the scheduler