Merge "Robustify service shutdown"

functions

@@ -1132,10 +1132,39 @@ function screen_it {
             sleep 1.5
 
             NL=`echo -ne '\015'`
-            screen -S $SCREEN_NAME -p $1 -X stuff "$2 || echo \"$1 failed to start\" | tee \"$SERVICE_DIR/$SCREEN_NAME/$1.failure\"$NL"
+            # This fun command does the following:
+            # - the passed server command is backgrounded
+            # - the pid of the background process is saved in the usual place
+            # - the server process is brought back to the foreground
+            # - if the server process exits prematurely the fg command errors
+            #   and a message is written to stdout and the service failure file
+            # The pid saved can be used in screen_stop() as a process group
+            # id to kill off all child processes
+            screen -S $SCREEN_NAME -p $1 -X stuff "$2 & echo \$! >$SERVICE_DIR/$SCREEN_NAME/$1.pid; fg || echo \"$1 failed to start\" | tee \"$SERVICE_DIR/$SCREEN_NAME/$1.failure\"$NL"
         else
             # Spawn directly without screen
-            run_process "$1" "$2" >$SERVICE_DIR/$SCREEN_NAME/$service.pid
+            run_process "$1" "$2" >$SERVICE_DIR/$SCREEN_NAME/$1.pid
+        fi
+    fi
+}
+
+
+# Stop a service in screen
+# screen_stop service
+function screen_stop() {
+    SCREEN_NAME=${SCREEN_NAME:-stack}
+    SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
+    USE_SCREEN=$(trueorfalse True $USE_SCREEN)
+
+    if is_service_enabled $1; then
+        # Kill via pid if we have one available
+        if [[ -r $SERVICE_DIR/$SCREEN_NAME/$1.pid ]]; then
+            pkill -TERM -P $(cat $SERVICE_DIR/$SCREEN_NAME/$1.pid)
+            rm $SERVICE_DIR/$SCREEN_NAME/$1.pid
+        fi
+        if [[ "$USE_SCREEN" = "True" ]]; then
+            # Clean up the screen window
+            screen -S $SCREEN_NAME -p $1 -X kill
         fi
     fi
 }

lib/ceilometer

@@ -162,7 +162,7 @@ function start_ceilometer() {
 function stop_ceilometer() {
     # Kill the ceilometer screen windows
     for serv in ceilometer-acompute ceilometer-acentral ceilometer-anotification ceilometer-collector ceilometer-api ceilometer-alarm-notifier ceilometer-alarm-evaluator; do
-        screen -S $SCREEN_NAME -p $serv -X kill
+        screen_stop $serv
     done
 }
 

lib/cinder

@@ -561,7 +561,7 @@ function start_cinder() {
 function stop_cinder() {
     # Kill the cinder screen windows
     for serv in c-api c-bak c-sch c-vol; do
-        screen -S $SCREEN_NAME -p $serv -X kill
+        screen_stop $serv
     done
 
     if is_service_enabled c-vol; then

lib/glance

@@ -206,8 +206,8 @@ function start_glance() {
 # stop_glance() - Stop running processes
 function stop_glance() {
     # Kill the Glance screen windows
-    screen -S $SCREEN_NAME -p g-api -X kill
-    screen -S $SCREEN_NAME -p g-reg -X kill
+    screen_stop g-api
+    screen_stop g-reg
 }
 
 

lib/heat

@@ -175,7 +175,7 @@ function start_heat() {
 function stop_heat() {
     # Kill the screen windows
     for serv in h-eng h-api h-api-cfn h-api-cw; do
-        screen -S $SCREEN_NAME -p $serv -X kill
+        screen_stop $serv
     done
 }
 

lib/keystone

@@ -429,7 +429,7 @@ function start_keystone() {
 # stop_keystone() - Stop running processes
 function stop_keystone() {
     # Kill the Keystone screen window
-    screen -S $SCREEN_NAME -p key -X kill
+    screen_stop key
 }
 
 

lib/nova

@@ -705,7 +705,7 @@ function stop_nova() {
     # Some services are listed here twice since more than one instance
     # of a service may be running in certain configs.
     for serv in n-api n-cpu n-crt n-net n-sch n-novnc n-xvnc n-cauth n-spice n-cond n-cell n-cell n-api-meta; do
-        screen -S $SCREEN_NAME -p $serv -X kill
+        screen_stop $serv
     done
     if is_service_enabled n-cpu && [[ -r $NOVA_PLUGINS/hypervisor-$VIRT_DRIVER ]]; then
         stop_nova_hypervisor

lib/trove

@@ -198,7 +198,7 @@ function start_trove() {
 function stop_trove() {
     # Kill the trove screen windows
     for serv in tr-api tr-tmgr tr-cond; do
-        screen -S $SCREEN_NAME -p $serv -X kill
+        screen_stop $serv
     done
 }
 

stackrc

@@ -9,6 +9,9 @@ DEST=/opt/stack
 # Destination for working data
 DATA_DIR=${DEST}/data
 
+# Destination for status files
+SERVICE_DIR=${DEST}/status
+
 # Determine stack user
 if [[ $EUID -eq 0 ]]; then
     STACK_USER=stack

unstack.sh

@@ -36,6 +36,9 @@ source $TOP_DIR/lib/apache
 # Get project function libraries
 source $TOP_DIR/lib/baremetal
 source $TOP_DIR/lib/cinder
+source $TOP_DIR/lib/keystone
+source $TOP_DIR/lib/glance
+source $TOP_DIR/lib/nova
 source $TOP_DIR/lib/horizon
 source $TOP_DIR/lib/swift
 source $TOP_DIR/lib/neutron
@@ -75,21 +78,29 @@ if [[ "$Q_USE_DEBUG_COMMAND" == "True" ]]; then
     teardown_neutron_debug
 fi
 
-# Shut down devstack's screen to get the bulk of OpenStack services in one shot
-SCREEN=$(which screen)
-if [[ -n "$SCREEN" ]]; then
-    SESSION=$(screen -ls | awk '/[0-9].stack/ { print $1 }')
-    if [[ -n "$SESSION" ]]; then
-        screen -X -S $SESSION quit
-    fi
+# Call service stop
+if is_service_enabled trove; then
+    stop_trove
 fi
 
-# Shut down Nova hypervisor plugins after Nova
-NOVA_PLUGINS=$TOP_DIR/lib/nova_plugins
-if is_service_enabled nova && [[ -r $NOVA_PLUGINS/hypervisor-$VIRT_DRIVER ]]; then
-    # Load plugin
-    source $NOVA_PLUGINS/hypervisor-$VIRT_DRIVER
-    stop_nova_hypervisor
+if is_service_enabled heat; then
+    stop_heat
+fi
+
+if is_service_enabled ceilometer; then
+    stop_ceilometer
+fi
+
+if is_service_enabled nova; then
+    stop_nova
+fi
+
+if is_service_enabled g-api g-reg; then
+    stop_glance
+fi
+
+if is_service_enabled key; then
+    stop_keystone
 fi
 
 # Swift runs daemons
@@ -123,6 +134,7 @@ SCSI_PERSIST_DIR=$CINDER_STATE_PATH/volumes/*
 
 # Get the iSCSI volumes
 if is_service_enabled cinder; then
+    stop_cinder
     cleanup_cinder
 fi
 
@@ -152,4 +164,13 @@ if is_service_enabled trove; then
     cleanup_trove
 fi
 
+# Clean up the remainder of the screen processes
+SCREEN=$(which screen)
+if [[ -n "$SCREEN" ]]; then
+    SESSION=$(screen -ls | awk '/[0-9].stack/ { print $1 }')
+    if [[ -n "$SESSION" ]]; then
+        screen -X -S $SESSION quit
+    fi
+fi
+
 cleanup_tmp