metal/devstack/lib/metal

#!/bin/bash
#
# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2019 Intel Corporation
#
# lib/metal
# Functions to control the configuration and operation of metal
#
# To add a new maintenance service, the following functions should be
# added/modified.
#
# install_maintenance   Build and install maintenance services regardless
#                       node personality. The services within this function
#                       will be installed on each node.
#
# install_mtce_{personality}
#                       Per personality build and install services on
#                       specific node. A new kind of personality installation
#                       function has to be added under 'install' of plugin.sh.
#
# configure_maintenance Add configuration for services, if a config file
#                       template is needed, add it to devstack/files/
#                       AIO-SX supported only for current configuration.
#
# start_{service_name}  The service start function, add to start_maintenance
#                       if the service is running on each node, or to
#                       start_mtce_{personality} if the service is running on one
#                       node per personality.
#                       Make sure the service is enabled in devstack when testing,
#                       otherwise run_process will return immediately.
#
# stop_{service_name}   Stop the service when unstack.sh is processed. Called by
#                       stop_maintenance or stop_mtce_{personality}.
#
# cleanup_metal         Remove the service files installed.

_XTRACE_STX_METAL=$(set +o | grep xtrace)
set -o xtrace

# Defaults
# --------

STX_METAL_DIR=${GITDIR[$STX_METAL_NAME]}

# STX_INST_DIR should be a non-root-writable place to install build artifacts
STX_INST_DIR=${STX_INST_DIR:-/usr/local}
STX_BIN_DIR=$STX_INST_DIR/bin

# Set up so we don't use sudo for installs when not necessary
STX_SUDO="sudo"
[[ -w $STX_INST_DIR ]] && STX_SUDO="env"

SYSCONFDIR=/etc
# Todo:
# For the time being, we use the same interface for oam and management
# network as a temporal solution. In the future, stx devstack common
# module should create multiple inteface, then we need make changes here
# accordingly.
OAM_ETH_NAME=$(ip -f inet route | awk '/default/ {print $5}' | head -1)
MGMT_ETH_NAME=$(ip -f inet route | awk '/default/ {print $5}' | head -1)
SW_VERSION=18.10

OCF_ROOT=/usr/lib/ocf

# Functions
# ---------

function build_mtce {
    pushd ${STX_METAL_DIR}/mtce/src

    local major minor version x

    if [[ -z $1 || -z $2 ]]; then
        # get mtce version
        read x version <<< $(grep '^Version:' ${STX_METAL_DIR}/mtce/PKG-INFO)
        major=${version%%.*}
        minor=${version##*.}
    else
        major=$1
        minor=$2
    fi

    # build
    CPATH=$STX_INST_DIR/include LIBRARY_PATH=$STX_INST_DIR/lib make \
        VER_MJR=${major} \
        VER=${major}.${minor} \
        CCFLAGS=' -g -O2 -Wall -Wextra -std=c++11 -DBUILDINFO="\"$$(date)\""' \
        EXTRALDFLAGS="-L$STX_INST_DIR"/lib64 \
        build

    popd
}

function build_mtce_common {
    pushd ${STX_METAL_DIR}/mtce-common/src

    # build
    CPATH=$STX_INST_DIR/include LIBRARY_PATH=$STX_INST_DIR/lib make \
        CCFLAGS=' -g -O2 -Wall -Wextra -std=c++11 -DBUILDINFO="\"$$(date)\""' \
        build

    popd
}

function install_metal {
    install_mtce_common
    # components could be seperately installed if
    # installation is well controlled in Makefile
    install_mtce

    if is_service_enabled mtce-compute; then
        install_mtce_compute
    fi
    if is_service_enabled mtce-control; then
        install_mtce_control
    fi
    if is_service_enabled mtce-storage; then
        install_mtce_storage
    fi
}

function install_mtce_common {
    pushd ${STX_METAL_DIR}/mtce-common/src

    local x version

    # get mtce-common version
    read x version <<< $(grep '^Version:' ${STX_METAL_DIR}/mtce-common/PKG-INFO)
    local major=${version%%.*}
    local minor=${version##*.}

    build_mtce_common

    local lib64_dir=${STX_INST_DIR}/lib64
    local inc_dir=${STX_INST_DIR}/include
    local inc_dir_common=${STX_INST_DIR}/include/mtce-common
    local inc_dir_daemon=${STX_INST_DIR}/include/mtce-daemon

    local libdaecom_file=( \
        "common/libcommon.a" \
        "common/libthreadUtil.a" \
        "common/libbmcUtils.a" \
        "common/libpingUtil.a" \
        "common/libnodeBase.a" \
        "common/libregexUtil.a" \
        "common/libhostUtil.a" \
        "daemon/libdaemon.a" \
        )
    $STX_SUDO install -m 755 -d ${lib64_dir}
    $STX_SUDO install -m 644 -t ${lib64_dir} ${libdaecom_file[*]}

    $STX_SUDO install -m 755 -d ${inc_dir}
    local commonhdr_file=( \
        "common/fitCodes.h" \
        "common/logMacros.h" \
        "common/returnCodes.h" \
        "common/nodeTimers.h" \
        "common/hostClass.h" \
        "common/httpUtil.h" \
        "common/jsonUtil.h" \
        "common/msgClass.h" \
        "common/nodeBase.h" \
        "common/nodeEvent.h" \
        "common/nodeMacro.h" \
        "common/nodeUtil.h" \
        "common/timeUtil.h" \
        "common/alarmUtil.h" \
        "common/hostUtil.h" \
        "common/bmcUtil.h" \
        "common/ipmiUtil.h" \
        "common/redfishUtil.h" \
        "common/nlEvent.h" \
        "common/pingUtil.h" \
        "common/regexUtil.h" \
        "common/threadUtil.h" \
        "common/tokenUtil.h" \
        "common/secretUtil.h" \
        )
    $STX_SUDO install -m 755 -d ${inc_dir}
    $STX_SUDO install -m 644 -t ${inc_dir} ${commonhdr_file[*]}

    local daemonhdr_file=( \
        "daemon/daemon_ini.h" \
        "daemon/daemon_common.h" \
        "daemon/daemon_option.h" \
        )
    $STX_SUDO install -m 755 -d ${inc_dir}
    $STX_SUDO install -m 644 -t ${inc_dir} ${daemonhdr_file[*]}
    popd
}

function install_mtce_compute {
    local sysconf_dir=${SYSCONFDIR}
    local unit_dir=${SYSCONFDIR}/systemd/system
    local local_etc_pmond=${SYSCONFDIR}/pmon.d
    local local_etc_goenabledd=${SYSCONFDIR}/goenabled.d

    # install
    pushd ${STX_METAL_DIR}/mtce-compute/src
    # Worker-Only Init Scripts
    sudo install -m 755 -p -D scripts/goenabled ${sysconf_dir}/init.d/goenabledWorker

    # Worker-Only Process Monitor Config files
    sudo install -m 755 -d ${local_etc_pmond}

    # Worker-Only Go Enabled Test
    sudo install -m 755 -d ${local_etc_goenabledd}
    sudo install -m 755 -p -D scripts/virt-support-goenabled.sh ${local_etc_goenabledd}/virt-support-goenabled.sh

    popd
}

function install_mtce_control {
    local sysconf_dir=${SYSCONFDIR}
    local unit_dir=${SYSCONFDIR}/systemd/system
    local local_etc_pmond=${SYSCONFDIR}/pmon.d
    local local_etc_goenabledd=${SYSCONFDIR}/goenabled.d

    # install
    pushd ${STX_METAL_DIR}/mtce-control/src
    # Controller-Only Init Scripts
    sudo install -m 755 -p -D scripts/goenabled ${sysconf_dir}/init.d/goenabledControl
    sudo install -m 755 -p -D scripts/hbsAgent ${sysconf_dir}/init.d/hbsAgent
    # Controller-Only Process Monitor Config files
    sudo install -m 755 -d ${local_etc_pmond}
    sudo install -m 644 -p -D scripts/hbsAgent.conf ${local_etc_pmond}/hbsAgent.conf
    # Controller-Only Heartbeat Service file
    sudo install -m 644 -p -D scripts/hbsAgent.service ${unit_dir}/devstack@hbsAgent.service
    # Controller-Only Go Enabled Test
    sudo install -m 755 -d ${local_etc_goenabledd}
    popd
}

function install_mtce_storage {
    local sysconf_dir=${SYSCONFDIR}
    local unit_dir=${SYSCONFDIR}/systemd/system
    local local_etc_pmond=${SYSCONFDIR}/pmon.d
    local local_etc_goenabledd=${SYSCONFDIR}/goenabled.d
    local local_etc_servicesd=${SYSCONFDIR}/services.d

    # install
    pushd ${STX_METAL_DIR}/mtce-storage/src
    # Storage-Only Init Scripts
    sudo install -m 755 -p -D scripts/goenabled ${sysconf_dir}/init.d/goenabledStorage
    # Storage-Only Process Monitor Config files
    sudo install -m 755 -d ${local_etc_pmond}
    # Storage-Only Go Enabled Tests
    sudo install -m 755 -d ${local_etc_goenabledd}
    # Storage-Only Services
    sudo install -m 755 -d ${local_etc_servicesd}/storage
    popd
}

function install_mtce {
    pushd ${STX_METAL_DIR}/mtce/src

    local x version

    # get mtce version
    read x version <<< $(grep '^Version:' ${STX_METAL_DIR}/mtce/PKG-INFO)
    local major=${version%%.*}
    local minor=${version##*.}

    build_mtce $major $minor

    local bin_dir=/usr/local/bin
    local sbin_dir=/usr/local/sbin
    local lib_dir=${STX_INST_DIR}/lib
    local lib64_dir=${STX_INST_DIR}/lib64
    local inc_dir=${STX_INST_DIR}/include
    local sysconf_dir=${SYSCONFDIR}
    local unit_dir=${SYSCONFDIR}/systemd/system

    local local_etc_pmond=${sysconf_dir}/pmon.d
    local local_etc_goenabledd=${sysconf_dir}/goenabled.d
    local local_etc_servicesd=${sysconf_dir}/services.d
    local local_etc_logrotated=${sysconf_dir}/logrotate.d

    # install
    if [ ! -d "/etc/rc.d" ]; then
        sudo mkdir -p /etc/rc.d
    fi
    sudo ln -sf /etc/init.d /etc/rc.d/init.d

    # TODO: follow install_files way in install_mtce_common to install files
    # Resource Agent Stuff
    sudo install -m 755 -d ${OCF_ROOT}/resource.d/platform
    sudo install -m 755 -p -D scripts/mtcAgent ${OCF_ROOT}/resource.d/platform/mtcAgent
    sudo install -m 755 -p -D hwmon/scripts/ocf/hwmon ${OCF_ROOT}/resource.d/platform/hwmon

    # TODO: fix this issue in multi-os task
    # The default shell of ubuntu is dash, causes loop syntax error
    if is_ubuntu; then
        sudo sed -i "s|#!/bin/sh|#!/bin/bash|g" ${OCF_ROOT}/resource.d/platform/mtcAgent
        sudo sed -i "s|#!/bin/sh|#!/bin/bash|g" ${OCF_ROOT}/resource.d/platform/hwmon
    fi

    # TODO: figure out why OCF_ROOT is not set when running stack.sh
    # workaround to hardcode OCF_ROOT
    # sed -i "s|\${OCF_ROOT}|\${OCF_ROOT}|g" ${OCF_ROOT}/resource.d/platform/mtcAgent
    # sed -i "s|\${OCF_ROOT}|\${OCF_ROOT}|g" ${OCF_ROOT}/resource.d/platform/hwmon

    # config files
    sudo install -m 755 -d ${sysconf_dir}/mtc
    sudo install -m 644 -p -D scripts/mtc.ini ${sysconf_dir}/mtc.ini
    sudo install -m 644 -p -D scripts/mtc.conf ${sysconf_dir}/mtc.conf
    sudo install -m 644 -p -D fsmon/scripts/fsmond.conf ${sysconf_dir}/mtc/fsmond.conf
    sudo install -m 644 -p -D hwmon/scripts/hwmond.conf ${sysconf_dir}/mtc/hwmond.conf
    sudo install -m 644 -p -D pmon/scripts/pmond.conf ${sysconf_dir}/mtc/pmond.conf
    sudo install -m 644 -p -D hostw/scripts/hostwd.conf ${sysconf_dir}/mtc/hostwd.conf

    sudo install -m 755 -d ${sysconf_dir}/bmc/server_profiles.d
    sudo install -m 644 -p -D scripts/sensor_hp360_v1_ilo_v4.profile ${sysconf_dir}/bmc/server_profiles.d/sensor_hp360_v1_ilo_v4.profile
    sudo install -m 644 -p -D scripts/sensor_hp380_v1_ilo_v4.profile ${sysconf_dir}/bmc/server_profiles.d/sensor_hp380_v1_ilo_v4.profile
    sudo install -m 644 -p -D scripts/sensor_quanta_v1_ilo_v4.profile ${sysconf_dir}/bmc/server_profiles.d/sensor_quanta_v1_ilo_v4.profile

    # binaries
    sudo install -m 755 -p -D maintenance/mtcAgent ${bin_dir}/mtcAgent
    sudo install -m 755 -p -D maintenance/mtcClient ${bin_dir}/mtcClient
    sudo install -m 755 -p -D heartbeat/hbsAgent ${bin_dir}/hbsAgent
    sudo install -m 755 -p -D heartbeat/hbsClient ${bin_dir}/hbsClient
    sudo install -m 755 -p -D pmon/pmond ${bin_dir}/pmond
    sudo install -m 755 -p -D hostw/hostwd ${bin_dir}/hostwd
    sudo install -m 755 -p -D fsmon/fsmond ${bin_dir}/fsmond
    sudo install -m 755 -p -D hwmon/hwmond ${bin_dir}/hwmond
    sudo install -m 755 -p -D mtclog/mtclogd ${bin_dir}/mtclogd
    sudo install -m 755 -p -D alarm/mtcalarmd ${bin_dir}/mtcalarmd
    sudo install -m 755 -p -D scripts/wipedisk ${bin_dir}/wipedisk
    $STX_SUDO install -m 755 -p -D fsync/fsync ${STX_INST_DIR}/sbin/fsync
    sudo install -m 700 -p -D pmon/scripts/pmon-restart ${sbin_dir}/pmon-restart
    sudo install -m 700 -p -D pmon/scripts/pmon-start ${sbin_dir}/pmon-start
    sudo install -m 700 -p -D pmon/scripts/pmon-stop ${sbin_dir}/pmon-stop

    # init script files
    sudo install -m 755 -p -D scripts/mtcClient ${sysconf_dir}/init.d/mtcClient
    sudo install -m 755 -p -D scripts/hbsClient ${sysconf_dir}/init.d/hbsClient
    sudo install -m 755 -p -D hwmon/scripts/lsb/hwmon ${sysconf_dir}/init.d/hwmon
    sudo install -m 755 -p -D fsmon/scripts/fsmon ${sysconf_dir}/init.d/fsmon
    sudo install -m 755 -p -D scripts/mtclog ${sysconf_dir}/init.d/mtclog
    sudo install -m 755 -p -D pmon/scripts/pmon ${sysconf_dir}/init.d/pmon
    sudo install -m 755 -p -D hostw/scripts/hostw ${sysconf_dir}/init.d/hostw
    sudo install -m 755 -p -D alarm/scripts/mtcalarm.init ${sysconf_dir}/init.d/mtcalarm

    sudo install -m 755 -p -D scripts/config ${sysconf_dir}/init.d/config
    sudo install -m 755 -p -D scripts/hwclock.sh ${sysconf_dir}/init.d/hwclock.sh
    sudo install -m 644 -p -D scripts/hwclock.service ${unit_dir}/devstack@hwclock.service

    # systemd service files
    sudo install -m 644 -p -D fsmon/scripts/fsmon.service ${unit_dir}/devstack@fsmon.service
    sudo install -m 644 -p -D hwmon/scripts/hwmon.service ${unit_dir}/devstack@hwmon.service
    sudo install -m 644 -p -D pmon/scripts/pmon.service ${unit_dir}/devstack@pmon.service
    sudo install -m 644 -p -D hostw/scripts/hostw.service ${unit_dir}/devstack@hostw.service
    sudo install -m 644 -p -D scripts/mtcClient.service ${unit_dir}/devstack@mtcClient.service
    sudo install -m 644 -p -D scripts/hbsClient.service ${unit_dir}/devstack@hbsClient.service
    sudo install -m 644 -p -D scripts/mtclog.service ${unit_dir}/devstack@mtclog.service
    sudo install -m 644 -p -D scripts/goenabled.service ${unit_dir}/devstack@goenabled.service
    sudo install -m 644 -p -D scripts/runservices.service ${unit_dir}/devstack@runservices.service
    sudo install -m 644 -p -D alarm/scripts/mtcalarm.service ${unit_dir}/devstack@mtcalarm.service

    # go enabled stuff
    sudo install -m 755 -d ${local_etc_goenabledd}
    sudo install -m 755 -p -D scripts/goenabled ${sysconf_dir}/init.d/goenabled

    # start or stop services test script
    sudo install -m 755 -d ${local_etc_servicesd}/controller
    sudo install -m 755 -d ${local_etc_servicesd}/storage
    sudo install -m 755 -d ${local_etc_servicesd}/worker
    sudo install -m 755 -p -D scripts/mtcTest ${local_etc_servicesd}/controller
    sudo install -m 755 -p -D scripts/mtcTest ${local_etc_servicesd}/storage
    sudo install -m 755 -p -D scripts/mtcTest ${local_etc_servicesd}/worker
    sudo install -m 755 -p -D scripts/runservices ${sysconf_dir}/init.d/runservices

    # test tools
    sudo install -m 755 -p -D scripts/dmemchk.sh ${sbin_dir}

    # process monitor config files
    sudo install -m 755 -d ${local_etc_pmond}
    sudo install -m 644 -p -D scripts/mtcClient.conf ${local_etc_pmond}/mtcClient.conf
    sudo install -m 644 -p -D scripts/hbsClient.conf ${local_etc_pmond}/hbsClient.conf
    sudo install -m 644 -p -D pmon/scripts/acpid.conf ${local_etc_pmond}/acpid.conf
    sudo install -m 644 -p -D pmon/scripts/sshd.conf ${local_etc_pmond}/sshd.conf
    sudo install -m 644 -p -D pmon/scripts/syslog-ng.conf ${local_etc_pmond}/syslog-ng.conf
    sudo install -m 644 -p -D pmon/scripts/nslcd.conf ${local_etc_pmond}/nslcd.conf
    sudo install -m 644 -p -D fsmon/scripts/fsmon.conf ${local_etc_pmond}/fsmon.conf
    sudo install -m 644 -p -D scripts/mtclogd.conf ${local_etc_pmond}/mtclogd.conf
    sudo install -m 644 -p -D alarm/scripts/mtcalarm.pmon.conf ${local_etc_pmond}/mtcalarm.conf

    # log rotation
    sudo install -m 755 -d ${local_etc_logrotated}
    sudo install -m 644 -p -D scripts/mtce.logrotate ${local_etc_logrotated}/mtce.logrotate
    sudo install -m 644 -p -D hostw/scripts/hostw.logrotate ${local_etc_logrotated}/hostw.logrotate
    sudo install -m 644 -p -D pmon/scripts/pmon.logrotate ${local_etc_logrotated}/pmon.logrotate
    sudo install -m 644 -p -D fsmon/scripts/fsmon.logrotate ${local_etc_logrotated}/fsmon.logrotate
    sudo install -m 644 -p -D hwmon/scripts/hwmon.logrotate ${local_etc_logrotated}/hwmon.logrotate
    sudo install -m 644 -p -D alarm/scripts/mtcalarm.logrotate ${local_etc_logrotated}/mtcalarm.logrotate

    # software development files
    $STX_SUDO install -m 644 -p -D heartbeat/mtceHbsCluster.h ${inc_dir}/mtceHbsCluster.h
    $STX_SUDO install -m 755 -p -D public/libamon.so.${major} ${lib64_dir}/libamon.so.${major}
    popd

    pushd ${lib64_dir}
    $STX_SUDO ln -sf libamon.so.${major} libamon.so.${major}.${minor}
    $STX_SUDO ln -sf libamon.so.${major} libamon.so
    popd
}

function configure_metal {
    echo "${STX_INST_DIR}/lib64" | sudo tee /etc/ld.so.conf.d/stx-metal.conf
    sudo ldconfig

    sudo mkdir -p /etc/platform

    sudo cp -p ${GITDIR["$STX_METAL_NAME"]}/devstack/files/platform.conf /etc/platform/

    # TODO: implement other configuration
    # All-in-one simplex config
    if is_service_enabled mtce-control -a is_service_enabled mtce-compute \
        -a is_service_enabled mtce-storage; then
        sudo sed -i "s|@SUBFUNCTION@|controller,worker,storage|g" /etc/platform/platform.conf
        sudo sed -i "s|@SYS_TYPE@|All-in-one|g" /etc/platform/platform.conf
        sudo sed -i "s|@SYS_MODE@|simplex|g" /etc/platform/platform.conf
        sudo sed -i "s|@SW_VERSION@|${SW_VERSION}|g" /etc/platform/platform.conf
        sudo sed -i "s|@MGMT_ETH@|${MGMT_ETH_NAME}|g" /etc/platform/platform.conf
        sudo sed -i "s|@OAM_ETH@|${OAM_ETH_NAME}|g" /etc/platform/platform.conf
    fi

    # TODO: enable controllerconfig to do this
    # Todo:
    # For the time being, we use the same ips as fixed and float ip.  In
    # the future, we need stx devstack common module to enable float ip and
    # write them in /etc/hosts, then we can remove below code.
    # add floating ip hostname in hosts
    if ! grep -q 'controller$' /etc/hosts; then
        echo "$HOST_IP controller" | sudo tee -a /etc/hosts
    fi
    # add controller-0
    if ! grep -q 'controller-0$' /etc/hosts; then
        echo "$HOST_IP controller-0" | sudo tee -a /etc/hosts
    fi

    # puppet configuration for mtce
    iniset -sudo /etc/mtc.ini "agent" "keystone_auth_username" "admin"
    iniset -sudo /etc/mtc.ini "agent" "keystone_auth_pw" "$SERVICE_PASSWORD"
    iniset -sudo /etc/mtc.ini "agent" "keystone_auth_project" "admin"
    iniset -sudo /etc/mtc.ini "agent" "keystone_user_domain" "$SERVICE_DOMAIN_NAME"
    iniset -sudo /etc/mtc.ini "agent" "keystone_project_domain" "$SERVICE_DOMAIN_NAME"
    iniset -sudo /etc/mtc.ini "agent" "keystone_auth_host" "127.0.0.1"
    iniset -sudo /etc/mtc.ini "agent" "keystone_auth_uri" "$KEYSTONE_SERVICE_URI"
    iniset -sudo /etc/mtc.ini "agent" "keystone_auth_port" "5000"
    iniset -sudo /etc/mtc.ini "agent" "keystone_region_name" "$REGION_NAME"
    iniset -sudo /etc/mtc.ini "agent" "keyring_directory" "/opt/platform/.keyring/${SW_VERSION}"
    # sm port add to config file
    iniset -sudo /etc/mtc.ini "agent" "sm_server_port" "2124"
    iniset -sudo /etc/mtc.ini "agent" "sm_client_port" "2224"

    # TODO(dtroyer): Do this with the devstack functions in lib/apache
    # # add port 5000 to keystone apache server
    # sudo sed -i '$aListen 5000' /etc/apache2/sites-available/keystone-wsgi-public.conf
    # sudo sed -i '$a<VirtualHost *:5000>' /etc/apache2/sites-available/keystone-wsgi-public.conf
    # sudo sed -i '$aProxyPass / "unix:/var/run/uwsgi/keystone-wsgi-public.socket|uwsgi://uwsgi-uds-keystone-wsgi-public/" retry=0"' /etc/apache2/sites-available/keystone-wsgi-public.conf
    # sudo sed -i '$a</VirtualHost>' /etc/apache2/sites-available/keystone-wsgi-public.conf
    # sudo systemctl restart apache2

    if is_ubuntu; then
        sudo ln -sf /lib/lsb/init-functions /etc/init.d/functions
    fi
}

function start_mtcAgent {
    # add admin compute endpoint
    if is_service_enabled nova; then
        source openrc admin admin
        openstack endpoint create --region RegionOne compute admin http://172.28.0.79/compute/v2.1
    fi

    # init mtcAgent
    iniset -sudo /etc/systemd/system/devstack@mtcAgent.service "Service" "Type" "forking"
    iniset -sudo /etc/systemd/system/devstack@mtcAgent.service "Service" "PIDFile" "/var/run/mtcAgent.pid"
    iniset -sudo /etc/systemd/system/devstack@mtcAgent.service "Service" "Environment" "OCF_ROOT=${OCF_ROOT}"

    sudo sed -i "s|OCF_RESKEY_state_default=\"standby\"|OCF_RESKEY_state_default=\"active\"|g" \
        ${OCF_ROOT}/resource.d/platform/mtcAgent

    run_process mtcAgent "${OCF_ROOT}/resource.d/platform/mtcAgent start" root root
}

function start_hbsAgent {
    run_process hbsAgent "${SYSCONFDIR}/rc.d/init.d/hbsAgent start" root root
}

function start_hwmon {
    # init hwmon
    iniset -sudo /etc/systemd/system/devstack@hwmon.service "Service" "Type" "forking"
    iniset -sudo /etc/systemd/system/devstack@hwmon.service "Service" "PIDFile" "/var/run/hwmond.pid"
    iniset -sudo /etc/systemd/system/devstack@hwmon.service "Service" "Environment" "OCF_ROOT=${OCF_ROOT}"

    run_process hwmon "${OCF_ROOT}/resource.d/platform/hwmon start" root root
}

function start_mtce_control {
    # Oneshot goenabled control
    sudo ${SYSCONFDIR}/init.d/goenabledControl start

    if is_service_enabled mtcAgent; then
        start_mtcAgent
    fi
    if is_service_enabled hbsAgent; then
        start_hbsAgent
    fi
    if is_service_enabled hwmon && is_service_enabled sysinv; then
        start_hwmon
    fi
}

function start_mtce_compute {
    # Oneshot goenabled worker
    sudo ${SYSCONFDIR}/init.d/goenabledWorker start
}

function start_mtce_storage {
    # Oneshot goenabled storage
    sudo ${SYSCONFDIR}/init.d/goenabledStorage start
}

function start_metal {
    if is_service_enabled mtce-control; then
        start_mtce_control
    fi
    if is_service_enabled mtce-compute; then
        start_mtce_compute
    fi
    if is_service_enabled mtce-storage; then
        start_mtce_storage
    fi

    start_goenabled

    if is_service_enabled hbsClient; then
        start_hbsClient
    fi
    if is_service_enabled mtcClient; then
        start_mtcClient
    fi
    if is_service_enabled mtclog; then
        start_mtclog
    fi
    if is_service_enabled mtcalarm; then
        start_mtcalarm
    fi
    if is_service_enabled fsmon; then
        start_fsmon
    fi
    if is_service_enabled pmon; then
        start_pmon
    fi
    if is_service_enabled hostw; then
        start_hostw
    fi
}

function start_mtcClient {
    run_process mtcClient "${SYSCONFDIR}/rc.d/init.d/mtcClient start" root root
}

function start_hbsClient {
    run_process hbsClient "${SYSCONFDIR}/rc.d/init.d/hbsClient start" root root
}

function start_pmon {
    run_process pmon "${SYSCONFDIR}/rc.d/init.d/pmon start" root root
}

function start_mtclog {
    run_process mtclog "${SYSCONFDIR}/rc.d/init.d/mtclog start" root root
}

function start_mtcalarm {
    run_process mtcalarm "${SYSCONFDIR}/rc.d/init.d/mtcalarm start" root root
}

function start_goenabled {
    sudo ${SYSCONFDIR}/init.d/goenabled start
}

function start_fsmon {
    run_process fsmon "${SYSCONFDIR}/rc.d/init.d/fsmon start" root root
}

function start_hostw {
    run_process hostw "${SYSCONFDIR}/rc.d/init.d/hostw start" root root
}

function stop_mtcAgent {
    stop_process mtcAgent
}

function stop_hbsAgent {
    stop_process hbsAgent
}

function stop_hwmon {
    stop_process hwmon
}

function stop_mtce_control {
    sudo ${SYSCONFDIR}/init.d/goenabledControl stop

    if is_service_enabled mtcAgent; then
        stop_mtcAgent
    fi
    if is_service_enabled hbsAgent; then
        stop_hbsAgent
    fi
    if is_service_enabled hwmon && is_service_enabled sysinv; then
        stop_hwmon
    fi
}

function stop_mtce_compute {
    sudo ${SYSCONFDIR}/init.d/goenabledWorker stop
}

function stop_mtce_storage {
    sudo ${SYSCONFDIR}/init.d/goenabledStorage stop
}

function stop_goenabled {
    sudo ${SYSCONFDIR}/init.d/goenabled stop
}

function stop_pmon {
    stop_process pmon
}

function stop_mtcClient {
    stop_process mtcClient
}

function stop_hbsClient {
    stop_process hbsClient
}

function stop_mtclog {
    stop_process mtclog
}

function stop_mtcalarm {
    stop_process mtcalarm
}

function stop_fsmon {
    stop_process fsmon
}

function stop_hostw {
    stop_process hostw
}

function stop_metal {
    stop_goenabled

    if is_service_enabled pmon; then
        stop_pmon
    fi
    if is_service_enabled mtce; then
        stop_mtcClient
    fi
    if is_service_enabled hbs; then
        stop_hbsClient
    fi
    if is_service_enabled mtclog; then
        stop_mtclog
    fi
    if is_service_enabled mtcalarm; then
        stop_mtcalarm
    fi
    if is_service_enabled fsmon; then
        stop_fsmon
    fi
    if is_service_enabled hostw; then
        stop_hostw
    fi

    if is_service_enabled mtce-control; then
        stop_mtce_control
    fi
    if is_service_enabled mtce-compute; then
        stop_mtce_compute
    fi
    if is_service_enabled mtce-storage; then
        stop_mtce_storage
    fi
}

function cleanup_metal {
    # TODO: move it to Makefile uninstall
    # remove mtce
    local x version

    # get mtce version
    read x version <<< $(grep '^Version:' ${STX_METAL_DIR}/mtce/PKG-INFO)
    local major=${version%%.*}
    local minor=${version##*.}

    local bin_dir=/usr/local/bin
    local sbin_dir=/usr/local/sbin
    local lib_dir=${STX_INST_DIR}/lib
    local lib64_dir=${STX_INST_DIR}/lib64
    local inc_dir=${STX_INST_DIR}/include
    local sysconf_dir=${SYSCONFDIR}
    local unit_dir=${SYSCONFDIR}/systemd/system

    local local_etc_pmond=${sysconf_dir}/pmon.d
    local local_etc_goenabledd=${sysconf_dir}/goenabled.d
    local local_etc_servicesd=${sysconf_dir}/services.d
    local local_etc_logrotated=${sysconf_dir}/logrotate.d

    sudo rm -rf ${OCF_ROOT}/resource.d/platform/mtcAgent
    sudo rm -rf ${OCF_ROOT}/resource.d/platform/hwmon

    sudo rm -rf ${sysconf_dir}/mtc.ini
    sudo rm -rf ${sysconf_dir}/mtc.conf
    sudo rm -rf ${sysconf_dir}/mtc/fsmond.conf
    sudo rm -rf ${sysconf_dir}/mtc/hwmond.conf
    sudo rm -rf ${sysconf_dir}/mtc/pmond.conf
    sudo rm -rf ${sysconf_dir}/mtc/hostwd.conf

    sudo rm -rf ${sysconf_dir}/bmc/server_profiles.d/sensor_hp360_v1_ilo_v4.profile
    sudo rm -rf ${sysconf_dir}/bmc/server_profiles.d/sensor_hp380_v1_ilo_v4.profile
    sudo rm -rf ${sysconf_dir}/bmc/server_profiles.d/sensor_quanta_v1_ilo_v4.profile

    # binaries
    sudo rm -rf ${bin_dir}/mtcAgent
    sudo rm -rf ${bin_dir}/mtcClient
    sudo rm -rf ${bin_dir}/hbsAgent
    sudo rm -rf ${bin_dir}/hbsClient
    sudo rm -rf ${bin_dir}/pmond
    sudo rm -rf ${bin_dir}/hostwd
    sudo rm -rf ${bin_dir}/fsmond
    sudo rm -rf ${bin_dir}/hwmond
    sudo rm -rf ${bin_dir}/mtclogd
    sudo rm -rf ${bin_dir}/mtcalarmd
    sudo rm -rf ${bin_dir}/wipedisk
    rm -rf ${STX_INST_DIR}/sbin/fsync
    sudo rm -rf ${sbin_dir}/pmon-restart
    sudo rm -rf ${sbin_dir}/pmon-start
    sudo rm -rf ${sbin_dir}/pmon-stop

    # init script files
    sudo rm -rf ${sysconf_dir}/init.d/mtcClient
    sudo rm -rf ${sysconf_dir}/init.d/hbsClient
    sudo rm -rf ${sysconf_dir}/init.d/hwmon
    sudo rm -rf ${sysconf_dir}/init.d/fsmon
    sudo rm -rf ${sysconf_dir}/init.d/mtclog
    sudo rm -rf ${sysconf_dir}/init.d/pmon
    sudo rm -rf ${sysconf_dir}/init.d/hostw
    sudo rm -rf ${sysconf_dir}/init.d/mtcalarm

    sudo rm -rf ${sysconf_dir}/init.d/config
    sudo rm -rf ${sysconf_dir}/init.d/hwclock.sh
    sudo rm -rf ${unit_dir}/devstack@hwclock.service

    # systemd service files
    sudo rm -rf ${unit_dir}/devstack@fsmon.service
    sudo rm -rf ${unit_dir}/devstack@hwmon.service
    sudo rm -rf ${unit_dir}/devstack@pmon.service
    sudo rm -rf ${unit_dir}/devstack@hostw.service
    sudo rm -rf ${unit_dir}/devstack@mtcClient.service
    sudo rm -rf ${unit_dir}/devstack@hbsClient.service
    sudo rm -rf ${unit_dir}/devstack@mtclog.service
    sudo rm -rf ${unit_dir}/devstack@goenabled.service
    sudo rm -rf ${unit_dir}/devstack@runservices.service
    sudo rm -rf ${unit_dir}/devstack@mtcalarm.service

    # go enabled stuff
    sudo rm -rf ${sysconf_dir}/init.d/goenabled

    # start or stop services test script
    sudo rm -rf ${local_etc_servicesd}/worker
    sudo rm -rf ${local_etc_servicesd}/controller
    sudo rm -rf ${local_etc_servicesd}/storage
    sudo rm -rf ${sysconf_dir}/init.d/runservices

    # test tools
    sudo rm -rf ${sbin_dir}/dmemchk.sh

    # process monitor config files
    sudo rm -rf ${local_etc_pmond}/mtcClient.conf
    sudo rm -rf ${local_etc_pmond}/hbsClient.conf
    sudo rm -rf ${local_etc_pmond}/acpid.conf
    sudo rm -rf ${local_etc_pmond}/sshd.conf
    sudo rm -rf ${local_etc_pmond}/syslog-ng.conf
    sudo rm -rf ${local_etc_pmond}/nslcd.conf
    sudo rm -rf ${local_etc_pmond}/fsmon.conf
    sudo rm -rf ${local_etc_pmond}/mtclogd.conf
    sudo rm -rf ${local_etc_pmond}/mtcalarm.conf

    # log rotation
    sudo rm -rf ${local_etc_logrotated}/mtce.logrotate
    sudo rm -rf ${local_etc_logrotated}/hostw.logrotate
    sudo rm -rf ${local_etc_logrotated}/pmon.logrotate
    sudo rm -rf ${local_etc_logrotated}/fsmon.logrotate
    sudo rm -rf ${local_etc_logrotated}/hwmon.logrotate
    sudo rm -rf ${local_etc_logrotated}/mtcalarm.logrotate

    # software development files
    $STX_SUDO rm -rf ${inc_dir}/mtceHbsCluster.h
    $STX_SUDO rm -rf ${lib64_dir}/libamon.so.${major}

    $STX_SUDO rm -rf ${lib64_dir}/libamon.so.${major}.${minor}
    $STX_SUDO rm -rf ${lib64_dir}/libamon.so

    #remove mtce_common
    local inc_dir_common=${STX_INST_DIR}/include/mtce-common
    local inc_dir_daemon=${STX_INST_DIR}/include/mtce-daemon

    local libdaecom_file=( \
        "libcommon.a" \
        "libthreadUtil.a" \
        "libBmcUtils.a" \
        "libpingUtil.a" \
        "libnodeBase.a" \
        "libregexUtil.a" \
        "libhostUtil.a" \
        "libdaemon.a" \
        )
    uninstall_files ${lib64_dir} "${libdaecom_file[*]}"

    local daecomhdr_file=( \
        "fitCodes.h" \
        "logMacros.h" \
        "returnCodes.h" \
        "nodeTimers.h" \
        "hostClass.h" \
        "httpUtil.h" \
        "jsonUtil.h" \
        "msgClass.h" \
        "nodeBase.h" \
        "nodeEvent.h" \
        "nodeMacro.h" \
        "nodeUtil.h" \
        "timeUtil.h" \
        "alarmUtil.h" \
        "hostUtil.h" \
        "ipmiUtil.h" \
        "nlEvent.h" \
        "pingUtil.h" \
        "regexUtil.h" \
        "threadUtil.h" \
        "tokenUtil.h" \
        "secretUtil.h" \
        "daemon_ini.h" \
        "daemon_common.h" \
        "daemon_option.h" \
        )
    uninstall_files ${inc_dir_common} "${daecomhdr_file[*]}"

    if is_service_enabled mtce-compute; then
        local sysconf_dir=${SYSCONFDIR}
        local unit_dir=${SYSCONFDIR}/systemd/system
        local local_etc_pmond=${SYSCONFDIR}/pmon.d
        local local_etc_goenabledd=${SYSCONFDIR}/goenabled.d

        sudo rm -rf ${sysconf_dir}/init.d/goenabledWorker

        # Worker-Only Go Enabled Test
        sudo rm -rf ${local_etc_goenabledd}/virt-support-goenabled.sh
    fi
    if is_service_enabled mtce-control; then
        local sysconf_dir=${SYSCONFDIR}
        local unit_dir=${SYSCONFDIR}/systemd/system
        local local_etc_pmond=${SYSCONFDIR}/pmon.d
        local local_etc_goenabledd=${SYSCONFDIR}/goenabled.d

        # Controller-Only Init Scripts
        sudo rm -rf  ${sysconf_dir}/init.d/goenabledControl
        sudo rm -rf  ${sysconf_dir}/init.d/hbsAgent
        sudo rm -rf  ${local_etc_pmond}/hbsAgent.conf
        sudo rm -rf  ${unit_dir}/devstack@hbsAgent.service
    fi
    if is_service_enabled mtce-storage; then
        local sysconf_dir=${SYSCONFDIR}
        local unit_dir=${SYSCONFDIR}/systemd/system
        local local_etc_pmond=${SYSCONFDIR}/pmon.d
        local local_etc_goenabledd=${SYSCONFDIR}/goenabled.d
        local local_etc_servicesd=${SYSCONFDIR}/services.d

        # Storage-Only Init Scripts
        sudo rm -rf ${sysconf_dir}/init.d/goenabledStorage
    fi

}

function uninstall_files {
    local dest_dir=$1
    local dest_file_array=($2)
    local file=""

    for (( i=0; i<${#dest_file_array[*]}; i++ )); do
        file=${dest_file_array[$i]##*/}
        $STX_SUDO rm -rf $dest_dir/$file
    done
}

$_XTRACE_STX_METAL