Add devstack job and fix linters

Add the base DevStack job and make sure bashate runs on
the devstack plugin files.

Begin to re-structure the plugin to match the common structure.

Add devstack/build.sh and split out the build steps into
separate functions in devstack/lib/stx-metal

This is complete, further work to be done in follow-up changes.

Change-Id: I05f6df758e18f182fb0a05731eddc6cb7f599e51
Signed-off-by: Dean Troyer <dtroyer@gmail.com>

.zuul.yaml

@@ -8,6 +8,8 @@
         - build-openstack-releasenotes
         - openstack-tox-linters
         - openstack-tox-pep8
+        - stx-devstack-metal:
+            voting: false
     gate:
       jobs:
         - build-openstack-api-ref
@@ -19,3 +21,36 @@
         - publish-stx-api-ref
         - publish-stx-tox
         - publish-stx-releasenotes
+
+- job:
+    name: stx-devstack-metal
+    parent: stx-devstack-base
+    timeout: 7800
+    required-projects:
+      - openstack/stx-update
+      - openstack/stx-fault
+      - openstack/stx-nfv
+    vars:
+      tox_envlist: functional
+      devstack_services:
+        horizon: false
+        # StarlingX services
+        fm-common: true
+        fm-api: true
+        mtce: true
+        mtce-compute: true
+        mtce-control: true
+        mtce-storage: true
+        fsmon: true
+        hbs: true
+        hwmon: true
+        mtcalarm: true
+        mtclog: true
+        pmon: true
+        rmon: true
+      devstack_plugins:
+        stx-fault: git://git.starlingx.io/stx-fault
+        stx-nfv: git://git.starlingx.io/stx-nfv
+        stx-update: git://git.starlingx.io/stx-update
+    files:
+      - ^devstack/.*

devstack/lib/stx-metal

@@ -48,18 +48,77 @@ SW_VERSION=18.10
 # Functions
 # ---------
 
-function uninstall_files() {
-    local dest_dir=$1
-    local dest_file_array=($2)
-    local file=""
 
-    for (( i=0; i<${#dest_file_array[*]}; i++ )); do
+function build_mtce {
-        file=${dest_file_array[$i]##*/}
+    pushd ${STX_METAL_DIR}/mtce/src
-        sudo rm -rf $dest_dir/$file
+
-    done
+    local major minor version x
+
+    if [[ -z $1 || -z $2 ]]; then
+        # get mtce version
+        read x version <<< $(grep '^Version:' ${STX_METAL_DIR}/mtce/PKG-INFO)
+        major=${version%%.*}
+        minor=${version##*.}
+    else
+        major=$1
+        minor=$2
+    fi
+
+    # build
+    make \
+        MAJOR=${major} \
+        MINOR=${minor} \
+        CCFLAGS=' -g -O2 -Wall -Wextra -std=c++11 -DBUILDINFO="\"$$(date)\""' \
+        build
+
+    popd
+}
+
+function build_mtce_common {
+    pushd ${STX_METAL_DIR}/mtce-common/src
+
+    local major minor version x
+
+    if [[ -z $1 || -z $2 ]]; then
+        # get mtce-common version
+        read x version <<< $(grep '^Version:' ${STX_METAL_DIR}/mtce/PKG-INFO)
+        major=${version%%.*}
+        minor=${version##*.}
+    else
+        major=$1
+        minor=$2
+    fi
+
+    # build
+    make \
+        MAJOR=${major} \
+        MINOR=${minor} \
+        CCFLAGS=' -g -O2 -Wall -Wextra -std=c++11 -DBUILDINFO="\"$$(date)\""' \
+        build
+
+    popd
+}
+
+function install_maintenance {
+    install_mtce_common
+    # components could be seperately installed if
+    # installation is well controlled in Makefile
+    install_mtce
+
+    if is_service_enabled mtce-compute; then
+        install_mtce_compute
+    fi
+    if is_service_enabled mtce-control; then
+        install_mtce_control
+    fi
+    if is_service_enabled mtce-storage; then
+        install_mtce_storage
+    fi
 }
 
 function install_mtce_common {
+    pushd ${STX_METAL_DIR}/mtce-common/src
+
     local x version
 
     # get mtce-common version
@@ -67,15 +126,13 @@ function install_mtce_common {
     local major=${version%%.*}
     local minor=${version##*.}
 
+    build_mtce_common
+
     local lib64_dir=${PREFIX}/lib64
     local inc_dir=${PREFIX}/include
     local inc_dir_common=${PREFIX}/include/mtce-common
     local inc_dir_daemon=${PREFIX}/include/mtce-daemon
 
-    # build
-    pushd ${STX_METAL_DIR}/mtce-common/src
-    make MAJOR=${major} MINOR=${minor} CCFLAGS=' -g -O2 -Wall -Wextra -std=c++11 -DBUILDINFO="\"$$(date)\""' build
-
     local libdaecom_file=( \
         "common/libcommon.a" \
         "common/libthreadUtil.a" \
@@ -200,6 +257,8 @@ function install_mtce_storage {
 }
 
 function install_mtce {
+    pushd ${STX_METAL_DIR}/mtce/src
+
     local x version
 
     # get mtce version
@@ -207,6 +266,8 @@ function install_mtce {
     local major=${version%%.*}
     local minor=${version##*.}
 
+    build_mtce $major $minor
+
     local bin_dir=${PREFIX}/local/bin
     local sbin_dir=${PREFIX}/local/sbin
     local lib_dir=${PREFIX}/lib
@@ -221,10 +282,6 @@ function install_mtce {
     local local_etc_servicesd=${sysconf_dir}/services.d
     local local_etc_logrotated=${sysconf_dir}/logrotate.d
 
-    # build
-    pushd ${STX_METAL_DIR}/mtce/src
-    make MAJOR=${major} MINOR=${minor} CCFLAGS=' -g -O2 -Wall -Wextra -std=c++11 -DBUILDINFO="\"$$(date)\""' build
-
     # install
     if [ ! -d "/etc/rc.d" ]; then
         sudo mkdir -p /etc/rc.d
@@ -354,7 +411,7 @@ function install_mtce {
     sudo install -m 644 -p -D rmon/scripts/oam_resource.conf ${sysconf_dir}/rmon_interfaces.d/oam_resource.conf
     sudo install -m 644 -p -D rmon/scripts/management_resource.conf ${sysconf_dir}/rmon_interfaces.d/management_resource.conf
     sudo install -m 644 -p -D rmon/scripts/infrastructure_resource.conf ${sysconf_dir}/rmon_interfaces.d/infrastructure_resource.conf
-    sudo install -m 755 -p -D rmon/scripts/query_ntp_servers.sh ${sysconf_dir}/rmonfiles.d/query_ntp_servers.sh
+    # sudo install -m 755 -p -D rmon/scripts/query_ntp_servers.sh ${sysconf_dir}/rmonfiles.d/query_ntp_servers.sh
     sudo install -m 755 -p -D rmon/scripts/rmon_reload_on_cpe.sh ${local_etc_goenabledd}/rmon_reload_on_cpe.sh
 
     # log rotation
@@ -380,14 +437,6 @@ function install_mtce {
     sudo ln -sf librmonapi.so.${major} librmonapi.so.${major}.${minor}
     sudo ln -sf librmonapi.so.${major} librmonapi.so
     popd
-
-}
-
-function install_maintenance {
-    install_mtce_common
-    # components could be seperatly installed if
-    # installation is well controlled in Makefile
-    install_mtce
 }
 
 function configure_maintenance {
@@ -452,7 +501,6 @@ function configure_maintenance {
 }
 
 function start_mtcAgent {
-    echo "trigger start mtc agent service"
     # add admin compute endpoint
     if is_service_enabled nova; then
         source openrc admin admin
@@ -470,8 +518,6 @@ function start_mtcAgent {
 }
 
 function start_hbsAgent {
-    echo "trigger start hbs agent service"
-
     # copy hbsAgent.service template to devstack
     sudo cp ${PREFIX}/lib/systemd/system/hbsAgent.service \
         ${SYSCONFDIR}/systemd/system/devstack@hbsAgent.service
@@ -479,7 +525,6 @@ function start_hbsAgent {
 }
 
 function start_hwmon {
-    echo "trigger start hwmon service"
     # init hwmon
     iniset -sudo /etc/systemd/system/devstack@hwmon.service "Service" "Type" "forking"
     iniset -sudo /etc/systemd/system/devstack@hwmon.service "Service" "PIDFile" "/var/run/hwmond.pid"
@@ -491,16 +536,13 @@ function start_mtce_control {
     # Oneshot goenabled control
     sudo ${SYSCONFDIR}/init.d/goenabledControl start
 
-    if is_service_enabled mtce; then
+    if is_service_enabled mtce-agent || is_service_enabled mtce; then
-        echo "start mtcAgent"
         start_mtcAgent
     fi
-    if is_service_enabled hbs; then
+    if is_service_enabled hbs-agent || is_service_enabled hbs; then
-        echo "start hbsAgent"
         start_hbsAgent
     fi
-    if is_service_enabled hwmon; then
+    if is_service_enabled hwmon && is_service_enabled sysinv; then
-        echo "start hwmon"
         start_hwmon
     fi
 }
@@ -524,45 +566,45 @@ function start_mtce_storage {
 }
 
 function start_maintenance {
+    if is_service_enabled mtce-control; then
+        start_mtce_control
+    fi
+    if is_service_enabled mtce-compute; then
+        start_mtce_compute
+    fi
+    if is_service_enabled mtce-storage; then
+        start_mtce_storage
+    fi
+
     start_goenabled
 
     if is_service_enabled hbs; then
-        echo "start hbs client service"
         start_hbsClient
     fi
     if is_service_enabled mtce; then
-        echo "start mtce client service"
         start_mtcClient
     fi
     if is_service_enabled rmon; then
-        echo "start rmon service"
         start_rmon
     fi
     if is_service_enabled mtclog; then
-        echo "start mtclog service"
         start_mtclog
     fi
     if is_service_enabled mtcalarm; then
-        echo "start mtcalarm service"
         start_mtcalarm
     fi
     if is_service_enabled fsmon; then
-        echo "start fsmon service"
         start_fsmon
     fi
     if is_service_enabled pmon; then
-        echo "start pmon service"
         start_pmon
     fi
     if is_service_enabled hostw; then
-        echo "start hostw service"
         start_hostw
     fi
 }
 
 function start_mtcClient {
-    echo "trigger start mtce client service"
-
     # copy mtcClient.service template to devstack
     sudo cp ${PREFIX}/lib/systemd/system/mtcClient.service \
         ${SYSCONFDIR}/systemd/system/devstack@mtcClient.service
@@ -570,8 +612,6 @@ function start_mtcClient {
 }
 
 function start_hbsClient {
-    echo "trigger start hbs client service"
-
     # copy hbsClient.service template to devstack
     sudo cp ${PREFIX}/lib/systemd/system/hbsClient.service \
         ${SYSCONFDIR}/systemd/system/devstack@hbsClient.service
@@ -579,8 +619,6 @@ function start_hbsClient {
 }
 
 function start_pmon {
-    echo "trigger start pmon service"
-
     # copy pmon.service template to devstack
     sudo cp ${PREFIX}/lib/systemd/system/pmon.service \
         ${SYSCONFDIR}/systemd/system/devstack@pmon.service
@@ -589,8 +627,6 @@ function start_pmon {
 }
 
 function start_rmon {
-    echo "trigger start rmon service"
-
     # copy rmon.service template to devstack
     sudo cp ${PREFIX}/lib/systemd/system/rmon.service \
         ${SYSCONFDIR}/systemd/system/devstack@rmon.service
@@ -599,8 +635,6 @@ function start_rmon {
 }
 
 function start_mtclog {
-    echo "trigger start mtclog service"
-
     # copy mtclog.service template to devstack
     sudo cp ${PREFIX}/lib/systemd/system/mtclog.service \
         ${SYSCONFDIR}/systemd/system/devstack@mtclog.service
@@ -609,8 +643,6 @@ function start_mtclog {
 }
 
 function start_mtcalarm {
-    echo "trigger start mtcalarm service"
-
     # copy mtcalarm.service template to devstack
     sudo cp ${PREFIX}/lib/systemd/system/mtcalarm.service \
         ${SYSCONFDIR}/systemd/system/devstack@mtcalarm.service
@@ -619,13 +651,10 @@ function start_mtcalarm {
 }
 
 function start_goenabled {
-    echo "trigger start goenabled service"
     sudo ${SYSCONFDIR}/init.d/goenabled start
 }
 
 function start_fsmon {
-    echo "trigger start fsmon service"
-
     # copy fsmon.service template to devstack
     sudo cp ${PREFIX}/lib/systemd/system/fsmon.service \
         ${SYSCONFDIR}/systemd/system/devstack@fsmon.service
@@ -634,8 +663,6 @@ function start_fsmon {
 }
 
 function start_hostw {
-    echo "trigger start hostw service"
-
     # copy hostw.service template to devstack
     sudo cp ${PREFIX}/lib/systemd/system/hostw.service \
         ${SYSCONFDIR}/systemd/system/devstack@hostw.service
@@ -658,16 +685,13 @@ function stop_hwmon {
 function stop_mtce_control {
     sudo ${SYSCONFDIR}/init.d/goenabledControl stop
 
-    if is_service_enabled mtce; then
+    if is_service_enabled mtce-agent || is_service_enabled mtce; then
-        echo "stop mtcAgent"
         stop_mtcAgent
     fi
-    if is_service_enabled hbs; then
+    if is_service_enabled hbs-agent || is_service_enabled hbs; then
-        echo "stop hbsAgent"
         stop_hbsAgent
     fi
-    if is_service_enabled hwmon; then
+    if is_service_enabled hwmon && is_service_enabled sysinv; then
-        echo "stop hwmon"
         stop_hwmon
     fi
 }
@@ -720,42 +744,42 @@ function stop_maintenance {
     stop_goenabled
 
     if is_service_enabled pmon; then
-        echo "stop pmon service"
         stop_pmon
     fi
     if is_service_enabled mtce; then
-        echo "stop mtce client service"
         stop_mtcClient
     fi
     if is_service_enabled hbs; then
-        echo "stop hbs client service"
         stop_hbsClient
     fi
     if is_service_enabled rmon; then
-        echo "stop rmon service"
         stop_rmon
     fi
     if is_service_enabled mtclog; then
-        echo "stop mtclog service"
         stop_mtclog
     fi
     if is_service_enabled mtcalarm; then
-        echo "stop mtcalarm service"
         stop_mtcalarm
     fi
     if is_service_enabled fsmon; then
-        echo "stop fsmon service"
         stop_fsmon
     fi
     if is_service_enabled hostw; then
-        echo "stop hostw service"
         stop_hostw
     fi
+
+    if is_service_enabled mtce-control; then
+        stop_mtce_control
+    fi
+    if is_service_enabled mtce-compute; then
+        stop_mtce_compute
+    fi
+    if is_service_enabled mtce-storage; then
+        stop_mtce_storage
+    fi
 }
 
 function cleanup_metal {
-    echo "cleanup_metal"
-
     # TODO: move it to Makefile uninstall
     # remove mtce
     local x version
@@ -987,4 +1011,15 @@ function cleanup_metal {
     fi
 }
 
+function uninstall_files {
+    local dest_dir=$1
+    local dest_file_array=($2)
+    local file=""
+
+    for (( i=0; i<${#dest_file_array[*]}; i++ )); do
+        file=${dest_file_array[$i]##*/}
+        sudo rm -rf $dest_dir/$file
+    done
+}
+
 $_XTRACE_STX_METAL

devstack/plugin.sh

@@ -12,21 +12,9 @@ if is_service_enabled stx-metal; then
         echo_summary "Pre-requires of stx-metal"
     elif [[ "$1" == "stack" && "$2" == "install" ]]; then
         # Perform installation of source
-        echo_summary "Install metal"
+        echo_summary "Install stx-metal"
         # maintenance components should be installed in each node
         install_maintenance
-        if is_service_enabled mtce-compute; then
-            echo_summary "Install mtce compute specific services"
-            install_mtce_compute
-        fi
-        if is_service_enabled mtce-control; then
-            echo_summary "Install mtce control specific services"
-            install_mtce_control
-        fi
-        if is_service_enabled mtce-storage; then
-            echo_summary "Install mtce storage specific services"
-            install_mtce_storage
-        fi
     elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
         # Configure after the other layer 1 and 2 services have been configured
         echo_summary "Configure metal"
@@ -34,19 +22,6 @@ if is_service_enabled stx-metal; then
     elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
         # Initialize and start the metal services
         echo_summary "Initialize and start metal "
-        if is_service_enabled mtce-control; then
-            echo_summary "Start mtce control specific services"
-            # Start Agents on controller node
-            start_mtce_control
-        fi
-        if is_service_enabled mtce-compute; then
-            echo_summary "Start mtce compute specific services"
-            start_mtce_compute
-        fi
-        if is_service_enabled mtce-storage; then
-            echo_summary "Start mtce storage specific services"
-            start_mtce_storage
-        fi
         # Start services on each node
         start_maintenance
     elif [[ "$1" == "stack" && "$2" == "test" ]]; then
@@ -59,19 +34,6 @@ if is_service_enabled stx-metal; then
         echo_summary "Stop metal services"
         # Stop client services on each node
         stop_maintenance
-        if is_service_enabled mtce-control; then
-            echo_summary "Stop mtce control specific services"
-            # Stop Agents on controller node
-            stop_mtce_control
-        fi
-        if is_service_enabled mtce-compute; then
-            echo_summary "Stop mtce compute specific services"
-            stop_mtce_compute
-        fi
-        if is_service_enabled mtce-storage; then
-            echo_summary "Stop mtce storage specific services"
-            stop_mtce_storage
-        fi
     fi
 
     if [[ "$1" == "clean" ]]; then

devstack/settings

@@ -11,17 +11,33 @@
 # functionality maintenance services.
 #
 # Current configuration supports All-in-One simplex mode.
+#
+# Servcies
+# stx-metal - Overall enable for this plugin
+#
+# mtce
+# mtce-control
+# mtce-compute
+# mtce-storage
+#
+# mtce-components (include all of the below)
+# fsmon
+# hbs
+# hwmon
+# mtcalarm
+# mtclog
+# pmon
+# rmon
 
 STX_METAL_NAME=stx-metal
 
 ######### Plugin Specific ##########
 # Enable service for node type
-enable_service $STX_METAL_NAME mtce-compute mtce-control mtce-storage
+enable_service $STX_METAL_NAME
 
-# TODO:remove the services enabled by default once
+if is_service_enabled mtce-components; then
-# stx-metal is completely enabled
+    enable_service fsmon hbs hwmon mtce mtcalarm mtclog pmon rmon
-# Enable service for components
+fi
-enable_service hbs mtce pmon rmon hwmon mtclog mtcalarm fsmon
 
 # Be careful to enable hostw, it will restart your host
 # if some config is not correct
@@ -42,6 +58,9 @@ if is_service_enabled mtce; then
     enable_service mtcAgent
     enable_service mtcClient
 fi
+if is_service_enabled hwmon && is_plugin_enabled stx-config; then
+    enable_service sysinv
+fi
 
 # Initial source of lib script
 source $DEST/stx-metal/devstack/lib/stx-metal

tox.ini

@@ -24,7 +24,11 @@ commands =
          -type f                              \
          -not -name \*~                       \
          -not -name \*.md                     \
-         -name \*.sh                          \
+         \(                                   \
+          -name \*.sh                         \
+          -or -not -wholename \*/devstack/files/\* \
+          -wholename \*/devstack/\*           \
+         \)                                   \
          -print0 | xargs -n 1 -0 bashate -v -i E010,E006"
   bash -c "find {toxinidir}        \
             \( -name middleware/io-monitor/recipes-common/io-monitor/io-monitor/io_monitor/test-tools/yaml/* -prune \) \
@@ -84,3 +88,8 @@ commands =
   rm -rf api-ref/build
   sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html
 whitelist_externals = rm
+
+[testenv:functional]
+basepython = python3
+whitelist_externals = cat
+commands = cat /etc/mtc.ini