starlingx/integ
Code Issues Proposed changes

Merge remote-tracking branch 'origin/master' into f/centos75

Browse Source 
Change-Id: I166b99c4ce2ed458ac79ce686064ab05ad3ca198
Signed-off-by: Dean Troyer <dtroyer@gmail.com>
This commit is contained in:
Dean Troyer 2018-09-14 10:55:30 -05:00
commit fe26a0ff84
87 changed files with 1318 additions and 1479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.zuul.yaml
View File

@ -4,7 +4,7 @@
jobs:
- openstack-tox-pep8:
voting: false
- openstack-tox-linters
gate:
jobs:
- noop
- openstack-tox-linters

1
base/cluster-resource-agents/centos/meta_patches/PATCH_ORDER
View File

@ -10,3 +10,4 @@ ipaddr2-if-down.patch
spec-add-ipaddr2-ignore-lo-state.patch
Disable-creation-of-the-debug-package.patch
metapatch-for-arp_bg.patch
ipaddr2-avoid-failing-svc-if-down-meta.patch

32
base/cluster-resource-agents/centos/meta_patches/ipaddr2-avoid-failing-svc-if-down-meta.patch Normal file
View File

@ -0,0 +1,32 @@
From a8fc00f7d84327284643f70638da2201327bdd10 Mon Sep 17 00:00:00 2001
From: Bin Qian <bin.qian@windriver.com>
Date: Wed, 29 Aug 2018 11:22:32 -0400
Subject: [PATCH 1/1] ipaddr2 avoid failing when svc i/f down
---
SPECS/resource-agents.spec | 2 ++
1 file changed, 2 insertions(+)
diff --git a/SPECS/resource-agents.spec b/SPECS/resource-agents.spec
index aa06c49..0eca09f 100644
--- a/SPECS/resource-agents.spec
+++ b/SPECS/resource-agents.spec
@@ -257,6 +257,7 @@ Patch1118: ipaddr2_if_down.patch
Patch1119: ipaddr2_ignore_lo_if_state.patch
Patch1120: Modify-error-code-of-bz1454699-fix-to-prevent-inactive-controller-reboot-loop.patch
Patch1121: Re-enable-background-execution-of-arp-commands.patch
+Patch1122: ipaddr2-avoid-failing-svc-if-down.patch
Obsoletes: heartbeat-resources <= %{version}
Provides: heartbeat-resources = %{version}
@@ -568,6 +569,7 @@ exit 1
%patch1119 -p1
%patch1120 -p1
%patch1121 -p1
+%patch1122 -p1
%build
if [ ! -f configure ]; then
--
1.8.3.1

61
base/cluster-resource-agents/centos/patches/ipaddr2-avoid-failing-svc-if-down.patch Normal file
View File

@ -0,0 +1,61 @@
From c3448b1536d50291dc5ca49dce5957c39403cc82 Mon Sep 17 00:00:00 2001
From: Bin Qian <bin.qian@windriver.com>
Date: Wed, 29 Aug 2018 11:00:22 -0400
Subject: [PATCH 1/1] avoid failing service when I/F is down
---
heartbeat/IPaddr2 | 24 ++++++------------------
1 file changed, 6 insertions(+), 18 deletions(-)
diff --git a/heartbeat/IPaddr2 b/heartbeat/IPaddr2
index 86009b9..2da5c5e 100755
--- a/heartbeat/IPaddr2
+++ b/heartbeat/IPaddr2
@@ -885,12 +885,8 @@ ip_start() {
then
exit $OCF_SUCCESS
else
- if [ "$OCF_RESKEY_dc" = "yes" ]; then
- ocf_log info "NIC $NIC is DOWN..."
- exit $OCF_SUCCESS
- else
- exit $OCF_ERR_GENERIC
- fi
+ ocf_log info "NIC $NIC is DOWN..."
+ exit $OCF_SUCCESS
fi
fi
@@ -954,12 +950,8 @@ ip_start() {
then
exit $OCF_SUCCESS
else
- if [ "$OCF_RESKEY_dc" = "yes" ]; then
- ocf_log info "NIC $NIC is DOWN"
- exit $OCF_SUCCESS
- else
- exit $OCF_ERR_GENERIC
- fi
+ ocf_log info "NIC $NIC is DOWN"
+ exit $OCF_SUCCESS
fi
}
@@ -1040,12 +1032,8 @@ ip_monitor() {
then
return $OCF_SUCCESS
else
- if [ "$OCF_RESKEY_dc" = "yes" ]; then
- ocf_log info "NIC $NIC is DOWN"
- return $OCF_SUCCESS
- else
- return $OCF_NOT_RUNNING
- fi
+ ocf_log info "NIC $NIC is DOWN"
+ return $OCF_SUCCESS
fi
;;
partial|no|partial2)
--
1.8.3.1

25
base/haproxy/haproxy/haproxy.sh
View File

@ -7,7 +7,7 @@
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: HA-Proxy TCP/HTTP reverse proxy
# Description: HA-Proxy is a TCP/HTTP reverse proxy
# Description: HA-Proxy is a TCP/HTTP reverse proxy
### END INIT INFO
PATH=/sbin:/bin:/usr/sbin:/usr/bin
@ -22,12 +22,12 @@ RETVAL=0
# This is only needed till TPM In-Kernel
# ResourceMgr comes in
remove_TPM_transients () {
_HANDLES=`find $TPM_DATA_DIR -type f -name "hp*.bin" -printf "%f "`
for handle in $_HANDLES; do
_HANDLES=`find $TPM_DATA_DIR -type f -name "hp*.bin" -printf "%f "`
for handle in $_HANDLES; do
handle_addr=`echo $handle | sed 's/hp\([0-9]*\)\.bin/\1/g'`
tss2_flushcontext -ha $handle_addr &> /dev/null
done
rm -f $TPM_DATA_DIR/*
done
rm -f $TPM_DATA_DIR/*
}
start() {
@ -59,10 +59,12 @@ start() {
}
stop() {
if [ ! -e $PIDFILE ]; then return; fi
if [ ! -e $PIDFILE ]; then
return
fi
echo -n "Stopping $DESC..."
start-stop-daemon --stop --quiet --retry 3 --oknodo --pidfile $PIDFILE -x "$DAEMON"
if [ -n "`pidof $DAEMON`" ] ; then
pkill -KILL -f $DAEMON
@ -73,8 +75,7 @@ stop() {
remove_TPM_transients
}
status()
{
status() {
pid=`cat $PIDFILE 2>/dev/null`
if [ -n "$pid" ]; then
if ps -p $pid &>/dev/null ; then
@ -90,7 +91,7 @@ status()
}
check() {
/usr/sbin/$NAME -c -q -V -f /etc/$NAME/$NAME.cfg
/usr/sbin/$NAME -c -q -V -f /etc/$NAME/$NAME.cfg
}
# See how we were called.
@ -116,5 +117,5 @@ case "$1" in
RETVAL=1
;;
esac
exit $RETVAL

103
base/initscripts/centos/files/mountnfs.sh
View File

@ -15,86 +15,83 @@ if [ "$1" = "stop" ]; then
fi
#
# Run in a subshell because of I/O redirection.
# Run in a subshell because of I/O redirection.
#
test -f /etc/fstab && (
#
# Read through fstab line by line. If it is NFS, set the flag
# for mounting NFS filesystems. If any NFS partition is found and it
# not mounted with the nolock option, we start the rpcbind.
# Read through fstab line by line. If it is NFS, set the flag
# for mounting NFS filesystems. If any NFS partition is found and it
# not mounted with the nolock option, we start the rpcbind.
#
rpcbind=no
mount_nfs=no
mount_smb=no
mount_ncp=no
mount_cifs=no
while read device mountpt fstype options
do
case "$device" in
""|\#*)
continue
;;
esac
while read device mountpt fstype options; do
case "$device" in
""|\#*)
continue
;;
esac
case "$options" in
*noauto*)
continue
;;
esac
case "$options" in
*noauto*)
continue
;;
esac
if test "$fstype" = nfs
then
mount_nfs=yes
case "$options" in
*nolock*)
;;
*)
rpcbind=yes
;;
esac
fi
if test "$fstype" = smbfs
then
mount_smb=yes
fi
if test "$fstype" = ncpfs
then
mount_ncp=yes
fi
if test "$fstype" = cifs
then
mount_cifs=yes
fi
if test "$fstype" = nfs
then
mount_nfs=yes
case "$options" in
*nolock*)
;;
*)
rpcbind=yes
;;
esac
fi
if test "$fstype" = smbfs
then
mount_smb=yes
fi
if test "$fstype" = ncpfs
then
mount_ncp=yes
fi
if test "$fstype" = cifs
then
mount_cifs=yes
fi
done
exec 0>&1
if test "$rpcbind" = yes
then
if test "$rpcbind" = yes; then
# WRL: Centos precheck: Dont start rpcbind in this init script.
# It is started by a systemd service file.
if test "/etc/centos-release" = no
then
if test -x /usr/sbin/rpcbind
then
service rpcbind status > /dev/null
if [ $? != 0 ]; then
echo -n "Starting rpcbind..."
start-stop-daemon --start --quiet --exec /usr/sbin/rpcbind
sleep 2
service rpcbind status > /dev/null
if [ $? != 0 ]; then
echo -n "Starting rpcbind..."
start-stop-daemon --start --quiet --exec /usr/sbin/rpcbind
sleep 2
fi
fi
fi
fi
if test "$mount_nfs" = yes || test "$mount_smb" = yes || test "$mount_ncp" = yes || test "$mount_cifs" = yes
then
echo "Mounting remote filesystems..."
test "$mount_nfs" = yes && mount -a -t nfs
test "$mount_smb" = yes && mount -a -t smbfs
test "$mount_ncp" = yes && mount -a -t ncpfs
test "$mount_cifs" = yes && mount -a -t cifs
if test "$mount_nfs" = yes || test "$mount_smb" = yes || test "$mount_ncp" = yes || test "$mount_cifs" = yes; then
echo "Mounting remote filesystems..."
test "$mount_nfs" = yes && mount -a -t nfs
test "$mount_smb" = yes && mount -a -t smbfs
test "$mount_ncp" = yes && mount -a -t ncpfs
test "$mount_cifs" = yes && mount -a -t cifs
fi
) < /etc/fstab

1
base/memcached/centos/build_srpm.data
View File

@ -1 +0,0 @@
TIS_PATCH_VER=1

25
base/memcached/centos/meta_patches/0001-Update-package-versioning-for-TIS-format.patch
View File

@ -1,25 +0,0 @@
From de355606dea0404c4ae92bad5ce00b841697c698 Mon Sep 17 00:00:00 2001
From: Jack Ding <jack.ding@windriver.com>
Date: Tue, 8 May 2018 14:29:14 -0400
Subject: [PATCH] Update package versioning for TIS format
---
SPECS/memcached.spec | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/SPECS/memcached.spec b/SPECS/memcached.spec
index 6008493..c8575c8 100644
--- a/SPECS/memcached.spec
+++ b/SPECS/memcached.spec
@@ -4,7 +4,7 @@
Name: memcached
Version: 1.4.39
-Release: 1%{?dist}
+Release: 1.el7%{?_tis_dist}.%{tis_patch_ver}
Epoch: 0
Summary: High Performance, Distributed Memory Object Cache
--
1.8.3.1

32
base/memcached/centos/meta_patches/0002-always-restart-memcached-service.patch
View File

@ -1,32 +0,0 @@
From f321c8a8b800a7c2ca9394d3c76bec72b98c0d77 Mon Sep 17 00:00:00 2001
From: Jack Ding <jack.ding@windriver.com>
Date: Fri, 11 May 2018 15:38:56 -0400
Subject: [PATCH] always restart memcached service
---
SPECS/memcached.spec | 2 ++
1 file changed, 2 insertions(+)
diff --git a/SPECS/memcached.spec b/SPECS/memcached.spec
index c8575c8..f389035 100644
--- a/SPECS/memcached.spec
+++ b/SPECS/memcached.spec
@@ -16,6 +16,7 @@ Source1: memcached.sysconfig
# https://github.com/memcached/memcached/issues/218
Patch1: 0001-systemd-fix-upstream-provided-service.patch
+Patch2: 0002-always-restart-memcached-service.patch
BuildRequires: libevent-devel systemd-units
BuildRequires: perl-generators
@@ -44,6 +45,7 @@ access to the memcached binary include files.
%prep
%setup -q
%patch1 -p1 -b .unit
+%patch2 -p1
%build
# compile with full RELRO
--
1.8.3.1

33
base/memcached/centos/meta_patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch
View File

@ -1,33 +0,0 @@
From bb6fd3da3ace960eb587e7ff01d5816ea2baaa54 Mon Sep 17 00:00:00 2001
From: Jack Ding <jack.ding@windriver.com>
Date: Sun, 13 May 2018 18:22:15 -0400
Subject: [PATCH] Add dependencies and comment out incompatible service
parameters
---
SPECS/memcached.spec | 2 ++
1 file changed, 2 insertions(+)
diff --git a/SPECS/memcached.spec b/SPECS/memcached.spec
index f389035..86653a1 100644
--- a/SPECS/memcached.spec
+++ b/SPECS/memcached.spec
@@ -17,6 +17,7 @@ Source1: memcached.sysconfig
# https://github.com/memcached/memcached/issues/218
Patch1: 0001-systemd-fix-upstream-provided-service.patch
Patch2: 0002-always-restart-memcached-service.patch
+Patch3: 0003-Add-dependencies-and-comment-out-incompatible-servic.patch
BuildRequires: libevent-devel systemd-units
BuildRequires: perl-generators
@@ -46,6 +47,7 @@ access to the memcached binary include files.
%setup -q
%patch1 -p1 -b .unit
%patch2 -p1
+%patch3 -p1
%build
# compile with full RELRO
--
1.8.3.1

3
base/memcached/centos/meta_patches/PATCH_ORDER
View File

@ -1,3 +0,0 @@
0001-Update-package-versioning-for-TIS-format.patch
0002-always-restart-memcached-service.patch
0003-Add-dependencies-and-comment-out-incompatible-servic.patch

26
base/memcached/centos/patches/0002-always-restart-memcached-service.patch
View File

@ -1,26 +0,0 @@
From bb7b75184f7037e6d8d844874ae248fce1d06736 Mon Sep 17 00:00:00 2001
From: Jack Ding <jack.ding@windriver.com>
Date: Fri, 11 May 2018 15:24:28 -0400
Subject: [PATCH] Always restart memcached service
---
scripts/memcached.service | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/scripts/memcached.service b/scripts/memcached.service
index 1bb9d33..8e58485 100644
--- a/scripts/memcached.service
+++ b/scripts/memcached.service
@@ -71,5 +71,9 @@ RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
# Takes away the ability to create or manage any kind of namespace
RestrictNamespaces=true
+# WRS
+Restart=always
+RestartSec=0
+
[Install]
WantedBy=multi-user.target
--
1.8.3.1

67
base/memcached/centos/patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch
View File

@ -1,67 +0,0 @@
From 1d9f43c5ecb20fe0a2a4abe9b94abd0d389edb40 Mon Sep 17 00:00:00 2001
From: Jack Ding <jack.ding@windriver.com>
Date: Mon, 14 May 2018 22:44:32 -0400
Subject: [PATCH 2/2] Add dependencies and comment out incompatible service
parameters
---
scripts/memcached.service | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/scripts/memcached.service b/scripts/memcached.service
index 8e58485..021b8b4 100644
--- a/scripts/memcached.service
+++ b/scripts/memcached.service
@@ -12,7 +12,7 @@
[Unit]
Description=memcached daemon
Before=httpd.service
-After=network.target
+After=network-online.target
[Service]
EnvironmentFile=/etc/sysconfig/memcached
@@ -46,34 +46,34 @@ LimitNOFILE=16384
# Explicit module loading will be denied. This allows to turn off module load and unload
# operations on modular kernels. It is recommended to turn this on for most services that
# do not need special file systems or extra kernel modules to work.
-ProtectKernelModules=true
+#ProtectKernelModules=true
# Kernel variables accessible through /proc/sys, /sys, /proc/sysrq-trigger, /proc/latency_stats,
# /proc/acpi, /proc/timer_stats, /proc/fs and /proc/irq will be made read-only to all processes
# of the unit. Usually, tunable kernel variables should only be written at boot-time, with the
# sysctl.d(5) mechanism. Almost no services need to write to these at runtime; it is hence
# recommended to turn this on for most services.
-ProtectKernelTunables=true
+#ProtectKernelTunables=true
# The Linux Control Groups (cgroups(7)) hierarchies accessible through /sys/fs/cgroup will be
# made read-only to all processes of the unit. Except for container managers no services should
# require write access to the control groups hierarchies; it is hence recommended to turn this on
# for most services
-ProtectControlGroups=true
+#ProtectControlGroups=true
# Any attempts to enable realtime scheduling in a process of the unit are refused.
-RestrictRealtime=true
+#RestrictRealtime=true
# Restricts the set of socket address families accessible to the processes of this unit.
# Protects against vulnerabilities such as CVE-2016-8655
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
# Takes away the ability to create or manage any kind of namespace
-RestrictNamespaces=true
+#RestrictNamespaces=true
# WRS
Restart=always
-RestartSec=0
+RestartSec=10
[Install]
WantedBy=multi-user.target
--
1.8.3.1

1
base/memcached/centos/srpm_path
View File

@ -1 +0,0 @@
mirror:Source/memcached-1.4.39-1.el7.src.rpm

1
base/novnc/centos/build_srpm.data
View File

@ -1 +0,0 @@
TIS_PATCH_VER=1

12
base/novnc/centos/meta_patches/0001-Update-package-0.6.2-versioning-for-TIS-format.patch
View File

@ -1,12 +0,0 @@
diff --git a/SPECS/novnc.spec b/SPECS/novnc.spec
index a43f435..640bf1f 100644
--- a/SPECS/novnc.spec
+++ b/SPECS/novnc.spec
@@ -1,6 +1,6 @@
Name: novnc
Version: 0.6.2
-Release: 1%{?dist}
+Release: 1.el7%{?_tis_dist}.%{tis_patch_ver}
Summary: VNC client using HTML5 (Web Sockets, Canvas) with encryption support
Requires: python-websockify

1
base/novnc/centos/meta_patches/PATCH_ORDER
View File

@ -1 +0,0 @@
0001-Update-package-0.6.2-versioning-for-TIS-format.patch

1
base/novnc/centos/srpm_path
View File

@ -1 +0,0 @@
mirror:Source/novnc-0.6.2-1.el7.src.rpm

1
centos_iso_image.inc
View File

@ -104,6 +104,7 @@ collector
# platform-util
platform-util
platform-util-noncontroller
platform-util-controller
# monitor-tools
monitor-tools

6
centos_pkg_dirs
View File

@ -32,7 +32,6 @@ base/shadow-utils
security/shim-unsigned
security/shim-signed
logging/syslog-ng
base/novnc
base/sanlock
base/sudo
virt/cloud-init
@ -88,13 +87,12 @@ python/python-requests
base/systemd
python/python-gunicorn
config/puppet-modules/openstack/puppet-oslo-11.3.0
strorage-drivers/python-3parclient
strorage-drivers/python-lefthandclient
storage-drivers/python-3parclient
storage-drivers/python-lefthandclient
security/tboot
networking/mellanox/libibverbs
kernel/kernel-modules/mlnx-ofa_kernel
networking/mellanox/rdma-core
base/memcached
config/puppet-modules/openstack/puppet-memcached-3.0.2
config/puppet-modules/openstack/puppet-horizon-9.5.0
config/puppet-modules/openstack/puppet-swift-11.3.0

3
database/mariadb/centos/files/clustercheck.sh
View File

@ -43,8 +43,7 @@ fi
WSREP_STATUS=$($MYSQL_CMDLINE -e "SHOW STATUS LIKE 'wsrep_local_state';" \
2>${ERR_FILE} | tail -1 2>>${ERR_FILE})
if [[ "${WSREP_STATUS}" == "4" ]] || [[ "${WSREP_STATUS}" == "2" && ${AVAILABLE_WHEN_DONOR} == 1 ]]
then
if [[ "${WSREP_STATUS}" == "4" ]] || [[ "${WSREP_STATUS}" == "2" && ${AVAILABLE_WHEN_DONOR} == 1 ]]; then
# Check only when set to 0 to avoid latency in response.
if [[ $AVAILABLE_WHEN_READONLY -eq 0 ]];then
READ_ONLY=$($MYSQL_CMDLINE -e "SHOW GLOBAL VARIABLES LIKE 'read_only';" \

6
database/mariadb/centos/files/mysql-check-upgrade.sh
View File

@ -27,9 +27,9 @@ if [ $version -ne $thisversion ] ; then
cat <<EOF >&2
The datadir located at $datadir needs to be upgraded using 'mysql_upgrade' tool. This can be done using the following steps:
1. Back-up your data before with 'mysql_upgrade'
2. Start the database daemon using 'service @DAEMON_NAME@ start'
3. Run 'mysql_upgrade' with a database user that has sufficient privileges
1. Back-up your data before with 'mysql_upgrade'
2. Start the database daemon using 'service @DAEMON_NAME@ start'
3. Run 'mysql_upgrade' with a database user that has sufficient privileges
Read more about 'mysql_upgrade' usage at:
https://mariadb.com/kb/en/mariadb/documentation/sql-commands/table-commands/mysql_upgrade/

19
database/mariadb/centos/files/mysql-prepare-db-dir.sh
View File

@ -27,30 +27,26 @@ should_initialize ()
# If two args given first is user, second is group
# otherwise the arg is the systemd service file
if [ "$#" -eq 2 ]
then
if [ "$#" -eq 2 ]; then
myuser="$1"
mygroup="$2"
else
# Absorb configuration settings from the specified systemd service file,
# or the default service if not specified
SERVICE_NAME="$1"
if [ x"$SERVICE_NAME" = x ]
then
if [ x"$SERVICE_NAME" = x ]; then
SERVICE_NAME=@DAEMON_NAME@.service
fi
myuser=`systemctl show -p User "${SERVICE_NAME}" |
sed 's/^User=//'`
if [ x"$myuser" = x ]
then
sed 's/^User=//'`
if [ x"$myuser" = x ]; then
myuser=mysql
fi
mygroup=`systemctl show -p Group "${SERVICE_NAME}" |
sed 's/^Group=//'`
if [ x"$mygroup" = x ]
then
sed 's/^Group=//'`
if [ x"$mygroup" = x ]; then
mygroup=mysql
fi
fi
@ -79,8 +75,7 @@ chmod 0640 "$errlogfile"
if should_initialize "$datadir" ; then
# First, make sure $datadir is there with correct permissions
# (note: if it's not, and we're not root, this'll fail ...)
if [ ! -e "$datadir" -a ! -h "$datadir" ]
then
if [ ! -e "$datadir" -a ! -h "$datadir" ]; then
mkdir -p "$datadir" || exit 1
fi
chown "$myuser:$mygroup" "$datadir"

24
database/mariadb/centos/files/mysql-scripts-common.sh
View File

@ -11,18 +11,18 @@ export LC_ALL=C
# We use my_print_defaults which prints all options from multiple files,
# with the more specific ones later; hence take the last match.
get_mysql_option(){
if [ $# -ne 3 ] ; then
echo "get_mysql_option requires 3 arguments: section option default_value"
return
fi
sections="$1"
option_name="$2"
default_value="$3"
result=`@bindir@/my_print_defaults $sections | sed -n "s/^--${option_name}=//p" | tail -n 1`
if [ -z "$result" ]; then
# not found, use default
result="${default_value}"
fi
if [ $# -ne 3 ] ; then
echo "get_mysql_option requires 3 arguments: section option default_value"
return
fi
sections="$1"
option_name="$2"
default_value="$3"
result=`@bindir@/my_print_defaults $sections | sed -n "s/^--${option_name}=//p" | tail -n 1`
if [ -z "$result" ]; then
# not found, use default
result="${default_value}"
fi
}
# Defaults here had better match what mysqld_safe will default to

42
database/mariadb/centos/files/mysql-wait-ready.sh
View File

@ -9,8 +9,8 @@ source "`dirname ${BASH_SOURCE[0]}`/mysql-scripts-common"
# "After mysqld" won't be started until the database is really ready.
if [ $# -ne 1 ] ; then
echo "You need to pass daemon pid as an argument for this script."
exit 20
echo "You need to pass daemon pid as an argument for this script."
exit 20
fi
# Service file passes us the daemon's PID (actually, mysqld_safe's PID)
@ -19,27 +19,27 @@ daemon_pid="$1"
# Wait for the server to come up or for the mysqld process to disappear
ret=0
while /bin/true; do
# Check process still exists
if ! [ -d "/proc/${daemon_pid}" ] ; then
ret=1
break
fi
RESPONSE=`@bindir@/mysqladmin --no-defaults --socket="$socketfile" --user=UNKNOWN_MYSQL_USER ping 2>&1`
mret=$?
if [ $mret -eq 0 ] ; then
break
fi
# exit codes 1, 11 (EXIT_CANNOT_CONNECT_TO_SERVICE) are expected,
# anything else suggests a configuration error
if [ $mret -ne 1 -a $mret -ne 11 ]; then
# Check process still exists
if ! [ -d "/proc/${daemon_pid}" ] ; then
ret=1
break
fi
RESPONSE=`@bindir@/mysqladmin --no-defaults --socket="$socketfile" --user=UNKNOWN_MYSQL_USER ping 2>&1`
mret=$?
if [ $mret -eq 0 ] ; then
break
fi
# exit codes 1, 11 (EXIT_CANNOT_CONNECT_TO_SERVICE) are expected,
# anything else suggests a configuration error
if [ $mret -ne 1 -a $mret -ne 11 ]; then
echo "Cannot check for @NICE_PROJECT_NAME@ Daemon startup because of mysqladmin failure." >&2
ret=$mret
break
fi
# "Access denied" also means the server is alive
echo "$RESPONSE" | grep -q "Access denied for user" && break
ret=$mret
break
fi
# "Access denied" also means the server is alive
echo "$RESPONSE" | grep -q "Access denied for user" && break
sleep 1
sleep 1
done
exit $ret

22
database/mariadb/centos/files/mysql-wait-stop.sh
View File

@ -13,24 +13,24 @@ TIMEOUT=60
SECONDS=0
if ! [ -f "$pidfile" ]; then
exit 0
exit 0
fi
MYSQLPID=`cat "$pidfile" 2>/dev/null`
if [ -z "$MYSQLPID" ] ; then
exit 2
exit 2
fi
while /bin/true; do
# Check process still exists
if ! [ -d "/proc/${MYSQLPID}" ] ; then
break
fi
if [ $SECONDS -gt $TIMEOUT ] ; then
ret=3
break
fi
sleep 1
# Check process still exists
if ! [ -d "/proc/${MYSQLPID}" ] ; then
break
fi
if [ $SECONDS -gt $TIMEOUT ] ; then
ret=3
break
fi
sleep 1
done
exit $ret

4
filesystem/nfscheck/files/nfscheck-init.sh
View File

@ -22,7 +22,7 @@ DESC="nfscheck"
DAEMON="/usr/bin/nfscheck"
PIDFILE="/var/run/nfscheck.pid"
start()
start()
{
if [ -e $PIDFILE ]; then
PIDDIR=/proc/$(cat $PIDFILE)
@ -47,7 +47,7 @@ start()
fi
}
stop()
stop()
{
echo -n "Stopping $DESC..."
start-stop-daemon --stop --quiet --pidfile $PIDFILE

15
filesystem/nfscheck/files/nfscheck.sh
View File

@ -12,12 +12,10 @@ MOUNT=/opt/platform
previous=1
delay=60
while :
do
while : ; do
# First, check that it's actually an NFS mount
mount | grep -q $MOUNT
if [ $? -ne 0 ]
then
if [ $? -ne 0 ]; then
logger -t NFSCHECK "$MOUNT is not mounted"
previous=1
sleep $delay
@ -31,18 +29,15 @@ do
# At this point, jobs will either report no jobs (empty) or Done,
# unless the job is still running/hung
rc=$(jobs)
if [[ -z "$rc" || $rc =~ "Done" ]]
then
if [[ -z "$rc" || $rc =~ "Done" ]]; then
# NFS is successful
if [ $previous -ne 0 ]
then
if [ $previous -ne 0 ]; then
logger -t NFSCHECK "NFS test of $MOUNT is ok"
previous=0
fi
else
# Keep waiting until the job is done
while ! [[ -z "$rc" || $rc =~ "Done" ]]
do
while ! [[ -z "$rc" || $rc =~ "Done" ]]; do
logger -t NFSCHECK "NFS test of $MOUNT is failed"
previous=1
sleep $delay

51
networking/lldpd/centos/files/i40e-lldp-configure.sh
View File

@ -25,20 +25,17 @@ DEBUGFS_PATH=/sys/kernel/debug
DEBUGFS_I40_DEVICES_PATH=$DEBUGFS_PATH/i40e
LLDP_COMMAND=lldp
function log()
{
function log {
local MSG="${PROGNAME}: $1"
logger -p notice "${MSG}"
logger -p notice "${MSG}"
}
function err()
{
function err {
local MSG="${PROGNAME}: $1"
logger -p error "${MSG}"
}
function configure_device()
{
function configure_device {
local DEVICE=$1
local ACTION=$2
local DEVICE_PATH=${DEBUGFS_I40_DEVICES}/${DEVICE}
@ -59,30 +56,29 @@ function configure_device()
return ${RET}
}
function is_debugfs_mounted() {
if grep -qs "${DEBUGFS_PATH}" /proc/mounts; then
return 0
fi
return 1
function is_debugfs_mounted {
if grep -qs "${DEBUGFS_PATH}" /proc/mounts; then
return 0
fi
return 1
}
function mount_debugfs() {
function mount_debugfs {
mount -t debugfs none ${DEBUGFS_PATH}
}
function unmount_debugfs() {
function unmount_debugfs {
umount ${DEBUGFS_PATH}
}
function scan_devices()
{
function scan_devices {
local ACTION=$1
local DEBUGFS_MOUNTED="false"
local DEVICES=${DEBUGFS_I40_DEVICES_PATH}/*
if is_debugfs_mounted; then
DEBUGFS_MOUNTED="true"
fi
fi
if [ ${DEBUGFS_MOUNTED} = "false" ]; then
mount_debugfs
@ -111,38 +107,35 @@ function scan_devices()
return 0
}
function start()
{
function start {
scan_devices start
return $?
}
function stop()
{
function stop {
scan_devices stop
return $?
return $?
}
function status()
{
function status {
return 0
}
case "$1" in
start)
start)
start
;;
stop)
stop)
stop
;;
restart)
restart)
stop
start
;;
status)
status)
status
;;
*)
*)
echo "Usage: $0 {start|stop|restart|status}"
exit 1
esac

30
networking/mellanox/mlx4-config/files/mlx4-configure.sh
View File

@ -15,17 +15,15 @@ DEBUG=${DEBUG:-0}
# all files beginning in "mlx4_port" and ending in one or more digits.
shopt -s extglob
function log()
{
function log {
local MSG="${PROGNAME}: $1"
if [ ${DEBUG} -ne 0 ]; then
echo "${MSG}"
echo "${MSG}"
fi
echo "${MSG}" >> /var/log/mlx4-configure.log
}
function configure_device()
{
function configure_device {
local DEVICE=$1
local DEVICE_PATH=${SYSFS_PCI_DEVICES}/${DEVICE}
@ -58,8 +56,7 @@ function configure_device()
}
function scan_devices()
{
function scan_devices {
local DEVICES=$(ls -1 ${SYSFS_PCI_DEVICES})
for DEVICE in ${DEVICES}; do
@ -83,37 +80,34 @@ function scan_devices()
}
function start()
{
function start {
scan_devices
return $?
}
function stop()
{
function stop {
return 0
}
function status()
{
function status {
return 0
}
case "$1" in
start)
start)
start
;;
stop)
stop)
stop
;;
restart)
restart)
stop
start
;;
status)
status)
status
;;
*)
*)
echo "Usage: $0 {start|stop|restart|status}"
exit 1
esac

2
networking/mellanox/mlx4-config/files/mlx4_core_config.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
################################################################################
# Copyright (c) 2015-2016 Wind River Systems, Inc.
#
#
# SPDX-License-Identifier: Apache-2.0
#
################################################################################

6
networking/mellanox/mlx4-config/files/mlx4_core_goenabled.sh
View File

@ -12,13 +12,11 @@ NAME=$(basename $0)
OPTIONS_CHANGED_FLAG=/var/run/.mlx4_cx3_reboot_required
COMPUTE_CONFIG_COMPLETE=/var/run/.compute_config_complete
function LOG()
{
function LOG {
logger "$NAME: $*"
}
if [ -f $OPTIONS_CHANGED_FLAG ] && [ -f $COMPUTE_CONFIG_COMPLETE ]
then
if [ -f $OPTIONS_CHANGED_FLAG ] && [ -f $COMPUTE_CONFIG_COMPLETE ]; then
LOG "mlx4_core options has been changed. Failing goenabled check."
exit 1
fi

0
strorage-drivers/python-3parclient/centos/build_srpm.data → storage-drivers/python-3parclient/centos/build_srpm.data
View File

0
strorage-drivers/python-3parclient/centos/python-3parclient.spec → storage-drivers/python-3parclient/centos/python-3parclient.spec
View File

0
strorage-drivers/python-lefthandclient/centos/build_srpm.data → storage-drivers/python-lefthandclient/centos/build_srpm.data
View File

0
strorage-drivers/python-lefthandclient/centos/python-lefthandclient.spec → storage-drivers/python-lefthandclient/centos/python-lefthandclient.spec
View File

2
tools/collector/centos/build_srpm.data
View File

@ -1,2 +1,2 @@
SRC_DIR="scripts"
TIS_PATCH_VER=24
TIS_PATCH_VER=25

6
tools/collector/scripts/collect_ceph.sh
View File

@ -14,8 +14,7 @@ SERVICE="ceph"
LOGFILE="${extradir}/ceph.info"
echo "${hostname}: Ceph Info .........: ${LOGFILE}"
function is_service_active()
{
function is_service_active {
active=`sm-query service management-ip | grep "enabled-active"`
if [ -z "$active" ] ; then
return 0
@ -24,8 +23,7 @@ function is_service_active()
fi
}
function exit_if_timeout()
{
function exit_if_timeout {
if [ "$?" = "124" ] ; then
echo "Exiting due to ceph command timeout" >> ${LOGFILE}
exit 0

3
tools/collector/scripts/collect_ima.sh
View File

@ -10,8 +10,7 @@
source /usr/local/sbin/collect_parms
source /usr/local/sbin/collect_utils
function is_extended_profile()
{
function is_extended_profile {
if [ ! -n "${security_profile}" ] || [ "${security_profile}" != "extended" ]; then
return 0
else

5
tools/collector/scripts/collect_nfv_vim.sh
View File

@ -12,8 +12,7 @@ source /usr/local/sbin/collect_utils
LOGFILE="${extradir}/nfv-vim.info"
echo "${hostname}: NFV-Vim Info ......: ${LOGFILE}"
function is_service_active()
{
function is_service_active {
active=`sm-query service vim | grep "enabled-active"`
if [ -z "$active" ] ; then
return 0
@ -36,7 +35,7 @@ if [ "$nodetype" = "controller" ] ; then
DATABASE_DIR=$(awk -F "=" '/database_dir/ {print $2}' /etc/nfv/vim/config.ini)
SQLITE_DUMP="/usr/bin/sqlite3 ${DATABASE_DIR}/vim_db_v1 .dump"
delimiter ${LOGFILE} "dump database"
timeout 30 ${SQLITE_DUMP} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG}
fi

5
tools/collector/scripts/collect_openstack.sh
View File

@ -10,8 +10,7 @@
source /usr/local/sbin/collect_parms
source /usr/local/sbin/collect_utils
function is_service_active()
{
function is_service_active {
active=`sm-query service rabbit-fs | grep "enabled-active"`
if [ -z "$active" ] ; then
return 0
@ -51,7 +50,7 @@ num_exchanges=$(rabbitmqctl list_exchanges | wc -l); ((num_exchanges-=2))
num_connections=$(rabbitmqctl list_connections | wc -l); ((num_connections-=2))
num_channels=$(rabbitmqctl list_channels | wc -l); ((num_channels-=2))
arr=($(rabbitmqctl list_queues messages consumers memory | \
awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}'))
awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}'))
messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]}
printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" "queues" "bindings" "exchanges" "connections" "channels" "messages" "consumers" "memory" >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG}
printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" $num_queues $num_bindings $num_exchanges $num_connections $num_channels $messages $consumers $memory >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG}

19
tools/collector/scripts/collect_psqldb.sh
View File

@ -19,8 +19,7 @@ DB_DIR="${extradir}/database"
LOGFILE="${extradir}/database.info"
echo "${hostname}: Database Info .....: ${LOGFILE}"
function is_service_active()
{
function is_service_active {
active=`sm-query service postgres | grep "enabled-active"`
if [ -z "$active" ] ; then
return 0
@ -34,11 +33,9 @@ function is_service_active()
###############################################################################
mkdir -p ${DB_DIR}
function log_database()
{
function log_database {
db_list=( $(${PSQL_CMD} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") )
for db in "${db_list[@]}"
do
for db in "${db_list[@]}"; do
echo "postgres database: ${db}"
${PSQL_CMD} -d ${db} -c "
SELECT
@ -75,15 +72,13 @@ function log_database()
DB_EXT=db.sql.txt
function database_dump()
{
function database_dump {
mkdir -p ${DB_DIR}
db_list=( $(${PSQL_CMD} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") )
for DB in "${db_list[@]}"
do
for DB in "${db_list[@]}"; do
if [ "$DB" != "keystone" -a "$DB" != "ceilometer" ] ; then
echo "${hostname}: Dumping Database ..: ${DB_DIR}/$DB.$DB_EXT"
(cd ${DB_DIR} ; sudo -u postgres pg_dump $DB > $DB.$DB_EXT)
(cd ${DB_DIR} ; sudo -u postgres pg_dump $DB > $DB.$DB_EXT)
fi
done
}
@ -107,7 +102,7 @@ if [ "$nodetype" = "controller" ] ; then
pg_size_pretty(pg_database_size(pg_database.datname))
FROM pg_database
ORDER BY pg_database_size DESC;
" >> ${LOGFILE}
" >> ${LOGFILE}
# Number of postgres connections
delimiter ${LOGFILE} "ps -C postgres -o cmd="

3
tools/collector/scripts/collect_sysinv.sh
View File

@ -14,8 +14,7 @@ SERVICE="inventory"
LOGFILE="${extradir}/${SERVICE}.info"
RPMLOG="${extradir}/rpm.info"
function is_service_active()
{
function is_service_active {
active=`sm-query service management-ip | grep "enabled-active"`
if [ -z "$active" ] ; then
return 0

12
tools/collector/scripts/collect_tc.sh
View File

@ -27,8 +27,8 @@ fi
delimiter ${LOGFILE} "ip link"
ip link >> ${LOGFILE}
for i in $(ip link | grep mtu | grep eth |awk '{print $2}' | sed 's#:##g');
do
for i in $(ip link | grep mtu | grep eth |awk '{print $2}' | sed 's#:##g'); do
delimiter ${LOGFILE} "ethtool ${i}"
ethtool ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG}
@ -55,8 +55,8 @@ fi
delimiter ${LOGFILE} "tc qdisc show"
tc qdisc show >> ${LOGFILE}
for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g');
do
for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); do
delimiter ${LOGFILE} "tc class show dev ${i}"
tc class show dev ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG}
@ -70,8 +70,8 @@ done
delimiter ${LOGFILE} "tc -s qdisc show"
tc -s qdisc show >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG}
for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g');
do
for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); do
delimiter ${LOGFILE} "tc -s class show dev ${i}"
tc -s class show dev ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG}

2
tools/collector/scripts/etc.exclude
View File

@ -35,6 +35,8 @@
/etc/bash_completion.d
/etc/pm
/etc/systemd/system/*.mount
/etc/systemd/system/*.socket
/etc/systemd/system/lvm2-lvmetad.service
/etc/systemd/system/ctrl-alt-del.target
/etc/ssl
/etc/mtc/tmp

10
tools/engtools/hostdata-collectors/patch-engtools.sh
View File

@ -3,15 +3,15 @@
# http://twiki.wrs.com/PBUeng/Patching
if [ -z $MY_WORKSPACE ] || [ -z $MY_REPO ]; then
echo "Some dev environment variables are not set."
echo "Refer to http://wiki.wrs.com/PBUeng/CentOSBuildProcess for instructions."
exit 1
echo "Some dev environment variables are not set."
echo "Refer to http://wiki.wrs.com/PBUeng/CentOSBuildProcess for instructions."
exit 1
fi
ENGTOOLS=$(ls ${MY_WORKSPACE}/std/rpmbuild/RPMS/engtools*noarch.rpm 2>/dev/null)
if [ $? -ne 0 ]; then
echo "Engtools RPM has not been built. Please run \"build-pkgs engtools\" first."
exit 1
echo "Engtools RPM has not been built. Please run \"build-pkgs engtools\" first."
exit 1
fi
source ${MY_REPO}/stx/middleware/recipes-common/build-info/release-info.inc

30
tools/engtools/hostdata-collectors/scripts/ceph.sh
View File

@ -6,27 +6,26 @@ TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
tools_init
if [ $? -ne 0 ]; then
echo "FATAL, tools_init - could not setup environment"
exit $?
echo "FATAL, tools_init - could not setup environment"
exit $?
fi
# Enable use of INTERVAL_SEC sample interval
OPT_USE_INTERVALS=1
# Print key ceph statistics
function print_ceph()
{
print_separator
TOOL_HIRES_TIME
function print_ceph {
print_separator
TOOL_HIRES_TIME
cmd='ceph -s'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='ceph -s'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='ceph osd tree'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='ceph osd tree'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='ceph df detail'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='ceph df detail'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
}
#-------------------------------------------------------------------------------
@ -47,10 +46,9 @@ tools_header
# Calculate number of sample repeats based on overall interval and sampling interval
((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC))
for ((rep=1; rep <= REPEATS ; rep++))
do
print_ceph
sleep ${INTERVAL_SEC}
for ((rep=1; rep <= REPEATS ; rep++)); do
print_ceph
sleep ${INTERVAL_SEC}
done
print_ceph
LOG "done"

31
tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh
View File

@ -6,8 +6,8 @@
TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
if [ $UID -ne 0 ]; then
ERRLOG "Require sudo/root access."
exit 1
ERRLOG "Require sudo/root access."
exit 1
fi
declare -a TOOLS
@ -32,25 +32,22 @@ LOG "Cleanup engtools:"
# ( be sure not to clobber /etc/init.d/collect-engtools.sh )
LOG "kill processes brute force"
pids=( $(pidof -x /usr/local/bin/collect-engtools.sh) )
if [ ${#pids[@]} -ne 0 ]
then
LOG "killing: ${pids[@]}"
for pid in ${pids[@]}
do
LOG "kill: [ ${pid} ] "
pkill -KILL -P ${pid}
kill -9 ${pid}
done
pkill -KILL iostat
pkill -KILL top
if [ ${#pids[@]} -ne 0 ]; then
LOG "killing: ${pids[@]}"
for pid in ${pids[@]}; do
LOG "kill: [ ${pid} ] "
pkill -KILL -P ${pid}
kill -9 ${pid}
done
pkill -KILL iostat
pkill -KILL top
else
LOG "no pids found"
LOG "no pids found"
fi
LOG "remove pidfiles"
for TOOL in "${TOOLS[@]}"
do
rm -f -v /var/run/${TOOL}.pid
for TOOL in "${TOOLS[@]}"; do
rm -f -v /var/run/${TOOL}.pid
done
LOG "done"

241
tools/engtools/hostdata-collectors/scripts/collect-engtools.sh
View File

@ -48,97 +48,92 @@ declare tgt_avail_bytes
declare tgt_used_bytes
# do_parallel_commands - launch parallel tools with separate output files
function do_parallel_commands()
{
parallel_outfiles=()
for elem in "${tlist[@]}"
do
tool=""; period=""; repeat=""; interval=""
my_hash="elem[*]"
local ${!my_hash}
if [ ! -z "${name}" ]; then
fname="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${name}"
parallel_outfiles+=( $fname )
LOG "collecting ${tool}, ${interval} second intervals, to: ${fname}"
if [ ! -z "${period}" ]; then
${STDBUF} ${tool} -p ${period} -i ${interval} > ${fname} 2>/dev/null &
elif [ ! -z "${repeat}" ]; then
${STDBUF} ${tool} --repeat=${repeat} --delay=${interval} > ${fname} 2>/dev/null &
fi
else
# run without file output (eg., ticker)
${STDBUF} ${tool} -p ${period} -i ${interval} 2>/dev/null &
fi
done
function do_parallel_commands {
parallel_outfiles=()
for elem in "${tlist[@]}"; do
tool=""; period=""; repeat=""; interval=""
my_hash="elem[*]"
local ${!my_hash}
if [ ! -z "${name}" ]; then
fname="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${name}"
parallel_outfiles+=( $fname )
LOG "collecting ${tool}, ${interval} second intervals, to: ${fname}"
if [ ! -z "${period}" ]; then
${STDBUF} ${tool} -p ${period} -i ${interval} > ${fname} 2>/dev/null &
elif [ ! -z "${repeat}" ]; then
${STDBUF} ${tool} --repeat=${repeat} --delay=${interval} > ${fname} 2>/dev/null &
fi
else
# run without file output (eg., ticker)
${STDBUF} ${tool} -p ${period} -i ${interval} 2>/dev/null &
fi
done
}
# get_current_avail_usage() - get output destination file-system usage and
# availability.
# - updates: df_size_bytes, df_avail_bytes, du_used_bytes
function get_current_avail_usage()
{
local -a df_arr_bytes=( $(df -P --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==2 {print $2, $4}') )
df_size_bytes=${df_arr_bytes[0]}
df_avail_bytes=${df_arr_bytes[1]}
du_used_bytes=$(du --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==1 {print $1}')
function get_current_avail_usage {
local -a df_arr_bytes=( $(df -P --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==2 {print $2, $4}') )
df_size_bytes=${df_arr_bytes[0]}
df_avail_bytes=${df_arr_bytes[1]}
du_used_bytes=$(du --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==1 {print $1}')
}
# purge_oldest_files() - remove oldest files based on file-system available space,
# and maximum collection size
function purge_oldest_files()
{
function purge_oldest_files {
# get current file-system usage
get_current_avail_usage
msg=$(printf "avail %d MB, headroom %d MB; used %d MB, max %d MB" \
get_current_avail_usage
msg=$(printf "avail %d MB, headroom %d MB; used %d MB, max %d MB" \
$[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] \
$[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024])
LOG "usage: ${msg}"
LOG "usage: ${msg}"
if [[ $df_avail_bytes -lt $tgt_avail_bytes ]] || \
[[ $du_used_bytes -gt $tgt_used_bytes ]]; then
# wait for compression to complete
wait
if [[ $df_avail_bytes -lt $tgt_avail_bytes ]] || \
[[ $du_used_bytes -gt $tgt_used_bytes ]]; then
# wait for compression to complete
wait
get_current_avail_usage
if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then
msg=$(printf "purge: avail %d MB < target %d MB" \
$[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] )
LOG "purge: ${msg}"
get_current_avail_usage
if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then
msg=$(printf "purge: avail %d MB < target %d MB" \
$[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] )
LOG "purge: ${msg}"
fi
if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then
msg=$(printf "purge: used %d MB > target %d MB" \
$[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024] )
LOG "purge: ${msg}"
fi
else
return
fi
if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then
msg=$(printf "purge: used %d MB > target %d MB" \
$[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024] )
LOG "purge: ${msg}"
fi
else
return
fi
# remove files in oldest time sorted order until we meet usage targets,
# incrementally updating usage as we remve files
for file in $( ls -rt ${TOOL_DEST_DIR}/${HOSTNAME}_* 2>/dev/null )
do
if [[ $df_avail_bytes -ge $tgt_avail_bytes ]] && \
[[ $du_used_bytes -le $tgt_used_bytes ]]; then
break
fi
for file in $( ls -rt ${TOOL_DEST_DIR}/${HOSTNAME}_* 2>/dev/null ); do
if [[ $df_avail_bytes -ge $tgt_avail_bytes ]] && \
[[ $du_used_bytes -le $tgt_used_bytes ]]; then
break
fi
if [ ${OPT_DEBUG} -eq 1 ]; then
msg="purge: file=$file"
if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then
msg="${msg}, < AVAIL"
fi
if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then
msg="${msg}, > MAXUSAGE"
fi
LOG "${msg}"
fi
if [ ${OPT_DEBUG} -eq 1 ]; then
msg="purge: file=$file"
if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then
msg="${msg}, < AVAIL"
fi
if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then
msg="${msg}, > MAXUSAGE"
fi
LOG "${msg}"
fi
sz_bytes=$(stat --printf="%s" $file)
((df_avail_bytes += sz_bytes))
((du_used_bytes -= sz_bytes))
rm -fv ${file}
done
sz_bytes=$(stat --printf="%s" $file)
((df_avail_bytes += sz_bytes))
((du_used_bytes -= sz_bytes))
rm -fv ${file}
done
}
#-------------------------------------------------------------------------------
@ -160,23 +155,23 @@ CPULIST=""
# Affine tools to NOVA pinned cores (i.e., non-cpu 0)
# - remove interference with cpu 0
if [ "${AFFINE_PINNED}" -eq 1 ]; then
NOVA_CONF=/etc/nova/compute_extend.conf
if [ -f "${NOVA_CONF}" ]; then
NOVA_CONF=/etc/nova/compute_extend.conf
if [ -f "${NOVA_CONF}" ]; then
source "${NOVA_CONF}"
CPULIST=${compute_pinned_cpulist}
else
else
CPULIST=""
fi
fi
fi
set_affinity ${CPULIST}
# Define output directory
if [[ "${HOSTNAME}" =~ "controller-" ]]; then
TOOL_DEST_DIR=/scratch/syseng_data/${HOSTNAME}
TOOL_DEST_DIR=/scratch/syseng_data/${HOSTNAME}
elif [[ "${HOSTNAME}" =~ "compute-" ]]; then
TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME}
TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME}
else
TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME}
TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME}
fi
mkdir -p ${TOOL_DEST_DIR}
@ -186,7 +181,7 @@ DAEMON_OUT="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${TOOLNAME}.log"
# Redirect stdout and append to log if not connected to TTY
if test ! -t 1 ; then
exec 1>> ${DAEMON_OUT}
exec 1>> ${DAEMON_OUT}
fi
# Get current availability and usage
@ -199,9 +194,9 @@ tgt_used_bytes=$[$PURGE_MAXUSAGE_MB*1024*1024]
# Set granularity based on duration
if [ $PERIOD_MIN -le 30 ]; then
GRAN_MIN=5
GRAN_MIN=5
else
GRAN_MIN=60
GRAN_MIN=60
fi
# Adjust repeats and intervals based on GRAN_MIN granularity
@ -209,7 +204,7 @@ PERIOD_MIN=$[($PERIOD_MIN+(GRAN_MIN-1))/GRAN_MIN*GRAN_MIN]
((REPEATS = PERIOD_MIN/GRAN_MIN))
GRAN_MIN_IN_SEC=$[$GRAN_MIN*60]
if [ ${INTERVAL_SEC} -gt ${GRAN_MIN_IN_SEC} ]; then
INTERVAL_SEC=${GRAN_MIN_IN_SEC}
INTERVAL_SEC=${GRAN_MIN_IN_SEC}
fi
# Define tools and options
@ -231,7 +226,7 @@ if [ ${OPT_SOAK} -eq 1 ]; then
ceph_interval=${DUR_15MIN_IN_SEC}
diskstats_interval=${DUR_15MIN_IN_SEC}
memstats_interval=${DUR_15MIN_IN_SEC}
filestats_interval=${DUR_15MIN_IN_SEC}
filestats_interval=${DUR_15MIN_IN_SEC}
elif [ ${OPT_SOAK} -eq 2 ]; then
# Assume much shorter collection (eg, < hours)
schedtop_interval=2 # i.e., 2 second interval
@ -243,7 +238,7 @@ elif [ ${OPT_SOAK} -eq 2 ]; then
ceph_interval=${DUR_5MIN_IN_SEC}
diskstats_interval=${DUR_5MIN_IN_SEC}
memstats_interval=${DUR_5MIN_IN_SEC}
filestats_interval=${DUR_5MIN_IN_SEC}
filestats_interval=${DUR_5MIN_IN_SEC}
else
# Assume shorter collection (eg, < a few hours)
schedtop_interval=5 # i.e., 5 second interval
@ -255,7 +250,7 @@ else
ceph_interval=${DUR_5MIN_IN_SEC}
diskstats_interval=${DUR_5MIN_IN_SEC}
memstats_interval=${DUR_5MIN_IN_SEC}
filestats_interval=${DUR_5MIN_IN_SEC}
filestats_interval=${DUR_5MIN_IN_SEC}
fi
schedtop_repeat=$[ $PERIOD_MIN * 60 / $schedtop_interval ]
occtop_repeat=$[ $PERIOD_MIN * 60 / $occtop_interval ]
@ -274,29 +269,29 @@ LBINDIR=/usr/local/bin
declare -a tlist
if [[ ${ENABLE_STATIC_COLLECTION} == "Y" ]] || [[ ${ENABLE_STATIC_COLLECTION} == "y" ]]; then
tlist+=( "tool=${LBINDIR}/top.sh name=top period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" )
tlist+=( "tool=${LBINDIR}/iostat.sh name=iostat period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" )
tlist+=( "tool=${LBINDIR}/netstats.sh name=netstats period=${PERIOD_MIN} interval=${netstats_interval}" )
tlist+=( "tool=${BINDIR}/occtop name=occtop repeat=${occtop_repeat} interval=${occtop_interval}" )
tlist+=( "tool=${BINDIR}/memtop name=memtop repeat=${memtop_repeat} interval=${memtop_interval}" )
tlist+=( "tool=${BINDIR}/schedtop name=schedtop repeat=${schedtop_repeat} interval=${schedtop_interval}" )
tlist+=( "tool=${LBINDIR}/diskstats.sh name=diskstats period=${PERIOD_MIN} interval=${diskstats_interval}" )
tlist+=( "tool=${LBINDIR}/memstats.sh name=memstats period=${PERIOD_MIN} interval=${memstats_interval}" )
tlist+=( "tool=${LBINDIR}/filestats.sh name=filestats period=${PERIOD_MIN} interval=${filestats_interval}" )
if [[ "${HOSTNAME}" =~ "controller-" ]]; then
tlist+=( "tool=${LBINDIR}/ceph.sh name=ceph period=${PERIOD_MIN} interval=${ceph_interval}" )
tlist+=( "tool=${LBINDIR}/postgres.sh name=postgres period=${PERIOD_MIN} interval=${postgres_interval}" )
tlist+=( "tool=${LBINDIR}/rabbitmq.sh name=rabbitmq period=${PERIOD_MIN} interval=${rabbitmq_interval}" )
elif [[ "${HOSTNAME}" =~ "compute-" ]]; then
tlist+=( "tool=${LBINDIR}/vswitch.sh name=vswitch period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" )
fi
tlist+=( "tool=${LBINDIR}/top.sh name=top period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" )
tlist+=( "tool=${LBINDIR}/iostat.sh name=iostat period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" )
tlist+=( "tool=${LBINDIR}/netstats.sh name=netstats period=${PERIOD_MIN} interval=${netstats_interval}" )
tlist+=( "tool=${BINDIR}/occtop name=occtop repeat=${occtop_repeat} interval=${occtop_interval}" )
tlist+=( "tool=${BINDIR}/memtop name=memtop repeat=${memtop_repeat} interval=${memtop_interval}" )
tlist+=( "tool=${BINDIR}/schedtop name=schedtop repeat=${schedtop_repeat} interval=${schedtop_interval}" )
tlist+=( "tool=${LBINDIR}/diskstats.sh name=diskstats period=${PERIOD_MIN} interval=${diskstats_interval}" )
tlist+=( "tool=${LBINDIR}/memstats.sh name=memstats period=${PERIOD_MIN} interval=${memstats_interval}" )
tlist+=( "tool=${LBINDIR}/filestats.sh name=filestats period=${PERIOD_MIN} interval=${filestats_interval}" )
if [[ "${HOSTNAME}" =~ "controller-" ]]; then
tlist+=( "tool=${LBINDIR}/ceph.sh name=ceph period=${PERIOD_MIN} interval=${ceph_interval}" )
tlist+=( "tool=${LBINDIR}/postgres.sh name=postgres period=${PERIOD_MIN} interval=${postgres_interval}" )
tlist+=( "tool=${LBINDIR}/rabbitmq.sh name=rabbitmq period=${PERIOD_MIN} interval=${rabbitmq_interval}" )
elif [[ "${HOSTNAME}" =~ "compute-" ]]; then
tlist+=( "tool=${LBINDIR}/vswitch.sh name=vswitch period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" )
fi
# ticker - shows progress on the screen
tlist+=( "tool=${LBINDIR}/ticker.sh name= period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" )
tlist+=( "tool=${LBINDIR}/ticker.sh name= period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" )
fi
if [[ ${ENABLE_LIVE_STREAM} == "Y" ]] || [[ ${ENABLE_LIVE_STREAM} == "y" ]]; then
${TOOLBIN}/live_stream.py &
${TOOLBIN}/live_stream.py &
fi
#-------------------------------------------------------------------------------
@ -306,32 +301,30 @@ OPT_DEBUG=0
REP=0
if [ ${#tlist[@]} -ne 0 ]; then
# Static stats collection is turned on
while [[ ${TOOL_USR1_SIGNAL} -eq 0 ]] &&
[[ ${OPT_FOREVER} -eq 1 || ${REP} -lt ${REPEATS} ]]
do
# increment loop counter
((REP++))
# Static stats collection is turned on
while [[ ${TOOL_USR1_SIGNAL} -eq 0 ]] && [[ ${OPT_FOREVER} -eq 1 || ${REP} -lt ${REPEATS} ]]; do
# increment loop counter
((REP++))
# purge oldest files
purge_oldest_files
# purge oldest files
purge_oldest_files
# define filename timestamp
timestamp=$( date +"%Y-%0m-%0e_%H%M" )
# define filename timestamp
timestamp=$( date +"%Y-%0m-%0e_%H%M" )
# collect tools in parallel to separate output files
LOG "collecting ${TOOLNAME} at ${timestamp} for ${PERIOD_MIN} mins, repeat=${REP}"
do_parallel_commands
wait
# collect tools in parallel to separate output files
LOG "collecting ${TOOLNAME} at ${timestamp} for ${PERIOD_MIN} mins, repeat=${REP}"
do_parallel_commands
wait
# Compress latest increment
LOG "compressing: ${parallel_outfiles[@]}"
${CMD_IDLE} bzip2 -q -f ${parallel_outfiles[@]} 2>/dev/null &
done
# Compress latest increment
LOG "compressing: ${parallel_outfiles[@]}"
${CMD_IDLE} bzip2 -q -f ${parallel_outfiles[@]} 2>/dev/null &
done
# Wait for the compression to complete
wait
tools_cleanup 0
wait
tools_cleanup 0
fi
# Should wait here in case live stats streaming is turned on.

117
tools/engtools/hostdata-collectors/scripts/diskstats.sh
View File

@ -6,86 +6,84 @@ TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
tools_init
if [ $? -ne 0 ]; then
echo "FATAL, tools_init - could not setup environment"
exit $?
echo "FATAL, tools_init - could not setup environment"
exit $?
fi
# Enable use of INTERVAL_SEC sample interval
OPT_USE_INTERVALS=1
# Print disk summary
function print_disk()
{
print_separator
TOOL_HIRES_TIME
function print_disk {
print_separator
TOOL_HIRES_TIME
# NOTES:
# --total (grand-total) is a new option, but don't necessarily want to add tmpfs
# or dummy filesystems.
# or dummy filesystems.
# - use -H to print in SI (eg, GB, vs GiB)
# - can use -a to print all filesystems including dummy filesystems, but then
# there can be double-counting:
print_separator
cmd='df -h -H -T --local -t ext2 -t ext3 -t ext4 -t xfs --total'
${ECHO} "Disk space usage ext2,ext3,ext4,xfs,tmpfs (SI):"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
print_separator
cmd='df -h -H -T --local -t ext2 -t ext3 -t ext4 -t xfs --total'
${ECHO} "Disk space usage ext2,ext3,ext4,xfs,tmpfs (SI):"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
print_separator
cmd='df -h -H -T --local -i -t ext2 -t ext3 -t ext4 -t xfs --total'
${ECHO} "Disk inodes usage ext2,ext3,ext4,xfs,tmpfs (SI):"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
print_separator
cmd='df -h -H -T --local -i -t ext2 -t ext3 -t ext4 -t xfs --total'
${ECHO} "Disk inodes usage ext2,ext3,ext4,xfs,tmpfs (SI):"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
print_separator
cmd='drbd-overview'
${ECHO} "drbd disk usage and status:"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
print_separator
cmd='drbd-overview'
${ECHO} "drbd disk usage and status:"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
print_separator
cmd='lvs'
${ECHO} "logical volumes usage and status:"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
print_separator
cmd='lvs'
${ECHO} "logical volumes usage and status:"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
print_separator
cmd='pvs'
${ECHO} "physical volumes usage and status:"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
print_separator
cmd='pvs'
${ECHO} "physical volumes usage and status:"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
print_separator
cmd='vgs'
${ECHO} "volume groups usage and status:"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
print_separator
cmd='vgs'
${ECHO} "volume groups usage and status:"
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
}
# Print disk static summary
function print_disk_static()
{
print_separator
cmd='cat /proc/scsi/scsi'
${ECHO} "Attached devices: ${cmd}"
${cmd}
${ECHO}
function print_disk_static {
print_separator
cmd='cat /proc/scsi/scsi'
${ECHO} "Attached devices: ${cmd}"
${cmd}
${ECHO}
# fdisk - requires sudo/root
print_separator
cmd='fdisk -l'
if [ $UID -eq 0 ]; then
${ECHO} "List disk devices: ${cmd}"
${cmd}
else
WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required"
fi
${ECHO}
print_separator
cmd='fdisk -l'
if [ $UID -eq 0 ]; then
${ECHO} "List disk devices: ${cmd}"
${cmd}
else
WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required"
fi
${ECHO}
# parted - requires sudo/root
print_separator
cmd='parted -l'
if [ $UID -eq 0 ]; then
${ECHO} "List disk devices: ${cmd}"
${cmd}
else
WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required"
fi
${ECHO}
print_separator
cmd='parted -l'
if [ $UID -eq 0 ]; then
${ECHO} "List disk devices: ${cmd}"
${cmd}
else
WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required"
fi
${ECHO}
}
#-------------------------------------------------------------------------------
@ -109,10 +107,9 @@ print_disk_static
# Calculate number of sample repeats based on overall interval and sampling interval
((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC))
for ((rep=1; rep <= REPEATS ; rep++))
do
print_disk
sleep ${INTERVAL_SEC}
for ((rep=1; rep <= REPEATS ; rep++)); do
print_disk
sleep ${INTERVAL_SEC}
done
print_disk
LOG "done"

601
tools/engtools/hostdata-collectors/scripts/engtools_util.sh
View File

@ -7,7 +7,7 @@ TOOL_USR1_SIGNAL=0
TOOL_USR2_SIGNAL=0
TOOL_TTY=0
if tty 1>/dev/null ; then
TOOL_TTY=1
TOOL_TTY=1
fi
# [ JGAULD : SHOULD RENAME TO TOOL_X ]
@ -19,145 +19,143 @@ CPULIST=0
# Include lsb functions
if [ -d /lib/lsb ]; then
. /lib/lsb/init-functions
. /lib/lsb/init-functions
else
. /etc/init.d/functions
. /etc/init.d/functions
fi
# Lightweight replacement for pidofproc -p <pid>
function check_pidfile ()
{
local pidfile pid
function check_pidfile {
local pidfile pid
OPTIND=1
while getopts p: opt ; do
OPTIND=1
while getopts p: opt ; do
case "$opt" in
p)
p)
pidfile="$OPTARG"
;;
esac
done
shift $(($OPTIND - 1))
done
shift $(($OPTIND - 1))
read pid < "${pidfile}"
if [ -n "${pid:-}" ]; then
if $(kill -0 "${pid:-}" 2> /dev/null); then
echo "$pid"
return 0
elif ps "${pid:-}" >/dev/null 2>&1; then
echo "$pid"
return 0 # program is running, but not owned by this user
else
return 1 # program is dead and /var/run pid file exists
read pid < "${pidfile}"
if [ -n "${pid:-}" ]; then
if $(kill -0 "${pid:-}" 2> /dev/null); then
echo "$pid"
return 0
elif ps "${pid:-}" >/dev/null 2>&1; then
echo "$pid"
return 0 # program is running, but not owned by this user
else
return 1 # program is dead and /var/run pid file exists
fi
fi
fi
}
# tools_init - initialize tool resources
function tools_init ()
{
local rc=0
local error=0
TOOLNAME=$(basename $0)
function tools_init {
local rc=0
local error=0
TOOLNAME=$(basename $0)
# Check for sufficient priviledges
if [ $UID -ne 0 ]; then
ERRLOG "${NAME} requires sudo/root access."
return 1
fi
if [ $UID -ne 0 ]; then
ERRLOG "${NAME} requires sudo/root access."
return 1
fi
# Check for essential binaries
ECHO=$(which echo 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ECHO=echo # use bash built-in echo
${ECHO} "FATAL, 'echo' not found, rc=$rc";
error=$rc
fi
DATE=$(which date 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
${ECHO} "FATAL, 'date' not found, rc=$rc";
error=$rc
fi
ECHO=$(which echo 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ECHO=echo # use bash built-in echo
${ECHO} "FATAL, 'echo' not found, rc=$rc";
error=$rc
fi
DATE=$(which date 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
${ECHO} "FATAL, 'date' not found, rc=$rc";
error=$rc
fi
# Check for standard linux binaries, at least can use LOG functions now
# - these are used in tools_header
CAT=$(which cat 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'cat' not found, rc=$rc";
error=$rc
fi
CAT=$(which cat 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'cat' not found, rc=$rc";
error=$rc
fi
ARCH=$(which arch 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'arch' not found, rc=$rc";
error=$rc
fi
ARCH=$(which arch 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'arch' not found, rc=$rc";
error=$rc
fi
SED=$(which sed 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'sed' not found, rc=$rc";
error=$rc
fi
SED=$(which sed 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'sed' not found, rc=$rc";
error=$rc
fi
GREP=$(which grep 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'grep' not found, rc=$rc";
error=$rc
fi
GREP=$(which grep 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'grep' not found, rc=$rc";
error=$rc
fi
WC=$(which wc 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'wc' not found, rc=$rc";
error=$rc
fi
WC=$(which wc 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'wc' not found, rc=$rc";
error=$rc
fi
UNAME=$(which uname 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'uname' not found, rc=$rc";
error=$rc
fi
UNAME=$(which uname 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'uname' not found, rc=$rc";
error=$rc
fi
SORT=$(which sort 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'sort' not found, rc=$rc";
error=$rc
fi
SORT=$(which sort 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'sort' not found, rc=$rc";
error=$rc
fi
TR=$(which tr 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'tr' not found, rc=$rc";
error=$rc
fi
TR=$(which tr 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'tr' not found, rc=$rc";
error=$rc
fi
AWK=$(which awk 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'awk' not found, rc=$rc";
error=$rc
fi
AWK=$(which awk 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'awk' not found, rc=$rc";
error=$rc
fi
PKILL=$(which pkill 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'pkill' not found, rc=$rc";
error=$rc
fi
PKILL=$(which pkill 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'pkill' not found, rc=$rc";
error=$rc
fi
LS=$(which ls 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'ls' not found, rc=$rc";
error=$rc
fi
LS=$(which ls 2>/dev/null)
rc=$?
if [ $rc -ne 0 ]; then
ERRLOG "'ls' not found, rc=$rc";
error=$rc
fi
# The following block is needed for LSB systems such as Windriver Linux.
# The utility is not available on CentOS so comment it out.
@ -170,168 +168,172 @@ function tools_init ()
# Let parent program decide what to do with the errors,
# give ominous warning
if [ $error -eq 1 ]; then
WARNLOG "possibly cannot continue, missing linux binaries"
fi
if [ $error -eq 1 ]; then
WARNLOG "possibly cannot continue, missing linux binaries"
fi
# Check if tool was previously running
if [ -e ${PIDFILE} ]; then
if [ -e ${PIDFILE} ]; then
# [ JGAULD - remove pidofproc() / LSB compatibility issue ]
if check_pidfile -p "${PIDFILE}" >/dev/null; then
ERRLOG "${PIDFILE} exists and ${TOOLNAME} is running"
return 1
ERRLOG "${PIDFILE} exists and ${TOOLNAME} is running"
return 1
else
# remove pid file
WARNLOG "${PIDFILE} exists but ${TOOLNAME} is not running; cleaning up"
rm -f ${PIDFILE}
WARNLOG "${PIDFILE} exists but ${TOOLNAME} is not running; cleaning up"
rm -f ${PIDFILE}
fi
fi
fi
# Create pid file
echo $$ > ${PIDFILE}
echo $$ > ${PIDFILE}
# Setup trap handler - these signals trigger child shutdown and cleanup
trap tools_exit_handler INT HUP TERM EXIT
trap tools_usr1_handler USR1
trap tools_usr2_handler USR2
trap tools_exit_handler INT HUP TERM EXIT
trap tools_usr1_handler USR1
trap tools_usr2_handler USR2
return ${rc}
return ${rc}
}
# tools_cleanup() - terminate child processes
function tools_cleanup() {
function tools_cleanup {
# restore signal handling to default behaviour
trap - INT HUP TERM EXIT
trap - USR1 USR2
trap - INT HUP TERM EXIT
trap - USR1 USR2
local VERBOSE_OPT=''
if [ "$1" -ne "0" ]; then
LOG "cleanup invoked with code: $1"
if [ ${TOOL_DEBUG} -ne 0 ]; then
VERBOSE_OPT='-v'
local VERBOSE_OPT=''
if [ "$1" -ne "0" ]; then
LOG "cleanup invoked with code: $1"
if [ ${TOOL_DEBUG} -ne 0 ]; then
VERBOSE_OPT='-v'
fi
fi
fi
# stop all processes launched from this process
pkill -TERM -P $$
if [ "$1" -ne "0" ]; then
sleep 1
fi
pkill -TERM -P $$
if [ "$1" -ne "0" ]; then
sleep 1
fi
# OK, if the above didn't work, use force
pkill -KILL -P $$
pkill -KILL -P $$
# remove pid file
if [ -e ${PIDFILE} ]; then
rm -f ${VERBOSE_OPT} ${PIDFILE}
fi
exit $1
if [ -e ${PIDFILE} ]; then
rm -f ${VERBOSE_OPT} ${PIDFILE}
fi
exit $1
}
# tools_exit_handler() - exit handler routine
function tools_exit_handler() {
TOOL_EXIT_SIGNAL=1
tools_cleanup 128
function tools_exit_handler {
TOOL_EXIT_SIGNAL=1
tools_cleanup 128
}
# tools_usr1_handler() - USR1 handler routine
function tools_usr1_handler() {
TOOL_USR1_SIGNAL=1
LOG "caught USR1"
function tools_usr1_handler {
TOOL_USR1_SIGNAL=1
LOG "caught USR1"
}
# tools_usr2_handler() - USR2 handler routine
function tools_usr2_handler() {
TOOL_USR2_SIGNAL=1
LOG "caught USR1"
function tools_usr2_handler {
TOOL_USR2_SIGNAL=1
LOG "caught USR1"
}
# LOG(), WARNLOG(), ERRLOG() - simple print log functions (not logger)
function LOG ()
{
local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" )
echo "${tstamp_H} ${HOSTNAME} $0($$): $@";
function LOG {
local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" )
echo "${tstamp_H} ${HOSTNAME} $0($$): $@";
}
function LOG_NOCR ()
{
local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" )
echo -n "${tstamp_H} ${HOSTNAME} $0($$): $@";
function LOG_NOCR {
local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" )
echo -n "${tstamp_H} ${HOSTNAME} $0($$): $@";
}
function WARNLOG {
LOG "WARN $@";
}
function ERRLOG {
LOG "ERROR $@";
}
function WARNLOG () { LOG "WARN $@"; }
function ERRLOG () { LOG "ERROR $@"; }
# TOOL_HIRES_TIME() - easily parsed date/timestamp and hi-resolution uptime
function TOOL_HIRES_TIME()
{
echo "time: " $( ${DATE} +"%a %F %H:%M:%S.%N %Z %z" ) "uptime: " $( cat /proc/uptime )
function TOOL_HIRES_TIME {
echo "time: " $( ${DATE} +"%a %F %H:%M:%S.%N %Z %z" ) "uptime: " $( cat /proc/uptime )
}
# set_affinity() - set affinity for current script if a a CPULIST is defined
function set_affinity() {
local CPULIST=$1
if [ -z "${CPULIST}" ]; then
return
fi
function set_affinity {
local CPULIST=$1
if [ -z "${CPULIST}" ]; then
return
fi
# Set cpu affinity for current program
local TASKSET=$(which taskset 2>/dev/null)
if [ -x "${TASKSET}" ]; then
${TASKSET} -pc ${CPULIST} $$ 2>/dev/null
fi
local TASKSET=$(which taskset 2>/dev/null)
if [ -x "${TASKSET}" ]; then
${TASKSET} -pc ${CPULIST} $$ 2>/dev/null
fi
}
# cmd_idle_priority() - command to set nice + ionice
function cmd_idle_priority() {
local NICE=""
local IONICE=""
function cmd_idle_priority {
local NICE=""
local IONICE=""
NICE=$( which nice 2>/dev/null )
if [ $? -eq 0 ]; then
NICE="${NICE} -n 19"
else
NICE=""
fi
IONICE=$( which ionice 2>/dev/null )
if [ $? -eq 0 ]; then
IONICE="${IONICE} -c 3"
else
IONICE=""
fi
echo "${NICE} ${IONICE}"
NICE=$( which nice 2>/dev/null )
if [ $? -eq 0 ]; then
NICE="${NICE} -n 19"
else
NICE=""
fi
IONICE=$( which ionice 2>/dev/null )
if [ $? -eq 0 ]; then
IONICE="${IONICE} -c 3"
else
IONICE=""
fi
echo "${NICE} ${IONICE}"
}
# print_separator() - print a horizontal separation line '\u002d' is '-'
function print_separator () {
printf '\u002d%.s' {1..80}
printf '\n'
function print_separator {
printf '\u002d%.s' {1..80}
printf '\n'
}
# tools_header() - print out common GenWare tools header
function tools_header() {
local TOOLNAME=$(basename $0)
function tools_header {
local TOOLNAME=$(basename $0)
# Get timestamp
#local tstamp=$( date +"%Y-%0m-%0e %H:%M:%S" 2>/dev/null )
local tstamp=$( date --rfc-3339=ns | cut -c1-23 2>/dev/null )
local tstamp=$( date --rfc-3339=ns | cut -c1-23 2>/dev/null )
# Linux Generic
local UPTIME=/proc/uptime
local UPTIME=/proc/uptime
# Get number of online cpus
local CPUINFO=/proc/cpuinfo
local online_cpus=$( cat ${CPUINFO} | grep -i ^processor | wc -l 2>/dev/null )
local CPUINFO=/proc/cpuinfo
local online_cpus=$( cat ${CPUINFO} | grep -i ^processor | wc -l 2>/dev/null )
# Get load average, run-queue size, and number of threads
local LOADAVG=/proc/loadavg
local LDAVG=( `cat ${LOADAVG} | sed -e 's#[/]# #g' 2>/dev/null` )
local LOADAVG=/proc/loadavg
local LDAVG=( `cat ${LOADAVG} | sed -e 's#[/]# #g' 2>/dev/null` )
# Get current architecture
local arch=$( uname -m )
local arch=$( uname -m )
# Determine processor name (there are many different formats... *sigh* )
# - build up info from multiple lines
local processor='unk'
local NAME=$( cat ${CPUINFO} | grep \
local processor='unk'
local NAME=$( cat ${CPUINFO} | grep \
-e '^cpu\W\W:' \
-e ^'cpu model' \
-e ^'model name' \
@ -340,139 +342,138 @@ function tools_header() {
-e ^[Mm]achine | \
sort -u | awk 'BEGIN{FS=":";} {print $2;}' | \
tr '\n' ' ' | tr -s [:blank:] 2>/dev/null )
if [ ! -z "${NAME}" ]; then
processor=${NAME}
fi
if [ ! -z "${NAME}" ]; then
processor=${NAME}
fi
# Determine processor speed (abort grep after first match)
local speed='unk'
local BOGO=$( cat ${CPUINFO} | grep -m1 -e ^BogoMIPS -e ^bogomips | \
local speed='unk'
local BOGO=$( cat ${CPUINFO} | grep -m1 -e ^BogoMIPS -e ^bogomips | \
awk 'BEGIN{FS=":";} {printf "%.1f", $2;}' 2>/dev/null )
local MHZ=$( cat ${CPUINFO} | grep -m1 -e ^'cpu MHz' -e ^clock | \
local MHZ=$( cat ${CPUINFO} | grep -m1 -e ^'cpu MHz' -e ^clock | \
awk 'BEGIN{FS=":";} {printf "%.1f", $2;}' 2>/dev/null )
local MHZ2=$( cat ${CPUINFO} | grep -m1 -e ^Cpu0ClkTck -e ^'cycle frequency' | \
local MHZ2=$( cat ${CPUINFO} | grep -m1 -e ^Cpu0ClkTck -e ^'cycle frequency' | \
awk 'BEGIN{FS=":";} {printf "%.1f", $2/1.0E6;}' 2>/dev/null )
if [ ! -z "${MHZ}" ]; then
speed=${MHZ}
elif [ ! -z "${MHZ2}" ]; then
speed=${MHZ2}
elif [ ! -z ${BOGO} ]; then
speed=${BOGO}
fi
if [ ! -z "${MHZ}" ]; then
speed=${MHZ}
elif [ ! -z "${MHZ2}" ]; then
speed=${MHZ2}
elif [ ! -z ${BOGO} ]; then
speed=${BOGO}
fi
# Determine OS and kernel version
local os_name=$( uname -s 2>/dev/null )
local os_release=$( uname -r 2>/dev/null )
local os_name=$( uname -s 2>/dev/null )
local os_release=$( uname -r 2>/dev/null )
declare -a arr
declare -a arr
local dist_id=""
local dist_id=""
# Determine OS distribution ID
if [ lsb_pres == "yes" ]; then
arr=( $( lsb_release -i 2>/dev/null ) )
dist_id=${arr[2]}
else
local dist_id=$(cat /etc/centos-release | awk '{print $1}' 2>/dev/null)
fi
local dist_rel=""
if [ lsb_pres == "yes" ]; then
if [ lsb_pres == "yes" ]; then
arr=( $( lsb_release -i 2>/dev/null ) )
dist_id=${arr[2]}
else
local dist_id=$(cat /etc/centos-release | awk '{print $1}' 2>/dev/null)
fi
local dist_rel=""
if [ lsb_pres == "yes" ]; then
# Determine OS distribution release
arr=( $( cat /proc/version | awk '{print $3}' 2>/dev/null ) )
local dist_rel=${arr[1]}
else
local dist_rel=$(cat /etc/centos-release | awk '{print $4}' 2>/dev/null)
fi
arr=( $( cat /proc/version | awk '{print $3}' 2>/dev/null ) )
local dist_rel=${arr[1]}
else
local dist_rel=$(cat /etc/centos-release | awk '{print $4}' 2>/dev/null)
fi
# Print generic header
echo "${TOOLNAME} -- ${tstamp} load average:${LDAVG[0]}, ${LDAVG[1]}, ${LDAVG[2]} runq:${LDAVG[3]} nproc:${LDAVG[4]}"
echo " host:${HOSTNAME} Distribution:${dist_id} ${dist_rel} ${os_name} ${os_release}"
echo " arch:${arch} processor:${processor} speed:${speed} MHz CPUs:${online_cpus}"
echo "${TOOLNAME} -- ${tstamp} load average:${LDAVG[0]}, ${LDAVG[1]}, ${LDAVG[2]} runq:${LDAVG[3]} nproc:${LDAVG[4]}"
echo " host:${HOSTNAME} Distribution:${dist_id} ${dist_rel} ${os_name} ${os_release}"
echo " arch:${arch} processor:${processor} speed:${speed} MHz CPUs:${online_cpus}"
}
# tools_usage() - show generic tools tool usage
function tools_usage() {
if [ ${OPT_USE_INTERVALS} -eq 1 ]; then
echo "usage: ${TOOLNAME} [-f] [-p <period_mins>] [-i <interval_seconds>] [-c <cpulist>] [-h]"
else
echo "Usage: ${TOOLNAME} [-f] [-p <period_mins>] [-c <cpulist>] [-h]"
fi
function tools_usage {
if [ ${OPT_USE_INTERVALS} -eq 1 ]; then
echo "usage: ${TOOLNAME} [-f] [-p <period_mins>] [-i <interval_seconds>] [-c <cpulist>] [-h]"
else
echo "Usage: ${TOOLNAME} [-f] [-p <period_mins>] [-c <cpulist>] [-h]"
fi
}
# tools_print_help() - print generic tool help
function tools_print_help() {
tools_usage
echo
echo "Options:";
echo " -f : collect forever : default: none"
echo " -p <period_minutes> : overall collection period (minutes) : default: ${DEFAULT_PERIOD_MIN}"
if [ ${OPT_USE_INTERVALS} -eq 1 ]; then
echo " -i <interval_seconds> : sample interval (seconds) : default: ${DEFAULT_INTERVAL_SEC}"
fi
echo " -c <cpulist> : cpu list where tool runs (e.g., 0-1,8) : default: none"
echo
if [ ${OPT_USE_INTERVALS} -eq 1 ]; then
echo "Example: collect 5 minute period, sample every 30 seconds interval"
echo " ${TOOLNAME} -p 5 -i 30"
else
echo "Example: collect 5 minute period"
echo " ${TOOLNAME} -p 5"
fi
function tools_print_help {
tools_usage
echo
echo "Options:";
echo " -f : collect forever : default: none"
echo " -p <period_minutes> : overall collection period (minutes) : default: ${DEFAULT_PERIOD_MIN}"
if [ ${OPT_USE_INTERVALS} -eq 1 ]; then
echo " -i <interval_seconds> : sample interval (seconds) : default: ${DEFAULT_INTERVAL_SEC}"
fi
echo " -c <cpulist> : cpu list where tool runs (e.g., 0-1,8) : default: none"
echo
if [ ${OPT_USE_INTERVALS} -eq 1 ]; then
echo "Example: collect 5 minute period, sample every 30 seconds interval"
echo " ${TOOLNAME} -p 5 -i 30"
else
echo "Example: collect 5 minute period"
echo " ${TOOLNAME} -p 5"
fi
}
# tools_parse_options() -- parse common options for tools scripts
function tools_parse_options() {
function tools_parse_options {
# check for no arguments, print usage
if [ $# -eq "0" ]; then
tools_usage
tools_cleanup 0
exit 0
fi
if [ $# -eq "0" ]; then
tools_usage
tools_cleanup 0
exit 0
fi
# parse the input arguments
while getopts "fp:i:c:h" Option
do
while getopts "fp:i:c:h" Option; do
case $Option in
f)
OPT_FOREVER=1
PERIOD_MIN=60
;;
OPT_FOREVER=1
PERIOD_MIN=60
;;
p) PERIOD_MIN=$OPTARG ;;
i)
OPT_USE_INTERVALS=1
INTERVAL_SEC=$OPTARG
;;
OPT_USE_INTERVALS=1
INTERVAL_SEC=$OPTARG
;;
c) CPULIST=$OPTARG ;;
h)
tools_print_help
tools_cleanup 0
exit 0
;;
tools_print_help
tools_cleanup 0
exit 0
;;
*)
tools_usage
tools_cleanup 0
exit 0
;;
tools_usage
tools_cleanup 0
exit 0
;;
esac
done
done
# validate input arguments
PERIOD_MAX=$[4*24*60]
INTERVAL_MAX=$[60*60]
PERIOD_MAX=$[4*24*60]
INTERVAL_MAX=$[60*60]
error=0
if [[ ${PERIOD_MIN} -lt 1 || ${PERIOD_MIN} -gt ${PERIOD_MAX} ]]; then
echo "-p <period_mid> must be > 0 and <= ${PERIOD_MAX}."
error=1
fi
if [[ ${INTERVAL_SEC} -lt 1 || ${INTERVAL_SEC} -gt ${INTERVAL_MAX} ]]; then
echo "-i <interval> must be > 0 and <= ${INTERVAL_MAX}."
error=1
fi
if [ ${error} -eq 1 ]; then
tools_cleanup 0
exit 1
fi
error=0
if [[ ${PERIOD_MIN} -lt 1 || ${PERIOD_MIN} -gt ${PERIOD_MAX} ]]; then
echo "-p <period_mid> must be > 0 and <= ${PERIOD_MAX}."
error=1
fi
if [[ ${INTERVAL_SEC} -lt 1 || ${INTERVAL_SEC} -gt ${INTERVAL_MAX} ]]; then
echo "-i <interval> must be > 0 and <= ${INTERVAL_MAX}."
error=1
fi
if [ ${error} -eq 1 ]; then
tools_cleanup 0
exit 1
fi
}

46
tools/engtools/hostdata-collectors/scripts/filestats.sh
View File

@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
tools_init
if [ $? -ne 0 ]; then
echo "FATAL, tools_init - could not setup environment"
exit $?
echo "FATAL, tools_init - could not setup environment"
exit $?
fi
PAGE_SIZE=$(getconf PAGE_SIZE)
@ -16,21 +16,20 @@ PAGE_SIZE=$(getconf PAGE_SIZE)
OPT_USE_INTERVALS=1
function print_files()
{
print_separator
function print_files {
print_separator
TOOL_HIRES_TIME
${ECHO} "# ls -l /proc/*/fd"
sudo ls -l /proc/*/fd 2>/dev/null | awk \
'$11 ~ /socket/ {a += 1} ; \
$11 ~ /null/ {b += 1} ; \
{c += 1} \
END {\
{printf "%-10s %-10s %-10s %-10s\n", "TOTAL", "FILES", "SOCKETS", "NULL PIPES"} \
{printf "%-10s %-10s %-10s %-10s\n", c, c-(a+b) , a, b}}'
${ECHO} "# ls -l /proc/*/fd"
sudo ls -l /proc/*/fd 2>/dev/null | awk \
'$11 ~ /socket/ {a += 1} ; \
$11 ~ /null/ {b += 1} ; \
{c += 1} \
END {\
{printf "%-10s %-10s %-10s %-10s\n", "TOTAL", "FILES", "SOCKETS", "NULL PIPES"} \
{printf "%-10s %-10s %-10s %-10s\n", c, c-(a+b) , a, b}}'
${ECHO}
${ECHO}
${ECHO} "# lsof"
printf "%-7s %-7s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %s\n" "PID" "TOTAL" "FD" "U" "W" "R" "CWD" "RTD" "TXT" "MEM" "DEL" "TCP" "CMD"
@ -46,7 +45,7 @@ function print_files()
{printf "%-7s %-7s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %s\n", \
pids[i]["PID"], \
pids[i]["TOTAL"],\
((pids[i]["u"]!="")? pids[i]["u"] : 0) + ((pids[i]["w"]!="")? pids[i]["w"] : 0 )+ ((pids[i]["r"]!="")? pids[i]["r"] : 0),\
((pids[i]["u"]!="")? pids[i]["u"] : 0) + ((pids[i]["w"]!="")? pids[i]["w"] : 0 )+ ((pids[i]["r"]!="")? pids[i]["r"] : 0),\
(pids[i]["u"]!="")? pids[i]["u"] : 0,\
(pids[i]["w"]!="")? pids[i]["w"] : 0,\
(pids[i]["r"]!="")? pids[i]["r"] : 0,\
@ -56,13 +55,13 @@ function print_files()
(pids[i]["mem"]!="")? pids[i]["mem"] : 0,\
(pids[i]["DEL"]!="")? pids[i]["DEL"] : 0,\
(pids[i]["TCP"]!="")? pids[i]["TCP"] : 0,\
pids[i]["COMMAND"]} }}}' | sort -n -r -k3
pids[i]["COMMAND"]} }}}' | sort -n -r -k3
${ECHO}
${ECHO}
${ECHO} "# lsof -nP +L1"
sudo lsof -nP +L1
${ECHO}
${ECHO} "# lsof -nP +L1"
sudo lsof -nP +L1
${ECHO}
}
@ -85,10 +84,9 @@ tools_header
# Calculate number of sample repeats based on overall interval and sampling interval
((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC))
for ((rep=1; rep <= REPEATS ; rep++))
do
print_files
sleep ${INTERVAL_SEC}
for ((rep=1; rep <= REPEATS ; rep++)); do
print_files
sleep ${INTERVAL_SEC}
done
print_files
LOG "done"

75
tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh
View File

@ -41,76 +41,75 @@ fi
# Check for sufficient priviledges
# [ JGAULD : possibly provide user = 'operator' option instead... ]
if [ $UID -ne 0 ]; then
log_daemon_msg "Starting ${NAME} requires sudo/root access."
exit 1
log_daemon_msg "Starting ${NAME} requires sudo/root access."
exit 1
fi
case $1 in
start)
start)
if [ -e ${PIDFILE} ]; then
pid=$(pidof -x ${NAME})
if test "${pid}" != ""
then
echo_success "${NAME} already running"
pid=$(pidof -x ${NAME})
if test "${pid}" != ""
then
echo_success "${NAME} already running"
exit
fi
fi
fi
log_daemon_msg "Starting ${NAME}"
if start-stop-daemon --start --background --quiet --oknodo --pidfile ${PIDFILE} \
--exec ${DAEMON} -- ${DAEMON_ARGS} ; then
log_end_msg 0
--exec ${DAEMON} -- ${DAEMON_ARGS} ; then
log_end_msg 0
else
log_end_msg 1
log_end_msg 1
fi
;;
stop)
stop)
if [ -e ${PIDFILE} ]; then
pids=$(pidof -x ${NAME})
if [[ ! -z "${pids}" ]]
then
echo_success "Stopping ${NAME} [$pid]"
start-stop-daemon --stop --quiet --oknodo --pidfile ${PIDFILE} --retry=TERM/3/KILL/5
# [ JGAULD: none of the following should be necessary ]
/usr/local/bin/cleanup-engtools.sh
else
echo_failure "${NAME} is not running"
fi
pids=$(pidof -x ${NAME})
if [[ ! -z "${pids}" ]]; then
echo_success "Stopping ${NAME} [$pid]"
start-stop-daemon --stop --quiet --oknodo --pidfile ${PIDFILE} --retry=TERM/3/KILL/5
# [ JGAULD: none of the following should be necessary ]
/usr/local/bin/cleanup-engtools.sh
else
echo_failure "${NAME} is not running"
fi
else
echo_failure "${PIDFILE} does not exist"
echo_failure "${PIDFILE} does not exist"
fi
;;
restart)
restart)
$0 stop && sleep 2 && $0 start
;;
status)
status)
if [ -e ${PIDFILE} ]; then
pid=$(pidof -x ${NAME})
if test "${pid}" != ""
then
echo_success "${NAME} is running"
else
echo_success "${NAME} is not running"
fi
pid=$(pidof -x ${NAME})
if test "${pid}" != ""
then
echo_success "${NAME} is running"
else
echo_success "${NAME} is not running"
fi
else
echo_success "${NAME} is not running"
echo_success "${NAME} is not running"
fi
;;
reload)
reload)
if [ -e ${PIDFILE} ]; then
start-stop-daemon --stop --signal USR1 --quiet --pidfile ${PIDFILE} --name ${NAME}
echo_success "${NAME} reloaded successfully"
start-stop-daemon --stop --signal USR1 --quiet --pidfile ${PIDFILE} --name ${NAME}
echo_success "${NAME} reloaded successfully"
else
echo_success "${PIDFILE} does not exist"
echo_success "${PIDFILE} does not exist"
fi
;;
*)
*)
echo "Usage: $0 {start|stop|restart|reload|status}"
exit 2
;;

10
tools/engtools/hostdata-collectors/scripts/iostat.sh
View File

@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
tools_init
if [ $? -ne 0 ]; then
echo "FATAL, tools_init - could not setup environment"
exit $?
echo "FATAL, tools_init - could not setup environment"
exit $?
fi
# Enable use of INTERVAL_SEC sample interval
@ -15,9 +15,9 @@ OPT_USE_INTERVALS=1
IOSTAT=$( which iostat 2>/dev/null )
if [ $? -ne 0 ]; then
print_separator
WARNLOG "iostat not available"
tools_cleanup 0
print_separator
WARNLOG "iostat not available"
tools_cleanup 0
fi
# MAIN Program:

42
tools/engtools/hostdata-collectors/scripts/linux_benchmark.sh
View File

@ -29,8 +29,7 @@ compute_ports=(8000 8001 8002)
traffic_types=(storage migration default drbd)
flow_ids=(1:20 1:30 1:40 1:50)
function exec_cmd ()
{
function exec_cmd {
node="$1"
cmd="$2"
@ -41,8 +40,7 @@ function exec_cmd ()
fi
}
function iperf3_server_start ()
{
function iperf3_server_start {
local server="$1"
local result="$2"
local port="$3"
@ -55,8 +53,7 @@ function iperf3_server_start ()
$(exec_cmd "${server}" "${cmd}")
}
function iperf3_client_tcp_start ()
{
function iperf3_client_tcp_start {
local result="${result_dir}/throughput"
local cmd=""
local client="$1"
@ -76,8 +73,7 @@ function iperf3_client_tcp_start ()
$(exec_cmd "${client}" "${cmd} > ${result} 2>&1")
}
function iperf3_client_udp_start ()
{
function iperf3_client_udp_start {
local result="${result_dir}/throughput_udp"
local cmd=""
local client="$1"
@ -102,20 +98,17 @@ function iperf3_client_udp_start ()
$(exec_cmd "${client}" "${cmd} -b ${bw} >> ${result} 2>&1" )
}
function iperf3_stop ()
{
function iperf3_stop {
local node="$1"
local cmd="pkill iperf3"
$(exec_cmd "${node}" "${cmd}")
}
function get_ip_addr ()
{
function get_ip_addr {
arp -a | grep -oP "(?<=$1 \()[^)]*" | head -n 1
}
function throughput_tcp_test()
{
function throughput_tcp_test {
for (( i = 0; i < ${#nodes[@]} ; i+=2 )); do
for interface in "${interfaces[@]}"; do
local interface_name="management"
@ -137,8 +130,7 @@ function throughput_tcp_test()
done
}
function throughput_udp_test ()
{
function throughput_udp_test {
for (( i = 0; i < ${#nodes[@]} ; i+=2 )); do
for interface in "${interfaces[@]}"; do
local interface_name="management"
@ -229,8 +221,7 @@ function throughput_udp_test ()
done
}
function throughput_parallel_test ()
{
function throughput_parallel_test {
local dev=""
local ip_addr=""
local interface_name=""
@ -304,8 +295,7 @@ function throughput_parallel_test ()
done
}
function latency_test ()
{
function latency_test {
for (( i = 0; i < ${#nodes[@]} ; i+=2 )); do
for interface in "${interfaces[@]}"; do
local interface_name="management"
@ -328,16 +318,14 @@ function latency_test ()
done
}
function setup ()
{
function setup {
for node in ${nodes[@]}; do
iperf3_stop "${node}"
$(exec_cmd "${node}" "rm -rf ${result_dir}; mkdir -p ${result_dir}")
done
}
function get_remote_results ()
{
function get_remote_results {
for node in ${nodes[@]}; do
if [ "${node}" != "${host}" ]; then
mkdir ${result_dir}/${node}
@ -346,8 +334,7 @@ function get_remote_results ()
done
}
function get_interface_info ()
{
function get_interface_info {
local dev=""
local ip_addr=""
printf "Network interfaces info\n" >> ${summary_file}
@ -365,8 +352,7 @@ function get_interface_info ()
done
}
function generate_summary ()
{
function generate_summary {
local header=""
local result=""
local result_file=""

90
tools/engtools/hostdata-collectors/scripts/memstats.sh
View File

@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
tools_init
if [ $? -ne 0 ]; then
echo "FATAL, tools_init - could not setup environment"
exit $?
echo "FATAL, tools_init - could not setup environment"
exit $?
fi
PAGE_SIZE=$(getconf PAGE_SIZE)
@ -16,69 +16,68 @@ PAGE_SIZE=$(getconf PAGE_SIZE)
OPT_USE_INTERVALS=1
# Print key networking device statistics
function print_memory()
{
function print_memory {
# Configuration for netcmds
MEMINFO=/proc/meminfo
NODEINFO=/sys/devices/system/node/node?/meminfo
BUDDYINFO=/proc/buddyinfo
SLABINFO=/proc/slabinfo
MEMINFO=/proc/meminfo
NODEINFO=/sys/devices/system/node/node?/meminfo
BUDDYINFO=/proc/buddyinfo
SLABINFO=/proc/slabinfo
print_separator
TOOL_HIRES_TIME
print_separator
TOOL_HIRES_TIME
${ECHO} "# ${MEMINFO}"
${CAT} ${MEMINFO}
${ECHO}
${ECHO} "# ${MEMINFO}"
${CAT} ${MEMINFO}
${ECHO}
${ECHO} "# ${NODEINFO}"
${CAT} ${NODEINFO}
${ECHO}
${ECHO} "# ${NODEINFO}"
${CAT} ${NODEINFO}
${ECHO}
${ECHO} "# ${BUDDYINFO}"
${CAT} ${BUDDYINFO}
${ECHO}
${ECHO} "# ${BUDDYINFO}"
${CAT} ${BUDDYINFO}
${ECHO}
${ECHO} "# PSS"
cat /proc/*/smaps 2>/dev/null | \
${ECHO} "# PSS"
cat /proc/*/smaps 2>/dev/null | \
awk '/^Pss:/ {a += $2;} END {printf "%d MiB\n", a/1024.0;}'
${ECHO}
${ECHO}
# use old slabinfo format (i.e. slub not enabled in kernel)
${ECHO} "# ${SLABINFO}"
${CAT} ${SLABINFO} | \
${ECHO} "# ${SLABINFO}"
${CAT} ${SLABINFO} | \
awk -v page_size_B=${PAGE_SIZE} '
BEGIN {page_KiB = page_size_B/1024; TOT_KiB = 0;}
(NF == 17) {
gsub(/[<>]/, "");
printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n",
gsub(/[<>]/, "");
printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n",
$2, $3, $4, $5, $6, $7, $8, $10, $11, $12, $13, $15, $16, $17, "KiB");
}
(NF == 16) {
num_objs=$3; obj_per_slab=$5; pages_per_slab=$6;
KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0;
TOT_KiB += KiB;
printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n",
num_objs=$3; obj_per_slab=$5; pages_per_slab=$6;
KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0;
TOT_KiB += KiB;
printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n",
$1, $2, $3, $4, $5, $6, $7, $9, $10, $11, $12, $14, $15, $16, KiB);
}
END {
printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n",
printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n",
"TOTAL", "-", "-", "-", "-", "-", ":", "-", "-", "-", ":", "-", "-", "-", TOT_KiB);
}
' 2>/dev/null
${ECHO}
${ECHO}
${ECHO} "# disk usage: rootfs, tmpfs"
cmd='df -h -H -T --local -t rootfs -t tmpfs'
${ECHO} "Disk space usage rootfs,tmpfs (SI):"
${ECHO} "${cmd}"
${cmd}
${ECHO}
${ECHO} "# disk usage: rootfs, tmpfs"
cmd='df -h -H -T --local -t rootfs -t tmpfs'
${ECHO} "Disk space usage rootfs,tmpfs (SI):"
${ECHO} "${cmd}"
${cmd}
${ECHO}
CMD='ps -e -o ppid,pid,nlwp,rss:10,vsz:10,cmd --sort=-rss'
${ECHO} "# ${CMD}"
${CMD}
${ECHO}
CMD='ps -e -o ppid,pid,nlwp,rss:10,vsz:10,cmd --sort=-rss'
${ECHO} "# ${CMD}"
${CMD}
${ECHO}
}
#-------------------------------------------------------------------------------
@ -99,10 +98,9 @@ tools_header
# Calculate number of sample repeats based on overall interval and sampling interval
((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC))
for ((rep=1; rep <= REPEATS ; rep++))
do
print_memory
sleep ${INTERVAL_SEC}
for ((rep=1; rep <= REPEATS ; rep++)); do
print_memory
sleep ${INTERVAL_SEC}
done
print_memory
LOG "done"

39
tools/engtools/hostdata-collectors/scripts/netstats.sh
View File

@ -6,33 +6,31 @@ TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
tools_init
if [ $? -ne 0 ]; then
echo "FATAL, tools_init - could not setup environment"
exit $?
echo "FATAL, tools_init - could not setup environment"
exit $?
fi
# Enable use of INTERVAL_SEC sample interval
OPT_USE_INTERVALS=1
# Print key networking device statistics
function print_netcmds()
{
function print_netcmds {
# Configuration for netcmds
DEV=/proc/net/dev
NETSTAT=/proc/net/netstat
DEV=/proc/net/dev
NETSTAT=/proc/net/netstat
print_separator
TOOL_HIRES_TIME
print_separator
TOOL_HIRES_TIME
for net in \
for net in \
${DEV} ${NETSTAT}
do
if [ -e "${net}" ]
then
${ECHO} "# ${net}"
${CAT} ${net}
${ECHO}
fi
done
do
if [ -e "${net}" ]; then
${ECHO} "# ${net}"
${CAT} ${net}
${ECHO}
fi
done
}
#-------------------------------------------------------------------------------
@ -53,10 +51,9 @@ tools_header
# Calculate number of sample repeats based on overall interval and sampling interval
((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC))
for ((rep=1; rep <= REPEATS ; rep++))
do
print_netcmds
sleep ${INTERVAL_SEC}
for ((rep=1; rep <= REPEATS ; rep++)); do
print_netcmds
sleep ${INTERVAL_SEC}
done
print_netcmds
LOG "done"

73
tools/engtools/hostdata-collectors/scripts/postgres.sh
View File

@ -6,42 +6,40 @@ TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
tools_init
if [ $? -ne 0 ]; then
echo "FATAL, tools_init - could not setup environment"
exit $?
echo "FATAL, tools_init - could not setup environment"
exit $?
fi
# Enable use of INTERVAL_SEC sample interval
OPT_USE_INTERVALS=1
# Print key networking device statistics
function print_postgres()
{
print_separator
TOOL_HIRES_TIME
function print_postgres {
print_separator
TOOL_HIRES_TIME
# postgressql command: set user, disable pagination, and be quiet
PSQL="sudo -u postgres psql --pset pager=off -q"
PSQL="sudo -u postgres psql --pset pager=off -q"
# List postgres databases
db_list=( $(${PSQL} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") )
${ECHO} "# postgres databases"
echo "db_list = ${db_list[@]}"
${ECHO}
db_list=( $(${PSQL} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") )
${ECHO} "# postgres databases"
echo "db_list = ${db_list[@]}"
${ECHO}
# List sizes of all postgres databases (similar to "\l+")
${ECHO} "# postgres database sizes"
${PSQL} -c "
${ECHO} "# postgres database sizes"
${PSQL} -c "
SELECT
pg_database.datname,
pg_database_size(pg_database.datname),
pg_size_pretty(pg_database_size(pg_database.datname))
pg_database.datname,
pg_database_size(pg_database.datname),
pg_size_pretty(pg_database_size(pg_database.datname))
FROM pg_database
ORDER BY pg_database_size DESC;
"
# For each database, list tables and their sizes (similar to "\dt+")
for db in "${db_list[@]}"
do
for db in "${db_list[@]}"; do
${ECHO} "# postgres database: ${db}"
${PSQL} -d ${db} -c "
SELECT
@ -85,29 +83,29 @@ SELECT
last_autoanalyze
FROM pg_stat_user_tables;
"
done
done
# Specific table counts (This is very SLOW, look at "live tuples" instead)
# Number of keystone tokens
#${ECHO} "# keystone token count"
# Number of postgres connections
${ECHO} "# postgres database connections"
CONN=$(ps -C postgres -o cmd= | wc -l)
CONN_T=$(ps -C postgres -o cmd= | awk '/postgres: / {print $3}' | awk '{for(i=1;i<=NF;i++) a[$i]++} END {for(k in a) print k, a[k]}' | sort -k 2 -nr )
${ECHO} "connections total = ${CONN}"
${ECHO}
${ECHO} "connections breakdown:"
${ECHO} "${CONN_T}"
${ECHO}
${ECHO} "# postgres database connections"
CONN=$(ps -C postgres -o cmd= | wc -l)
CONN_T=$(ps -C postgres -o cmd= | awk '/postgres: / {print $3}' | awk '{for(i=1;i<=NF;i++) a[$i]++} END {for(k in a) print k, a[k]}' | sort -k 2 -nr )
${ECHO} "connections total = ${CONN}"
${ECHO}
${ECHO} "connections breakdown:"
${ECHO} "${CONN_T}"
${ECHO}
${ECHO} "connections breakdown (query):"
${PSQL} -c "SELECT datname,state,count(*) from pg_stat_activity group by datname,state;"
${ECHO}
${ECHO} "connections breakdown (query):"
${PSQL} -c "SELECT datname,state,count(*) from pg_stat_activity group by datname,state;"
${ECHO}
${ECHO} "connections idle age:"
${PSQL} -c "SELECT datname,age(now(),state_change) from pg_stat_activity where state='idle';"
${ECHO}
${ECHO} "connections idle age:"
${PSQL} -c "SELECT datname,age(now(),state_change) from pg_stat_activity where state='idle';"
${ECHO}
}
#-------------------------------------------------------------------------------
@ -128,10 +126,9 @@ tools_header
# Calculate number of sample repeats based on overall interval and sampling interval
((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC))
for ((rep=1; rep <= REPEATS ; rep++))
do
print_postgres
sleep ${INTERVAL_SEC}
for ((rep=1; rep <= REPEATS ; rep++)); do
print_postgres
sleep ${INTERVAL_SEC}
done
print_postgres
LOG "done"

56
tools/engtools/hostdata-collectors/scripts/rabbitmq.sh
View File

@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
tools_init
if [ $? -ne 0 ]; then
echo "FATAL, tools_init - could not setup environment"
exit $?
echo "FATAL, tools_init - could not setup environment"
exit $?
fi
# Enable use of INTERVAL_SEC sample interval
@ -15,10 +15,9 @@ OPT_USE_INTERVALS=1
#Need this workaround
MQOPT="-n rabbit@localhost"
# Print key networking device statistics
function print_rabbitmq()
{
print_separator
TOOL_HIRES_TIME
function print_rabbitmq {
print_separator
TOOL_HIRES_TIME
# IMPORTANT:
# - Difficulty getting rabbitmqctl to work from init.d script;
@ -27,31 +26,31 @@ function print_rabbitmq()
# - WORKAROUND: run command using 'sudo', even if you are 'root'
# Dump various rabbitmq related stats
MQ_STATUS="rabbitmqctl ${MQOPT} status"
${ECHO} "# ${MQ_STATUS}"
sudo ${MQ_STATUS} | grep -e '{memory' -A30
${ECHO}
MQ_STATUS="rabbitmqctl ${MQOPT} status"
${ECHO} "# ${MQ_STATUS}"
sudo ${MQ_STATUS} | grep -e '{memory' -A30
${ECHO}
# THe following is useful in diagnosing rabbit memory leaks
# when end-users do not drain their queues (eg, due to RPC timeout issues, etc)
MQ_QUEUES="rabbitmqctl ${MQOPT} list_queues messages name pid messages_ready messages_unacknowledged memory consumers"
${ECHO} "# ${MQ_QUEUES}"
sudo ${MQ_QUEUES}
${ECHO}
MQ_QUEUES="rabbitmqctl ${MQOPT} list_queues messages name pid messages_ready messages_unacknowledged memory consumers"
${ECHO} "# ${MQ_QUEUES}"
sudo ${MQ_QUEUES}
${ECHO}
num_queues=$(sudo rabbitmqctl ${MQOPT} list_queues | wc -l); ((num_queues-=2))
num_bindings=$(sudo rabbitmqctl ${MQOPT} list_bindings | wc -l); ((num_bindings-=2))
num_exchanges=$(sudo rabbitmqctl ${MQOPT} list_exchanges | wc -l); ((num_exchanges-=2))
num_connections=$(sudo rabbitmqctl ${MQOPT} list_connections | wc -l); ((num_connections-=2))
num_channels=$(sudo rabbitmqctl ${MQOPT} list_channels | wc -l); ((num_channels-=2))
arr=($(sudo rabbitmqctl ${MQOPT} list_queues messages consumers memory | \
awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}'))
messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]}
printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" \
num_queues=$(sudo rabbitmqctl ${MQOPT} list_queues | wc -l); ((num_queues-=2))
num_bindings=$(sudo rabbitmqctl ${MQOPT} list_bindings | wc -l); ((num_bindings-=2))
num_exchanges=$(sudo rabbitmqctl ${MQOPT} list_exchanges | wc -l); ((num_exchanges-=2))
num_connections=$(sudo rabbitmqctl ${MQOPT} list_connections | wc -l); ((num_connections-=2))
num_channels=$(sudo rabbitmqctl ${MQOPT} list_channels | wc -l); ((num_channels-=2))
arr=($(sudo rabbitmqctl ${MQOPT} list_queues messages consumers memory | \
awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}'))
messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]}
printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" \
"queues" "bindings" "exchanges" "connections" "channels" "messages" "consumers" "memory"
printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" \
printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" \
$num_queues $num_bindings $num_exchanges $num_connections $num_channels $messages $consumers $memory
${ECHO}
${ECHO}
}
#-------------------------------------------------------------------------------
@ -72,10 +71,9 @@ tools_header
# Calculate number of sample repeats based on overall interval and sampling interval
((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC))
for ((rep=1; rep <= REPEATS ; rep++))
do
print_rabbitmq
sleep ${INTERVAL_SEC}
for ((rep=1; rep <= REPEATS ; rep++)); do
print_rabbitmq
sleep ${INTERVAL_SEC}
done
print_rabbitmq
LOG "done"

36
tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh
View File

@ -6,8 +6,8 @@
TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
if [ $UID -eq 0 ]; then
ERRLOG "Do not start $0 using sudo/root access."
exit 1
ERRLOG "Do not start $0 using sudo/root access."
exit 1
fi
# environment for system commands
@ -22,24 +22,24 @@ STORAGE=( $(system host-list | awk '(/storage/) {print $4;}') )
LOG "Remote bzip2 engtools data on all blades:"
for blade in ${CONTROLLER[@]}; do
ping -c1 ${blade} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "bzip2 on $blade:"
ssh -q -t -o StrictHostKeyChecking=no \
${blade} sudo bzip2 /scratch/syseng_data/${blade}/*
else
WARNLOG "cannot ping: ${blade}"
fi
ping -c1 ${blade} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "bzip2 on $blade:"
ssh -q -t -o StrictHostKeyChecking=no \
${blade} sudo bzip2 /scratch/syseng_data/${blade}/*
else
WARNLOG "cannot ping: ${blade}"
fi
done
for blade in ${STORAGE[@]} ${COMPUTE[@]} ; do
ping -c1 ${blade} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "bzip2 on $blade:"
ssh -q -t -o StrictHostKeyChecking=no \
${blade} sudo bzip2 /tmp/syseng_data/${blade}/*
else
WARNLOG "cannot ping: ${blade}"
fi
ping -c1 ${blade} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "bzip2 on $blade:"
ssh -q -t -o StrictHostKeyChecking=no \
${blade} sudo bzip2 /tmp/syseng_data/${blade}/*
else
WARNLOG "cannot ping: ${blade}"
fi
done
LOG "done"

26
tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh
View File

@ -6,8 +6,8 @@
TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
if [ $UID -eq 0 ]; then
ERRLOG "Do not start $0 using sudo/root access."
exit 1
ERRLOG "Do not start $0 using sudo/root access."
exit 1
fi
# environment for system commands
@ -18,19 +18,19 @@ BLADES=( $(system host-list | awk '(/compute|controller|storage/) {print $4;}')
LOG "Remote start engtools on all blades:"
for blade in ${BLADES[@]}; do
if [ "${blade}" == "${HOSTNAME}" ]; then
LOG "start on $blade:"
sudo service collect-engtools.sh start
else
ping -c1 ${blade} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "start on $blade:"
ssh -q -t -o StrictHostKeyChecking=no \
${blade} sudo service collect-engtools.sh start
if [ "${blade}" == "${HOSTNAME}" ]; then
LOG "start on $blade:"
sudo service collect-engtools.sh start
else
WARNLOG "cannot ping: ${blade}"
ping -c1 ${blade} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "start on $blade:"
ssh -q -t -o StrictHostKeyChecking=no \
${blade} sudo service collect-engtools.sh start
else
WARNLOG "cannot ping: ${blade}"
fi
fi
fi
done
LOG "done"

26
tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh
View File

@ -6,8 +6,8 @@
TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
if [ $UID -eq 0 ]; then
ERRLOG "Do not start $0 using sudo/root access."
exit 1
ERRLOG "Do not start $0 using sudo/root access."
exit 1
fi
# environment for system commands
@ -18,19 +18,19 @@ BLADES=( $(system host-list | awk '(/compute|controller|storage/) {print $4;}')
LOG "Remote stop engtools on all blades:"
for blade in ${BLADES[@]}; do
if [ "${blade}" == "${HOSTNAME}" ]; then
LOG "stop on $blade:"
sudo service collect-engtools.sh stop
else
ping -c1 ${blade} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "stop on $blade:"
ssh -q -t -o StrictHostKeyChecking=no \
${blade} sudo service collect-engtools.sh stop
if [ "${blade}" == "${HOSTNAME}" ]; then
LOG "stop on $blade:"
sudo service collect-engtools.sh stop
else
WARNLOG "cannot ping: ${blade}"
ping -c1 ${blade} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "stop on $blade:"
ssh -q -t -o StrictHostKeyChecking=no \
${blade} sudo service collect-engtools.sh stop
else
WARNLOG "cannot ping: ${blade}"
fi
fi
fi
done
LOG "done"

44
tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh
View File

@ -6,8 +6,8 @@
TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
if [ $UID -eq 0 ]; then
ERRLOG "Do not start $0 using sudo/root access."
exit 1
ERRLOG "Do not start $0 using sudo/root access."
exit 1
fi
# environment for system commands
@ -24,10 +24,10 @@ STORAGE=( $(system host-list | awk '(/storage/) {print $4;}') )
DEST=/opt/backups/syseng_data/
if [[ "${HOSTNAME}" =~ "controller-" ]]; then
LOG "rsync DEST=${DEST}"
LOG "rsync DEST=${DEST}"
else
LOG "*ERROR* only run this on controller"
exit 1
LOG "*ERROR* only run this on controller"
exit 1
fi
sudo mkdir -p ${DEST}
@ -41,29 +41,27 @@ LOG "rsync engtools data from all blades:"
# controllers
SRC=/scratch/syseng_data/
DEST=/opt/backups/syseng_data/
for HOST in ${CONTROLLER[@]}
do
ping -c1 ${HOST} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}"
sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}
else
WARNLOG "cannot ping: ${HOST}"
fi
for HOST in ${CONTROLLER[@]}; do
ping -c1 ${HOST} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}"
sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}
else
WARNLOG "cannot ping: ${HOST}"
fi
done
# computes & storage
SRC=/tmp/syseng_data/
DEST=/opt/backups/syseng_data/
for HOST in ${STORAGE[@]} ${COMPUTE[@]}
do
ping -c1 ${HOST} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}"
sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}
else
WARNLOG "cannot ping: ${HOST}"
fi
for HOST in ${STORAGE[@]} ${COMPUTE[@]}; do
ping -c1 ${HOST} 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}"
sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}
else
WARNLOG "cannot ping: ${HOST}"
fi
done
LOG 'done'

14
tools/engtools/hostdata-collectors/scripts/slab.sh
View File

@ -3,19 +3,19 @@ PAGE_SIZE=$(getconf PAGE_SIZE)
cat /proc/slabinfo | awk -v page_size_B=${PAGE_SIZE} '
BEGIN {page_KiB = page_size_B/1024; TOT_KiB = 0;}
(NF == 17) {
gsub(/[<>]/, "");
printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n",
gsub(/[<>]/, "");
printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n",
$2, $3, $4, $5, $6, $7, $8, $10, $11, $12, $13, $15, $16, $17, "KiB");
}
(NF == 16) {
num_objs=$3; obj_per_slab=$5; pages_per_slab=$6;
KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0;
TOT_KiB += KiB;
printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n",
num_objs=$3; obj_per_slab=$5; pages_per_slab=$6;
KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0;
TOT_KiB += KiB;
printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n",
$1, $2, $3, $4, $5, $6, $7, $9, $10, $11, $12, $14, $15, $16, KiB);
}
END {
printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n",
printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n",
"TOTAL", "-", "-", "-", "-", "-", ":", "-", "-", "-", ":", "-", "-", "-", TOT_KiB);
}
' 2>/dev/null

25
tools/engtools/hostdata-collectors/scripts/ticker.sh
View File

@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
tools_init
if [ $? -ne 0 ]; then
echo "FATAL, tools_init - could not setup environment"
exit $?
echo "FATAL, tools_init - could not setup environment"
exit $?
fi
# Enable use of INTERVAL_SEC sample interval
@ -29,17 +29,16 @@ set_affinity ${CPULIST}
LOG_NOCR "collecting "
t=0
for ((rep=1; rep <= REPEATS ; rep++))
do
((t++))
sleep ${INTERVAL_SEC}
if [ ${t} -ge ${REP_LOG} ]; then
t=0
echo "."
LOG_NOCR "collecting "
else
echo -n "."
fi
for ((rep=1; rep <= REPEATS ; rep++)); do
((t++))
sleep ${INTERVAL_SEC}
if [ ${t} -ge ${REP_LOG} ]; then
t=0
echo "."
LOG_NOCR "collecting "
else
echo -n "."
fi
done
echo "."

4
tools/engtools/hostdata-collectors/scripts/top.sh
View File

@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
tools_init
if [ $? -ne 0 ]; then
echo "FATAL, tools_init - could not setup environment"
exit $?
echo "FATAL, tools_init - could not setup environment"
exit $?
fi
# Enable use of INTERVAL_SEC sample interval

50
tools/engtools/hostdata-collectors/scripts/vswitch.sh
View File

@ -6,35 +6,34 @@ TOOLBIN=$(dirname $0)
. ${TOOLBIN}/engtools_util.sh
tools_init
if [ $? -ne 0 ]; then
echo "FATAL, tools_init - could not setup environment"
exit $?
echo "FATAL, tools_init - could not setup environment"
exit $?
fi
# Enable use of INTERVAL_SEC sample interval
OPT_USE_INTERVALS=1
# Print key networking device statistics
function print_vswitch()
{
print_separator
TOOL_HIRES_TIME
function print_vswitch {
print_separator
TOOL_HIRES_TIME
cmd='vshell engine-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell engine-stats-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell port-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell port-stats-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell network-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell network-stats-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell interface-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell interface-stats-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell engine-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell engine-stats-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell port-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell port-stats-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell network-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell network-stats-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell interface-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
cmd='vshell interface-stats-list'
${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO}
}
#-------------------------------------------------------------------------------
@ -55,10 +54,9 @@ tools_header
# Calculate number of sample repeats based on overall interval and sampling interval
((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC))
for ((rep=1; rep <= REPEATS ; rep++))
do
print_vswitch
sleep ${INTERVAL_SEC}
for ((rep=1; rep <= REPEATS ; rep++)); do
print_vswitch
sleep ${INTERVAL_SEC}
done
print_vswitch
LOG "done"

15
tools/engtools/parsers/common/cleanup-uncompressed.sh
View File

@ -5,15 +5,15 @@
#SPDX-License-Identifier: Apache-2.0
#
# This script removes uncompressed file. It can save a huge amount of disk space
# on the analysis server. Run this script after the very last time the data is parsed
# and BEFORE running parse-daily.sh script.
# If it is run after each intermediary parse, the download-data.sh script will download the
# on the analysis server. Run this script after the very last time the data is parsed
# and BEFORE running parse-daily.sh script.
# If it is run after each intermediary parse, the download-data.sh script will download the
# uncompressed files again.
if [ ! -f lab.conf ]; then
echo "Lab configuration file is missing."
echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info."
exit 1
echo "Lab configuration file is missing."
echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info."
exit 1
fi
source ./lab.conf
@ -22,8 +22,7 @@ YEAR=`date +'%Y'`
files="${FILE_LIST// /, }"
read -p "Are you sure you want to remove all uncompressed $files files? [Y/N]: " -n 1 -r
echo
if [[ $REPLY =~ ^[Y]$ ]]
then
if [[ $REPLY =~ ^[Y]$ ]]; then
for FILE in ${FILE_LIST}; do
rm -v */*_${YEAR}-*${FILE}
done

8
tools/engtools/parsers/common/download-data.sh
View File

@ -14,9 +14,9 @@
# This script is to be run after running download-computes.sh on one of the controllers.
if [ ! -f lab.conf ]; then
echo "Lab configuration file is missing."
echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info."
exit 1
echo "Lab configuration file is missing."
echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info."
exit 1
fi
source ./lab.conf
@ -38,6 +38,6 @@ for HOST in ${ALL_HOSTS}; do
bzip2 ${HOST}*
cd ${CURDIR}
else
echo "${HOST} not found"
echo "${HOST} not found"
fi
done

116
tools/engtools/parsers/common/parse-all.sh
View File

@ -8,7 +8,7 @@
# This script is used to parse all stats data. It is designed to be called by either
# parse-controllers.sh or parse-computes.sh and not used as a standalone script.
# If the input node is a controller, it will parse controller specific postgres &
# and rabbitmq stats first. If the input node is a compute, it will pars the compute
# and rabbitmq stats first. If the input node is a compute, it will pars the compute
# specific vswitch stats first.
#
# The following parsing steps are common to all hosts and are executed in the specified order:
@ -22,12 +22,12 @@
# - Parse filestats (summary)
# - Parse process level schedtop (optional step, configured in lab.conf)
# - Generate tarball
if [[ $# != 1 ]]; then
echo "ERROR: This script is meant to be called by either parse-controllers.sh or parse-computes.sh script."
echo "To run it separately, copy the script to the host directory that contains *.bz2 files."
echo "It takes a single argument - the name of the host directory (e.g. ./parse-all.sh controller-0)."
exit 1
echo "ERROR: This script is meant to be called by either parse-controllers.sh or parse-computes.sh script."
echo "To run it separately, copy the script to the host directory that contains *.bz2 files."
echo "It takes a single argument - the name of the host directory (e.g. ./parse-all.sh controller-0)."
exit 1
fi
source ../lab.conf
@ -41,17 +41,15 @@ NODE=$1
CURDATE=$(date)
DATESTAMP=$(date +%b-%d)
function sedit()
{
function sedit {
local FILETOSED=$1
sed -i -e "s/ */ /g" ${FILETOSED}
sed -i -e "s/ /,/g" ${FILETOSED}
# Remove any trailing comma
sed -i "s/,$//" ${FILETOSED}
}
function get_filename_from_mountname()
{
function get_filename_from_mountname {
local name=$1
local fname
if test "${name#*"scratch"}" != "${name}"; then
@ -82,8 +80,7 @@ function get_filename_from_mountname()
echo $fname
}
function parse_process_schedtop_data()
{
function parse_process_schedtop_data {
# Logic has been moved to a separate script so that parsing process level schedtop
# can be run either as part of parse-all.sh script or independently.
LOG "Process level schedtop parsing is turned on in lab.conf. Parsing schedtop detail..."
@ -92,8 +89,7 @@ function parse_process_schedtop_data()
cd ${NODE}
}
function parse_controller_specific()
{
function parse_controller_specific {
# Parsing Postgres data, removing data from previous run if there are any. Generate summary
# data for each database and detail data for specified tables
LOG "Parsing postgres data for ${NODE}"
@ -123,14 +119,12 @@ function parse_controller_specific()
done
}
function parse_compute_specific()
{
function parse_compute_specific {
LOG "Parsing vswitch data for ${NODE}"
../parse-vswitch.sh ${NODE}
}
function parse_occtop_data()
{
function parse_occtop_data {
LOG "Parsing occtop data for ${NODE}"
bzcat *occtop.bz2 >occtop-${NODE}-${DATESTAMP}.txt
cp occtop-${NODE}-${DATESTAMP}.txt tmp.txt
@ -153,7 +147,7 @@ function parse_occtop_data()
echo "${header}" > occtop-${NODE}-detailed.csv
cat tmp2.txt >> occtop-${NODE}-detailed.csv
# Generate simple CSV file which is used to generate host CPU occupancy chart. Platform cores are
# Generate simple CSV file which is used to generate host CPU occupancy chart. Platform cores are
# defined in the host.conf. The simple CSV contains only the Date/Time and Total platform CPU occupancy e.g.
# Date/Time,Total
# 2016-11-22 00:29:16.523,94.9
@ -181,13 +175,12 @@ function parse_occtop_data()
rm tmp.txt tmp2.txt tmpdate.txt tmpcore.txt
}
function parse_memtop_data()
{
function parse_memtop_data {
LOG "Parsing memtop data for ${NODE}"
bzcat *memtop.bz2 > memtop-${NODE}-${DATESTAMP}.txt
cp memtop-${NODE}-${DATESTAMP}.txt tmp.txt
sedit tmp.txt
# After dumping all memtop bz2 output into one text file and in-place sed, grab only relevant data
# for CSV output. Generate both detailed and simple CSV files. Simple output will be used to generate
# chart.
@ -200,8 +193,7 @@ function parse_memtop_data()
rm tmp.txt tmp2.txt
}
function parse_netstats_data()
{
function parse_netstats_data {
LOG "Parsing netstats data for ${NODE}"
# First generate the summary data then detail data for specified interfaces
../parse_netstats *netstats.bz2 > netstats-summary-${NODE}-${DATESTAMP}.txt
@ -222,11 +214,10 @@ function parse_netstats_data()
done < tmp.txt
done
rm tmp.txt
fi
fi
}
function parse_iostats_data()
{
function parse_iostats_data {
LOG "Parsing iostat data for ${NODE}"
if [ -z "${IOSTATS_DEVICE_LIST}" ]; then
ERRLOG "IOSTAT_DEVICE_LIST is not set in host.conf. Skipping iostats..."
@ -236,30 +227,29 @@ function parse_iostats_data()
echo "Date/Time,${DEVICE},rqm/s,wrqm/s,r/s,w/s,rkB/s,wkB/s,avgrq-sz,avgqu-sz,await,r_await,w_await,svctm,%util" > iostat-${NODE}-${DEVICE}.csv
# Dumping iostat content to tmp file
bzcat *iostat.bz2 | grep -E "/2015|/2016|/2017|${DEVICE}" | awk '{print $1","$2","$3","$4","$5","$6","$7","$8","$9","$10","$11","$12","$13","$14}' > tmp.txt
while IFS= read -r current
do
if test "${current#*Linux}" != "$current"
then
# Skip the line that contains the word "Linux"
continue
else
if test "${current#*$DEVICE}" == "$current"
then
# It's a date entry, look ahead
read -r next
if test "${next#*$DEVICE}" != "${next}"
then
# This next line contains the device stats
# Combine date and time fields
current="${current//2016,/2016 }"
current="${current//2017,/2017 }"
# Combine time and AM/PM fields
current="${current//,AM/ AM}"
current="${current//,PM/ PM}"
# Write both lines to intermediate file
echo "${current}" >> tmp2.txt
echo "${next}" >> tmp2.txt
fi
while IFS= read -r current; do
if test "${current#*Linux}" != "$current"
then
# Skip the line that contains the word "Linux"
continue
else
if test "${current#*$DEVICE}" == "$current"
then
# It's a date entry, look ahead
read -r next
if test "${next#*$DEVICE}" != "${next}"
then
# This next line contains the device stats
# Combine date and time fields
current="${current//2016,/2016 }"
current="${current//2017,/2017 }"
# Combine time and AM/PM fields
current="${current//,AM/ AM}"
current="${current//,PM/ PM}"
# Write both lines to intermediate file
echo "${current}" >> tmp2.txt
echo "${next}" >> tmp2.txt
fi
fi
fi
done < tmp.txt
@ -272,13 +262,12 @@ function parse_iostats_data()
cut -d, -f2-11 --complement tmp2.txt > tmp.txt
# Write final content to output csv
cat tmp.txt >> iostat-${NODE}-${DEVICE}.csv
rm tmp.txt tmp2.txt
rm tmp.txt tmp2.txt
done
fi
}
function parse_diskstats_data()
{
function parse_diskstats_data {
LOG "Parsing diskstats data for ${NODE}"
if [ -z "${DISKSTATS_FILESYSTEM_LIST}" ]; then
@ -317,9 +306,9 @@ parse_occtop_data
# Parsing memtop data
parse_memtop_data
# Parsing memstats data to generate the high level report. The most important piece of info is the list of
# Parsing memstats data to generate the high level report. The most important piece of info is the list of
# hi-runners at the end of the file. If there is a leak, run parse-daily.sh script to generate the time
# series data for the offending processes only. Use process name, not PID as most Titanium Cloud processes have
# series data for the offending processes only. Use process name, not PID as most Titanium Cloud processes have
# workers.
LOG "Parsing memstats summary for ${NODE}"
../parse_memstats --report *memstats.bz2 > memstats-summary-${NODE}-${DATESTAMP}.txt
@ -331,7 +320,7 @@ rm pid-*.csv
parse_netstats_data
# Parsing schedtop data to generate the high level report. Leave the process level schedtop parsing till
# the end as it is a long running task.
# the end as it is a long running task.
LOG "Parsing schedtop summary for ${NODE}"
FILES=$(ls *schedtop.bz2)
../parse_schedtop ${FILES} > schedtop-summary-${NODE}-${DATESTAMP}.txt
@ -342,17 +331,17 @@ parse_iostats_data
# Parsing diskstats data
parse_diskstats_data
# Parsing filestats data to generate the high level report. If there is a file descriptor leak, run parse-daily.sh
# script to generate the time series data for the offending processes only. Use process name, not PID as most
# Parsing filestats data to generate the high level report. If there is a file descriptor leak, run parse-daily.sh
# script to generate the time series data for the offending processes only. Use process name, not PID as most
# Titanium Cloud processes have workers.
LOG "Parsing filestats summary for ${NODE}"
../parse_filestats --all *filestats.bz2 > filestats-summary-${NODE}-${DATESTAMP}.txt
../parse_filestats --all *filestats.bz2 > filestats-summary-${NODE}-${DATESTAMP}.txt
# Parsing process level schedtop data. This is a long running task. To skip this step or generate data for
# only specific processes, update the lab.conf and host.conf files.
[[ ${GENERATE_PROCESS_SCHEDTOP} == Y ]] && parse_process_schedtop_data || WARNLOG "Parsing process level schedtop is skipped."
# Done parsing for this host. If it's a controller host, check if the parsing of postgres connection stats which is run in
# Done parsing for this host. If it's a controller host, check if the parsing of postgres connection stats which is run in
# parallel is done before creating a tar file.
if test "${NODE#*"controller"}" != "${NODE}"; then
# If postgres-conns.csv file has not been created which is highly unlikely, wait a couple of minutes
@ -362,8 +351,7 @@ if test "${NODE#*"controller"}" != "${NODE}"; then
# is to use inotify which requires another inotify-tools package.
oldsize=0
newsize=0
while true
do
while true; do
newsize=$(stat -c %s postgres-conns.csv)
if [ "$oldsize" == "$newsize" ]; then
break

10
tools/engtools/parsers/common/parse-controllers.sh
View File

@ -14,9 +14,9 @@ PARSERDIR=$(dirname $0)
. ${PARSERDIR}/parse-util.sh
if [ ! -f lab.conf ]; then
echo "Lab configuration file is missing."
echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info."
exit 1
echo "Lab configuration file is missing."
echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info."
exit 1
fi
source ./lab.conf
@ -36,12 +36,12 @@ for HOST in ${CONTROLLER_LIST}; do
sleep 120
cd ..
else
ERRLOG "${HOST} does not exist. Parsing skipped."
ERRLOG "${HOST} does not exist. Parsing skipped."
fi
done
# Parsing postgres connection stats is a time consuming step, run it in parallel with parse-all
# script.
# script.
for HOST in ${CONTROLLER_LIST}; do
if [ -d ${HOST} ]; then
LOG "Parsing postgres connection stats data for controller host ${HOST}"

10
tools/engtools/parsers/common/parse-daily.sh
View File

@ -19,16 +19,14 @@
# e.g. >./parse-daily.sh memstats sm-eru
# >./parse-daily.sh controller-0 filestats postgress
function print_usage()
{
function print_usage {
echo "Usage: ./parse-daily.sh <parser-name> <process-name> will parse daily data for all hosts."
echo "Usage: ./parse-daily.sh <host-name> <parser-name> <process-name> will parse daily data for specified host."
echo "Valid parsers for daily stats are: memstats & filestats."
exit 1
}
function parse_daily_stats()
{
function parse_daily_stats {
local PARSER_NAME=$1
local PROCESS_NAME=$2
local TMPFILE="tmp.txt"
@ -102,8 +100,8 @@ elif [[ $# -eq 3 ]]; then
cd $1
parse_daily_stats $2 $3
else
echo "ERROR: Specified host $1 does not exist."
exit 1
echo "ERROR: Specified host $1 does not exist."
exit 1
fi
else
echo "Specified parser $2 is not a valid parser."

14
tools/engtools/parsers/common/parse-postgres.sh
View File

@ -25,8 +25,7 @@ else
fi
LOG "Parsing postgres files ${FILES}"
function generate_header()
{
function generate_header {
local header="Date/Time,Total"
for DB in ${DATABASE_LIST}; do
if [ ${DB} == "nova_api" ]; then
@ -59,18 +58,16 @@ function generate_header()
echo $header
}
function generate_grep_str()
{
function generate_grep_str {
local grepstr="time:"
for DB in ${DATABASE_LIST}; do
grepstr="${grepstr}|${DB}"
grepstr="${grepstr}|${DB}"
done
grepstr="${grepstr}|breakdown|connections total|rows"
echo $grepstr
}
function init_variables()
{
function init_variables {
CONN_TOTAL="0"
CONN_ACTIVE_TOTAL="0"
CONN_IDLE_TOTAL="0"
@ -85,8 +82,7 @@ function init_variables()
done
}
function output_values()
{
function output_values {
local result="${DATEVAL} ${TIMEVAL},${CONN_TOTAL}"
for DB in ${DATABASE_LIST}; do
val=$(eval echo \${CONN_${DB^^}})

2
tools/engtools/parsers/common/parse-rabbitmq.sh
View File

@ -8,7 +8,7 @@ OUTFILE=$1
FILES=$(ls *rabbitmq.bz2 | sort)
[ -e tmp.txt ] && rm tmp.txt
[ -e tmp.txt ] && rm tmp.txt
echo "Time/Date,Memory Total,Connection Readers,Connection Writers,Connection Channels,Connection Other,Queue Procs,Queue Slave Procs,Plugins,Other Proc,Mnesia,Mgmt DB,Msg Index,Other ETS,Binary,Code,Atom,Other System,VM Memory High Watermark,VM Memory Limit,Disk Free Limit,Disk Free,Descriptor Limit,Descriptors Used,Sockets Limit,Sockets Used,Processes Limit,Processes Used,Run Queue,Uptime" >${OUTFILE}

11
tools/engtools/parsers/common/parse-schedtop.sh
View File

@ -17,16 +17,14 @@
PARSERDIR=$(dirname $0)
. ${PARSERDIR}/parse-util.sh
function print_usage()
{
function print_usage {
echo "Usage: ./parse-schedtop.sh <host-name>"
echo " ./parse-schedtop.sh <host-name> <service-name>"
echo "e.g. >./parse-schedtop.sh controller-0 nova-conductor"
exit 1
}
function sedit()
{
function sedit {
local FILETOSED=$1
sed -i -e "s/ */ /g" ${FILETOSED}
sed -i -e "s/ /,/2g" ${FILETOSED}
@ -34,8 +32,7 @@ function sedit()
sed -i "s/,$//" ${FILETOSED}
}
function parse_schedtop_data()
{
function parse_schedtop_data {
HOST=$1
SERVICE=$2
LOG "Parsing ${SERVICE} schedtop for host ${HOST}"
@ -48,7 +45,7 @@ function parse_schedtop_data()
if [[ $# -eq 0 ]]; then
# Parsing detail schedtop stats for all services configured in host.conf for all hosts would
# take a very long time and is often unnecessary. Until the performance issue with parse_schedtop
# take a very long time and is often unnecessary. Until the performance issue with parse_schedtop
# is addressed, this is not supported.
print_usage
else

17
tools/engtools/parsers/common/parse-util.sh
View File

@ -8,19 +8,16 @@
#LOGFILE="${PARSERDIR}/parserlog.txt"
LOGFILE="parserlog.txt"
function LOG ()
{
local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" )
echo -e "${tstamp_H} $0($$): $@" >> ${LOGFILE}
function LOG {
local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" )
echo -e "${tstamp_H} $0($$): $@" >> ${LOGFILE}
}
function ERRLOG ()
{
LOG "ERROR: $@"
function ERRLOG {
LOG "ERROR: $@"
}
function WARNLOG ()
{
LOG "WARN: $@"
function WARNLOG {
LOG "WARN: $@"
}

8
tools/engtools/parsers/large_office/download-computes.sh
View File

@ -6,14 +6,14 @@
#
# Create /opt/backups/tmp/syseng-data directory on controller 0, change mode of this
# directory to 777 and place this script and the lab.conf files there. It is recommended
# to set up password-less login from the controller to all storage and compute hosts
# to set up password-less login from the controller to all storage and compute hosts
# before running the script.
#
if [ ! -f lab.conf ]; then
echo "Lab configuration file is missing."
echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info."
exit 1
echo "Lab configuration file is missing."
echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info."
exit 1
fi
source ./lab.conf

16
tools/engtools/parsers/large_office/parse-computes.sh
View File

@ -7,22 +7,22 @@
# This script is used to parse stats data for storage and compute hosts. It is not
# relevant for CPE. For large office, it is called by parse-everything.sh script.
# File lab.conf must exist for the script to run. The STORAGE_LIST and COMPUTE_LIST
# config parameters can be set to suit the parsing needs.
# File lab.conf must exist for the script to run. The STORAGE_LIST and COMPUTE_LIST
# config parameters can be set to suit the parsing needs.
PARSERDIR=$(dirname $0)
. ${PARSERDIR}/parse-util.sh
if [ ! -f lab.conf ]; then
echo "Lab configuration file is missing."
echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info."
exit 1
echo "Lab configuration file is missing."
echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info."
exit 1
fi
source ./lab.conf
if [ -z "${STORAGE_LIST}" ]; then
# This script could be invoked from another script or run separately so write to both
# This script could be invoked from another script or run separately so write to both
# console and log file.
echo "STORAGE_LIST is not set in lab.conf file. Skipping stats parsing for all storage."
WARNLOG "STORAGE_LIST is not set in lab.conf file. Skipping stats parsing for all storage."
@ -35,7 +35,7 @@ else
../parse-all.sh ${HOST} > /dev/null 2>&1 &
cd ..
else
ERRLOG "${HOST} does not exist. Parsing skipped."
ERRLOG "${HOST} does not exist. Parsing skipped."
fi
done
fi
@ -45,7 +45,7 @@ if [ -z "${COMPUTE_LIST}" ]; then
WARNLOG "COMPUTE_LIST is not set in lab.conf file. Skipping stats parsing for all computes."
exit 1
else
# If there is a large number of computes, they need to be parsed one batch at a time,
# If there is a large number of computes, they need to be parsed one batch at a time,
# otherwise, the analysis server will be brought down to a crawl. Set the number of
# computes to process in parallel as batches of 25 if it's not set in lab.conf
BATCH_SIZE=${BATCH_SIZE:-25}

8
tools/vm-topology/vm-topology/vm_topology/exec/vm_topology.py
View File

@ -444,7 +444,7 @@ def _mask_to_cpulist(mask=0):
# Assume max number of cpus for now...
max_cpus = 128
for cpu in xrange(max_cpus):
for cpu in range(max_cpus):
if ((1 << cpu) & mask):
cpulist.append(cpu)
return cpulist
@ -492,9 +492,9 @@ def range_to_list(csv_range=None):
"""
if not csv_range:
return []
xranges = [(lambda L: xrange(L[0], L[-1] + 1))(map(int, r.split('-')))
ranges = [(lambda L: range(L[0], L[-1] + 1))(map(int, r.split('-')))
for r in csv_range.split(',')]
return [y for x in xranges for y in x]
return [y for x in ranges for y in x]
class TimeoutError(Exception):
@ -613,7 +613,7 @@ def do_libvirt_domain_info((host)):
cpulist_d = {}
cpuset_total = 0
up_total = 0
for vcpu in xrange(d_nrVirtCpu):
for vcpu in range(d_nrVirtCpu):
cpuset_b = d_vcpus[1][vcpu]
cpuset = 0
for cpu, up in enumerate(cpuset_b):

12
tox.ini
View File

@ -14,6 +14,12 @@ deps = -r{toxinidir}/test-requirements.txt
[testenv:linters]
whitelist_externals = bash
#bashate ignore
#E006 Line too long
#E041 Arithmetic expansion using $[ is deprecated for $((
#E042 local declaration hides errors
#E043 Arithmetic compound has inconsistent return semantics
#E044 Use [[ for non-POSIX comparisions
commands =
bash -c "find {toxinidir} \
-not \( -type d -name .?\* -prune \) \
@ -21,10 +27,10 @@ commands =
-not -name \*~ \
-not -name \*.md \
-name \*.sh \
-print0 | xargs -0 bashate -v"
-print0 | xargs -0 bashate -v \
-i E006,E041,E042,E043,E044 -e E*"
bash -c "find {toxinidir} \
\( -name middleware/io-monitor/recipes-common/io-monitor/io-monitor/io_monitor/test-tools/yaml/* -prune \) \
-o \( -name .tox -prune \) \
\( -path {toxinidir}/.tox \) -a -prune \
-o -type f -name '*.yaml' \
-print0 | xargs -0 yamllint"

18
utilities/build-info/build-info-1.0/collect.sh
View File

@ -24,7 +24,7 @@ if [ ! -e $jenkinsBuildFile ]; then
fi
if [ -e $releaseInfoFile ]; then
source $releaseInfoFile
source $releaseInfoFile
fi
if [ "${REPO}x" == "x" ]; then
@ -35,12 +35,12 @@ if [ -e $jenkinsBuildFile ]; then
cp $jenkinsBuildFile $destFile
source $jenkinsBuildFile
else
# PLATFORM_RELEASE should be set in release-info.inc
if [ "x${PLATFORM_RELEASE}" == "x" ]; then
SW_VERSION="Unknown"
else
SW_VERSION="${PLATFORM_RELEASE}"
fi
# PLATFORM_RELEASE should be set in release-info.inc
if [ "x${PLATFORM_RELEASE}" == "x" ]; then
SW_VERSION="Unknown"
else
SW_VERSION="${PLATFORM_RELEASE}"
fi
BUILD_TARGET="Unknown"
BUILD_TYPE="Informal"
@ -89,8 +89,8 @@ fi
echo "#ifndef _BUILD_INFO_H_" > $destH
echo "#define _BUILD_INFO_H_" >> $destH
echo "" >> $destH
echo "#define RELEASE_NAME \"$RELEASE_NAME\"" >> $destH
echo "#define SW_VERSION \"$SW_VERSION\"" >> $destH
echo "#define RELEASE_NAME \"$RELEASE_NAME\"" >> $destH
echo "#define SW_VERSION \"$SW_VERSION\"" >> $destH
echo "" >> $destH
echo "#define BUILD_TARGET \"$BUILD_TARGET\"" >> $destH
echo "#define BUILD_TYPE \"$BUILD_TYPE\"" >> $destH

2
utilities/platform-util/centos/build_srpm.data
View File

@ -1,4 +1,4 @@
SRC_DIR="platform-util"
COPY_LIST_TO_TAR="scripts"
TIS_PATCH_VER=12
TIS_PATCH_VER=13

11
utilities/platform-util/centos/platform-util.spec
View File

@ -20,6 +20,12 @@ Summary: non controller platform utilities
%description -n platform-util-noncontroller
Platform utilities that don't get packaged on controller hosts
%package -n platform-util-controller
Summary: controller platform utilities
%description -n platform-util-controller
Platform utilities that packaged on controllers or one node system
%define local_dir /usr/local
%define local_bindir %{local_dir}/bin
%define local_sbindir %{local_dir}/sbin
@ -52,6 +58,7 @@ install -m 700 -p -D %{_buildsubdir}/scripts/patch-restart-haproxy %{buildroot}%
install -d %{buildroot}/etc/systemd/system
install -m 644 -p -D %{_buildsubdir}/scripts/opt-platform.mount %{buildroot}/etc/systemd/system
install -m 644 -p -D %{_buildsubdir}/scripts/opt-platform.service %{buildroot}/etc/systemd/system
install -m 644 -p -D %{_buildsubdir}/scripts/memcached.service %{buildroot}/etc/systemd/system
# Mask the systemd ctrl-alt-delete.target, to disable reboot on ctrl-alt-del
ln -sf /dev/null %{buildroot}/etc/systemd/system/ctrl-alt-del.target
@ -85,3 +92,7 @@ systemctl enable opt-platform.service
# from parsing the fstab is not used by systemd.
/etc/systemd/system/opt-platform.mount
/etc/systemd/system/opt-platform.service
%files -n platform-util-controller
%defattr(-,root,root,-)
/etc/systemd/system/memcached.service

43
utilities/platform-util/scripts/cgcs_tc_setup.sh
View File

@ -22,17 +22,15 @@ fi
# network link to autonegotiate link speed. Re-run the script in
# the background so the parent can return right away and init can
# continue.
if [ $# -eq 3 ]
then
if [ $# -eq 3 ]; then
$0 $DEV $NETWORKTYPE $NETWORKSPEED dummy &
disown
exit 0
fi
function test_valid_speed
{
function test_valid_speed {
# After the link is enabled but before the autonegotiation is complete
# the link speed may be read as either -1 or as 4294967295 (which is
# the link speed may be read as either -1 or as 4294967295 (which is
# uint(-1) in twos-complement) depending on the kernel. Neither one is valid.
if (( $1 > 0 )) && (( $1 != 4294967295 ))
then
@ -42,51 +40,43 @@ function test_valid_speed
fi
}
function log
{
function log {
# It seems that syslog isn't yet running, so append directly to the syslog file
echo `date +%FT%T.%3N` `hostname` CGCS_TC_SETUP: $@ >> /var/log/platform.log
}
function infra_exists
{
if [ -z "$infrastructure_interface" ]
then
function infra_exists {
if [ -z "$infrastructure_interface" ]; then
return 1
else
return 0
fi
}
function is_consolidated
{
function is_consolidated {
if ! infra_exists
then
return 1
return 1
else
# determine whether the management interface is a parent of the
# infrastructure interface based on name.
# eg. this matches enp0s8 to enp0s8.10 but not enp0s88
if [[ $infrastructure_interface =~ $management_interface[\.][0-9]+$ ]]
then
return 0
fi
return 1
if [[ $infrastructure_interface =~ $management_interface[\.][0-9]+$ ]]; then
return 0
fi
return 1
fi
}
function is_vlan
{
if [ -f /proc/net/vlan/$DEV ]
then
function is_vlan {
if [ -f /proc/net/vlan/$DEV ]; then
return 0
else
return 1
fi
}
function is_loopback
{
function is_loopback {
# (from include/uapi/linux/if.h)
IFF_LOOPBACK=$((1<<3))
@ -101,8 +91,7 @@ function is_loopback
fi
}
function setup_tc_port_filter
{
function setup_tc_port_filter {
local PORT=$1
local PORTMASK=$2
local FLOWID=$3

55
utilities/platform-util/scripts/memcached.service Normal file
View File

@ -0,0 +1,55 @@
#
# This service file is a customized version in platform-util package from
# openstack/stx-integ project
[Unit]
Description=memcached daemon
Before=httpd.service
After=network-online.target
[Service]
EnvironmentFile=/etc/sysconfig/memcached
ExecStart=/usr/bin/memcached -p ${PORT} -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS
# Set up a new file system namespace and mounts private /tmp and /var/tmp directories
# so this service cannot access the global directories and other processes cannot
# access this service's directories.
PrivateTmp=true
# Mounts the /usr, /boot, and /etc directories read-only for processes invoked by this unit.
ProtectSystem=full
# Ensures that the service process and all its children can never gain new privileges
NoNewPrivileges=true
# Sets up a new /dev namespace for the executed processes and only adds API pseudo devices
# such as /dev/null, /dev/zero or /dev/random (as well as the pseudo TTY subsystem) to it,
# but no physical devices such as /dev/sda.
PrivateDevices=true
# Required for dropping privileges and running as a different user
CapabilityBoundingSet=CAP_SETGID CAP_SETUID CAP_SYS_RESOURCE
LimitNOFILE=16384
# Attempts to create memory mappings that are writable and executable at the same time,
# or to change existing memory mappings to become executable are prohibited.
# XXX: this property is supported with systemd 231+ which is not yet on EL7
# MemoryDenyWriteExecute=true
# Restricts the set of socket address families accessible to the processes of this unit.
# Protects against vulnerabilities such as CVE-2016-8655
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
# These service parameters are commented out since they are incompatible with
# Centos 7 and generate warning messages when included.
#ProtectKernelModules=true
#ProtectKernelTunables=true
#ProtectControlGroups=true
#RestrictRealtime=true
#RestrictNamespaces=true
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target

14
utilities/platform-util/scripts/remotelogging_tc_setup.sh
View File

@ -9,8 +9,7 @@
# $1 - listening port of remote log server
PORT=$1
function is_loopback
{
function is_loopback {
# (from include/uapi/linux/if.h)
IFF_LOOPBACK=$((1<<3))
@ -25,16 +24,14 @@ function is_loopback
fi
}
function log
{
function log {
# It seems that syslog isn't yet running, so append directly to the syslog file
echo `date +%FT%T.%3N` `hostname` CGCS_TC_SETUP: $@ >> /var/log/platform.log
}
function test_valid_speed
{
function test_valid_speed {
# After the link is enabled but before the autonegotiation is complete
# the link speed may be read as either -1 or as 4294967295 (which is
# the link speed may be read as either -1 or as 4294967295 (which is
# uint(-1) in twos-complement) depending on the kernel. Neither one is valid.
if (( $1 > 0 )) && (( $1 != 4294967295 ))
then
@ -44,8 +41,7 @@ function test_valid_speed
fi
}
function get_dev_speed
{
function get_dev_speed {
# If the link doesn't come up we won't go enabled, so here we can
# afford to wait forever for the link.
while true

4
virt/qemu/centos/files/build_configure.sh
View File

@ -41,11 +41,11 @@ shift
if [ "$have_rbd" == "enable" ]; then
rbd_driver=rbd,
rbd_driver=rbd,
fi
if [ "$have_gluster" == "enable" ]; then
gluster_driver=gluster,
gluster_driver=gluster,
fi
./configure \