#!/bin/bash
set -o xtrace
set -o errexit
set -o pipefail
# Enable unbuffered output
export PYTHONUNBUFFERED=1
function check_kibana {
# Query kibana, and check that the returned page looks like a kibana page.
KIBANA_URL=${OS_AUTH_URL%:*}:5601
output_path=$1
kibana_password=$(awk '$1 == "kibana_password:" { print $2 }' /etc/kolla/passwords.yml)
args=(
--include
--location
--fail
--user
kibana:$kibana_password
)
if [[ "$TLS_ENABLED" = "True" ]]; then
args+=(--cacert $OS_CACERT)
fi
if ! curl "${args[@]}" $KIBANA_URL > $output_path; then
return 1
fi
if ! grep '
Kibana' $output_path >/dev/null; then
return 1
fi
}
function check_grafana {
# Query grafana, and check that the returned page looks like a grafana page.
GRAFANA_URL=${OS_AUTH_URL%:*}:3000
output_path=$1
grafana_password=$(awk '$1 == "grafana_admin_password:" { print $2 }' /etc/kolla/passwords.yml)
args=(
--include
--location
--fail
--user
admin:$grafana_password
)
if [[ "$TLS_ENABLED" = "True" ]]; then
args+=(--cacert $OS_CACERT)
fi
if ! curl "${args[@]}" $GRAFANA_URL > $output_path; then
return 1
fi
if ! grep 'Grafana' $output_path >/dev/null; then
return 1
fi
}
function check_prometheus {
# Query prometheus graph, and check that the returned page looks like a
# prometheus page.
PROMETHEUS_URL=${OS_AUTH_URL%:*}:9091/graph
output_path=$1
args=(
--include
--location
--fail
)
if [[ "$TLS_ENABLED" = "True" ]]; then
args+=(--cacert $OS_CACERT)
fi
if ! curl "${args[@]}" $PROMETHEUS_URL > $output_path; then
return 1
fi
if ! grep 'Prometheus' $output_path >/dev/null; then
return 1
fi
}
function test_kibana {
# TODO(mgoddard): Query elasticsearch for logs.
echo "TESTING: Kibana"
output_path=$(mktemp)
attempt=1
while ! check_kibana $output_path; do
echo "Kibana not accessible yet"
attempt=$((attempt+1))
if [[ $attempt -eq 12 ]]; then
echo "FAILED: Kibana did not become accessible. Response:"
cat $output_path
return 1
fi
sleep 10
done
echo "SUCCESS: Kibana"
}
function test_grafana {
echo "TESTING: Grafana"
output_path=$(mktemp)
attempt=1
while ! check_grafana $output_path; do
echo "Grafana not accessible yet"
attempt=$((attempt+1))
if [[ $attempt -eq 12 ]]; then
echo "FAILED: Grafana did not become accessible. Response:"
cat $output_path
return 1
fi
sleep 10
done
echo "SUCCESS: Grafana"
}
function test_prometheus {
# TODO(mgoddard): Query metrics.
echo "TESTING: Prometheus"
output_path=$(mktemp)
attempt=1
while ! check_prometheus $output_path; do
echo "Prometheus not accessible yet"
attempt=$((attempt+1))
if [[ $attempt -eq 12 ]]; then
echo "FAILED: Prometheus did not become accessible. Response:"
cat $output_path
return 1
fi
sleep 10
done
echo "SUCCESS: Prometheus"
}
function test_prometheus_efk_logged {
. /etc/kolla/admin-openrc.sh
test_kibana
test_grafana
test_prometheus
}
function test_prometheus_efk {
echo "Testing prometheus and EFK"
test_prometheus_efk_logged > /tmp/logs/ansible/test-prometheus-efk 2>&1
result=$?
if [[ $result != 0 ]]; then
echo "Testing prometheus and EFK failed. See ansible/test-prometheus-efk for details"
else
echo "Successfully tested prometheus and EFK. See ansible/test-prometheus-efk for details"
fi
return $result
}
test_prometheus_efk