From 6db35a5230578e296d9f493b28e6330e22569c8f Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Wed, 4 Sep 2019 22:06:01 +0200 Subject: Integrate also nagios-based monitoring here --- lib/adei/adei.sh | 287 +++++++++++++++++++++++++++++++++++++++++++++++ lib/adei/adei_error.sh | 74 ++++++++++++ lib/adei/adei_print.sh | 25 +++++ lib/adei/adei_version.sh | 51 +++++++++ lib/helpers.sh | 9 ++ lib/nagios/nagios.sh | 125 +++++++++++++++++++++ lib/remote/README.txt | 34 ++++++ lib/remote/print.sh | 168 +++++++++++++++++++++++++++ lib/remote/report.sh | 51 +++++++++ lib/remote/run.sh | 13 +++ lib/remote/status.sh | 196 ++++++++++++++++++++++++++++++++ 11 files changed, 1033 insertions(+) create mode 100644 lib/adei/adei.sh create mode 100644 lib/adei/adei_error.sh create mode 100644 lib/adei/adei_print.sh create mode 100644 lib/adei/adei_version.sh create mode 100644 lib/helpers.sh create mode 100755 lib/nagios/nagios.sh create mode 100644 lib/remote/README.txt create mode 100644 lib/remote/print.sh create mode 100644 lib/remote/report.sh create mode 100644 lib/remote/run.sh create mode 100644 lib/remote/status.sh (limited to 'lib') diff --git a/lib/adei/adei.sh b/lib/adei/adei.sh new file mode 100644 index 0000000..3a0aeb6 --- /dev/null +++ b/lib/adei/adei.sh @@ -0,0 +1,287 @@ +cd $(dirname $0) + +MAX_SOURCE_OFFSET=3600 +MAX_MASTER_OFFSET=300 +MAX_SLAVE_OFFSET=300 + +adei_default_timeout=${default_timeout:-120} + +. lib/adei/adei_version.sh +. lib/adei/adei_print.sh +. lib/adei/adei_error.sh +. lib/nagios/nagios.sh + +shopt -s nocasematch; + +# Auth with redirect (-L/--location-trusted) not working. Credentials are lost on hops. +function adei_query { + local resp + local timeout=${3:-$adei_default_timeout} + + local err=0 + if [ "$2" == "ecode" ]; then + url="$1&mysql=master" +# resp=$(curl --proxy "" -f -m "$timeout" "$url" 2>&1 | grep "returned error") + resp=$(curl --proxy "" --location-trusted -sf -m "$timeout" -w "%{http_code}" -o /dev/null "$url" 2>&1); + [ -z "$resp" -o "$resp" = "000" ] && resp="10" # error code 1 + elif [ "$2" == "emsg" ]; then + url="$1&mysql=master" + resp=$(curl --proxy "" --location-trusted -f -m "$timeout" "$url" 2>&1 | grep -o "curl.*") + [ -z "$resp" ] && resp="No response from" + else + if [ "$2" == "slave" ]; then + url="$1&mysql=slave" + else + url="$1&mysql=master" + fi + + + resp=$(curl --proxy "" --location-trusted -sf -m "$timeout" "$url"); err=$? + if [ $err -ne 0 ]; then + resp=$(adei_query "$1" "ecode" "$timeout") + err=$(($(($resp / 10)) + $(($resp % 10)))) + + resp=$(adei_query "$1" "emsg" "$timeout") + + [ -n "$debug" ] && echo "$(date) $timeout Failed $url" >> /tmp/adei.log + else + [ -n "$debug" ] && echo "$(date) $timeout OK $url" >> /tmp/adei.log + fi + fi + + echo -n $resp + return $err +} + +function adei_format_query { + local with_auth="${1:-1}" + local query="$2" + local timeout=${3:-$adei_default_timeout} + local source=${4:-"$adei_source"} + local url="${5:-$adei_url}" + + auth="" + [ $with_auth -gt 0 ] && auth="$adei_auth" + if [[ "$query" =~ \? ]]; then + echo "http://${auth}${url}/${query}${source}" + else + echo "http://${auth}${url}/${query}?xxxxx${source}" + fi +} + +function adei_simple_query { + local req=$(adei_format_query 1 "$@") + adei_query "$req" "master" "$timeout"; err=$? + return $err +} + +function adei_text_query { + local out # Local on the same string breaks error reporting + out="$(adei_simple_query "$@")"; local err=$? + + if [ $err -gt 0 -o -z "$out" ]; then +# echo "Error sending ADEI request: $(adei_format_query 0 "$@")" + [ $err -eq 0 ] && local err=1 + [ -n "$out" ] && echo -n "$out" + return $err + elif [[ "$out" =~ "Error:" ]]; then + echo -n "$out" + return 7 + else + echo -n "$out" + return 0 + fi +} + +function adei_xml_query { + local out # Local on the same string breaks error reporting + out="$(adei_simple_query "$@")"; local err=$? + + if [ $err -ne 0 -o -z "$out" ]; then +# echo "Error sending ADEI request: $(adei_format_query 0 '$@')" + [ $err -eq 0 ] && err=1 + [ -n "$out" ] && echo "$out" + return $err + fi + + local xml="$(echo "$out" | xmllint --format - 2>/dev/null)" + if [ $err -ne 0 -o -z "$out" ]; then + echo "$out" + if [[ "$out" =~ "Error:" ]]; then + return 7 + else + return 4 + fi + fi + + error=$(echo "$xml" | grep "") + if [ -n "$error" ]; then + echo $error | sed -e "s|||g" + return 7 + fi + + echo "$xml" + return $err +} + +function adei_value_query { + local out # Local on the same string breaks error reporting + out="$(adei_xml_query "$@")"; local err=$? + [ $err -ne 0 ] && { echo "$out"; return $err; } + + local values + values="$(echo "$out" | grep "Value")"; err=$? + [ $err -ne 0 ] && return 6 + + echo "$values" +} + + +function adei_get_databases { + local out + out="$(adei_xml_query "list.php?target=databases" "$@")" + [ $err -gt 0 ] && { echo -n "$out"; return $err; } + echo "$out" | grep "Value" | sed -e "s/^.*db_name=\"\([^\"]*\)\".*$/\\1/" | sed -e "s/ /::space::/" +} + +function adei_query_version { + local version # Local on the same string breaks error reporting + + version="$(adei_text_query "info.php?target=version&encoding=text" "$@")"; local err=$? + [ $err -gt 0 ] && { echo "$version"; return $err; } + + adei_version="$version" + + adei_revision=$(echo $adei_version | cut -d '-' -f 1) + if [ "$adei_revision" == "$adei_version" ]; then + adei_date="" + else + adei_date=$(echo $adei_version | cut -d '-' -f 2) + fi +} + +function adei_resolve_id { + fn="$1" + id="$2" + host="$3" + setup="$4" + + local var=$(cat "$fn" | grep "^$id" | awk '{ print $2 }') + [ -z "$var" -a -n "$host" ] && var=$(cat "$fn" | grep "^$host" | awk '{ print $2 }') + [ -z "$var" -a -n "$setup" ] && var=$(cat "$fn" | grep "^$setup" | awk '{ print $2 }') + + [ -n "$debug" ] && echo "$(date) resolved ($var) from $fn (id=$id, host=$host, setup=$setup) pwd ($(pwd))" >> /tmp/resolv.log + echo "$var" +} + +function adei_init_ { + local id="$1" && shift # Either URL or [setup]@[host] +# local url="$1" && shift + local server="$1" && shift + local database="$1" && [[ ! "$server" =~ \&|= ]] && shift + adei_args=( "$@" ) + + [ -z "$id" ] && { echo "ADEI ID is not specified" && exit 8 ; } + + + local url + local host + local setup + if [[ $id =~ http.*:// ]]; then # url + url="$id" + host="$(echo ${url#http*://} | cut -d '/' -f 1)" # parse port, maybe + + unset $setup + elif [[ $id =~ : ]]; then # [setup@]host + local seho + IFS='@' read -ra seho <<< "$id" && shift + host="${seho[1]:-${seho[0]}}" # parse port, for sure + setup="${seho[1]:+${seho[0]}}" + + unset $url + else # [setup]@[host] or both, no port + local seho + IFS='@' read -ra seho <<< "$id" && shift # techincally we can try to resolve to decide which is which + + host="${seho[1]:-${seho[0]}}" # This could be mixed up, but it is either not important, or correct, or misconfigured anyway + setup="${seho[1]:+${seho[0]}}" # Only error if only setup is provided. But we handle it in resolution code which agnostic and treats them as ids + + url=$(adei_resolve_id "setup/adei.txt" "$id" "$host" "$setup") + + if [ -n "$url" ]; then # Get URL, now we can determine host correctly and ignore the ids. + host="$(echo ${url#http*://} | cut -d '/' -f 1)" + unset $setup + else # only 'setup' is not allowed if URL is not configured, so it should be host (or error in configuration) + unset $url # so either we have both (correctly) or only host (correctly) + fi + fi + + local hopo + IFS=':' read -ra hopo <<< "$host" && shift + host="${hopo[0]}" # only non fqdn if also url not set + port=":${hopo[1]:-80}" + [ $port = ":80" ] && port="" + + local fqdn + if [ -z "$url" ]; then + url=$(adei_resolve_id "setup/adei.txt" "$id" "$host" "$setup") + if [ -n "$url" ]; then + fqdn="$(echo ${url#http*://} | cut -d '/' -f 1)" + + IFS=':' read -ra hopo <<< "$fqdn" && shift + host="${hopo[0]}" # again fqdn + port=":${hopo[1]:-80}" + [ $port = ":80" ] && port="" + else + fqdn=$(resolve_fqdn "$host") # this may be not fqdn + [ -n "$fqdn" ] && host="$fqdn" + url="${host}${port}/adei" + fi + fi + + # In some case we miss here "setup", but surely then the password is resolved host-based + adei_auth=$(adei_resolve_id "security/adei.txt" "$id" "$host" "$setup") + [ -n "$adei_auth" ] && adei_auth="$adei_auth@" + + # Now check port + adei_online=$(scripts/ping.pl "$host" "$port") + [ $adei_online -ne 1 ] && return 2 + + # Now build ADEI url + + adei_setup="$setup" + adei_host="$host" + adei_port="$port" + adei_url="${url#http*://}/services" + + + adei_query_version; local err=$? + [ $err -gt 0 ] && return $err + + + if [[ "$server" =~ \&|= ]]; then + adei_source="&$server" + else + adei_source="" + [ -n "$adei_setup" -a "$adei_setup" != "*" ] && adei_source+="&setup=$adei_setup" + [ -n "$server" -a "$server" != "-" ] && adei_source+="&db_server=$server" + + if [ "$database" == "#1" ]; then + databases="$(adei_get_databases)"; err=$? + [ $err -gt 0 ] && { echo "Failed to query ADEI databases: $databases"; return $err; } + [ -z "$databases" ] && { echo "No databases reported by ADEI"; return 6; } + database=$(echo "$databases" | head -n 1) + fi + + [ -n "$database" -a "$database" != "-" ] && adei_source+="&db_name=$database" + fi + + return 0 +} + +function adei_init { + adei_init_ "$@"; local code=$? + + adei_healthy=$(($code == 0)) + adei_process_error "$code" "" "*" "$(adei_print_status "$0" $adei_online $adei_healthy)" +} diff --git a/lib/adei/adei_error.sh b/lib/adei/adei_error.sh new file mode 100644 index 0000000..9de4795 --- /dev/null +++ b/lib/adei/adei_error.sh @@ -0,0 +1,74 @@ +# Error codes: 1 - unknown network, 2 - unreachable, 3 - timeout, 4 - invalid XML/JSON, 5 - parsing error, 6 - set is empty, 7 - service reported error, 8 - parameter error, 9 - unknown, 10-59 - HTTP errors +# http code (xyz) is converted to error code (xz), e.g. 404 -> 44 + +function adei_format_http_error { + err=$1 + echo -n "$(($err / 10))0$(($err % 10))" +} + +# Returns 1 if printing text is recommended +function adei_print_error { + local ret=0 + local code=${1:-0} + local text="$2" + + local service="${3#\*}" + service="${service:-ADEI}" + + local msg + if [ $code -ge 30 -a $code -lt 60 ]; then + msg="HTTP Error: $(adei_format_http_error $code)" + [ $code -eq 41 ] && msg+=" (Authentication Requested)" + [ $code -eq 43 ] && msg+=" (Access Denied)" + elif [ $code -ge 20 -a $code -lt 30 ]; then + msg="HTTP Status: $(adei_format_http_error $code)" + ret=1 + elif [ $code -eq 1 ]; then + msg="Error communicating with $service" + ret=1 + elif [ $code -eq 2 ]; then + msg="ADEI host ($adei_host:$adei_port) unreachable" + elif [ $code -eq 3 ]; then + msg="Timeout waiting for $service" + elif [ $code -eq 4 ]; then + msg="Non-XML response from $service" + ret=1 + elif [ $code -eq 5 ]; then + msg="Error parsing response from $service" + ret=1 + elif [ $code -eq 6 ]; then + msg="Empty response from $service" + ret=1 + elif [ $code -eq 7 ]; then + rep="$(echo "$text" | head -n 1 | sed 's/Error:\?[[:space:]]\+//i')" + if [ ${#rep} -lt 40 ]; then + msg="$service Error: $rep" + else + msg="$service returned a error" + ret=1 + fi + elif [ $code -eq 8 ]; then + msg="Invalid parameter" + else + msg="Unknown Error $code while accessing $service" + fi + + echo -n "$msg" + return $ret +} + +function adei_process_error { + local code=${1:-0}; shift + local text="$1"; shift + local service="$1"; shift + local status="${1:-0}" + + if [ $code -ne 0 ]; then + local msg # local on the same line break error reporting + msg="$(adei_print_error $code "$text" "$service")" + [ $? -eq 1 ] && echo "$text" + echo "$status $msg" + exit + fi +} + diff --git a/lib/adei/adei_print.sh b/lib/adei/adei_print.sh new file mode 100644 index 0000000..aca7735 --- /dev/null +++ b/lib/adei/adei_print.sh @@ -0,0 +1,25 @@ +function adei_print_status { + cmd="$1" + online="$2" + healthy="$3" + + if [[ "$cmd" =~ check_adei_ ]]; then + echo "$healthy" + else + echo "$online $healthy" + fi +} + +function format_time { + offset=$1 + if [ $offset -ge 86400 ]; then + echo "$((offset / 86400))d" + elif [ $offset -ge 3600 ]; then + echo "$((offset / 3600))h" + elif [ $offset -ge 60 ]; then + echo "$((offset / 60))m" + else + echo "${offset}s" + fi +} + diff --git a/lib/adei/adei_version.sh b/lib/adei/adei_version.sh new file mode 100644 index 0000000..b71b1e1 --- /dev/null +++ b/lib/adei/adei_version.sh @@ -0,0 +1,51 @@ +vercomp() { + if [[ $1 == $2 ]] + then + return 0 + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + return 2 + fi + done + return 0 +} + +adei_version_check() { + vercomp $adei_revision $1 + res=$? + if [ $res -eq 1 ]; then + return 0 + elif [ $res -eq 2 ]; then + return 1; + fi + + if [ -z "$2" -o -z "$adei_date" ]; then + return 0 + fi + + if [ $adei_date -ge $2 ]; then + return 0 + fi + + return 1 +} + diff --git a/lib/helpers.sh b/lib/helpers.sh new file mode 100644 index 0000000..b562eb6 --- /dev/null +++ b/lib/helpers.sh @@ -0,0 +1,9 @@ +function in_array() { + local hay needle=$1 + shift + for hay; do + [[ $hay == $needle ]] && return 0 + done + + return 1 +} diff --git a/lib/nagios/nagios.sh b/lib/nagios/nagios.sh new file mode 100755 index 0000000..40631c1 --- /dev/null +++ b/lib/nagios/nagios.sh @@ -0,0 +1,125 @@ +cd $(dirname "$0") + +. setup/parameters.sh + +function nagios_find_host { + echo "$@" | grep -oP '\-H\s*\K[^\s]*' +} + +function find_host_param { + local id="$1" && shift + local pname="$1" && shift + local defval="$1" && shift + + local val=$(cat setup/${pname}.txt | grep -P "^$id" | awk '{ print $2 }') + [ -n "$val" ] || val="$defval" + + echo -n "$val" +} + +function nagios_find_host_param { + local pname="$1" && shift + local defval="$1" && shift + local host=$(nagios_find_host "$@") + + find_host_param "$host" "$pname" "$defval" +} + +function resolve_fqdn { + local host="$1" && shift + + if [[ ! $host =~ \. ]]; then + local domain=$(find_host_param "$host" domains "$default_domain") + host+=".$domain" + fi + + echo -n $host +} + +function resolve_ip { + local host="$1" && shift + + if [[ ! "$host" =~ ^[[:digit:]] ]]; then + local fqdn=$(resolve_fqdn "$host") + host=$(dig +short "$fqdn" A | head -n 1 | tr -d '\n') + if [ $? -ne 0 -o -z "$host" ]; then + echo "DNS resolution for host ($fqdn) failed: $(dig +short "$fqdn" A 2>&1)" + exit 2 + fi + fi + + echo -n "$host" +} + +function nagios_run { + if [[ "$1" =~ ^- ]]; then + cmd=${nagios_plugins}/$(basename $0) + else + cmd=$1 && shift + [[ "$cmd" =~ ^/ ]] || cmd="${nagios_plugins}/$cmd" + fi + + ARGS="" + while [[ $# -gt 0 ]]; do + key="$1" && shift + + case "$key" in + -H) + host="$1" && shift + ip=$(resolve_ip "$host") + if [ $? -eq 0 -a -n "$ip" ]; then + ARGS+=" ${key} $ip" + else + echo "$ip" + exit $? + fi + ;; + *) + ARGS+=" $key" + esac + done + + [ -n "$debug" ] && echo "$(date) $cmd" "$ARGS" >> /tmp/nagios.log + eval "$cmd" "$ARGS" +} + +function nagios_parse { + cmd="$1"; shift + + output=$($cmd "$@") + exit_code=$? + + if [ $exit_code -eq 0 ]; then + last_line=$(echo "$output" | tail -1) + + status=$(echo $last_line | cut -d ' ' -f 1) + [[ "$status" =~ ^[0-9]$ ]] || status=0 + echo -n ${last_line#$status} + else + status=0 + fi + + if [ $status -ne 1 ]; then + echo -n " -- " + + lines=$(echo "$output" | sed '/^[[:space:]]*$/d' | wc -l) + if [ $lines -gt 1 ]; then + echo "$output" | sed '/^[[:space:]]*$/d' | head -n 1 | tr -d '\n' + echo " ... " + echo "------------------" + echo "$output" | head -n -1 + else + echo "$output" | sed '/^[[:space:]]*$/d' | head -n -1 + fi + else + echo + fi + + if [ "$status" -eq 0 ]; then + exit "2" + elif [ "$status" -eq 1 ]; then + exit "0" + else + exit "1" + fi +} diff --git a/lib/remote/README.txt b/lib/remote/README.txt new file mode 100644 index 0000000..c5998f8 --- /dev/null +++ b/lib/remote/README.txt @@ -0,0 +1,34 @@ +conky format +============ + - Server status + * ping works + * no output from 'check_server_status.sh' and additional listed checks + [*] Checks for additional listed services 'check__status.sh'. + - The optional status is in the beginning of first line (0 - red, 1 - green, 2 - yellow). Otherwise, 'red' if any output (also if first word is non-number on the first string). + - If first line starts from the status, the rest of the line is considered a header and reported on the status line. Everything else goes to textual section bellow. + [headers] + + ... Textual information reported by 'check_server_status.sh' and check_*_status.sh scripts. The format + server:port[:service] text.... + + - Extra info for DarkSoft servers + * VPN to the server is working (checked with the ping). This not included in the scripted version, instead server traffic reported directly here. + # # Traffic on VPN interface from/to + + - Server traffic + in/out/forw. Yellow/Red if above specified threshold. No additional errors are reported here. + + - Services (operated by local scripts in 'services' directory) + * Online + * Healthy + [headers] Information about service operation + [errors] Short information on critical errors are reported here (in conky, we push everything in the text section) + + ... Larger textual information about the problems. + + + Service format is following (status is on the last line and mandatory): + * Critical Lines starting with '*' contain critical errors and highlighted (remote) / reported in the service table (conky) + Text This is normal problem description reported in the textual area + # # [header] Two statuses (0 - error, 1 - ok, 2 - warning) and optional header with information about service operation + \ No newline at end of file diff --git a/lib/remote/print.sh b/lib/remote/print.sh new file mode 100644 index 0000000..a0fdcbe --- /dev/null +++ b/lib/remote/print.sh @@ -0,0 +1,168 @@ +# underlines 4;30,... blinking 5:30,... +# backgrounds: 40,... + +declare -A colors=( + [black]='0;30' + [red]='0;31' + [green]='0;32' + [orange]='0;33' + [blue]='0;34' + [purple]='0;35' + [cyan]='0;36' + [lightgray]='0;37' + [gray]='1;30' + [lightred]='1;31' + [lightgreen]='1;32' + [yellow]='1;33' + [lightblue]='1;34' + [lightpurple]='1;35' + [lightcyan]='1;36' + [white]='1;37' +) + +background="" +foreground="" + + + + +# Fixing color map +#echo -ne "\e[44m" +#echo -ne '\x1B[K' + +function set_background { + if [ -n "$background" ]; then + echo -ne "$background" + echo -ne '\x1B[K' + fi +} + +function set_color { + local color=${1:-""} + local effect=${2:-""} + + c=${colors[$color]} + + + case "$effect" in + 'u') + c=$(echo $c | sed -r "s/^0/4/ ; s/^1/1;4/") + ;; + esac + + [ -n "$color" ] && echo -ne "\033[${c}m" + set_background +} + +function reset_color { + echo -ne "\033[0m" + + set_background + [ -n "$foreground" ] && set_color "$foreground" +} + +function finish { + echo -ne "\033[0m" + echo +} + +function configure_palete { + case $1 in + blue) + background="\e[1;44m" + foreground="white" + ;; + *) + # Lightgray and Yellow are almost invisible on white + colors[yellow]=${colors[orange]} + colors[white]=${colors[black]} + colors[lightgray]=${colors[gray]} + colors[cyan]=${colors[blue]} + ;; + esac + + set_background + reset_color +} + +function decorate { + local output="$1" + local color=${2:-""} + local effect=${3:-""} + + if [ -n "$color" ]; then + set_color $color $effect + echo -ne "${output}" + reset_color + else + echo -ne "$output" + fi +} + +function eol { + sed 's/$/\x1B[K/' +} + +function print_eol { + echo -e '\x1B[K' +} + + + +function print { + local output="$1" + local color=${2:-""} + local char_limit=${3:-$fold} + local line_limit=${4:-0} + + local cmd="decorate \"${output}\" \"$color\" | fold -s -w $char_limit" + [ $line_limit -gt 0 ] && cmd="$cmd | head -n $line_limit" + + eval "$cmd" | eol +} + + +function print_hline { + local color=${1:-"gray"} + local cols=${COLUMNS:-$(tput cols)} + + decorate $(printf '%*s' $cols '' | tr ' ' -) "$color" + echo | eol +} + +function print_table { + # The problem here is that all escapes for colors are treated as normal visible symbols + sed "s/ ::: /@/g" | column -t -s "@" -o " " | eol +} + +function decorate_table { + print_hline + cat - + print_hline +} + +function print_status { + local status="$1" + + if [[ $status =~ ^.$ ]]; then + case $status in + 0) + decorate "*" "red" + ;; + 1) + decorate "*" "green" + ;; + 2) + decorate "*" "yellow" + ;; + *) + decorate "$status" "red" + ;; + esac + else + decorate "?" "red" + fi + echo -n " " +} + +clear diff --git a/lib/remote/report.sh b/lib/remote/report.sh new file mode 100644 index 0000000..e6766a4 --- /dev/null +++ b/lib/remote/report.sh @@ -0,0 +1,51 @@ +function simple_report { + local generator="$1" + local filter="${2:-cat}" + { + eval "$generator 12>/dev/fd/12 | $filter" + report=$(cat<&12) + } 12<&1 ) + + case "$service" in + 'traffic') + check_server_traffic "$result" "$@" + ;; + *) + print "$result" + esac + +} + + +function check_server_status { + local hopo + IFS=':' read -ra hopo <<< $1 && shift + local host=${hopo[0]} + local port=${hopo[1]:-22} + local services="$@" + + local ping_output=$(scripts/ping.pl $host $port $timeout) + print_status "$ping_output" + + if [ "$services" == "-" ]; then return; fi + + if [ "$ping_output" == "1" ]; then + local output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_status.sh 2>&1) + + local status=1 + [ -n "$output" ] && status=0 && output="\n$(decorate "$host:$port" "cyan" "u")\n$output\n" + print_status $status + + # Check Additional services, error is reported if anything provides information + local headers="" + if [ -n "$services" ]; then + for service in "$services"; do + if [[ "$service" =~ ^vpn/(.*)$ ]]; then + local ip=${BASH_REMATCH[1]} + [ $(hostname) == "styx" ] && check_server_status "$ip" "-" + else + local service_status=1 + local service_output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_${service}_status.sh 2>&1 | sed 's/^[[:space:]]*//;/^$/d') + if [ -n "$service_output" ]; then + first_line=$(echo "$service_output" | head -n 1) + if [[ "$first_line" =~ ^([0-9]+)[[:space:]](.*)$ ]]; then + service_status=${BASH_REMATCH[1]} + service_output=$(sed "1 d" <<< "$service_output") + if [ -n "${BASH_REMATCH[2]}" ]; then + service_header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "${BASH_REMATCH[2]}") #" + headers="$headers $service_header" + fi + else + service_status=0 + fi + + if [ -n "$service_output" ]; then + [ -z "$output" ] && output="\n$(decorate "$host:$port" "cyan" "u")" + output="${output}\n $(decorate "$service" "gray" "u")\n$(echo ${service_output} | sed 's/^/ /')" + fi + fi + fi + + done + fi + + print " ::: ${headers}" + #report="$report
$output" + if [ -n "$output" ]; then + flock -x $0 echo "${output}" >&12 + fi + else + print_status "x" + for service in "$services"; do + print_status "x" + done + fi +} + +function check_service { + local service="$1" && shift + local id="$1" && shift + local args="$@" + + local output=$(service/check_${service}.sh ${id} "$@" 2>&1 | sed 's/^[[:space:]]*//;/^$/d') + + local info=$(echo "$output" | tail -n 1) + + local online=0 + local health=0 + local header="" + if [[ "$info" =~ ^([0-9]+)[[:space:]]+([0-9]+)[[:space:]]+(.*)$ ]]; then + online=${BASH_REMATCH[1]} + health=${BASH_REMATCH[2]} + header=${BASH_REMATCH[3]} + + output=$(sed '$ d' <<< "$output") + fi + + print_status "$online" + print_status "$health" + + header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "$header") # " + print " ::: ${header}" + + + important=$(grep "^\*" <<< "$output") + messages=$(grep -v "^\*" <<< "$output") + + if [ -n "$output" ]; then + output="\n$(decorate "$service:$id" "cyan" "u")\n$important\n$(set_color gray)$messages$(reset_color)\n" + flock -x $0 echo "${output}" >&12 + fi +} + + +function check__ { + local args + local title="$1" && shift + read -ra args <<< "$1" && shift + local host=${args[0]}; + + print "$(decorate $title "purple") ::: " | sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g; " + + local service + for service in "$@"; do + service=$(sed -r "s/<([0-9]+)>/\${args[\\1]}/g; s/<([^>]+)>/\${\\1}/g" <<< "$service") # " + ( eval "$service" ) + done + print_eol +} + +function check_ { + # Buffer the output + local output=$(check__ "$@") + echo -e "$output" +} + +function check { + if [ $parallel -gt 0 ]; then + check_ "$@" & + else + check_ "$@" + fi +} -- cgit v1.2.1