#!/bin/bash
#
# $Id$
#
# logs the load to /var/log/montools/load/YYYY-MM-DD.log
#
# if the load is unusually high, further information is gathered in
# /var/log/montools/loadreasons/YYYY-MM-DD_hh:mm_$curload


# prepare

. /usr/lib/montools/mt-prepare

RESNAME="Load"

LOCKFILE=/tmp/mt-log-load.lck
if ( set -o noclobber; echo "Locked" > "$LOCKFILE") 2> /dev/null; then
  trap 'rm -f "$LOCKFILE"; exit $?' INT TERM EXIT
else
  echo "Failed to lock $LOCKFILE, aborting." >&2
  add2warnings \
    "mt-log-load aborting on $HOSTNAME because $LOCKFILE still exists from former run"
  . /usr/lib/montools/mt-notify-exit
  exit 1
fi

# work

#. /root/.profile

DS=$(date +%Y-%m-%d\ %H%M)
DSS=$(echo "$DS" |cut -f1 -d' ')
LOAD=$(cat /proc/loadavg)
mkdir -p /var/log/montools/load
echo "$DS load $LOAD" >>/var/log/montools/load/${DSS}.log
#echo "$DS load $LOAD"

CURLOAD=$(echo "$LOAD" |cut -f1 -d'.')

if [ $CURLOAD -lt $WARNINGLOAD ] ; then
  add2infos "Load OK at ${CURLOAD}"
  #. /usr/lib/montools/mt-notify-exit
  exit 0
fi

if [ $CURLOAD -gt $PANICLOAD ] ; then
  add2criticals \
     "Load over PANIC treshhold ${PANICLOAD}: ${CURLOAD}." \
     "Suppressing further analytical activities to protect system."
  . /usr/lib/montools/mt-notify-exit
fi
 
mkdir -p /var/log/montools/loadreasons
#LOGDIR="/var/log/montools/loadreasons/"$(date  +%Y-%m-%d\_%H%M)"_"${CURLOAD}
LOGDIR="/var/log/montools/loadreasons/$(date  +%Y-%m-%d\_%H%M)_${CURLOAD}"
#echo "LOGDIR=$LOGDIR"

if [ $CURLOAD -gt $CRITICALLOAD ] ; then
  add2criticals \
    "Load over CRITICAL treshhold ${CRITICALLOAD}: ${LOAD}" \
    "See ${LOGDIR}/"
else
  add2warnings \
    "Load over WARNING treshhold ${WARNINGLOAD}: ${LOAD}" \
    "See ${LOGDIR}/"
fi

# notifications BEFORE further analytical activities, those might hold me up considerable!

export RC=0
. /usr/lib/montools/mt-notify


mkdir -p $LOGDIR

free -h >$LOGDIR/free 2>&1

top -b -n1 >$LOGDIR/top.out 2>&1

if [ -x /usr/bin/mysqladmin ] ; then
  mysqladmin --verbose processlist >$LOGDIR/mysqladmin_processlist.out 2>&1
else
  echo "No mysqladmin. For container it might be required to run $0 in the container." >$LOGDIR/mysqladmin_not_found.txt
fi

# with recent kernels this is restricted to sockets in host, not in container
lsof -i -n -l -P -b -w |gzip >$LOGDIR/lsof-i.out.gz 2>&1

# this never provided any help, and does not show files open sockets in containers any more:
#lsof -n -l -P -b -w |gzip >$LOGDIR/lsof.out.gz 2>&1

ps -ef >$LOGDIR/ps-ef.out 2>&1

vmstat 1 3  >$LOGDIR/vmstat.log 2>&1

if [ -x /etc/montools/log-extra-load-reasons ] ; then
  /etc/montools/log-extra-load-reasons $LOGDIR
fi

# log /proc/*/stack too
#cd /proc
#
#PDDIR=$LOGDIR/proc_details
#mkdir -p $PDDIR
#
#PL=$(find . -mindepth 1 -maxdepth 1 -type d |cut -c3- |egrep "[0-9]+$")
#for PID in $PL ; do
#  if [ -e "/proc/$PID/stack" ] ; then
#    mkdir -p $PDDIR/$PID
#    cat /proc/$PID/stack >$PDDIR/$PID/stack
#  #else
#  #  NOP="1"
#  #  #echo "No /proc/$PID/stack" >$PDDIR/$PID/no_stack.err
#  fi
#done

exit 0

