#!/bin/bash
# 
# 0. sources /usr/lib/bsclient/load_environment
#    which guarantees $BSCLIENT_ETC_DIR and $BSCLIENT_ENV_FILE beeing set,
#    usally to /etc/bsclient resp. /etc/bsclient/environment
# 1. executes all commands in $BSCLIENT_ETC_DIR/prep-steps
#    and lists them with OK or FAILURE
# 2. calls bsc-backup for all directories listed in $BSCLIENT_ETC_DIR/dirs2backup
# 3. executes all commands in $BSCLIENT_ETC_DIR/post-steps

MYPID="$$"

echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: bsc-daily starting with PID ${MYPID}"
echo " "

. /usr/lib/bsclient/load_environment

# hacks with set -C or flock are weird, and we don't need hi-res timing, so just go with -f, trap, echo ...
MYPID=$$
if [ -f /run/bsc-daily.pid ] ; then
  OTHERPID=$(cat /run/bsc-daily.pid)
  MSG=$(
    echo "$(date +%Y-%m-%d\ %H:%M:%S): ERROR: Another bsc-daily might still be running on $(hostname) with PID ${OTHERPID}, aborting myself ($MYPID)."
    echo ""
    echo 'ps -ef |egrep "bsc-|'"${OTHERPID}"'" says:'
    ps -ef |egrep "bsc-|${OTHERPID}" |grep -v "grep |fmt -s"
    echo ""
    if [ -f /proc/${OTHERPID}/cgroup ] ; then
      echo 'Content of /proc/'"${OTHERPID}"'/cgroup:'
      cat /proc/${OTHERPID}/cgroup
    else
      echo 'There is no /proc/'"${OTHERPID}"'/cgroup, maybe /run/bsc-daily.pid is obsolete.'
    fi
    echo ""
  )
  echo "$MSG"
  if [ -x /usr/bin/mail.mailutils  ] ; then
    echo "$MSG" |/usr/bin/mail.mailutils -s "bsc-daily error on $(hostname)" root
  elif [ -x /usr/bin/mail -o -x /bin/mail ] ; then
    echo "$MSG" |mail root
  elif [ -x /usr/sbin/sendmail ] ; then
    ( echo "To: root@$(hostname)" ; echo "Subject: bsc-daily error on $(hostname)" ; echo "" ; echo "$MSG" ) |/usr/sbin/sendmail root
  fi
  exit 1
fi
trap 'rm -f /run/bsc-daily.pid ; exit $?' INT TERM EXIT KILL
echo $MYPID >/run/bsc-daily.pid

FAILURESFILE=/tmp/bsc-daily_$$_failures.txt 
test -f ${FAILURESFILE} && mv ${FAILURESFILE} ${FAILURESFILE}.bak

echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Making sure .bsclient/ subdir exists on backup target ..."
if [ "$BAKMKDIRMODE" = "sftp" ] ; then
  #echo 'Calling   echo mkdir -pv ${BAKDESTBASE}/.bsclient |SSH_AUTH_SOCK="" SSH_CLIENT="" SSH_CONNECTION="" SSH_TTY=""'" sftp -i ${BAKSSHIDFILE} ${BAKUSER}@${BAKHOST} ..."
  echo mkdir -pv ${BAKDESTBASE}/.bslcient |SSH_AUTH_SOCK="" SSH_CLIENT="" SSH_CONNECTION="" SSH_TTY="" sftp -i ${BAKSSHIDFILE} ${BAKUSER}@${BAKHOST}
else
  #echo 'Calling   SSH_AUTH_SOCK="" SSH_CLIENT="" SSH_CONNECTION="" SSH_TTY=""'" ssh -i ${BAKSSHIDFILE} ${BAKUSER}@${BAKHOST} mkdir -pv ${BAKDESTBASE}/.bsclient ..."
  SSH_AUTH_SOCK="" SSH_CLIENT="" SSH_CONNECTION="" SSH_TTY="" ssh -i ${BAKSSHIDFILE} ${BAKUSER}@${BAKHOST} mkdir -pv ${BAKDESTBASE}/.bsclient
fi

echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Touching .bsclient/bsc-daily-beginning.txt on backup target ..."
if [ "$BAKMKDIRMODE" = "sftp" ] ; then
  echo touch ${BAKDESTBASE}/.bsclient/bsc-daily-beginning.txt |SSH_AUTH_SOCK="" SSH_CLIENT="" SSH_CONNECTION="" SSH_TTY="" sftp -i ${BAKSSHIDFILE} ${BAKUSER}@${BAKHOST}
else
  SSH_AUTH_SOCK="" SSH_CLIENT="" SSH_CONNECTION="" SSH_TTY="" ssh -i ${BAKSSHIDFILE} ${BAKUSER}@${BAKHOST} touch ${BAKDESTBASE}/.bsclient/bsc-daily-beginning.txt
fi
echo " "

PREPSTEPFILES=$(ls -1d ${BSCLIENT_ETC_DIR}/prep-steps ${BSCLIENT_ETC_DIR}/bsclient-etc-dirs.d/*/prep-steps 2>/dev/null |sort)
if [ -n "${PREPSTEPFILES}" ] ; then
  echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Found the following prep-steps files:"
  echo "${PREPSTEPFILES}"
  echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Doing pre-backup work from these prep-steps files ..."
  #cat ${PREPSTEPFILES} |egrep -v '^\s*(#|;|$)' |while read SCRIPTLINE
  egrep -H -n -v '^\s*(#|;|$)' ${PREPSTEPFILES} |while read FILELINESCRIPT
  do
    #echo "DEBUG: FILELINESCRIPT=$FILELINESCRIPT"
    STEPFILE=$(echo "$FILELINESCRIPT" |cut -f1 -d ':')
    #echo "DEBUG: STEPFILE=$STEPFILE"
    STEPLINENR=$(echo "$FILELINESCRIPT" |cut -f2 -d ':')
    #echo "DEBUG: STEPLINENR=$STEPLINENR"
    SCRIPTLINE=$(echo "$FILELINESCRIPT" |cut -f3- -d ':')
    #echo "DEBUG: SCRIPTLINE=$SCRIPTLINE"
    export STEPOK="unknown"
    SCRIPTCMD=$(basename $(echo "$SCRIPTLINE" |cut -f1 -d ' '))
    #echo "DEBUG: SCRIPTCMD=$SCRIPTCMD"
    #STEPFILEKEY=$(echo "${STEPFILE}" |sed -e 's|/etc/bsclient/prep-steps|main|g' -e 's|/etc/bsclient/bsclient-etc-dirs.d/\([^/]\+\)/prep-steps|\1|g')
    STEPFILEKEY=$(echo "${STEPFILE}" |cut -f5 -d '/') # |sed -e 's|^$|main|g') # 'main' could be used in bsclient-etc-dirs.d/ too, and double __ is visible too
    #echo "DEBUG: SCRIPTFILEKEY=$SCRIPTFILEKEY"
    SCRIPTLOGFILE="${BSCLIENT_LOGBASE}/bsc-prep-step_${STEPFILEKEY}_${STEPLINENR}_${SCRIPTCMD}.log"
    #echo "DEBUG: SCRIPTLOGFILE=$SCRIPTLOGFILE"
    test -f ${SCRIPTLOGFILE} && mv -v ${SCRIPTLOGFILE} ${SCRIPTLOGFILE}.bak
    STEPOK="NO"
    echo -n "$(date +%Y-%m-%d\ %H:%M:%S): INFO: For ${STEPFILE} line ${STEPLINENR} calling $SCRIPTLINE" |tee ${SCRIPTLOGFILE}
    echo "" >>${SCRIPTLOGFILE}
    $SCRIPTLINE >>${SCRIPTLOGFILE} 2>&1 && STEPOK="YES"
    if [ "$STEPOK" = "YES" ] 
    then
      echo " - OK" 
      echo "$(date +%Y-%m-%d\ %H:%M:%S): GOOD: Back from script, seems it went OK" |tee -a ${SCRIPTLOGFILE}
    else
      echo " - FAILURE"
      echo "Please check ${SCRIPTLOGFILE}"
      echo "$(date +%Y-%m-%d\ %H:%M:%S): Prep-step ${SCRIPTLINE} FAILED, see ${SCRIPTLOGFILE}" >>${FAILURESFILE}
      echo "$(date +%Y-%m-%d\ %H:%M:%S): ERROR: Back from script, seems it FAILED" |tee -a ${SCRIPTLOGFILE}
    fi
  done
else
  echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: No ${BSCLIENT_ETC_DIR}/prep-steps or ${BSCLIENT_ETC_DIR}/bsclient-etc-dirs.d/*/prep-steps, not doing any preparation work."
fi

echo " "

DIRLISTFILES=$(ls -1d ${BSCLIENT_ETC_DIR}/dirs2backup ${BSCLIENT_ETC_DIR}/bsclient-etc-dirs.d/*/dirs2backup 2>/dev/null |sort)
export DIRLIST=""
if [ -n "${DIRLISTFILES}" ] ; then
  echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Found the following dirs2backup files:"
  echo "${DIRLISTFILES}"
  echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Backuping up directories from these dirs2backup files ..."
  DIRLIST=$(cat ${DIRLISTFILES} |cut -f1 -d ' ' |egrep -v '^\s*(#|;|$)')
else
  echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: No ${BSCLIENT_ETC_DIR}/dirs2backup or ${BSCLIENT_ETC_DIR}/bsclient-etc-dirs.d/*/dirs2backup, not backing up any directories."
fi
for DIR in $DIRLIST
do
  LOGFILENAME=$(echo "$DIR" |cut -c2- |tr '/' '_')
  test -f $BSCLIENT_LOGBASE/bsc-backup_$LOGFILENAME.log && mv -v $BSCLIENT_LOGBASE/bsc-backup_$LOGFILENAME.log $BSCLIENT_LOGBASE/bsc-backup_$LOGFILENAME.log.bak
  export SYNCOK="NO"
  echo -n "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Synching $DIR"
  /usr/sbin/bsc-backup $DIR >$BSCLIENT_LOGBASE/bsc-backup_$LOGFILENAME.log 2>&1 && SYNCOK="YES"
  if [ "$SYNCOK" = "YES" ] 
  then 
    echo " - OK" 
    echo "$(date +%Y-%m-%d\ %H:%M:%S): GOOD: Back from sync, seems it went OK" |tee -a $BSCLIENT_LOGBASE/bsc-backup_$LOGFILENAME.log
  else 
    echo " - FAILURE"
    echo "Please check $BSCLIENT_LOGBASE/bsc-backup_$LOGFILENAME.log"
    echo "$(date +%Y-%m-%d\ %H:%M:%S): Syncing $DIR FAILED, see $BSCLIENT_LOGBASE/bsc-backup_$LOGFILENAME.log" >>${FAILURESFILE}
    echo "$(date +%Y-%m-%d\ %H:%M:%S): ERROR: Back from sync, seems it FAILED" |tee -a $BSCLIENT_LOGBASE/bsc-backup_$LOGFILENAME.log
  fi
done
echo " "

POSTSTEPFILES=$(ls -1d ${BSCLIENT_ETC_DIR}/post-steps ${BSCLIENT_ETC_DIR}/bsclient-etc-dirs.d/*/post-steps 2>/dev/null |sort)
if [ -n "${POSTSTEPFILES}" ] ; then
  echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Found the following post-steps files:"
  echo "${POSTSTEPFILES}"
  echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Doing post-backup work from these post-steps files ..."
  #cat ${POSTSTEPFILES} |egrep -v '^\s*(#|;|$)' |while read SCRIPTLINE
  egrep -H -n -v '^\s*(#|;|$)' ${POSTSTEPFILES} |while read FILELINESCRIPT
  do
    STEPFILE=$(echo "$FILELINESCRIPT" |cut -f1 -d ':')
    STEPLINENR=$(echo "$FILELINESCRIPT" |cut -f2 -d ':')
    SCRIPTLINE=$(echo "$FILELINESCRIPT" |cut -f3- -d ':')
    export STEPOK="unknown"
    SCRIPTCMD=$(basename $(echo "$SCRIPTLINE" |cut -f1 -d ' '))
    #echo "SCRIPTCMD=$SCRIPTCMD"
    STEPFILEKEY=$(echo "${STEPFILE}" |cut -f5 -d '/') # |sed -e 's|^$|main|g')
    SCRIPTLOGFILE="${BSCLIENT_LOGBASE}/bsc-post-step_${STEPFILEKEY}_${STEPLINENR}_${SCRIPTCMD}.log"
    test -f ${SCRIPTLOGFILE} && mv -v ${SCRIPTLOGFILE} ${SCRIPTLOGFILE}.bak
    STEPOK="NO"
    echo -n "$(date +%Y-%m-%d\ %H:%M:%S): INFO: For ${STEPFILE} line ${STEPLINENR} calling $SCRIPTLINE" |tee ${SCRIPTLOGFILE}
    echo "" >>${SCRIPTLOGFILE}
    $SCRIPTLINE >>${SCRIPTLOGFILE} 2>&1 && STEPOK="YES"
    if [ "$STEPOK" = "YES" ] 
    then 
      echo " - OK" 
      echo "$(date +%Y-%m-%d\ %H:%M:%S): GOOD: Back from script, seems it went OK" |tee -a ${SCRIPTLOGFILE}
    else 
      echo " - FAILURE"
      echo "Please check ${SCRIPTLOGFILE}"
      echo "$(date +%Y-%m-%d\ %H:%M:%S): Post-step script ${SCRIPTLINE} FAILED, see ${SCRIPTLOGFILE}" >>${FAILURESFILE}
      echo "$(date +%Y-%m-%d\ %H:%M:%S): ERROR: Back from script, seems it FAILED" |tee -a ${SCRIPTLOGFILE}
    fi
  done
else
  echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: No ${BSCLIENT_ETC_DIR}/post-steps or ${BSCLIENT_ETC_DIR}/bsclient-etc-dirs.d/*/post-steps, not doing any post-backup work."
fi

echo " "

echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Touching .bsclient/bsc-daily-done.txt on backup target ..."
if [ "$BAKMKDIRMODE" = "sftp" ] ; then
  echo touch ${BAKDESTBASE}/.bsclient/bsc-daily-done.txt |SSH_AUTH_SOCK="" SSH_CLIENT="" SSH_CONNECTION="" SSH_TTY="" sftp -i ${BAKSSHIDFILE} ${BAKUSER}@${BAKHOST}
else
  SSH_AUTH_SOCK="" SSH_CLIENT="" SSH_CONNECTION="" SSH_TTY="" ssh -i ${BAKSSHIDFILE} ${BAKUSER}@${BAKHOST} touch ${BAKDESTBASE}/.bsclient/bsc-daily-done.txt
fi

echo " "

UPDATELISTFILE="${BSCLIENT_LOGBASE}/available_updates.txt"
# |wc -l |tr -d ' ')
if [ -z "$UPDATELISTFILE" ] ; then
  echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Update checks did not find any updates - OK"
else
  echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: Update checker suggest these updates:"
  cat $UPDATELISTFILE
fi

#fping $BAKHOST || exit 1

#echo "Press ENTER to continue" ; read ENTER

echo " "

test -f ${FAILURESFILE} || exit 0
chmod go-rwx ${FAILURESFILE}

echo "$(date +%Y-%m-%d\ %H:%M:%S): INFO: There were FAILURES:"
cat ${FAILURESFILE}

echo $MSG
if [ -x /usr/bin/mail.mailutils  ] ; then
  ( echo "There were FAILURES on $(hostname):" ; cat ${FAILURESFILE} ) |/usr/bin/mail.mailutils -s "bsc-daily error on $(hostname)" root
elif [ -x /usr/bin/bsd-mailx ] ; then
  ( echo "There were FAILURES on $(hostname):" ; cat ${FAILURESFILE} ) |/usr/bin/bsd-mailx -s "bsc-daily error on $(hostname)" root
elif [ -x /usr/sbin/sendmail ] ; then
  ( echo "To: root@$(hostname)" ; echo "Subject: bsc-daily error on $(hostname)" ; echo "" ; echo "There were FAILURES on $(hostname):" ; cat ${FAILURESFILE} ) |/usr/sbin/sendmail root
elif [ -x /usr/bin/mail -o -x /bin/mail ] ; then
  ( echo "There were FAILURES on $(hostname):" ; cat ${FAILURESFILE} ) |mail root
fi

rm -f ${FAILURESFILE}

exit 1

