#!/bin/sh
#
# Copyright 2011 (C) VMware, Inc.  All rights reserved.
#
# hbrsrv-stats:
#	Gather some basic statistics about the hbrserver and the
#       appliance VM it is running in and record it for later
#       analysis.
#
# The sysstat service is used to gather most of the statistics on the system
# but it doesn't quite get all the data that's necessary. This script is
# meant to be run periodically gather the remaining bits of data.
#

PROG_NAME=$(basename $0)

CONFIG_FILE="/etc/hbrsrv-stats.config"
STAT_FILE_PREFIX="hbrsrv-"
STAT_FILE_SUFFIX=".stats"

HBRSRV_BIN="hbrsrv-bin"

#
# This is the format of the data in the file. This number should be
# bumped if one of the versions changes so the stats parsing program
# can be sure it understands the file's contents.
#
DATA_FORMAT_VERSION="2.0"

#
# Defaults
#
LOG_DIR="/var/log/vmware"
ROTATE=""
COMPRESS=""
MAXFILESIZE=""

Log() {
   LEVEL=$1
   shift
   logger -p daemon.${LEVEL} -t "${PROG_NAME}" "$*"
}

GetConfigOptions() {
   # Use defaults if there's no config file.
   if [ ! -r "${CONFIG_FILE}" ]
   then
      return
   fi
   
   # Get the value of the last line that contains "logdir=<dir>"
   LINE=$(grep '^[ \t]*logdir[ \t]*=[ \t]*[^ \t]\+[ \t]*$' ${CONFIG_FILE} | \
          sed -e 's@[ \t]@@g' | cut -d '=' -f 2- | tail -n 1)
   if [ -n "${LOG_DIR}" ] && [ -d "${LOG_DIR}" ]
   then
      LOG_DIR="${LINE}"
   fi
   
   # Get the value of the last line that contains "rotate=<number>"
   LINE=$(grep '^[ \t]*rotate[ \t]*=[ \t]*[0-9]\+[ \t]*$' ${CONFIG_FILE} | \
          sed -e 's@[ \t]@@g' | cut -d '=' -f 2- | tail -n 1)
   if [ -n "${LINE}" ]
   then
      ROTATE="${LINE}"
   fi
   
   # Check if there's a "compress" line in the file.
   LINE=$(grep '^[ \t]*compress[ \t]*$' ${CONFIG_FILE})
   if [ -n "${LINE}" ]
   then
      COMPRESS="y"
   fi
   
   # Get the value of the last line that contains "maxfilesize=<number>"
   LINE=$(grep '^[ \t]*maxfilesize[ \t]*=[ \t]*[0-9]\+[ \t]*$' ${CONFIG_FILE} | \
          sed -e 's@[ \t]@@g' | cut -d '=' -f 2- | tail -n 1)
   if [ -n "${LINE}" ]
   then
      MAXFILESIZE=${LINE}
   fi
}

GetCurrentStatsFile()
{
   ls ${LOG_DIR}/${STAT_FILE_PREFIX}*${STAT_FILE_SUFFIX} 2>/dev/null | \
      grep -v 'int.stats' | sort -r | head -n 1
}

CreateNewStatsFile()
{
   TIMESTAMP=$(date +"%s")
   STATS_FILE=${LOG_DIR}/${STAT_FILE_PREFIX}${TIMESTAMP}${STAT_FILE_SUFFIX}
   echo "0 FILEVERSION ${DATA_FORMAT_VERSION}" > ${STATS_FILE}
}

RotateFiles()
{
   # Check if there are enough items to bother rotating.
   NUM_FILES=$(find ${LOG_DIR} -maxdepth 1 -type f \
               -name "${STAT_FILE_PREFIX}*${STAT_FILE_SUFFIX}*" | \
               grep -v 'int.stats' | wc -l)
   if [ ${NUM_FILES} -lt ${ROTATE} ]
   then
      return
   fi

   # Remove the extras + 1 for the new stats
   NUM_RMS=$(expr ${NUM_FILES} - ${ROTATE} + 1)
   if [ "${NUM_RMS}" -le "0" ]
   then
      return
   fi
   
   # Remove the oldest files that fall outside the rotation.
   find ${LOG_DIR} -maxdepth 1 -type f \
         -name "${STAT_FILE_PREFIX}*${STAT_FILE_SUFFIX}*" \
         -printf "%T@ %p\n" | grep -v 'int.stats' | sort | head -n ${NUM_RMS} | cut -d ' ' -f 2- | \
   while read f
   do
      Log info "Rotating out old stat file $f"
      rm -f "$f"
   done
}

CompressFiles()
{
   find ${LOG_DIR} -maxdepth 1 -type f \
      -name "${STAT_FILE_PREFIX}*${STAT_FILE_SUFFIX}" -exec gzip -f '{}' \;
}

GetHbrFdData()
{
   PID=$(pidof ${HBRSRV_BIN})
   PROC_PATH=/proc/${PID}/fd/

   if [ -z "${PID}" ] || [ ! -e "${PROC_PATH}" ]
   then
      # If there's no hbrsrv active. Just bail.
      return
   fi
   
   NUM_FDS=$(ls ${PROC_PATH} 2>/dev/null | wc -l)
   NUM_SOCKETS=$(find ${PROC_PATH} -exec readlink \{\} \; | \
                 grep '^socket:' | wc -l)
   
   TIMESTAMP=$(date +"%s")
   echo "${TIMESTAMP} HBRSRVFD ${NUM_FDS} ${NUM_SOCKETS}"
}

GetFSUsageData()
{
   # Get the usage stats on the local filesystems
   DATA=$(df -l | sed -e '/^Filesystem.*$/d' -e 's/[ \t]\+/ /g' | \
          cut -d ' ' -f 1,2,3,4,6 | sed -e 's/$/ /g' | tr -d '\n')
   TIMESTAMP=$(date +"%s")
   echo "${TIMESTAMP} FSUSAGE ${DATA}"
}

GetHbrDbSizeData()
{
   # Get the sizes of the database files
   SIZES=$(stat -c "%n %s" /etc/vmware/hbrsrv*.db | \
           sed -e 's@/etc/vmware/@@g' -e 's/$/ /g' | tr -d '\n')
   if [ -n "${SIZES}" ]
   then
      TIMESTAMP=$(date +"%s")
      echo "${TIMESTAMP} HBRDBSIZE ${SIZES}"
   fi
}

GetHbrNumThreadsData()
{
   # Get the number of threads in the hbrsrv
   THREADS=$(ls /proc/$(pidof hbrsrv-bin)/task 2>/dev/null | wc -l)
   TIMESTAMP=$(date +"%s")
   echo "${TIMESTAMP} HBRSRVTHREADS ${THREADS}"
}

GetHbrMemStatsData()
{
   STATS=$(cut -d ' ' -f 1,2 /proc/$(pidof hbrsrv-bin)/statm)
   if [ -n "${STATS}" ]
   then
      TIMESTAMP=$(date +"%s")
      echo "${TIMESTAMP} HBRVM ${STATS}"
   fi
}

GetSysLoadAvg()
{
   STATS=$(cut -d ' ' -f 1-3 /proc/loadavg)
   if [ -n "${STATS}" ]
   then
      TIMESTAMP=$(date +"%s")
      echo "${TIMESTAMP} LOADAVG ${STATS}"
   fi
}

GetSysSwap()
{
   STATS=$(grep pgpg /proc/vmstat | cut -d ' ' -f 2-2 | tr -s '\n' ' ' | \
           sed -e 's/ $/\n/g')
   if [ -n "${STATS}" ]
   then
      TIMESTAMP=$(date +"%s")
      echo "${TIMESTAMP} SWAP ${STATS}"
   fi
}

GetSysNetTcp()
{
   STATS=$(grep 'TcpExt: [0-9]' /proc/net/netstat | cut -d' ' -f 2- )
   if [ -n "${STATS}" ]
   then
      TIMESTAMP=$(date +"%s")
      echo "${TIMESTAMP} NETSTATTCP ${STATS}"
   fi
}

GetSysNetIp()
{
   STATS=$(grep 'IpExt: [0-9]' /proc/net/netstat | cut -d' ' -f 2- )
   if [ -n "${STATS}" ]
   then
      TIMESTAMP=$(date +"%s")
      echo "${TIMESTAMP} NETSTATIP ${STATS}"
   fi
}

##
## Main
##

GetConfigOptions

# If no hbrsrv is running. Don't collect any data.
if [ -z "$(pidof hbrsrv-bin)" ]
then
   exit 0
fi

# Gather all the data
HBRFD_DATA=$(GetHbrFdData)
FSUSAGE_DATA=$(GetFSUsageData)
HBRDBSIZE_DATA=$(GetHbrDbSizeData)
HBRSRVTHREADS_DATA=$(GetHbrNumThreadsData)
HBRMEMSTATS_DATA=$(GetHbrMemStatsData)
SYSLOADAVG_DATA=$(GetSysLoadAvg)
SYSSWAP_DATA=$(GetSysSwap)
SYSNETTCP_DATA=$(GetSysNetTcp)
SYSNETIP_DATA=$(GetSysNetIp)

# Find a place to write the data.
STATS_FILE=$(GetCurrentStatsFile)
if [ -w "${STATS_FILE}" ]
then
   # See if we need to rotate ahead of the write.
   if [ -n "$MAXFILESIZE" ]
   then
      # Compute the total length of the data to write.
      HBRFD_LEN=$(echo "${HBRFD_DATA}" | wc -c)
      FSUSAGE_LEN=$(echo "${FSUSAGE_DATA}" | wc -c)
      HBRDBSIZE_LEN=$(echo "${HBRDBSIZE_DATA}" | wc -c)
      HBRSRVTHREADS_LEN=$(echo "${HBRSRVTHREADS_DATA}" | wc -c)
      HBRMEMSTATS_LEN=$(echo "${HBRMEMSTATS_DATA}" | wc -c)
      SYSLOADAVG_LEN=$(echo "${SYSLOADAVG_DATA}" | wc -c)
      SYSSWAP_LEN=$(echo "${SYSSWAP_DATA}" | wc -c)
      SYSNETTCP_LEN=$(echo "${SYSNETTCP_DATA}" | wc -c)
      SYSNETIP_LEN=$(echo "${SYSNETIP_DATA}" | wc -c)
      
      TOTAL_LEN=$(expr ${HBRFD_LEN} + ${FSUSAGE_LEN} + ${HBRDBSIZE_LEN} +\
                       ${HBRSRVTHREADS_LEN} + ${HBRMEMSTATS_LEN} + \
                       ${SYSLOADAVG_LEN} + ${SYSSWAP_LEN} + \
                       ${SYSNETTCP_LEN} + ${SYSNETIP_LEN})

      # Make sure the current file supports our data format
      FILEVER=$(head -n 1 ${STATS_FILE} | cut -d' ' -f 3-3)
      if [ -z "${FILEVER}" ] || [ "${FILEVER}" != "${DATA_FORMAT_VERSION}" ]
      then
         NEEDS_ROTATE="y"
      else
         # Get the length of the current file
         STATS_FILE_LEN=$(stat -c "%s" ${STATS_FILE})

         # Make sure what will be written can fit in the file.
         TOTAL_LEN=$(expr ${TOTAL_LEN} + ${STATS_FILE_LEN})
         if [ "${TOTAL_LEN}" -gt "${MAXFILESIZE}" ]
         then
            NEEDS_ROTATE="Y"
         fi
      fi
      
      if [ -n "${NEEDS_ROTATE}" ]
      then
         # Rotate/compress/create         
         if [ -n "${ROTATE}" ]
         then
            RotateFiles
         fi
         
         if [ -n "${COMPRESS}" ]
         then
            CompressFiles
         fi

         CreateNewStatsFile
      fi
   fi
else
   # A clean slate. Generate a new file.
   CreateNewStatsFile
fi

# It is now safe to write the stats out to the file.
echo "${HBRFD_DATA}" >> ${STATS_FILE}
echo "${FSUSAGE_DATA}" >> ${STATS_FILE}
echo "${HBRDBSIZE_DATA}" >> ${STATS_FILE}
echo "${HBRSRVTHREADS_DATA}" >> ${STATS_FILE}
echo "${HBRMEMSTATS_DATA}" >> ${STATS_FILE}
echo "${SYSLOADAVG_DATA}" >> ${STATS_FILE}
echo "${SYSSWAP_DATA}" >> ${STATS_FILE}
echo "${SYSNETTCP_DATA}" >> ${STATS_FILE}
echo "${SYSNETIP_DATA}" >> ${STATS_FILE}
