#!/bin/bash
#
# Command to check how long ago a cron job successfully ran, and potentially
# provide helpful context for the user as to why.
#
# Utilizes the check_file_age nagios plugin.
set -u

function get_context {
  # Scans syslog for a given logger tag where we might have additional
  # information about why a cron job failed.
  echo "Some potentially helpful context from /var/log/messages: (may be empty)"
  grep "${1}: " /var/log/messages | tail | head -c 1024
}

name=$1
threshold_s=$2

<%= @check_file_age_path %> "/nail/run/success_wrapper/${name}" -w ${threshold_s} -c ${threshold_s} >/dev/null
return_code=$?

last_success=$(stat -c %z "/nail/run/success_wrapper/${name}")

if [ $return_code -ne 0 ]; then
  echo "${name} hasn't completed successfully since ${last_success}"
  get_context ${name}
else
  echo "${name} last completed successfully ${last_success}"
fi

exit $return_code

#!/bin/bash
#
# Command to check how long ago a cron job successfully ran, and potentially
# provide helpful context for the user as to why.
#
# Utilizes the check_file_age nagios plugin.
set -u

function get_context {
  # Scans syslog for a given logger tag where we might have additional
  # information about why a cron job failed.
  echo "Some potentially helpful context from /var/log/messages: (may be empty)"
  grep "${1}: " /var/log/messages | tail | head -c 1024
}

function uptime_less_than {
  local threshold=$1
  local uptime_seconds="$(cat /proc/uptime | grep -o '^[0-9]\+')"
  [[ $uptime_seconds -lt $threshold ]]
}

name=$1
threshold_s=$2

# If the server hasn't been up for long enough for the cron job to run in the
# first place, we don't need to alert people. This suppresses alerts that come
# from servers that are brought back up after reboots, downtime, or from being
# recently launched/provisioned.
if uptime_less_than $threshold_s; then
  echo "OK: Our uptime is less than $threshold_s seconds:"
  uptime
  echo "Assuming things will be OK once the script has had a chance to run."
  exit 0
fi

<%= @check_file_age_path %> "/nail/run/success_wrapper/${name}" -w ${threshold_s} -c ${threshold_s} >/dev/null
return_code=$?

last_success=$(stat -c %z "/nail/run/success_wrapper/${name}")

if [ $return_code -ne 0 ]; then
  echo "${name} hasn't completed successfully since ${last_success}"
  get_context ${name}
else
  echo "${name} last completed successfully ${last_success}"
fi

exit $return_code

Make the staleness_check wrapper 'uptime-aware'
Make the staleness_check wrapper 'uptime-aware'
