#!/bin/bash
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
 #
 #    http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
 
 set -o errexit
 
 # time to sleep between checks
 SLEEP_TIME=20
 
 # MemAvailable is the best estimation and has built-in heuristics
 # around reclaimable memory.  However, it is not available until 3.14
 # kernel (i.e. Ubuntu LTS Trusty misses it).  In that case, we fall
 # back to free+buffers+cache as the available memory.
 USE_MEM_AVAILBLE=0
 if grep -q '^MemAvailable:' /proc/meminfo; then
     USE_MEM_AVAILABLE=1
 fi
 
 function get_mem_available {
     if [[ $USE_MEM_AVAILABLE -eq 1 ]]; then
         awk '/^MemAvailable:/ {print $2}' /proc/meminfo
     else
         awk '/^MemFree:/ {free=$2}
             /^Buffers:/ {buffers=$2}
             /^Cached:/  {cached=$2}
             END { print free+buffers+cached }' /proc/meminfo
     fi
 }
 
 # whenever we see less memory available than last time, dump the
 # snapshot of current usage; i.e. checking the latest entry in the
 # file will give the peak-memory usage
 function tracker {
     local low_point=$(get_mem_available)
     while [ 1 ]; do
 
         local mem_available=$(get_mem_available)
 
         if [[ $mem_available -lt $low_point ]]; then
             low_point=$mem_available
             echo "[[["
             date
             echo "---"
             # always available greppable output; given difference in
             # meminfo output as described above...
             echo "peakmem_tracker low_point: $mem_available"
             echo "---"
             cat /proc/meminfo
             echo "---"
             # would hierarchial view be more useful (-H)?  output is
             # not sorted by usage then, however, and the first
             # question is "what's using up the memory"
             #
             # there are a lot of kernel threads, especially on a 8-cpu
             # system.  do a best-effort removal to improve
             # signal/noise ratio of output.
             ps --sort=-pmem -eo pid:10,pmem:6,rss:15,ppid:10,cputime:10,nlwp:8,wchan:25,args:100 |
                 grep -v ']$'
             echo "]]]"
         fi
 
         sleep $SLEEP_TIME
     done
 }
 
 function usage {
     echo "Usage: $0 [-x] [-s N]" 1>&2
     exit 1
 }
 
 while getopts ":s:x" opt; do
     case $opt in
         s)
             SLEEP_TIME=$OPTARG
             ;;
         x)
             set -o xtrace
             ;;
         *)
             usage
             ;;
     esac
 done
 shift $((OPTIND-1))
 
 tracker