#!/bin/bash
# This utility file contains functions that wrap commands to be tested. All wrapper functions run commands
# in a sub-shell and redirect all output. Tests in test-cmd *must* use these functions for testing.
# We assume ${OS_ROOT} is set
source "${OS_ROOT}/hack/text.sh"
source "${OS_ROOT}/hack/util.sh"
# expect_success runs the cmd and expects an exit code of 0
function os::cmd::expect_success() {
if [[ $# -ne 1 ]]; then echo "os::cmd::expect_success expects only one argument, got $#"; exit 1; fi
local cmd=$1
os::cmd::internal::expect_exit_code_run_grep "${cmd}"
}
# expect_failure runs the cmd and expects a non-zero exit code
function os::cmd::expect_failure() {
if [[ $# -ne 1 ]]; then echo "os::cmd::expect_failure expects only one argument, got $#"; exit 1; fi
local cmd=$1
os::cmd::internal::expect_exit_code_run_grep "${cmd}" "os::cmd::internal::failure_func"
}
# expect_success_and_text runs the cmd and expects an exit code of 0
# as well as running a grep test to find the given string in the output
function os::cmd::expect_success_and_text() {
if [[ $# -ne 2 ]]; then echo "os::cmd::expect_success_and_text expects two arguments, got $#"; exit 1; fi
local cmd=$1
local expected_text=$2
os::cmd::internal::expect_exit_code_run_grep "${cmd}" "os::cmd::internal::success_func" "${expected_text}"
}
# expect_failure_and_text runs the cmd and expects a non-zero exit code
# as well as running a grep test to find the given string in the output
function os::cmd::expect_failure_and_text() {
if [[ $# -ne 2 ]]; then echo "os::cmd::expect_failure_and_text expects two arguments, got $#"; exit 1; fi
local cmd=$1
local expected_text=$2
os::cmd::internal::expect_exit_code_run_grep "${cmd}" "os::cmd::internal::failure_func" "${expected_text}"
}
# expect_success_and_not_text runs the cmd and expects an exit code of 0
# as well as running a grep test to ensure the given string is not in the output
function os::cmd::expect_success_and_not_text() {
if [[ $# -ne 2 ]]; then echo "os::cmd::expect_success_and_not_text expects two arguments, got $#"; exit 1; fi
local cmd=$1
local expected_text=$2
os::cmd::internal::expect_exit_code_run_grep "${cmd}" "os::cmd::internal::success_func" "${expected_text}" "os::cmd::internal::failure_func"
}
# expect_failure_and_not_text runs the cmd and expects a non-zero exit code
# as well as running a grep test to ensure the given string is not in the output
function os::cmd::expect_failure_and_not_text() {
if [[ $# -ne 2 ]]; then echo "os::cmd::expect_failure_and_not_text expects two arguments, got $#"; exit 1; fi
local cmd=$1
local expected_text=$2
os::cmd::internal::expect_exit_code_run_grep "${cmd}" "os::cmd::internal::failure_func" "${expected_text}" "os::cmd::internal::failure_func"
}
# expect_code runs the cmd and expects a given exit code
function os::cmd::expect_code() {
if [[ $# -ne 2 ]]; then echo "os::cmd::expect_code expects two arguments, got $#"; exit 1; fi
local cmd=$1
local expected_cmd_code=$2
os::cmd::internal::expect_exit_code_run_grep "${cmd}" "os::cmd::internal::specific_code_func ${expected_cmd_code}"
}
# expect_code_and_text runs the cmd and expects the given exit code
# as well as running a grep test to find the given string in the output
function os::cmd::expect_code_and_text() {
if [[ $# -ne 3 ]]; then echo "os::cmd::expect_code_and_text expects three arguments, got $#"; exit 1; fi
local cmd=$1
local expected_cmd_code=$2
local expected_text=$3
os::cmd::internal::expect_exit_code_run_grep "${cmd}" "os::cmd::internal::specific_code_func ${expected_cmd_code}" "${expected_text}"
}
# expect_code_and_not_text runs the cmd and expects the given exit code
# as well as running a grep test to ensure the given string is not in the output
function os::cmd::expect_code_and_not_text() {
if [[ $# -ne 3 ]]; then echo "os::cmd::expect_code_and_not_text expects three arguments, got $#"; exit 1; fi
local cmd=$1
local expected_cmd_code=$2
local expected_text=$3
os::cmd::internal::expect_exit_code_run_grep "${cmd}" "os::cmd::internal::specific_code_func ${expected_cmd_code}" "${expected_text}" "os::cmd::internal::failure_func"
}
millisecond=1
second=$(( 1000 * millisecond ))
minute=$(( 60 * second ))
# os::cmd::try_until_success runs the cmd in a small interval until either the command succeeds or times out
# the default time-out for os::cmd::try_until_success is 60 seconds.
# the default interval for os::cmd::try_until_success is 200ms
function os::cmd::try_until_success() {
if [[ $# -lt 1 ]]; then echo "os::cmd::try_until_success expects at least one arguments, got $#"; exit 1; fi
local cmd=$1
local duration=${2:-minute}
local interval=${3:-0.2}
os::cmd::internal::run_until_exit_code "${cmd}" "os::cmd::internal::success_func" "${duration}" "${interval}"
}
# os::cmd::try_until_failure runs the cmd until either the command fails or times out
# the default time-out for os::cmd::try_until_failure is 60 seconds.
function os::cmd::try_until_failure() {
if [[ $# -lt 1 ]]; then echo "os::cmd::try_until_success expects at least one argument, got $#"; exit 1; fi
local cmd=$1
local duration=${2:-$minute}
local interval=${3:-0.2}
os::cmd::internal::run_until_exit_code "${cmd}" "os::cmd::internal::failure_func" "${duration}" "${interval}"
}
# os::cmd::try_until_text runs the cmd until either the command outputs the desired text or times out
# the default time-out for os::cmd::try_until_text is 60 seconds.
function os::cmd::try_until_text() {
if [[ $# -lt 2 ]]; then echo "os::cmd::try_until_success expects at least two arguments, got $#"; exit 1; fi
local cmd=$1
local text=$2
local duration=${3:-minute}
local interval=${4:-0.2}
os::cmd::internal::run_until_text "${cmd}" "${text}" "${duration}" "${interval}"
}
# Functions in the os::cmd::internal namespace are discouraged from being used outside of os::cmd
# In order to harvest stderr and stdout at the same time into different buckets, we need to stick them into files
# in an intermediate step
BASETMPDIR="${TMPDIR:-"/tmp"}/openshift"
os_cmd_internal_tmpdir="${BASETMPDIR}/test-cmd"
os_cmd_internal_tmpout="${os_cmd_internal_tmpdir}/tmp_stdout.log"
os_cmd_internal_tmperr="${os_cmd_internal_tmpdir}/tmp_stderr.log"
# os::cmd::internal::expect_exit_code_run_grep runs the provided test command and expects a specific
# exit code from that command as well as the success of a specified `grep` invocation. Output from the
# command to be tested is suppressed unless either `VERBOSE=1` or the test fails. This function bypasses
# any error exiting settings or traps set by upstream callers by masking the return code of the command
# with the return code of setting the result variable on failure.
function os::cmd::internal::expect_exit_code_run_grep() {
local cmd=$1
# default expected cmd code to 0 for success
local cmd_eval_func=${2:-os::cmd::internal::success_func}
# default to nothing
local grep_args=${3:-}
# default expected test code to 0 for success
local test_eval_func=${4:-os::cmd::internal::success_func}
os::cmd::internal::init_tempdir
local name=$(os::cmd::internal::describe_call "${cmd}" "${cmd_eval_func}" "${grep_args}" "${test_eval_func}")
echo "Running ${name}..."
local start_time=$(os::cmd::internal::seconds_since_epoch)
local cmd_result=$( os::cmd::internal::run_collecting_output "${cmd}"; echo $? )
local cmd_succeeded=$( ${cmd_eval_func} "${cmd_result}"; echo $? )
local test_result=0
if [[ -n "${grep_args}" ]]; then
test_result=$( os::cmd::internal::run_collecting_output 'os::cmd::internal::get_results | grep -Eq "${grep_args}"'; echo $? )
fi
local test_succeeded=$( ${test_eval_func} "${test_result}"; echo $? )
local end_time=$(os::cmd::internal::seconds_since_epoch)
local time_elapsed=$(echo "scale=3; ${end_time} - ${start_time}" | bc | xargs printf '%5.3f') # in decimal seconds, we need leading zeroes for parsing later
# some commands are multi-line, so we may need to clear more than just the previous line
local cmd_length=$(echo "${cmd}" | wc -l)
for (( i=0; i<${cmd_length}; i++ )); do
os::text::clear_last_line
done
if (( cmd_succeeded && test_succeeded )); then
os::text::print_green "SUCCESS after ${time_elapsed}s: ${name}"
if [[ -n ${VERBOSE-} ]]; then
os::cmd::internal::print_results
fi
return 0
else
local cause=$(os::cmd::internal::assemble_causes "${cmd_succeeded}" "${test_succeeded}")
os::text::print_red_bold "FAILURE after ${time_elapsed}s: ${name}: ${cause}"
os::text::print_red "$(os::cmd::internal::print_results)"
return 1
fi
}
# os::cmd::internal::init_tempdir initializes the temporary directory
function os::cmd::internal::init_tempdir() {
mkdir -p "${os_cmd_internal_tmpdir}"
rm -f "${os_cmd_internal_tmpdir}"/tmp_std{out,err}.log
}
# os::cmd::internal::describe_call determines the file:line of the latest function call made
# from outside of this file in the call stack, and the name of the function being called from
# that line, returning a string describing the call
function os::cmd::internal::describe_call() {
local cmd=$1
local cmd_eval_func=$2
local grep_args=${3:-}
local test_eval_func=${4:-}
local caller_id=$(os::cmd::internal::determine_caller)
local full_name="${caller_id}: executing '${cmd}'"
local cmd_expectation=$(os::cmd::internal::describe_expectation "${cmd_eval_func}")
local full_name="${full_name} expecting ${cmd_expectation}"
if [[ -n "${grep_args}" ]]; then
local text_expecting=
case "${test_eval_func}" in
"os::cmd::internal::success_func")
text_expecting="text" ;;
"os::cmd::internal::failure_func")
text_expecting="not text" ;;
esac
full_name="${full_name} and ${text_expecting} '${grep_args}'"
fi
echo "${full_name}"
}
# os::cmd::internal::determine_caller determines the file and line number of the function call to
# the outer os::cmd wrapper function
function os::cmd::internal::determine_caller() {
local call_depth=
local len_sources="${#BASH_SOURCE[@]}"
for (( i=0; i<${len_sources}; i++ )); do
if [ ! $(echo "${BASH_SOURCE[i]}" | grep "hack/cmd_util\.sh$") ]; then
call_depth=i
break
fi
done
local caller_file="${BASH_SOURCE[${call_depth}]}"
local caller_line="${BASH_LINENO[${call_depth}-1]}"
echo "${caller_file}:${caller_line}"
}
# os::cmd::internal::describe_expectation describes a command return code evaluation function
function os::cmd::internal::describe_expectation() {
local func=$1
case "${func}" in
"os::cmd::internal::success_func")
echo "success" ;;
"os::cmd::internal::failure_func")
echo "failure" ;;
"os::cmd::internal::specific_code_func"*[0-9])
local code=$(echo "${func}" | grep -Eo "[0-9]+$")
echo "exit code ${code}" ;;
"")
echo "any result"
esac
}
# os::cmd::internal::seconds_since_epoch returns the number of seconds elapsed since the epoch
# with milli-second precision
function os::cmd::internal::seconds_since_epoch() {
local ns=$(date +%s%N)
# if `date` doesn't support nanoseconds, return second precision
if [[ "$ns" == *N ]]; then
date "+%s.000"
return
fi
echo $(bc <<< "scale=3; ${ns}/1000000000")
}
# os::cmd::internal::run_collecting_output runs the command given, piping stdout and stderr into
# the given files, and returning the exit code of the command
function os::cmd::internal::run_collecting_output() {
local cmd=$1
local result=
$( eval "${cmd}" 1>>"${os_cmd_internal_tmpout}" 2>>"${os_cmd_internal_tmperr}" ) || result=$?
local result=${result:-0} # if we haven't set result yet, the command succeeded
return "${result}"
}
# os::cmd::internal::success_func determines if the input exit code denotes success
# this function returns 0 for false and 1 for true to be compatible with arithmetic tests
function os::cmd::internal::success_func() {
local exit_code=$1
# use a negated test to get output correct for (( ))
[[ "${exit_code}" -ne "0" ]]
return $?
}
# os::cmd::internal::failure_func determines if the input exit code denotes failure
# this function returns 0 for false and 1 for true to be compatible with arithmetic tests
function os::cmd::internal::failure_func() {
local exit_code=$1
# use a negated test to get output correct for (( ))
[[ "${exit_code}" -eq "0" ]]
return $?
}
# os::cmd::internal::specific_code_func determines if the input exit code matches the given code
# this function returns 0 for false and 1 for true to be compatible with arithmetic tests
function os::cmd::internal::specific_code_func() {
local expected_code=$1
local exit_code=$2
# use a negated test to get output correct for (( ))
[[ "${exit_code}" -ne "${expected_code}" ]]
return $?
}
# os::cmd::internal::get_results prints the stderr and stdout files
function os::cmd::internal::get_results() {
cat "${os_cmd_internal_tmpout}" "${os_cmd_internal_tmperr}"
}
# os::cmd::internal::get_try_until_results returns a concise view of the stdout and stderr output files
# using a timeline format, where consecutive output lines that are the same are condensed into one line
# with a counter
function os::cmd::internal::print_try_until_results() {
if grep -vq $'\x1e' "${os_cmd_internal_tmpout}"; then
echo "Standard output from the command:"
os::cmd::internal::compress_output "${os_cmd_internal_tmpout}"
else
echo "There was no output from the command."
fi
if grep -vq $'\x1e' "${os_cmd_internal_tmperr}"; then
echo "Standard error from the command:"
os::cmd::internal::compress_output "${os_cmd_internal_tmperr}"
else
echo "There was no error output from the command."
fi
}
# os::cmd::internal::mark_attempt marks the end of an attempt in the stdout and stderr log files
# this is used to make the try_until_* output more concise
function os::cmd::internal::mark_attempt() {
echo -e '\x1e' >> "${os_cmd_internal_tmpout}" | tee "${os_cmd_internal_tmperr}"
}
# os::cmd::internal::compress_output compresses an output file into timeline representation
function os::cmd::internal::compress_output() {
local logfile=$1
awk -f ${OS_ROOT}/hack/compress.awk $logfile
}
# os::cmd::internal::print_results pretty-prints the stderr and stdout files
function os::cmd::internal::print_results() {
if [[ -s "${os_cmd_internal_tmpout}" ]]; then
echo "Standard output from the command:"
cat "${os_cmd_internal_tmpout}"
else
echo "There was no output from the command."
fi
if [[ -s "${os_cmd_internal_tmperr}" ]]; then
echo "Standard error from the command:"
cat "${os_cmd_internal_tmperr}"
else
echo "There was no error output from the command."
fi
}
# os::cmd::internal::assemble_causes determines from the two input booleans which part of the test
# failed and generates a nice delimited list of failure causes
function os::cmd::internal::assemble_causes() {
local cmd_succeeded=$1
local test_succeeded=$2
local causes=()
if (( ! cmd_succeeded )); then
causes+=("the command returned the wrong error code")
fi
if (( ! test_succeeded )); then
causes+=("the output content test failed")
fi
local list=$(printf '; %s' "${causes[@]}")
echo "${list:2}"
}
# os::cmd::internal::run_until_exit_code runs the provided command until the exit code test given
# succeeds or the timeout given runs out. Output from the command to be tested is suppressed unless
# either `VERBOSE=1` or the test fails. This function bypasses any error exiting settings or traps
# set by upstream callers by masking the return code of the command with the return code of setting
# the result variable on failure.
function os::cmd::internal::run_until_exit_code() {
local cmd=$1
local cmd_eval_func=$2
local duration=$3
local interval=$4
os::cmd::internal::init_tempdir
local description=$(os::cmd::internal::describe_call "${cmd}" "${cmd_eval_func}")
local duration_seconds=$(echo "scale=3; $(( duration )) / 1000" | bc | xargs printf '%5.3f')
local description="${description}; re-trying every ${interval}s until completion or ${duration_seconds}s"
echo "Running ${description}..."
local start_time=$(os::cmd::internal::seconds_since_epoch)
local deadline=$(( $(date +%s000) + $duration ))
while [ $(date +%s000) -lt $deadline ]; do
local cmd_result=$( os::cmd::internal::run_collecting_output "${cmd}"; echo $? )
local cmd_succeeded=$( ${cmd_eval_func} "${cmd_result}"; echo $? )
if (( cmd_succeeded )); then
break
fi
sleep "${interval}"
os::cmd::internal::mark_attempt
done
local end_time=$(os::cmd::internal::seconds_since_epoch)
local time_elapsed=$(echo "scale=9; ${end_time} - ${start_time}" | bc | xargs printf '%5.3f') # in decimal seconds, we need leading zeroes for parsing later
# some commands are multi-line, so we may need to clear more than just the previous line
local cmd_length=$(echo "${cmd}" | wc -l)
for (( i=0; i<${cmd_length}; i++ )); do
os::text::clear_last_line
done
if (( cmd_succeeded )); then
os::text::print_green "SUCCESS after ${time_elapsed}s: ${description}"
if [[ -n ${VERBOSE-} ]]; then
os::cmd::internal::print_try_until_results
fi
return 0
else
os::text::print_red_bold "FAILURE after ${time_elapsed}s: ${description}: the command timed out"
os::text::print_red "$(os::cmd::internal::print_try_until_results)"
return 1
fi
}
# os::cmd::internal::run_until_text runs the provided command until the command output contains the
# given text or the timeout given runs out. Output from the command to be tested is suppressed unless
# either `VERBOSE=1` or the test fails. This function bypasses any error exiting settings or traps
# set by upstream callers by masking the return code of the command with the return code of setting
# the result variable on failure.
function os::cmd::internal::run_until_text() {
local cmd=$1
local text=$2
local duration=$3
local interval=$4
os::cmd::internal::init_tempdir
local description=$(os::cmd::internal::describe_call "${cmd}" "" "${text}" "os::cmd::internal::success_func")
local duration_seconds=$(echo "scale=3; $(( duration )) / 1000" | bc | xargs printf '%5.3f')
local description="${description}; re-trying every ${interval}s until completion or ${duration_seconds}s"
echo "Running ${description}..."
local start_time=$(os::cmd::internal::seconds_since_epoch)
local deadline=$(( $(date +%s000) + $duration ))
while [ $(date +%s000) -lt $deadline ]; do
local cmd_result=$( os::cmd::internal::run_collecting_output "${cmd}"; echo $? )
local test_result=$( os::cmd::internal::run_collecting_output 'os::cmd::internal::get_results | grep -Eq "${text}"'; echo $? )
local test_succeeded=$( os::cmd::internal::success_func "${test_result}"; echo $? )
if (( test_succeeded )); then
break
fi
sleep "${interval}"
os::cmd::internal::mark_attempt
done
local end_time=$(os::cmd::internal::seconds_since_epoch)
local time_elapsed=$(echo "scale=9; ${end_time} - ${start_time}" | bc | xargs printf '%5.3f') # in decimal seconds, we need leading zeroes for parsing later
# some commands are multi-line, so we may need to clear more than just the previous line
local cmd_length=$(echo "${cmd}" | wc -l)
for (( i=0; i<${cmd_length}; i++ )); do
os::text::clear_last_line
done
if (( test_succeeded )); then
os::text::print_green "SUCCESS after ${time_elapsed}s: ${description}"
if [[ -n ${VERBOSE-} ]]; then
os::cmd::internal::print_try_until_results
fi
return 0
else
os::text::print_red_bold "FAILURE after ${time_elapsed}s: ${description}: the command timed out"
os::text::print_red "$(os::cmd::internal::print_try_until_results)"
return 1
fi
}