#!/bin/bash # Provides simple utility functions # kill_all_processes function will kill all # all processes created by the test script. function kill_all_processes() { local sudo="${USE_SUDO:+sudo}" pids=($(jobs -pr)) for i in ${pids[@]-}; do pgrep -P "${i}" | xargs $sudo kill &> /dev/null $sudo kill ${i} &> /dev/null done } readonly -f kill_all_processes # dump_container_logs writes container logs to $LOG_DIR function dump_container_logs() { if ! docker version >/dev/null 2>&1; then return fi mkdir -p ${LOG_DIR} os::log::info "Dumping container logs to ${LOG_DIR}" for container in $(docker ps -aq); do container_name=$(docker inspect -f "{{.Name}}" $container) # strip off leading / container_name=${container_name:1} if [[ "$container_name" =~ ^k8s_ ]]; then pod_name=$(echo $container_name | awk 'BEGIN { FS="[_.]+" }; { print $4 }') container_name=${pod_name}-$(echo $container_name | awk 'BEGIN { FS="[_.]+" }; { print $2 }') fi docker logs "$container" >&"${LOG_DIR}/container-${container_name}.log" done } readonly -f dump_container_logs # delete_empty_logs deletes empty logs function delete_empty_logs() { # Clean up zero byte log files find "${ARTIFACT_DIR}" "${LOG_DIR}" -type f -name '*.log' \( -empty \) -delete } readonly -f delete_empty_logs # truncate_large_logs truncates large logs function truncate_large_logs() { # Clean up large log files so they don't end up on jenkins local max_file_size="100M" local large_files=$(find "${ARTIFACT_DIR}" "${LOG_DIR}" -type f -name '*.log' \( -size +${max_file_size} \)) for file in ${large_files}; do mv "${file}" "${file}.tmp" echo "LOGFILE TOO LONG ($(du -h "${file}.tmp")), PREVIOUS BYTES TRUNCATED. LAST ${max_file_size} OF LOGFILE:" > "${file}" tail -c ${max_file_size} "${file}.tmp" >> "${file}" rm "${file}.tmp" done } readonly -f truncate_large_logs ###### # start of common functions for extended test group's run.sh scripts ###### # cleanup_openshift saves container logs, saves resources, and kills all processes and containers function cleanup_openshift() { LOG_DIR="${LOG_DIR:-${BASETMPDIR}/logs}" ARTIFACT_DIR="${ARTIFACT_DIR:-${LOG_DIR}}" API_HOST="${API_HOST:-127.0.0.1}" API_SCHEME="${API_SCHEME:-https}" ETCD_PORT="${ETCD_PORT:-4001}" set +e dump_container_logs # pull information out of the server log so that we can get failure management in jenkins to highlight it and # really have it smack people in their logs. This is a severe correctness problem grep -a5 "CACHE.*ALTERED" ${LOG_DIR}/openshift.log os::cleanup::dump_etcd if [[ -z "${SKIP_TEARDOWN-}" ]]; then os::log::info "Tearing down test" kill_all_processes if docker version >/dev/null 2>&1; then os::log::info "Stopping k8s docker containers"; docker ps | awk 'index($NF,"k8s_")==1 { print $1 }' | xargs -l -r docker stop -t 1 >/dev/null if [[ -z "${SKIP_IMAGE_CLEANUP-}" ]]; then os::log::info "Removing k8s docker containers"; docker ps -a | awk 'index($NF,"k8s_")==1 { print $1 }' | xargs -l -r docker rm -v >/dev/null fi fi os::log::info "Pruning etcd data directory..." local sudo="${USE_SUDO:+sudo}" ${sudo} rm -rf "${ETCD_DATA_DIR}" set -u fi if grep -q 'no Docker socket found' "${LOG_DIR}/openshift.log" && command -v journalctl >/dev/null 2>&1; then # the Docker daemon crashed, we need the logs journalctl --unit docker.service --since -4hours > "${LOG_DIR}/docker.log" fi delete_empty_logs truncate_large_logs os::log::info "Cleanup complete" set -e } readonly -f cleanup_openshift ###### # end of common functions for extended test group's run.sh scripts ###### function find_files() { find . -not \( \ \( \ -wholename './_output' \ -o -wholename './.*' \ -o -wholename './pkg/assets/bindata.go' \ -o -wholename './pkg/assets/*/bindata.go' \ -o -wholename './pkg/bootstrap/bindata.go' \ -o -wholename './openshift.local.*' \ -o -wholename '*/vendor/*' \ -o -wholename './assets/bower_components/*' \ \) -prune \ \) -name '*.go' | sort -u } readonly -f find_files