@@ -180,6 +180,9 @@ For example, start the OpenShift server, create a "test" project, and then run

     $ oc new-project test

     $ test/cmd/newapp.sh

+In order to run the suite, generate a jUnit XML report, and see a summary of the test suite, use:

+

+    $ JUNIT_REPORT='true' hack/test-cmd.sh

 ### End-to-End (e2e) and Extended Tests

@@ -5,6 +5,7 @@

 # We assume ${OS_ROOT} is set

 source "${OS_ROOT}/hack/text.sh"

 source "${OS_ROOT}/hack/util.sh"

+source "${OS_ROOT}/hack/lib/test/junit.sh"

 # expect_success runs the cmd and expects an exit code of 0

 function os::cmd::expect_success() {

@@ -146,6 +147,18 @@ os_cmd_internal_tmperr="${os_cmd_internal_tmpdir}/tmp_stderr.log"

 # command to be tested is suppressed unless either `VERBOSE=1` or the test fails. This function bypasses

 # any error exiting settings or traps set by upstream callers by masking the return code of the command

 # with the return code of setting the result variable on failure.

+#

+# Globals:

+#  - JUNIT_REPORT_OUTPUT

+#  - VERBOSE

+# Arguments:

+#  - 1: the command to run

+#  - 2: command evaluation assertion to use

+#  - 3: text to test for

+#  - 4: text assertion to use

+# Returns:

+#  - 0: if all assertions met

+#  - 1: if any assertions fail

 function os::cmd::internal::expect_exit_code_run_grep() {

 	local cmd=$1

 	# default expected cmd code to 0 for success

@@ -155,10 +168,15 @@ function os::cmd::internal::expect_exit_code_run_grep() {

 	# default expected test code to 0 for success

 	local test_eval_func=${4:-os::cmd::internal::success_func}

+	local -a junit_log

+

 	os::cmd::internal::init_tempdir

+	os::test::junit::declare_test_start

 	local name=$(os::cmd::internal::describe_call "${cmd}" "${cmd_eval_func}" "${grep_args}" "${test_eval_func}")

 	echo "Running ${name}..."

+	# for ease of parsing, we want the entire declaration on one line, so we replace '\n' with ';'	

+	junit_log+=( "${name//$'\n'/;}" )

 	local start_time=$(os::cmd::internal::seconds_since_epoch)

@@ -181,19 +199,31 @@ function os::cmd::internal::expect_exit_code_run_grep() {

 		os::text::clear_last_line

 	done

+	local return_code

 	if (( cmd_succeeded && test_succeeded )); then

 		os::text::print_green "SUCCESS after ${time_elapsed}s: ${name}"

+		junit_log+=( "SUCCESS after ${time_elapsed}s: ${name//$'\n'/;}" )

+

 		if [[ -n ${VERBOSE-} ]]; then

 			os::cmd::internal::print_results

 		fi

-		return 0

+		return_code=0

 	else

 		local cause=$(os::cmd::internal::assemble_causes "${cmd_succeeded}" "${test_succeeded}")

 		os::text::print_red_bold "FAILURE after ${time_elapsed}s: ${name}: ${cause}"

+		junit_log+=( "FAILURE after ${time_elapsed}s: ${name//$'\n'/;}: ${cause}" )

+

 		os::text::print_red "$(os::cmd::internal::print_results)"

-		return 1

+		return_code=1

 	fi

+

+	junit_log+=( "$(os::cmd::internal::print_results)" )

+	# append inside of a subshell so that IFS doesn't get propagated out

+	( IFS=$'\n'; echo "${junit_log[*]}" >> "${JUNIT_REPORT_OUTPUT:-/dev/null}" )

+	os::test::junit::declare_test_end

+	return "${return_code}"

+

 }

 # os::cmd::internal::init_tempdir initializes the temporary directory

@@ -367,14 +397,14 @@ function os::cmd::internal::compress_output() {

 function os::cmd::internal::print_results() {

 	if [[ -s "${os_cmd_internal_tmpout}" ]]; then

 		echo "Standard output from the command:"

-		cat "${os_cmd_internal_tmpout}"

+		cat "${os_cmd_internal_tmpout}"; echo

 	else

 		echo "There was no output from the command."

 	fi

 	if [[ -s "${os_cmd_internal_tmperr}" ]]; then

 		echo "Standard error from the command:"

-		cat "${os_cmd_internal_tmperr}"

+		cat "${os_cmd_internal_tmperr}"; echo

 	else

 		echo "There was no error output from the command."

 	fi

@@ -404,19 +434,36 @@ function os::cmd::internal::assemble_causes() {

 # either `VERBOSE=1` or the test fails. This function bypasses any error exiting settings or traps

 # set by upstream callers by masking the return code of the command with the return code of setting

 # the result variable on failure.

+#

+# Globals:

+#  - JUNIT_REPORT_OUTPUT

+#  - VERBOSE

+# Arguments:

+#  - 1: the command to run

+#  - 2: command evaluation assertion to use

+#  - 3: timeout duration

+#  - 4: interval duration

+# Returns:

+#  - 0: if all assertions met before timeout

+#  - 1: if timeout occurs

 function os::cmd::internal::run_until_exit_code() {

 	local cmd=$1

 	local cmd_eval_func=$2

 	local duration=$3

 	local interval=$4

+	local -a junit_log

+

 	os::cmd::internal::init_tempdir

+	os::test::junit::declare_test_start

 	local description=$(os::cmd::internal::describe_call "${cmd}" "${cmd_eval_func}")

 	local duration_seconds=$(echo "scale=3; $(( duration )) / 1000" | bc | xargs printf '%5.3f')

 	local description="${description}; re-trying every ${interval}s until completion or ${duration_seconds}s"

 	echo "Running ${description}..."

-

+	# for ease of parsing, we want the entire declaration on one line, so we replace '\n' with ';'

+	junit_log+=( "${description//$'\n'/;}" )

+	

 	local start_time=$(os::cmd::internal::seconds_since_epoch)

 	local deadline=$(( $(date +%s000) + $duration ))

@@ -440,18 +487,27 @@ function os::cmd::internal::run_until_exit_code() {

 		os::text::clear_last_line

 	done

+	local return_code

 	if (( cmd_succeeded )); then

-

 		os::text::print_green "SUCCESS after ${time_elapsed}s: ${description}"

+		junit_log+=( "SUCCESS after ${time_elapsed}s: ${description//$'\n'/;}" )

+

 		if [[ -n ${VERBOSE-} ]]; then

 			os::cmd::internal::print_try_until_results

 		fi

-		return 0

+		return_code=0

 	else

 		os::text::print_red_bold "FAILURE after ${time_elapsed}s: ${description}: the command timed out"

+		junit_log+=( "FAILURE after ${time_elapsed}s: ${description//$'\n'/;}: the command timed out" )

+

 		os::text::print_red "$(os::cmd::internal::print_try_until_results)"

-		return 1

+		return_code=1

 	fi

+

+	junit_log+=( "$(os::cmd::internal::print_try_until_results)" )

+	( IFS=$'\n'; echo "${junit_log[*]}" >> "${JUNIT_REPORT_OUTPUT:-/dev/null}" )

+	os::test::junit::declare_test_end

+	return "${return_code}"

 }

 # os::cmd::internal::run_until_text runs the provided command until the command output contains the

@@ -459,19 +515,36 @@ function os::cmd::internal::run_until_exit_code() {

 # either `VERBOSE=1` or the test fails. This function bypasses any error exiting settings or traps

 # set by upstream callers by masking the return code of the command with the return code of setting

 # the result variable on failure.

+#

+# Globals:

+#  - JUNIT_REPORT_OUTPUT

+#  - VERBOSE

+# Arguments:

+#  - 1: the command to run

+#  - 2: text to test for

+#  - 3: timeout duration

+#  - 4: interval duration

+# Returns:

+#  - 0: if all assertions met before timeout

+#  - 1: if timeout occurs

 function os::cmd::internal::run_until_text() {

 	local cmd=$1

 	local text=$2

 	local duration=$3

 	local interval=$4

+	local -a junit_log

+

 	os::cmd::internal::init_tempdir

+	os::test::junit::declare_test_start

 	local description=$(os::cmd::internal::describe_call "${cmd}" "" "${text}" "os::cmd::internal::success_func")

 	local duration_seconds=$(echo "scale=3; $(( duration )) / 1000" | bc | xargs printf '%5.3f')

 	local description="${description}; re-trying every ${interval}s until completion or ${duration_seconds}s"

 	echo "Running ${description}..."

-

+	# for ease of parsing, we want the entire declaration on one line, so we replace '\n' with ';'

+	junit_log+=( "${description//$'\n'/;}" )

+	

 	local start_time=$(os::cmd::internal::seconds_since_epoch)

 	local deadline=$(( $(date +%s000) + $duration ))

@@ -497,16 +570,26 @@ function os::cmd::internal::run_until_text() {

 		os::text::clear_last_line

 	done

+	local return_code

 	if (( test_succeeded )); then

-

 		os::text::print_green "SUCCESS after ${time_elapsed}s: ${description}"

+		junit_log+=( "SUCCESS after ${time_elapsed}s: ${description//$'\n'/;}" )

+

 		if [[ -n ${VERBOSE-} ]]; then

 			os::cmd::internal::print_try_until_results

 		fi

-		return 0

+		return_code=0

 	else

 		os::text::print_red_bold "FAILURE after ${time_elapsed}s: ${description}: the command timed out"

+		junit_log+=( "FAILURE after ${time_elapsed}s: ${description//$'\n'/;}: the command timed out" )

+

 		os::text::print_red "$(os::cmd::internal::print_try_until_results)"

-		return 1

+		os::test::junit::declare_test_end

+		return_code=1

 	fi

+

+	junit_log+=( "$(os::cmd::internal::print_try_until_results)" )

+	( IFS=$'\n'; echo "${junit_log[*]}" >> "${JUNIT_REPORT_OUTPUT:-/dev/null}" )

+	os::test::junit::declare_test_end

+	return "${return_code}"

 }

new file mode 100644

@@ -0,0 +1,139 @@

+#!/bin/bash

+# This utility file contains functions that format test output to be parsed into jUnit XML

+

+# os::test::junit::declare_suite_start prints a message declaring the start of a test suite

+# Any number of suites can be in flight at any time, so there is no failure condition for this

+# script based on the number of suites in flight.

+#

+# Globals:

+#  - JUNIT_REPORT_OUTPUT

+#  - NUM_OS_JUNIT_SUITES_IN_FLIGHT

+# Arguments:

+#  - 1: the suite name that is starting

+# Returns:

+#  - increment NUM_OS_JUNIT_SUITES_IN_FLIGHT

+function os::test::junit::declare_suite_start() {

+    local suite_name=$1

+    local num_suites=${NUM_OS_JUNIT_SUITES_IN_FLIGHT:-0}

+    

+    echo "=== BEGIN TEST SUITE github.com/openshift/origin/test/${suite_name} ===" >> "${JUNIT_REPORT_OUTPUT:-/dev/null}"

+    NUM_OS_JUNIT_SUITES_IN_FLIGHT=$(( ${num_suites} + 1 ))

+    export NUM_OS_JUNIT_SUITES_IN_FLIGHT

+}

+

+# os::test::junit::declare_suite_end prints a message declaring the end of a test suite

+# If there aren't any suites in flight, this function will fail.

+#

+# Globals:

+#  - JUNIT_REPORT_OUTPUT

+#  - NUM_OS_JUNIT_SUITES_IN_FLIGHT

+# Arguments:

+#  - 1: the suite name that is starting

+# Returns:

+#  - export/decrement NUM_OS_JUNIT_SUITES_IN_FLIGHT

+function os::test::junit::declare_suite_end() {

+    local num_suites=${NUM_OS_JUNIT_SUITES_IN_FLIGHT:-0}

+    if [[ "${num_suites}" -lt "1" ]]; then

+        # we can't end a suite if none have been started yet

+        echo "[ERROR] jUnit suite marker could not be placed, expected suites in flight, got ${num_suites}"

+        return 1

+    fi

+

+    echo "=== END TEST SUITE ===" >> "${JUNIT_REPORT_OUTPUT:-/dev/null}"

+    NUM_OS_JUNIT_SUITES_IN_FLIGHT=$(( ${num_suites} - 1 ))

+    export NUM_OS_JUNIT_SUITES_IN_FLIGHT

+}

+

+# os::test::junit::declare_test_start prints a message declaring the start of a test case

+# If there is already a test marked as being in flight, this function will fail.

+#

+# Globals:

+#  - JUNIT_REPORT_OUTPUT

+#  - NUM_OS_JUNIT_TESTS_IN_FLIGHT

+# Arguments:

+#  None

+# Returns:

+#  - increment NUM_OS_JUNIT_TESTS_IN_FLIGHT

+function os::test::junit::declare_test_start() {

+    local num_tests=${NUM_OS_JUNIT_TESTS_IN_FLIGHT:-0}

+    if [[ "${num_tests}" -ne "0" ]]; then

+        # someone's declaring the starting of a test when a test is already in flight

+        echo "[ERROR] jUnit test marker could not be placed, expected no tests in flight, got ${num_tests}"

+        return 1

+    fi

+

+    local num_suites=${NUM_OS_JUNIT_SUITES_IN_FLIGHT:-0}

+    if [[ "${num_suites}" -lt "1" ]]; then

+        # we can't end a test if no suites are in flight

+        echo "[ERROR] jUnit test marker could not be placed, expected suites in flight, got ${num_suites}"

+        return 1

+    fi

+

+    echo "=== BEGIN TEST CASE ===" >> "${JUNIT_REPORT_OUTPUT:-/dev/null}"

+    NUM_OS_JUNIT_TESTS_IN_FLIGHT=$(( ${num_tests} + 1 ))

+    export NUM_OS_JUNIT_TESTS_IN_FLIGHT

+}

+

+# os::test::junit::declare_test_end prints a message declaring the end of a test case

+# If there is no test marked as being in flight, this function will fail.

+#

+# Globals:

+#  - JUNIT_REPORT_OUTPUT

+#  - NUM_OS_JUNIT_TESTS_IN_FLIGHT

+# Arguments:

+#  None

+# Returns:

+#  - decrement NUM_OS_JUNIT_TESTS_IN_FLIGHT

+function os::test::junit::declare_test_end() {

+    local num_tests=${NUM_OS_JUNIT_TESTS_IN_FLIGHT:-0}

+    if [[ "${num_tests}" -ne "1" ]]; then

+        # someone's declaring the end of a test when a test is not in flight

+        echo "[ERROR] jUnit test marker could not be placed, expected one test in flight, got ${num_tests}"

+        return 1

+    fi

+

+    echo "=== END TEST CASE ===" >> "${JUNIT_REPORT_OUTPUT:-/dev/null}"

+    NUM_OS_JUNIT_TESTS_IN_FLIGHT=$(( ${num_tests} - 1 ))

+    export NUM_OS_JUNIT_TESTS_IN_FLIGHT

+}

+

+# os::test::junit::check_test_counters checks that we do not have any test suites or test cases in flight

+# This function should be called at the very end of any test script using jUnit markers to make sure no error in 

+# marking has occured.

+#

+# Globals:

+#  - NUM_OS_JUNIT_SUITES_IN_FLIGHT

+#  - NUM_OS_JUNIT_TESTS_IN_FLIGHT

+# Arguments:

+#  None

+# Returns:

+#  None

+function os::test::junit::check_test_counters() {

+    if [[ "${NUM_OS_JUNIT_SUITES_IN_FLIGHT-}" -ne "0" ]]; then

+        echo "[ERROR] Expected no test suites to be marked as in-flight at the end of testing, got ${NUM_OS_JUNIT_SUITES_IN_FLIGHT-}"

+        return 1

+    elif [[ "${NUM_OS_JUNIT_TESTS_IN_FLIGHT-}" -ne "0" ]]; then

+        echo "[ERROR] Expected no test cases to be marked as in-flight at the end of testing, got ${NUM_OS_JUNIT_TESTS_IN_FLIGHT-}"

+        return 1

+    fi

+}

+

+# os::test::junit::reconcile_output appends the necessary suite and test end statements to the jUnit output file

+# in order to ensure that the file is in a consistent state to allow for parsing

+#

+# Globals:

+#  - NUM_OS_JUNIT_SUITES_IN_FLIGHT

+#  - NUM_OS_JUNIT_TESTS_IN_FLIGHT

+# Arguments:

+#  None

+# Returns:

+#  None

+function os::test::junit::reconcile_output() {

+    if [[ "${NUM_OS_JUNIT_TESTS_IN_FLIGHT:-0}" = "1" ]]; then

+        os::test::junit::declare_test_end

+    fi

+

+    for (( i = 0; i < ${NUM_OS_JUNIT_SUITES_IN_FLIGHT:-0}; i++ )); do

+        os::test::junit::declare_suite_end

+    done

+}

\ No newline at end of file

@@ -14,6 +14,7 @@ source "${OS_ROOT}/hack/util.sh"

 source "${OS_ROOT}/hack/lib/log.sh"

 source "${OS_ROOT}/hack/lib/util/environment.sh"

 source "${OS_ROOT}/hack/cmd_util.sh"

+source "${OS_ROOT}/hack/lib/test/junit.sh"

 os::log::install_errexit

 os::util::environment::setup_time_vars

@@ -78,6 +79,11 @@ export ETCD_PEER_PORT=${ETCD_PEER_PORT:-27001}

 os::util::environment::setup_all_server_vars "test-cmd/"

 reset_tmp_dir

+# Allow setting $JUNIT_REPORT to toggle output behavior

+if [[ -n "${JUNIT_REPORT}" ]]; then

+  export JUNIT_REPORT_OUTPUT="${LOG_DIR}/raw_test_output.log"

+fi

+

 echo "Logging to ${LOG_DIR}..."

 os::log::start_system_logger

@@ -145,19 +151,6 @@ openshift start \

   --etcd-dir="${ETCD_DATA_DIR}" \

   --images="${USE_IMAGES}"

-# validate config that was generated

-os::cmd::expect_success_and_text "openshift ex validate master-config ${MASTER_CONFIG_DIR}/master-config.yaml" 'SUCCESS'

-os::cmd::expect_success_and_text "openshift ex validate node-config ${NODE_CONFIG_DIR}/node-config.yaml" 'SUCCESS'

-# breaking the config fails the validation check

-cp ${MASTER_CONFIG_DIR}/master-config.yaml ${BASETMPDIR}/master-config-broken.yaml

-echo "kubernetesMasterConfig: {}" >> ${BASETMPDIR}/master-config-broken.yaml

-os::cmd::expect_failure_and_text "openshift ex validate master-config ${BASETMPDIR}/master-config-broken.yaml" 'ERROR'

-

-cp ${NODE_CONFIG_DIR}/node-config.yaml ${BASETMPDIR}/node-config-broken.yaml

-os::util::sed '5,10d' ${BASETMPDIR}/node-config-broken.yaml

-os::cmd::expect_failure_and_text "openshift ex validate node-config ${BASETMPDIR}/node-config-broken.yaml" 'ERROR'

-echo "validation: ok"

-

 # Don't try this at home.  We don't have flags for setting etcd ports in the config, but we want deconflicted ones.  Use sed to replace defaults in a completely unsafe way

 os::util::sed "s/:4001$/:${ETCD_PORT}/g" ${SERVER_CONFIG_DIR}/master/master-config.yaml

 os::util::sed "s/:7001$/:${ETCD_PEER_PORT}/g" ${SERVER_CONFIG_DIR}/master/master-config.yaml

@@ -196,6 +189,22 @@ atomic-enterprise start \

   --etcd-dir="${ETCD_DATA_DIR}" \

   --images="${USE_IMAGES}"

+os::test::junit::declare_suite_start "cmd/validatation"

+# validate config that was generated

+os::cmd::expect_success_and_text "openshift ex validate master-config ${MASTER_CONFIG_DIR}/master-config.yaml" 'SUCCESS'

+os::cmd::expect_success_and_text "openshift ex validate node-config ${NODE_CONFIG_DIR}/node-config.yaml" 'SUCCESS'

+# breaking the config fails the validation check

+cp ${MASTER_CONFIG_DIR}/master-config.yaml ${BASETMPDIR}/master-config-broken.yaml

+os::util::sed '7,12d' ${BASETMPDIR}/master-config-broken.yaml

+os::cmd::expect_failure_and_text "openshift ex validate master-config ${BASETMPDIR}/master-config-broken.yaml" 'ERROR'

+

+cp ${NODE_CONFIG_DIR}/node-config.yaml ${BASETMPDIR}/node-config-broken.yaml

+os::util::sed '5,10d' ${BASETMPDIR}/node-config-broken.yaml

+os::cmd::expect_failure_and_text "openshift ex validate node-config ${BASETMPDIR}/node-config-broken.yaml" 'ERROR'

+echo "validation: ok"

+os::test::junit::declare_suite_end

+

+os::test::junit::declare_suite_start "cmd/config"

 # ensure that DisabledFeatures aren't written to config files

 os::cmd::expect_success_and_text "cat ${MASTER_CONFIG_DIR}/master-config.yaml" 'disabledFeatures: null'

 os::cmd::expect_success_and_text "cat ${BASETMPDIR}/atomic.local.config/master/master-config.yaml" 'disabledFeatures: null'

@@ -217,7 +226,6 @@ if [[ "${API_SCHEME}" == "https" ]]; then

     os::cmd::expect_failure_and_text "oc get services" 'certificate signed by unknown authority'

 fi

-

 # login and logout tests

 # bad token should error

 os::cmd::expect_failure_and_text "oc login ${KUBERNETES_MASTER} --certificate-authority='${MASTER_CONFIG_DIR}/ca.crt' --token=badvalue" 'The token provided is invalid or expired'

@@ -279,6 +287,7 @@ cp ${MASTER_CONFIG_DIR}/admin.kubeconfig ${HOME}/.kube/config

 os::cmd::expect_success 'oc get services'

 mv ${HOME}/.kube/config ${HOME}/.kube/non-default-config

 echo "config files: ok"

+os::test::junit::declare_suite_end

 # from this point every command will use config from the KUBECONFIG env var

 export NODECONFIG="${NODE_CONFIG_DIR}/node-config.yaml"

@@ -303,6 +312,8 @@ for test in "${tests[@]}"; do

   cp ${KUBECONFIG}{.bak,}  # since nothing ever gets deleted from kubeconfig, reset it

 done

+# check that we didn't mangle jUnit output

+os::test::junit::check_test_counters

 # Done

 echo

new file mode 100755

@@ -0,0 +1,41 @@

+#!/bin/bash

+

+# This script runs all of the test written for our Bash libraries.

+

+set -o errexit

+set -o nounset

+set -o pipefail

+

+function exit_trap() {

+    local return_code=$?

+

+    end_time=$(date +%s)

+    

+    if [[ "${return_code}" -eq "0" ]]; then

+        verb="succeeded"

+    else

+        verb="failed"

+    fi

+

+    echo "$0 ${verb} after $((${end_time} - ${start_time})) seconds"

+    exit "${return_code}"

+}

+

+trap exit_trap EXIT

+

+start_time=$(date +%s)

+OS_ROOT=$(dirname "${BASH_SOURCE}")/..

+source "${OS_ROOT}/hack/common.sh"

+source "${OS_ROOT}/hack/util.sh"

+source "${OS_ROOT}/hack/lib/util/environment.sh"

+os::log::install_errexit

+os::util::environment::setup_tmpdir_vars "test-lib"

+

+cd "${OS_ROOT}"

+

+library_tests="$( find 'hack/test-lib/' -type f -executable )"

+for test in ${library_tests}; do

+	# run each library test found in a subshell so that we can isolate them

+	( ${test} )

+	echo "$(basename "${test//.sh}"): ok"

+done

\ No newline at end of file

new file mode 100755

@@ -0,0 +1,82 @@

+#!/bin/bash

+#

+# This script tests os::test::junit functionality.

+

+set -o errexit

+set -o nounset

+set -o pipefail

+

+function exit_trap() {

+    local return_code=$?

+

+    end_time=$(date +%s)

+    

+    if [[ "${return_code}" -eq "0" ]]; then

+        verb="succeeded"

+    else

+        verb="failed"

+    fi

+

+    echo "$0 ${verb} after $((${end_time} - ${start_time})) seconds"

+    exit "${return_code}"

+}

+

+trap exit_trap EXIT

+

+start_time=$(date +%s)

+OS_ROOT="$( dirname "${BASH_SOURCE}" )"/../../..

+source "${OS_ROOT}/hack/lib/log.sh"

+source "${OS_ROOT}/hack/cmd_util.sh"

+os::log::install_errexit

+

+# envars used to track these interactions are not propagated out of the subshells used to run these commands

+# therefore each os::cmd call is its own sandbox and complicated scenarios need to play out inside one call

+# however, envars from this scope *are* propagated into each subshell, so they need to be cleared in each call

+

+os::test::junit::declare_suite_start 'lib/test/junit'

+

+# shouldn't be able to end a suite straight away

+os::cmd::expect_failure_and_text 'unset NUM_OS_JUNIT_SUITES_IN_FLIGHT NUM_OS_JUNIT_TESTS_IN_FLIGHT JUNIT_REPORT_OUTPUT

+os::test::junit::declare_suite_end' '\[ERROR\] jUnit suite marker could not be placed, expected suites in flight, got 0'

+# should be able to start one straight away

+os::cmd::expect_success 'unset NUM_OS_JUNIT_SUITES_IN_FLIGHT NUM_OS_JUNIT_TESTS_IN_FLIGHT JUNIT_REPORT_OUTPUT

+os::test::junit::declare_suite_start whatever'

+# should be able to start and end a suite

+os::cmd::expect_success 'unset NUM_OS_JUNIT_SUITES_IN_FLIGHT NUM_OS_JUNIT_TESTS_IN_FLIGHT JUNIT_REPORT_OUTPUT

+os::test::junit::declare_suite_start whatever

+os::test::junit::declare_suite_end'

+# should not be able to end more suites than are in flight

+os::cmd::expect_failure_and_text 'unset NUM_OS_JUNIT_SUITES_IN_FLIGHT NUM_OS_JUNIT_TESTS_IN_FLIGHT JUNIT_REPORT_OUTPUT

+os::test::junit::declare_suite_start whatever

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_end' '\[ERROR\] jUnit suite marker could not be placed, expected suites in flight, got 0'

+# should not be able to end more suites than are in flight

+os::cmd::expect_failure_and_text 'unset NUM_OS_JUNIT_SUITES_IN_FLIGHT NUM_OS_JUNIT_TESTS_IN_FLIGHT JUNIT_REPORT_OUTPUT

+os::test::junit::declare_suite_start whatever

+os::test::junit::declare_suite_start whateverelse

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_end' '\[ERROR\] jUnit suite marker could not be placed, expected suites in flight, got 0'

+# should be able to staart a test

+os::cmd::expect_success 'unset NUM_OS_JUNIT_SUITES_IN_FLIGHT NUM_OS_JUNIT_TESTS_IN_FLIGHT JUNIT_REPORT_OUTPUT

+os::test::junit::declare_suite_start whatever

+os::test::junit::declare_test_start'

+# shouldn't be able to end a test that hasn't been started

+os::cmd::expect_failure_and_text 'unset NUM_OS_JUNIT_SUITES_IN_FLIGHT NUM_OS_JUNIT_TESTS_IN_FLIGHT JUNIT_REPORT_OUTPUT

+os::test::junit::declare_test_end' '\[ERROR\] jUnit test marker could not be placed, expected one test in flight, got 0'

+# should be able to start and end a test case

+os::cmd::expect_success 'unset NUM_OS_JUNIT_SUITES_IN_FLIGHT NUM_OS_JUNIT_TESTS_IN_FLIGHT JUNIT_REPORT_OUTPUT

+os::test::junit::declare_suite_start whatever

+os::test::junit::declare_test_start

+os::test::junit::declare_test_end'

+# shouldn't be able to end too many test cases

+os::cmd::expect_failure_and_text 'unset NUM_OS_JUNIT_SUITES_IN_FLIGHT NUM_OS_JUNIT_TESTS_IN_FLIGHT JUNIT_REPORT_OUTPUT

+os::test::junit::declare_suite_start whatever

+os::test::junit::declare_test_start

+os::test::junit::declare_test_end

+os::test::junit::declare_test_end' '\[ERROR\] jUnit test marker could not be placed, expected one test in flight, got 0'

+# shouldn't be able to start a test without a suite

+os::cmd::expect_failure_and_text 'unset NUM_OS_JUNIT_SUITES_IN_FLIGHT NUM_OS_JUNIT_TESTS_IN_FLIGHT JUNIT_REPORT_OUTPUT

+os::test::junit::declare_test_start' '\[ERROR\] jUnit test marker could not be placed, expected suites in flight, got 0'

+

+os::test::junit::declare_suite_end

\ No newline at end of file

@@ -9,14 +9,20 @@ set -o pipefail

 OS_ROOT=$(dirname "${BASH_SOURCE}")/..

 source "${OS_ROOT}/hack/util.sh"

 source "${OS_ROOT}/hack/cmd_util.sh"

+source "${OS_ROOT}/hack/lib/test/junit.sh"

 os::log::install_errexit

 BASETMPDIR="${TMPDIR:-/tmp}/openshift/test-tools"

-JUNIT_OUTPUT_FILE="${BASETMPDIR}/junit_output.txt"

+JUNIT_REPORT_OUTPUT="${BASETMPDIR}/junit_output.txt"

+mkdir -p "${BASETMPDIR}"

+touch "${JUNIT_REPORT_OUTPUT}"

 # set verbosity so we can see that command output renders correctly

 VERBOSE=1

+os::test::junit::declare_suite_start "cmd/util"

+

+os::test::junit::declare_suite_start "cmd/util/positive"

 # positive tests

 os::cmd::expect_success 'exit 0'

@@ -37,7 +43,9 @@ os::cmd::expect_code_and_text 'echo "hello" && exit 213' '213' 'hello'

 os::cmd::expect_code_and_not_text 'echo "goodbye" && exit 213' '213' 'hello'

 echo "positive tests: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/util/negative"

 # negative tests

 if os::cmd::expect_success 'exit 1'; then

@@ -125,7 +133,9 @@ if os::cmd::expect_code_and_not_text 'echo "hello" && exit 0' '1' 'hello'; then

 fi

 echo "negative tests: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/util/complex"

 # complex input tests

 # pipes

@@ -170,8 +180,9 @@ EOF

 os::cmd::expect_success 'grep hello <<< hello'

 echo "complex tests: ok"

+os::test::junit::declare_suite_end

-

+os::test::junit::declare_suite_start "cmd/util/output"

 # test for output correctness

 # expect_code

@@ -333,7 +344,9 @@ echo "${output}" | grep -q '; the output content test failed'

 echo "${output}" | grep -q 'hello' 

 echo "output tests: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/util/tryuntil"

 function current_time_millis_mod_1000() {

 	mod=$(expr $(date +%s000) % 1000)

 	if [ $mod -eq 0 ]; then

@@ -379,7 +392,9 @@ if os::cmd::try_until_failure 'exit 0' $(( 1 * second )); then

 fi

 echo "try_until: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/util/compression"

 TMPDIR="${TMPDIR:-"/tmp"}"

 TEST_DIR=${TMPDIR}/openshift/origin/test/cmd

 rm -rf ${TEST_DIR} || true

@@ -432,4 +447,9 @@ line 2

 os::cmd::internal::compress_output ${TEST_DIR}//compress_test.txt > ${TEST_DIR}/actual-compressed.out

 diff ${TEST_DIR}/expected-compressed.out ${TEST_DIR}/actual-compressed.out

-echo "compression: ok"

\ No newline at end of file

+echo "compression: ok"

+os::test::junit::declare_suite_end

+

+os::test::junit::declare_suite_end

+

+os::test::junit::check_test_counters

\ No newline at end of file

@@ -704,7 +704,7 @@ os::log::errexit() {

 }

 os::log::install_errexit() {

-	# trap ERR to provide an error handler whenever a command exits nonzero	this

+	# trap ERR to provide an error handler whenever a command exits nonzero this

 	# is a more verbose version of set -o errexit

 	trap 'os::log::errexit' ERR

@@ -7,7 +7,9 @@ set -o pipefail

 OS_ROOT=$(dirname "${BASH_SOURCE}")/../..

 source "${OS_ROOT}/hack/util.sh"

 source "${OS_ROOT}/hack/cmd_util.sh"

+source "${OS_ROOT}/hack/lib/test/junit.sh"

 os::log::install_errexit

+trap os::test::junit::reconcile_output EXIT

 # Cleanup cluster resources created by this test

 (

@@ -30,8 +32,10 @@ os::log::install_errexit

 defaultimage="openshift/origin-\${component}:latest"

 USE_IMAGES=${USE_IMAGES:-$defaultimage}

+os::test::junit::declare_suite_start "cmd/admin"

 # This test validates admin level commands including system policy

+os::test::junit::declare_suite_start "cmd/admin/start"

 # Check failure modes of various system commands

 os::cmd::expect_failure_and_text 'openshift start network' 'kubeconfig must be set'

 os::cmd::expect_failure_and_text 'openshift start network --config=${NODECONFIG} --enable=kubelet' 'the following components are not recognized: kubelet'

@@ -41,7 +45,9 @@ os::cmd::expect_failure_and_text 'openshift start network --config=${NODECONFIG}

 os::cmd::expect_failure_and_text 'openshift start node' 'kubeconfig must be set'

 os::cmd::expect_failure_and_text 'openshift start node --config=${NODECONFIG} --disable=other' 'the following components are not recognized: other'

 os::cmd::expect_failure_and_text 'openshift start node --config=${NODECONFIG} --disable=kubelet,proxy,plugins' 'at least one node component must be enabled \(kubelet, plugins, proxy\)'

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/admin/manage-node"

 # Test admin manage-node operations

 os::cmd::expect_success_and_text 'openshift admin manage-node --help' 'Manage nodes'

@@ -65,7 +71,10 @@ status:

 os::cmd::expect_success_and_text 'oadm manage-node --selector= --schedulable=true' 'Ready'

 os::cmd::expect_success_and_not_text 'oadm manage-node --selector= --schedulable=true' 'Sched'

+echo "manage-node: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/admin/certs"

 # check create-master-certs validation

 os::cmd::expect_failure_and_text 'oadm ca create-master-certs --hostnames=example.com --master='                                                'master must be provided'

 os::cmd::expect_failure_and_text 'oadm ca create-master-certs --hostnames=example.com --master=example.com'                                     'master must be a valid URL'

@@ -93,7 +102,10 @@ os::cmd::expect_success "oadm ca decrypt --key='${ARTIFACT_DIR}/secret.key'    <

 os::cmd::expect_success "diff '${ARTIFACT_DIR}/secret.data' '${ARTIFACT_DIR}/secret.file-in-file-out.decrypted'"

 os::cmd::expect_success "diff '${ARTIFACT_DIR}/secret.data' '${ARTIFACT_DIR}/secret.file-in-pipe-out.decrypted'"

 os::cmd::expect_success "diff '${ARTIFACT_DIR}/secret.data' '${ARTIFACT_DIR}/secret.pipe-in-pipe-out.decrypted'"

+echo "certs: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/admin/groups"

 os::cmd::expect_success 'oadm groups new group1 foo bar'

 os::cmd::expect_success_and_text 'oc get groups/group1 --no-headers' 'foo, bar'

 os::cmd::expect_success 'oadm groups add-users group1 baz'

@@ -101,7 +113,9 @@ os::cmd::expect_success_and_text 'oc get groups/group1 --no-headers' 'baz'

 os::cmd::expect_success 'oadm groups remove-users group1 bar'

 os::cmd::expect_success_and_not_text 'oc get groups/group1 --no-headers' 'bar'

 echo "groups: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/admin/admin-scc"

 os::cmd::expect_success 'oadm policy who-can get pods'

 os::cmd::expect_success 'oadm policy who-can get pods -n default'

 os::cmd::expect_success 'oadm policy who-can get pods --all-namespaces'

@@ -145,7 +159,9 @@ os::cmd::expect_success_and_not_text 'oc get scc/privileged -o yaml' "system:ser

 os::cmd::expect_success 'oadm policy remove-scc-from-group privileged fake-group'

 os::cmd::expect_success_and_not_text 'oc get scc/privileged -o yaml' 'fake-group'

 echo "admin-scc: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/admin/reconcile-cluster-roles"

 os::cmd::expect_success 'oc delete clusterrole/cluster-status --cascade=false'

 os::cmd::expect_failure 'oc get clusterrole/cluster-status'

 os::cmd::expect_success 'oadm policy reconcile-cluster-roles'

@@ -172,7 +188,9 @@ os::cmd::expect_success_and_text 'oc get clusterroles/basic-user -o json' 'group

 os::cmd::expect_success 'oadm policy reconcile-cluster-roles --confirm'

 os::cmd::expect_success_and_not_text 'oc get clusterroles/basic-user -o yaml' 'groups'

 echo "admin-reconcile-cluster-roles: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/admin/reconcile-cluster-role-bindings"

 # Ensure a removed binding gets re-added

 os::cmd::expect_success 'oc delete clusterrolebinding/cluster-status-binding'

 os::cmd::expect_failure 'oc get clusterrolebinding/cluster-status-binding'

@@ -203,7 +221,9 @@ os::cmd::expect_failure 'oc get clusterrolebinding/basic-users'

 os::cmd::expect_success 'oadm policy reconcile-cluster-role-bindings basic-user --confirm'

 os::cmd::expect_success 'oc get clusterrolebinding/basic-users'

 echo "admin-reconcile-cluster-role-bindings: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/admin/role-reapers"

 os::cmd::expect_success "oc create -f test/extended/fixtures/roles/policy-roles.yaml"

 os::cmd::expect_success "oc get rolebinding/basic-users"

 os::cmd::expect_success "oc delete role/basic-user"

@@ -219,9 +239,9 @@ os::cmd::expect_failure "oc get rolebinding/edit"

 os::cmd::expect_success "oadm policy reconcile-cluster-roles --confirm"

 os::cmd::expect_success "oadm policy reconcile-cluster-role-bindings --confirm"

 echo "admin-role-reapers: ok"

+os::test::junit::declare_suite_end

-echo "admin-policy: ok"

-

+os::test::junit::declare_suite_start "cmd/admin/ui-project-commands"

 # Test the commands the UI projects page tells users to run

 # These should match what is described in projects.html

 os::cmd::expect_success 'oadm new-project ui-test-project --admin="createuser"'

@@ -232,8 +252,9 @@ os::cmd::try_until_text 'oc get projects' 'ui\-test\-project'

 os::cmd::expect_success_and_text "oc describe policybinding ':default' -n ui-test-project" 'createuser'

 os::cmd::expect_success_and_text "oc describe policybinding ':default' -n ui-test-project" 'adduser'

 echo "ui-project-commands: ok"

+os::test::junit::declare_suite_end

-

+os::test::junit::declare_suite_start "cmd/admin/new-project"

 # Test deleting and recreating a project

 os::cmd::expect_success 'oadm new-project recreated-project --admin="createuser1"'

 os::cmd::expect_success 'oc delete project recreated-project'

@@ -241,7 +262,22 @@ os::cmd::try_until_failure 'oc get project recreated-project'

 os::cmd::expect_success 'oadm new-project recreated-project --admin="createuser2"'

 os::cmd::expect_success_and_text "oc describe policybinding ':default' -n recreated-project" 'createuser2'

 echo "new-project: ok"

-

+os::test::junit::declare_suite_end

+

+os::test::junit::declare_suite_start "cmd/admin/router"

+# Test running a router

+os::cmd::expect_failure_and_text 'oadm router --dry-run' 'does not exist'

+encoded_json='{"kind":"ServiceAccount","apiVersion":"v1","metadata":{"name":"router"}}'

+os::cmd::expect_success "echo '${encoded_json}' | oc create -f - -n default"

+os::cmd::expect_success "oadm policy add-scc-to-user privileged system:serviceaccount:default:router"

+os::cmd::expect_success_and_text "oadm router -o yaml --credentials=${KUBECONFIG} --service-account=router -n default" 'image:.*-haproxy-router:'

+os::cmd::expect_success "oadm router --credentials=${KUBECONFIG} --images='${USE_IMAGES}' --service-account=router -n default"

+os::cmd::expect_success_and_text 'oadm router -n default' 'service exists'

+os::cmd::expect_success_and_text 'oc get dc/router -o yaml -n default' 'readinessProbe'

+echo "router: ok"

+os::test::junit::declare_suite_end

+

+os::test::junit::declare_suite_start "cmd/admin/registry"

 # Test running a registry

 os::cmd::expect_failure_and_text 'oadm registry --dry-run' 'does not exist'

 os::cmd::expect_success_and_text "oadm registry -o yaml --credentials=${KUBECONFIG}" 'image:.*-docker-registry'

@@ -250,14 +286,18 @@ os::cmd::expect_success_and_text 'oadm registry' 'service exists'

 os::cmd::expect_success_and_text 'oc describe svc/docker-registry' 'Session Affinity:\s*ClientIP'

 os::cmd::expect_success_and_text 'oc get dc/docker-registry -o yaml' 'readinessProbe'

 echo "registry: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/admin/apply"

 workingdir=$(mktemp -d)

 os::cmd::expect_success "oadm registry --credentials=${KUBECONFIG} -o yaml > ${workingdir}/oadm_registry.yaml"

 os::util::sed "s/5000/6000/g" ${workingdir}/oadm_registry.yaml

 os::cmd::expect_success "oc apply -f ${workingdir}/oadm_registry.yaml"

 os::cmd::expect_success_and_text 'oc get dc/docker-registry -o yaml' '6000'

 echo "apply: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/admin/build-chain"

 # Test building a dependency tree

 os::cmd::expect_success 'oc process -f examples/sample-app/application-template-stibuild.json -l build=sti | oc create -f -'

 # Test both the type/name resource syntax and the fact that istag/origin-ruby-sample:latest is still

@@ -266,7 +306,9 @@ os::cmd::expect_success_and_text 'oadm build-chain istag/origin-ruby-sample' 'is

 os::cmd::expect_success_and_text 'oadm build-chain ruby-22-centos7 -o dot' 'digraph'

 os::cmd::expect_success 'oc delete all -l build=sti'

 echo "ex build-chain: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/admin/complex-scenarios"

 os::cmd::expect_success 'oadm new-project example --admin="createuser"'

 os::cmd::expect_success 'oc project example'

 os::cmd::try_until_success 'oc get serviceaccount default'

@@ -274,7 +316,9 @@ os::cmd::expect_success 'oc create -f test/fixtures/app-scenarios'

 os::cmd::expect_success 'oc status'

 os::cmd::expect_success 'oc status -o dot'

 echo "complex-scenarios: ok"

+os::test::junit::declare_suite_end

+os::test::junit::declare_suite_start "cmd/admin/reconcile-security-context-constraints"

 # Test reconciling SCCs

 os::cmd::expect_success 'oc delete scc/restricted'

 os::cmd::expect_failure 'oc get scc/restricted'

@@ -307,8 +351,9 @@ os::cmd::expect_success_and_text 'oc get scc/restricted -o yaml' 'topic: my-foo-

 os::cmd::expect_success 'oadm policy reconcile-sccs --confirm --additive-only=false'

 os::cmd::expect_success_and_not_text 'oc get scc/restricted -o yaml' 'topic: my-foo-bar'

 echo "reconcile-scc: ok"

+os::test::junit::declare_suite_end