Browse code

make test_with_retry a function

We have this pattern of timeout with while tests for a non infinite
while loop condition. It's enough of a pattern that we should probably
extract it into a function to make it more widely used.

Change-Id: I11afcda9fac9709acf2f52d256d6e97644d4727c

Sean Dague authored on 2015/06/25 02:24:02
Showing 2 changed files
... ...
@@ -1967,6 +1967,19 @@ function stop_service {
1967 1967
     fi
1968 1968
 }
1969 1969
 
1970
+# Test with a finite retry loop.
1971
+#
1972
+function test_with_retry {
1973
+    local testcmd=$1
1974
+    local failmsg=$2
1975
+    local until=${3:-10}
1976
+    local sleep=${4:-0.5}
1977
+
1978
+    if ! timeout $until sh -c "while ! $testcmd; do sleep $sleep; done"; then
1979
+        die $LINENO "$failmsg"
1980
+    fi
1981
+}
1982
+
1970 1983
 
1971 1984
 # Restore xtrace
1972 1985
 $XTRACE
... ...
@@ -696,9 +696,10 @@ function start_neutron_service_and_check {
696 696
     if is_ssl_enabled_service "neutron"; then
697 697
         ssl_ca="--ca-certificate=${SSL_BUNDLE_FILE}"
698 698
     fi
699
-    if ! timeout $SERVICE_TIMEOUT sh -c "while ! wget ${ssl_ca} --no-proxy -q -O- $service_protocol://$Q_HOST:$service_port; do sleep 1; done"; then
700
-        die $LINENO "Neutron did not start"
701
-    fi
699
+
700
+    local testcmd="wget ${ssl_ca} --no-proxy -q -O- $service_protocol://$Q_HOST:$service_port"
701
+    test_with_retry "$testcmd" "Neutron did not start" $SERVICE_TIMEOUT
702
+
702 703
     # Start proxy if enabled
703 704
     if is_service_enabled tls-proxy; then
704 705
         start_tls_proxy '*' $Q_PORT $Q_HOST $Q_PORT_INT &
... ...
@@ -1380,9 +1381,8 @@ function _ssh_check_neutron {
1380 1380
     local timeout_sec=$5
1381 1381
     local probe_cmd = ""
1382 1382
     probe_cmd=`_get_probe_cmd_prefix $from_net`
1383
-    if ! timeout $timeout_sec sh -c "while ! $probe_cmd ssh -o StrictHostKeyChecking=no -i $key_file ${user}@$ip echo success; do sleep 1; done"; then
1384
-        die $LINENO "server didn't become ssh-able!"
1385
-    fi
1383
+    local testcmd="$probe_cmd ssh -o StrictHostKeyChecking=no -i $key_file ${user}@$ip echo success"
1384
+    test_with_retry "$testcmd" "server $ip didn't become ssh-able" $timeout_sec
1386 1385
 }
1387 1386
 
1388 1387
 # Neutron 3rd party programs