Browse code

Wait for hypervisor-stats to reflect ironic nodes

When enrolling nodes into Ironic, poll nova's hypervisor-stats until
the hypervisor count reflects the number of enrolled nodes. This
eliminates a race where devstack completes and an instance is spawned
before the first post-enrollment periodic task ticks on the Nova side,
which has recently started popping up in the gate.

Conflicts:
lib/ironic

Change-Id: Ib3d8005e0094ee8af2d5fcb65aca6cd92736da90
Closes-bug: #1398128
(cherry picked from commit c78e4671098f08c2eaf0b033777d3c01082141cd)

Adam Gandelman authored on 2014/12/02 04:24:37
Showing 1 changed files
... ...
@@ -462,9 +462,23 @@ function create_bridge_and_vms {
462 462
     create_ovs_taps
463 463
 }
464 464
 
465
+function wait_for_nova_resources {
466
+    # After nodes have been enrolled, we need to wait for n-cpu's periodic
467
+    # task populate the resource tracker with available nodes.  Wait for 2
468
+    # minutes before timing out.
469
+    local expected_count=$1
470
+    echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $expected_count Ironic nodes"
471
+    for i in $(seq 1 120); do
472
+        if [ $(nova hypervisor-stats | grep " count " | get_field 2) -ge $expected_count ]; then
473
+            return 0
474
+        fi
475
+        sleep 1
476
+    done
477
+    die $LINENO "Nova hypervisor-stats did not register at least $expected_count nodes"
478
+}
479
+
465 480
 function enroll_vms {
466 481
     local chassis_id=$(ironic chassis-create -d "ironic test chassis" | grep " uuid " | get_field 2)
467
-    local idx=0
468 482
 
469 483
     if [[ "$IRONIC_DEPLOY_DRIVER" == "pxe_ssh" ]] ; then
470 484
         local _IRONIC_DEPLOY_KERNEL_KEY=pxe_deploy_kernel
... ...
@@ -474,6 +488,7 @@ function enroll_vms {
474 474
         local _IRONIC_DEPLOY_RAMDISK_KEY=deploy_ramdisk
475 475
     fi
476 476
 
477
+    local total_nodes=0
477 478
     while read MAC; do
478 479
 
479 480
         local node_id=$(ironic node-create --chassis_uuid $chassis_id \
... ...
@@ -493,7 +508,7 @@ function enroll_vms {
493 493
 
494 494
         ironic port-create --address $MAC --node_uuid $node_id
495 495
 
496
-        idx=$((idx+1))
496
+        total_nodes=$((total_nodes+1))
497 497
     done < $IRONIC_VM_MACS_CSV_FILE
498 498
 
499 499
     # create the nova flavor
... ...
@@ -508,6 +523,10 @@ function enroll_vms {
508 508
     # from the flavor after the completion of
509 509
     # https://blueprints.launchpad.net/ironic/+spec/add-node-instance-info
510 510
     nova flavor-key baremetal set "cpu_arch"="x86_64" "baremetal:deploy_kernel_id"="$IRONIC_DEPLOY_KERNEL_ID" "baremetal:deploy_ramdisk_id"="$IRONIC_DEPLOY_RAMDISK_ID"
511
+
512
+    if [ "$VIRT_DRIVER" == "ironic" ]; then
513
+        wait_for_nova_resources $total_nodes
514
+    fi
511 515
 }
512 516
 
513 517
 function configure_iptables {