When enrolling nodes into Ironic, poll nova's hypervisor-stats until
the hypervisor count reflects the number of enrolled nodes. This
eliminates a race where devstack completes and an instance is spawned
before the first post-enrollment periodic task ticks on the Nova side,
which has recently started popping up in the gate.
Conflicts:
lib/ironic
Change-Id: Ib3d8005e0094ee8af2d5fcb65aca6cd92736da90
Closes-bug: #1398128
(cherry picked from commit c78e4671098f08c2eaf0b033777d3c01082141cd)
| ... | ... |
@@ -462,9 +462,23 @@ function create_bridge_and_vms {
|
| 462 | 462 |
create_ovs_taps |
| 463 | 463 |
} |
| 464 | 464 |
|
| 465 |
+function wait_for_nova_resources {
|
|
| 466 |
+ # After nodes have been enrolled, we need to wait for n-cpu's periodic |
|
| 467 |
+ # task populate the resource tracker with available nodes. Wait for 2 |
|
| 468 |
+ # minutes before timing out. |
|
| 469 |
+ local expected_count=$1 |
|
| 470 |
+ echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $expected_count Ironic nodes" |
|
| 471 |
+ for i in $(seq 1 120); do |
|
| 472 |
+ if [ $(nova hypervisor-stats | grep " count " | get_field 2) -ge $expected_count ]; then |
|
| 473 |
+ return 0 |
|
| 474 |
+ fi |
|
| 475 |
+ sleep 1 |
|
| 476 |
+ done |
|
| 477 |
+ die $LINENO "Nova hypervisor-stats did not register at least $expected_count nodes" |
|
| 478 |
+} |
|
| 479 |
+ |
|
| 465 | 480 |
function enroll_vms {
|
| 466 | 481 |
local chassis_id=$(ironic chassis-create -d "ironic test chassis" | grep " uuid " | get_field 2) |
| 467 |
- local idx=0 |
|
| 468 | 482 |
|
| 469 | 483 |
if [[ "$IRONIC_DEPLOY_DRIVER" == "pxe_ssh" ]] ; then |
| 470 | 484 |
local _IRONIC_DEPLOY_KERNEL_KEY=pxe_deploy_kernel |
| ... | ... |
@@ -474,6 +488,7 @@ function enroll_vms {
|
| 474 | 474 |
local _IRONIC_DEPLOY_RAMDISK_KEY=deploy_ramdisk |
| 475 | 475 |
fi |
| 476 | 476 |
|
| 477 |
+ local total_nodes=0 |
|
| 477 | 478 |
while read MAC; do |
| 478 | 479 |
|
| 479 | 480 |
local node_id=$(ironic node-create --chassis_uuid $chassis_id \ |
| ... | ... |
@@ -493,7 +508,7 @@ function enroll_vms {
|
| 493 | 493 |
|
| 494 | 494 |
ironic port-create --address $MAC --node_uuid $node_id |
| 495 | 495 |
|
| 496 |
- idx=$((idx+1)) |
|
| 496 |
+ total_nodes=$((total_nodes+1)) |
|
| 497 | 497 |
done < $IRONIC_VM_MACS_CSV_FILE |
| 498 | 498 |
|
| 499 | 499 |
# create the nova flavor |
| ... | ... |
@@ -508,6 +523,10 @@ function enroll_vms {
|
| 508 | 508 |
# from the flavor after the completion of |
| 509 | 509 |
# https://blueprints.launchpad.net/ironic/+spec/add-node-instance-info |
| 510 | 510 |
nova flavor-key baremetal set "cpu_arch"="x86_64" "baremetal:deploy_kernel_id"="$IRONIC_DEPLOY_KERNEL_ID" "baremetal:deploy_ramdisk_id"="$IRONIC_DEPLOY_RAMDISK_ID" |
| 511 |
+ |
|
| 512 |
+ if [ "$VIRT_DRIVER" == "ironic" ]; then |
|
| 513 |
+ wait_for_nova_resources $total_nodes |
|
| 514 |
+ fi |
|
| 511 | 515 |
} |
| 512 | 516 |
|
| 513 | 517 |
function configure_iptables {
|