When enrolling nodes into Ironic, poll nova's hypervisor-stats until
the hypervisor count reflects the number of enrolled nodes. This
eliminates a race where devstack completes and an instance is spawned
before the first post-enrollment periodic task ticks on the Nova side,
which has recently started popping up in the gate.
Change-Id: Ib3d8005e0094ee8af2d5fcb65aca6cd92736da90
Closes-bug: #1398128
| ... | ... |
@@ -500,9 +500,23 @@ function create_bridge_and_vms {
|
| 500 | 500 |
create_ovs_taps |
| 501 | 501 |
} |
| 502 | 502 |
|
| 503 |
+function wait_for_nova_resources {
|
|
| 504 |
+ # After nodes have been enrolled, we need to wait for n-cpu's periodic |
|
| 505 |
+ # task populate the resource tracker with available nodes. Wait for 2 |
|
| 506 |
+ # minutes before timing out. |
|
| 507 |
+ local expected_count=$1 |
|
| 508 |
+ echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $expected_count Ironic nodes" |
|
| 509 |
+ for i in $(seq 1 120); do |
|
| 510 |
+ if [ $(nova hypervisor-stats | grep " count " | get_field 2) -ge $expected_count ]; then |
|
| 511 |
+ return 0 |
|
| 512 |
+ fi |
|
| 513 |
+ sleep 1 |
|
| 514 |
+ done |
|
| 515 |
+ die $LINENO "Nova hypervisor-stats did not register at least $expected_count nodes" |
|
| 516 |
+} |
|
| 517 |
+ |
|
| 503 | 518 |
function enroll_nodes {
|
| 504 | 519 |
local chassis_id=$(ironic chassis-create -d "ironic test chassis" | grep " uuid " | get_field 2) |
| 505 |
- local idx=0 |
|
| 506 | 520 |
|
| 507 | 521 |
if [[ "$IRONIC_DEPLOY_DRIVER" == "pxe_ssh" ]] ; then |
| 508 | 522 |
local _IRONIC_DEPLOY_KERNEL_KEY=pxe_deploy_kernel |
| ... | ... |
@@ -536,6 +550,7 @@ function enroll_nodes {
|
| 536 | 536 |
fi |
| 537 | 537 |
fi |
| 538 | 538 |
|
| 539 |
+ local total_nodes=0 |
|
| 539 | 540 |
while read hardware_info; do |
| 540 | 541 |
if ! is_ironic_hardware; then |
| 541 | 542 |
local mac_address=$hardware_info |
| ... | ... |
@@ -566,7 +581,7 @@ function enroll_nodes {
|
| 566 | 566 |
|
| 567 | 567 |
ironic port-create --address $mac_address --node_uuid $node_id |
| 568 | 568 |
|
| 569 |
- idx=$((idx+1)) |
|
| 569 |
+ total_nodes=$((total_nodes+1)) |
|
| 570 | 570 |
done < $ironic_hwinfo_file |
| 571 | 571 |
|
| 572 | 572 |
# create the nova flavor |
| ... | ... |
@@ -581,6 +596,10 @@ function enroll_nodes {
|
| 581 | 581 |
# from the flavor after the completion of |
| 582 | 582 |
# https://blueprints.launchpad.net/ironic/+spec/add-node-instance-info |
| 583 | 583 |
nova flavor-key baremetal set "cpu_arch"="x86_64" "baremetal:deploy_kernel_id"="$IRONIC_DEPLOY_KERNEL_ID" "baremetal:deploy_ramdisk_id"="$IRONIC_DEPLOY_RAMDISK_ID" |
| 584 |
+ |
|
| 585 |
+ if [ "$VIRT_DRIVER" == "ironic" ]; then |
|
| 586 |
+ wait_for_nova_resources $total_nodes |
|
| 587 |
+ fi |
|
| 584 | 588 |
} |
| 585 | 589 |
|
| 586 | 590 |
function configure_iptables {
|