Browse code

Poll resource tracker for ironic cpus as well as count

When ironic nodes are enrolled, their resources are not available
to the nova scheduler until after a round of ironic and nova periodic
tasks have run In addition to waiting for ironic nodes to show up in
the resource tracker, also wait for associated CPU resources. In
the worst case, this means waiting for 3 total rounds of periodic
tasks.

Change-Id: Idbbc43bf74ff5fff3d50f3494148454bb51e378f
Closes-bug: #1398128

Adam Gandelman authored on 2014/12/10 07:44:24
Showing 1 changed files
... ...
@@ -501,18 +501,20 @@ function create_bridge_and_vms {
501 501
 }
502 502
 
503 503
 function wait_for_nova_resources {
504
-    # After nodes have been enrolled, we need to wait for n-cpu's periodic
505
-    # task populate the resource tracker with available nodes.  Wait for 2
506
-    # minutes before timing out.
507
-    local expected_count=$1
508
-    echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $expected_count Ironic nodes"
504
+    # After nodes have been enrolled, we need to wait for both ironic and
505
+    # nova's periodic tasks to populate the resource tracker with available
506
+    # nodes and resources. Wait up to 2 minutes for a given resource before
507
+    # timing out.
508
+    local resource=$1
509
+    local expected_count=$2
510
+    echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $resource >= $expected_count"
509 511
     for i in $(seq 1 120); do
510
-        if [ $(nova hypervisor-stats | grep " count " | get_field 2) -ge $expected_count ]; then
512
+        if [ $(nova hypervisor-stats | grep " $resource " | get_field 2) -ge $expected_count ]; then
511 513
             return 0
512 514
         fi
513 515
         sleep 1
514 516
     done
515
-    die $LINENO "Nova hypervisor-stats did not register at least $expected_count nodes"
517
+    die $LINENO "Timed out waiting for Nova hypervisor-stats $resource >= $expected_count"
516 518
 }
517 519
 
518 520
 function enroll_nodes {
... ...
@@ -551,6 +553,7 @@ function enroll_nodes {
551 551
     fi
552 552
 
553 553
     local total_nodes=0
554
+    local total_cpus=0
554 555
     while read hardware_info; do
555 556
         if ! is_ironic_hardware; then
556 557
             local mac_address=$hardware_info
... ...
@@ -582,6 +585,7 @@ function enroll_nodes {
582 582
         ironic port-create --address $mac_address --node_uuid $node_id
583 583
 
584 584
         total_nodes=$((total_nodes+1))
585
+        total_cpus=$((total_cpus+$ironic_node_cpu))
585 586
     done < $ironic_hwinfo_file
586 587
 
587 588
     # create the nova flavor
... ...
@@ -598,7 +602,8 @@ function enroll_nodes {
598 598
     nova flavor-key baremetal set "cpu_arch"="x86_64" "baremetal:deploy_kernel_id"="$IRONIC_DEPLOY_KERNEL_ID" "baremetal:deploy_ramdisk_id"="$IRONIC_DEPLOY_RAMDISK_ID"
599 599
 
600 600
     if [ "$VIRT_DRIVER" == "ironic" ]; then
601
-        wait_for_nova_resources $total_nodes
601
+        wait_for_nova_resources "count" $total_nodes
602
+        wait_for_nova_resources "vcpus" $total_cpus
602 603
     fi
603 604
 }
604 605