This makes us start two levels of nova-conductor processes, and one per cell.
Note that this also sets the notification transport_url to the top-level mq
so that we continue to get a unified stream of notifications.
Related-Bug: #1700496
Change-Id: I08d7da843d18b426dda8a8a231039d950a4c0ce5
Depends-On: I64b600b30f6e54db0ec9083c6c176e895c6d0cc2
Depends-On: If59453f1899e99040c554bcb9ad54c8a506adc56
... | ... |
@@ -51,6 +51,7 @@ NOVA_AUTH_CACHE_DIR=${NOVA_AUTH_CACHE_DIR:-/var/cache/nova} |
51 | 51 |
NOVA_CONF_DIR=/etc/nova |
52 | 52 |
NOVA_CONF=$NOVA_CONF_DIR/nova.conf |
53 | 53 |
NOVA_CELLS_CONF=$NOVA_CONF_DIR/nova-cells.conf |
54 |
+NOVA_CPU_CONF=$NOVA_CONF_DIR/nova-cpu.conf |
|
54 | 55 |
NOVA_FAKE_CONF=$NOVA_CONF_DIR/nova-fake.conf |
55 | 56 |
NOVA_CELLS_DB=${NOVA_CELLS_DB:-nova_cell} |
56 | 57 |
NOVA_API_DB=${NOVA_API_DB:-nova_api} |
... | ... |
@@ -59,6 +60,13 @@ NOVA_METADATA_UWSGI=$NOVA_BIN_DIR/nova-metadata-wsgi |
59 | 59 |
NOVA_UWSGI_CONF=$NOVA_CONF_DIR/nova-api-uwsgi.ini |
60 | 60 |
NOVA_METADATA_UWSGI_CONF=$NOVA_CONF_DIR/nova-metadata-uwsgi.ini |
61 | 61 |
|
62 |
+# The total number of cells we expect. Must be greater than one and doesn't |
|
63 |
+# count cell0. |
|
64 |
+NOVA_NUM_CELLS=${NOVA_NUM_CELLS:-1} |
|
65 |
+# Our cell index, so we know what rabbit vhost to connect to. |
|
66 |
+# This should be in the range of 1-$NOVA_NUM_CELLS |
|
67 |
+NOVA_CPU_CELL=${NOVA_CPU_CELL:-1} |
|
68 |
+ |
|
62 | 69 |
NOVA_API_PASTE_INI=${NOVA_API_PASTE_INI:-$NOVA_CONF_DIR/api-paste.ini} |
63 | 70 |
|
64 | 71 |
# Toggle for deploying Nova-API under a wsgi server. We default to |
... | ... |
@@ -424,7 +432,7 @@ function create_nova_conf { |
424 | 424 |
# require them running on the host. The ensures that n-cpu doesn't |
425 | 425 |
# leak a need to use the db in a multinode scenario. |
426 | 426 |
if is_service_enabled n-api n-cond n-sched; then |
427 |
- iniset $NOVA_CONF database connection `database_connection_url nova` |
|
427 |
+ iniset $NOVA_CONF database connection `database_connection_url nova_cell0` |
|
428 | 428 |
iniset $NOVA_CONF api_database connection `database_connection_url nova_api` |
429 | 429 |
fi |
430 | 430 |
|
... | ... |
@@ -518,6 +526,7 @@ function create_nova_conf { |
518 | 518 |
# Set the oslo messaging driver to the typical default. This does not |
519 | 519 |
# enable notifications, but it will allow them to function when enabled. |
520 | 520 |
iniset $NOVA_CONF oslo_messaging_notifications driver "messagingv2" |
521 |
+ iniset $NOVA_CONF oslo_messaging_notifications transport_url $(get_transport_url) |
|
521 | 522 |
iniset_rpc_backend nova $NOVA_CONF |
522 | 523 |
iniset $NOVA_CONF glance api_servers "$GLANCE_URL" |
523 | 524 |
|
... | ... |
@@ -558,6 +567,20 @@ function create_nova_conf { |
558 | 558 |
if [ "$NOVA_USE_SERVICE_TOKEN" == "True" ]; then |
559 | 559 |
init_nova_service_user_conf |
560 | 560 |
fi |
561 |
+ |
|
562 |
+ if is_service_enabled n-cond; then |
|
563 |
+ for i in $(seq 1 $NOVA_NUM_CELLS); do |
|
564 |
+ local conf |
|
565 |
+ local vhost |
|
566 |
+ conf=$(conductor_conf $i) |
|
567 |
+ vhost="nova_cell${i}" |
|
568 |
+ iniset $conf database connection `database_connection_url nova_cell${i}` |
|
569 |
+ iniset $conf conductor workers "$API_WORKERS" |
|
570 |
+ iniset $conf DEFAULT debug "$ENABLE_DEBUG_LOG_LEVEL" |
|
571 |
+ rpc_backend_add_vhost $vhost |
|
572 |
+ iniset_rpc_backend nova $conf DEFAULT $vhost |
|
573 |
+ done |
|
574 |
+ fi |
|
561 | 575 |
} |
562 | 576 |
|
563 | 577 |
function init_nova_service_user_conf { |
... | ... |
@@ -572,6 +595,11 @@ function init_nova_service_user_conf { |
572 | 572 |
iniset $NOVA_CONF service_user auth_strategy keystone |
573 | 573 |
} |
574 | 574 |
|
575 |
+function conductor_conf { |
|
576 |
+ local cell="$1" |
|
577 |
+ echo "${NOVA_CONF_DIR}/nova_cell${cell}.conf" |
|
578 |
+} |
|
579 |
+ |
|
575 | 580 |
function init_nova_cells { |
576 | 581 |
if is_service_enabled n-cell; then |
577 | 582 |
cp $NOVA_CONF $NOVA_CELLS_CONF |
... | ... |
@@ -638,8 +666,6 @@ function init_nova { |
638 | 638 |
recreate_database $NOVA_API_DB |
639 | 639 |
$NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF api_db sync |
640 | 640 |
|
641 |
- # (Re)create nova databases |
|
642 |
- recreate_database nova |
|
643 | 641 |
recreate_database nova_cell0 |
644 | 642 |
|
645 | 643 |
# map_cell0 will create the cell mapping record in the nova_api DB so |
... | ... |
@@ -651,6 +677,12 @@ function init_nova { |
651 | 651 |
# Migrate nova and nova_cell0 databases. |
652 | 652 |
$NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF db sync |
653 | 653 |
|
654 |
+ # (Re)create nova databases |
|
655 |
+ for i in $(seq 1 $NOVA_NUM_CELLS); do |
|
656 |
+ recreate_database nova_cell${i} |
|
657 |
+ $NOVA_BIN_DIR/nova-manage --config-file $(conductor_conf $i) db sync |
|
658 |
+ done |
|
659 |
+ |
|
654 | 660 |
if is_service_enabled n-cell; then |
655 | 661 |
recreate_database $NOVA_CELLS_DB |
656 | 662 |
fi |
... | ... |
@@ -660,8 +692,9 @@ function init_nova { |
660 | 660 |
$NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF db online_data_migrations |
661 | 661 |
|
662 | 662 |
# create the cell1 cell for the main nova db where the hosts live |
663 |
- nova-manage cell_v2 create_cell --transport-url $(get_transport_url) \ |
|
664 |
- --name 'cell1' |
|
663 |
+ for i in $(seq 1 $NOVA_NUM_CELLS); do |
|
664 |
+ nova-manage --config-file $NOVA_CONF --config-file $(conductor_conf $i) cell_v2 create_cell --name "cell$i" |
|
665 |
+ done |
|
665 | 666 |
fi |
666 | 667 |
|
667 | 668 |
create_nova_cache_dir |
... | ... |
@@ -760,25 +793,40 @@ function start_nova_api { |
760 | 760 |
|
761 | 761 |
# start_nova_compute() - Start the compute process |
762 | 762 |
function start_nova_compute { |
763 |
+ local nomulticellflag="$1" |
|
763 | 764 |
# Hack to set the path for rootwrap |
764 | 765 |
local old_path=$PATH |
765 | 766 |
export PATH=$NOVA_BIN_DIR:$PATH |
766 | 767 |
|
767 | 768 |
if is_service_enabled n-cell; then |
768 | 769 |
local compute_cell_conf=$NOVA_CELLS_CONF |
770 |
+ # NOTE(danms): Don't setup conductor fleet for cellsv1 |
|
771 |
+ nomulticellflag='nomulticell' |
|
769 | 772 |
else |
770 | 773 |
local compute_cell_conf=$NOVA_CONF |
771 | 774 |
fi |
772 | 775 |
|
776 |
+ if [ "$nomulticellflag" = 'nomulticell' ]; then |
|
777 |
+ # NOTE(danms): Grenade doesn't setup multi-cell rabbit, so |
|
778 |
+ # skip these bits and use the normal config. |
|
779 |
+ NOVA_CPU_CONF=$compute_cell_conf |
|
780 |
+ echo "Skipping multi-cell conductor fleet setup" |
|
781 |
+ else |
|
782 |
+ cp $compute_cell_conf $NOVA_CPU_CONF |
|
783 |
+ # FIXME(danms): Should this be configurable? |
|
784 |
+ iniset $NOVA_CPU_CONF workarounds disable_group_policy_check_upcall True |
|
785 |
+ iniset_rpc_backend nova $NOVA_CPU_CONF DEFAULT "nova_cell${NOVA_CPU_CELL}" |
|
786 |
+ fi |
|
787 |
+ |
|
773 | 788 |
if [[ "$VIRT_DRIVER" = 'libvirt' ]]; then |
774 | 789 |
# The group **$LIBVIRT_GROUP** is added to the current user in this script. |
775 | 790 |
# ``sg`` is used in run_process to execute nova-compute as a member of the |
776 | 791 |
# **$LIBVIRT_GROUP** group. |
777 |
- run_process n-cpu "$NOVA_BIN_DIR/nova-compute --config-file $compute_cell_conf" $LIBVIRT_GROUP |
|
792 |
+ run_process n-cpu "$NOVA_BIN_DIR/nova-compute --config-file $NOVA_CPU_CONF" $LIBVIRT_GROUP |
|
778 | 793 |
elif [[ "$VIRT_DRIVER" = 'lxd' ]]; then |
779 |
- run_process n-cpu "$NOVA_BIN_DIR/nova-compute --config-file $compute_cell_conf" $LXD_GROUP |
|
794 |
+ run_process n-cpu "$NOVA_BIN_DIR/nova-compute --config-file $NOVA_CPU_CONF" $LXD_GROUP |
|
780 | 795 |
elif [[ "$VIRT_DRIVER" = 'docker' || "$VIRT_DRIVER" = 'zun' ]]; then |
781 |
- run_process n-cpu "$NOVA_BIN_DIR/nova-compute --config-file $compute_cell_conf" $DOCKER_GROUP |
|
796 |
+ run_process n-cpu "$NOVA_BIN_DIR/nova-compute --config-file $NOVA_CPU_CONF" $DOCKER_GROUP |
|
782 | 797 |
elif [[ "$VIRT_DRIVER" = 'fake' ]]; then |
783 | 798 |
local i |
784 | 799 |
for i in `seq 1 $NUMBER_FAKE_NOVA_COMPUTE`; do |
... | ... |
@@ -787,13 +835,13 @@ function start_nova_compute { |
787 | 787 |
# gets its own configuration and own log file. |
788 | 788 |
local fake_conf="${NOVA_FAKE_CONF}-${i}" |
789 | 789 |
iniset $fake_conf DEFAULT nhost "${HOSTNAME}${i}" |
790 |
- run_process "n-cpu-${i}" "$NOVA_BIN_DIR/nova-compute --config-file $compute_cell_conf --config-file $fake_conf" |
|
790 |
+ run_process "n-cpu-${i}" "$NOVA_BIN_DIR/nova-compute --config-file $NOVA_CPU_CONF --config-file $fake_conf" |
|
791 | 791 |
done |
792 | 792 |
else |
793 | 793 |
if is_service_enabled n-cpu && [[ -r $NOVA_PLUGINS/hypervisor-$VIRT_DRIVER ]]; then |
794 | 794 |
start_nova_hypervisor |
795 | 795 |
fi |
796 |
- run_process n-cpu "$NOVA_BIN_DIR/nova-compute --config-file $compute_cell_conf" |
|
796 |
+ run_process n-cpu "$NOVA_BIN_DIR/nova-compute --config-file $NOVA_CPU_CONF" |
|
797 | 797 |
fi |
798 | 798 |
|
799 | 799 |
export PATH=$old_path |
... | ... |
@@ -813,7 +861,6 @@ function start_nova_rest { |
813 | 813 |
fi |
814 | 814 |
|
815 | 815 |
# ``run_process`` checks ``is_service_enabled``, it is not needed here |
816 |
- run_process n-cond "$NOVA_BIN_DIR/nova-conductor --config-file $compute_cell_conf" |
|
817 | 816 |
run_process n-cell-region "$NOVA_BIN_DIR/nova-cells --config-file $api_cell_conf" |
818 | 817 |
run_process n-cell-child "$NOVA_BIN_DIR/nova-cells --config-file $compute_cell_conf" |
819 | 818 |
|
... | ... |
@@ -840,8 +887,38 @@ function start_nova_rest { |
840 | 840 |
export PATH=$old_path |
841 | 841 |
} |
842 | 842 |
|
843 |
+function enable_nova_fleet { |
|
844 |
+ if is_service_enabled n-cond; then |
|
845 |
+ enable_service n-super-cond |
|
846 |
+ for i in $(seq 1 $NOVA_NUM_CELLS); do |
|
847 |
+ enable_service n-cond-cell${i} |
|
848 |
+ done |
|
849 |
+ fi |
|
850 |
+} |
|
851 |
+ |
|
852 |
+function start_nova_conductor { |
|
853 |
+ if is_service_enabled n-cell; then |
|
854 |
+ echo "Starting nova-conductor in a cellsv1-compatible way" |
|
855 |
+ run_process n-cond "$NOVA_BIN_DIR/nova-conductor --config-file $NOVA_CELLS_CONF" |
|
856 |
+ return |
|
857 |
+ fi |
|
858 |
+ |
|
859 |
+ enable_nova_fleet |
|
860 |
+ if is_service_enabled n-super-cond; then |
|
861 |
+ run_process n-super-cond "$NOVA_BIN_DIR/nova-conductor --config-file $NOVA_CONF" |
|
862 |
+ fi |
|
863 |
+ for i in $(seq 1 $NOVA_NUM_CELLS); do |
|
864 |
+ if is_service_enabled n-cond-cell${i}; then |
|
865 |
+ local conf |
|
866 |
+ conf=$(conductor_conf $i) |
|
867 |
+ run_process n-cond-cell${i} "$NOVA_BIN_DIR/nova-conductor --config-file $conf" |
|
868 |
+ fi |
|
869 |
+ done |
|
870 |
+} |
|
871 |
+ |
|
843 | 872 |
function start_nova { |
844 | 873 |
start_nova_rest |
874 |
+ start_nova_conductor |
|
845 | 875 |
start_nova_compute |
846 | 876 |
} |
847 | 877 |
|
... | ... |
@@ -861,14 +938,24 @@ function stop_nova_compute { |
861 | 861 |
|
862 | 862 |
function stop_nova_rest { |
863 | 863 |
# Kill the non-compute nova processes |
864 |
- for serv in n-api n-api-meta n-net n-sch n-novnc n-xvnc n-cauth n-spice n-cond n-cell n-cell n-sproxy; do |
|
864 |
+ for serv in n-api n-api-meta n-net n-sch n-novnc n-xvnc n-cauth n-spice n-cell n-cell n-sproxy; do |
|
865 | 865 |
stop_process $serv |
866 | 866 |
done |
867 | 867 |
} |
868 | 868 |
|
869 |
+function stop_nova_conductor { |
|
870 |
+ enable_nova_fleet |
|
871 |
+ for srv in n-super-cond $(seq -f n-cond-cell%0.f 1 $NOVA_NUM_CELLS); do |
|
872 |
+ if is_service_enabled $srv; then |
|
873 |
+ stop_process $srv |
|
874 |
+ fi |
|
875 |
+ done |
|
876 |
+} |
|
877 |
+ |
|
869 | 878 |
# stop_nova() - Stop running processes (non-screen) |
870 | 879 |
function stop_nova { |
871 | 880 |
stop_nova_rest |
881 |
+ stop_nova_conductor |
|
872 | 882 |
stop_nova_compute |
873 | 883 |
} |
874 | 884 |
|
... | ... |
@@ -1304,7 +1304,9 @@ fi |
1304 | 1304 |
# Unable to use LUKS passphrase that is exactly 16 bytes long |
1305 | 1305 |
# https://bugzilla.redhat.com/show_bug.cgi?id=1447297 |
1306 | 1306 |
if is_service_enabled nova; then |
1307 |
- iniset $NOVA_CONF key_manager fixed_key $(generate_hex_string 36) |
|
1307 |
+ key=$(generate_hex_string 36) |
|
1308 |
+ iniset $NOVA_CONF key_manager fixed_key "$key" |
|
1309 |
+ iniset $NOVA_CPU_CONF key_manager fixed_key "$key" |
|
1308 | 1310 |
fi |
1309 | 1311 |
|
1310 | 1312 |
# Launch the nova-api and wait for it to answer before continuing |