Browse code

enable quantum multi-node scenarios.

Let's you run with a centralized quantum service node running the plugin
and quantum agents running on one or more hosts.

Also:
- make OVS plugin work on precise by configuring cgroup_device_acl
- consolidate ovs + linux-bridge config to avoid duplicate code
- support configuring tunnel-mode for OVS plugin
- add additional build packages for OVS apt-get
- remove backward compat support for old DB config for linux-bridge plugin

Change-Id: Ifab268f739b004db13024633e8abeb17691b9e46

Dan Wendlandt authored on 2012/05/19 05:37:47
Showing 1 changed files
... ...
@@ -689,7 +689,7 @@ fi
689 689
 if is_service_enabled quantum; then
690 690
     git_clone $QUANTUM_CLIENT_REPO $QUANTUM_CLIENT_DIR $QUANTUM_CLIENT_BRANCH
691 691
 fi
692
-if is_service_enabled q-svc; then
692
+if is_service_enabled quantum; then
693 693
     # quantum
694 694
     git_clone $QUANTUM_REPO $QUANTUM_DIR $QUANTUM_BRANCH
695 695
 fi
... ...
@@ -727,7 +727,7 @@ fi
727 727
 if is_service_enabled quantum; then
728 728
     cd $QUANTUM_CLIENT_DIR; sudo python setup.py develop
729 729
 fi
730
-if is_service_enabled q-svc; then
730
+if is_service_enabled quantum; then
731 731
     cd $QUANTUM_DIR; sudo python setup.py develop
732 732
 fi
733 733
 if is_service_enabled m-svc; then
... ...
@@ -1027,128 +1027,111 @@ if is_service_enabled g-reg; then
1027 1027
     cp $GLANCE_DIR/etc/policy.json $GLANCE_POLICY_JSON
1028 1028
 fi
1029 1029
 
1030
-# Quantum
1030
+# Quantum (for controller or agent nodes)
1031 1031
 # -------
1032 1032
 if is_service_enabled quantum; then
1033 1033
     # Put config files in /etc/quantum for everyone to find
1034
-    QUANTUM_CONF_DIR=/etc/quantum
1035
-    if [[ ! -d $QUANTUM_CONF_DIR ]]; then
1036
-        sudo mkdir -p $QUANTUM_CONF_DIR
1034
+    if [[ ! -d /etc/quantum ]]; then
1035
+        sudo mkdir -p /etc/quantum
1036
+    fi
1037
+    sudo chown `whoami` /etc/quantum
1038
+
1039
+    if [[ "$Q_PLUGIN" = "openvswitch" ]]; then
1040
+        Q_PLUGIN_CONF_PATH=etc/quantum/plugins/openvswitch
1041
+        Q_PLUGIN_CONF_FILENAME=ovs_quantum_plugin.ini
1042
+        Q_DB_NAME="ovs_quantum"
1043
+        Q_PLUGIN_CLASS="quantum.plugins.openvswitch.ovs_quantum_plugin.OVSQuantumPlugin"
1044
+    elif [[ "$Q_PLUGIN" = "linuxbridge" ]]; then
1045
+        # Install deps
1046
+        # FIXME add to files/apts/quantum, but don't install if not needed!
1047
+        install_package python-configobj
1048
+        Q_PLUGIN_CONF_PATH=etc/quantum/plugins/linuxbridge
1049
+        Q_PLUGIN_CONF_FILENAME=linuxbridge_conf.ini
1050
+        Q_DB_NAME="quantum_linux_bridge"
1051
+        Q_PLUGIN_CLASS="quantum.plugins.linuxbridge.LinuxBridgePlugin.LinuxBridgePlugin"
1052
+    else
1053
+        echo "Unknown Quantum plugin '$Q_PLUGIN'.. exiting"
1054
+        exit 1
1055
+    fi
1056
+
1057
+    # if needed, move config file from $QUANTUM_DIR/etc/quantum to /etc/quantum
1058
+    mkdir -p /$Q_PLUGIN_CONF_PATH
1059
+    Q_PLUGIN_CONF_FILE=$Q_PLUGIN_CONF_PATH/$Q_PLUGIN_CONF_FILENAME
1060
+    if [[ -e $QUANTUM_DIR/$Q_PLUGIN_CONF_FILE ]]; then
1061
+            sudo mv $QUANTUM_DIR/$Q_PLUGIN_CONF_FILE /$Q_PLUGIN_CONF_FILE
1037 1062
     fi
1038
-    sudo chown `whoami` $QUANTUM_CONF_DIR
1039
-
1040
-    # Set default values when using Linux Bridge plugin
1041
-    if [[ "$Q_PLUGIN" = "linuxbridge" ]]; then
1042
-        # set the config file
1043
-        QUANTUM_LB_CONF_DIR=$QUANTUM_CONF_DIR/plugins/linuxbridge
1044
-        mkdir -p $QUANTUM_LB_CONF_DIR
1045
-        QUANTUM_LB_CONFIG_FILE=$QUANTUM_LB_CONF_DIR/linuxbridge_conf.ini
1046
-        # must remove this file from existing location, otherwise Quantum will prefer it
1047
-        if [[ -e $QUANTUM_DIR/etc/quantum/plugins/linuxbridge/linuxbridge_conf.ini ]]; then
1048
-            sudo mv $QUANTUM_DIR/etc/quantum/plugins/linuxbridge/linuxbridge_conf.ini $QUANTUM_LB_CONFIG_FILE
1063
+    sudo sed -i -e "s/^sql_connection =.*$/sql_connection = mysql:\/\/$MYSQL_USER:$MYSQL_PASSWORD@$MYSQL_HOST\/$Q_DB_NAME?charset=utf8/g" /$Q_PLUGIN_CONF_FILE
1064
+
1065
+    OVS_ENABLE_TUNNELING=${OVS_ENABLE_TUNNELING:-True}
1066
+    if [[ "$Q_PLUGIN" = "openvswitch" && $OVS_ENABLE_TUNNELING = "True" ]]; then
1067
+        OVS_VERSION=`ovs-vsctl --version | head -n 1 | awk '{print $4;}'`
1068
+        if [ $OVS_VERSION \< "1.4" ] && ! is_service_enabled q-svc ; then
1069
+            echo "You are running OVS version $OVS_VERSION."
1070
+            echo "OVS 1.4+ is required for tunneling between multiple hosts."
1071
+            exit 1
1049 1072
         fi
1050
-        #set the default network interface
1051
-        QUANTUM_LB_PRIVATE_INTERFACE=${QUANTUM_LB_PRIVATE_INTERFACE:-$GUEST_INTERFACE_DEFAULT}
1073
+        sudo sed -i -e "s/.*enable-tunneling = .*$/enable-tunneling = $OVS_ENABLE_TUNNELING/g" /$Q_PLUGIN_CONF_FILE
1052 1074
     fi
1053 1075
 fi
1054
-# Quantum service
1076
+
1077
+# Quantum service (for controller node)
1055 1078
 if is_service_enabled q-svc; then
1056
-    QUANTUM_PLUGIN_INI_FILE=$QUANTUM_CONF_DIR/plugins.ini
1079
+    Q_PLUGIN_INI_FILE=/etc/quantum/plugins.ini
1080
+    Q_CONF_FILE=/etc/quantum/quantum.conf
1057 1081
     # must remove this file from existing location, otherwise Quantum will prefer it
1058 1082
     if [[ -e $QUANTUM_DIR/etc/plugins.ini ]]; then
1059
-        sudo mv $QUANTUM_DIR/etc/plugins.ini $QUANTUM_PLUGIN_INI_FILE
1083
+        sudo mv $QUANTUM_DIR/etc/plugins.ini $Q_PLUGIN_INI_FILE
1060 1084
     fi
1061 1085
 
1062
-    if [[ "$Q_PLUGIN" = "openvswitch" ]]; then
1063
-        # Install deps
1064
-        # FIXME add to files/apts/quantum, but don't install if not needed!
1065
-        if [[ "$os_PACKAGE" = "deb" ]]; then
1066
-            kernel_version=`cat /proc/version | cut -d " " -f3`
1067
-            install_package openvswitch-switch openvswitch-datapath-dkms linux-headers-$kernel_version
1068
-        else
1069
-            ### FIXME(dtroyer): Find RPMs for OpenVSwitch
1070
-            echo "OpenVSwitch packages need to be located"
1071
-        fi
1072
-
1073
-        QUANTUM_OVS_CONF_DIR=$QUANTUM_CONF_DIR/plugins/openvswitch
1074
-        QUANTUM_OVS_CONFIG_FILE=$QUANTUM_OVS_CONF_DIR/ovs_quantum_plugin.ini
1086
+    if [[ -e $QUANTUM_DIR/etc/quantum.conf ]]; then
1087
+      sudo mv $QUANTUM_DIR/etc/quantum.conf $Q_CONF_FILE
1088
+    fi
1075 1089
 
1076
-        # Create database for the plugin/agent
1077
-        if is_service_enabled mysql; then
1078
-            mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'DROP DATABASE IF EXISTS ovs_quantum;'
1079
-            mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'CREATE DATABASE IF NOT EXISTS ovs_quantum CHARACTER SET utf8;'
1080
-        else
1081
-            echo "mysql must be enabled in order to use the $Q_PLUGIN Quantum plugin."
1082
-            exit 1
1083
-        fi
1084
-        # Make sure we're using the openvswitch plugin
1085
-        sudo sed -i -e "s/^provider =.*$/provider = quantum.plugins.openvswitch.ovs_quantum_plugin.OVSQuantumPlugin/g" $QUANTUM_PLUGIN_INI_FILE
1086
-    elif [[ "$Q_PLUGIN" = "linuxbridge" ]]; then
1087
-        # Install deps
1088
-        # FIXME add to files/apts/quantum, but don't install if not needed!
1089
-        install_package python-configobj
1090
-        # Create database for the plugin/agent
1091
-        if is_service_enabled mysql; then
1092
-            mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'DROP DATABASE IF EXISTS quantum_linux_bridge;'
1093
-            mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'CREATE DATABASE IF NOT EXISTS quantum_linux_bridge;'
1094
-            if grep -Fxq "user = " $QUANTUM_LB_CONFIG_FILE
1095
-            then
1096
-                sudo sed -i -e "s/^connection = sqlite$/#connection = sqlite/g" $QUANTUM_LB_CONFIG_FILE
1097
-                sudo sed -i -e "s/^#connection = mysql$/connection = mysql/g" $QUANTUM_LB_CONFIG_FILE
1098
-                sudo sed -i -e "s/^user = .*$/user = $MYSQL_USER/g" $QUANTUM_LB_CONFIG_FILE
1099
-                sudo sed -i -e "s/^pass = .*$/pass = $MYSQL_PASSWORD/g" $QUANTUM_LB_CONFIG_FILE
1100
-                sudo sed -i -e "s/^host = .*$/host = $MYSQL_HOST/g" $QUANTUM_LB_CONFIG_FILE
1101
-            else
1102
-                sudo sed -i -e "s/^sql_connection =.*$/sql_connection = mysql:\/\/$MYSQL_USER:$MYSQL_PASSWORD@$MYSQL_HOST\/quantum_linux_bridge?charset=utf8/g" $QUANTUM_LB_CONFIG_FILE
1103
-            fi
1090
+    if is_service_enabled mysql; then
1091
+            mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e "DROP DATABASE IF EXISTS $Q_DB_NAME;"
1092
+            mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e "CREATE DATABASE IF NOT EXISTS $Q_DB_NAME CHARACTER SET utf8;"
1104 1093
         else
1105 1094
             echo "mysql must be enabled in order to use the $Q_PLUGIN Quantum plugin."
1106 1095
             exit 1
1107
-        fi
1108
-        # Make sure we're using the linuxbridge plugin
1109
-        sudo sed -i -e "s/^provider =.*$/provider = quantum.plugins.linuxbridge.LinuxBridgePlugin.LinuxBridgePlugin/g" $QUANTUM_PLUGIN_INI_FILE
1110
-    fi
1111
-    if [[ -e $QUANTUM_DIR/etc/quantum.conf ]]; then
1112
-        sudo mv $QUANTUM_DIR/etc/quantum.conf $QUANTUM_CONF_DIR/quantum.conf
1113 1096
     fi
1114
-    screen_it q-svc "cd $QUANTUM_DIR && PYTHONPATH=.:$QUANTUM_CLIENT_DIR:$PYTHONPATH python $QUANTUM_DIR/bin/quantum-server $QUANTUM_CONF_DIR/quantum.conf"
1097
+    sudo sed -i -e "s/^provider =.*$/provider = $Q_PLUGIN_CLASS/g" $Q_PLUGIN_INI_FILE
1098
+
1099
+    screen_it q-svc "cd $QUANTUM_DIR && python $QUANTUM_DIR/bin/quantum-server $Q_CONF_FILE"
1115 1100
 fi
1116 1101
 
1117 1102
 # Quantum agent (for compute nodes)
1118 1103
 if is_service_enabled q-agt; then
1119 1104
     if [[ "$Q_PLUGIN" = "openvswitch" ]]; then
1105
+        # Install deps
1106
+        # FIXME add to files/apts/quantum, but don't install if not needed!
1107
+        if [[ "$os_PACKAGE" = "deb" ]]; then
1108
+            kernel_version=`cat /proc/version | cut -d " " -f3`
1109
+            install_package make fakeroot dkms openvswitch-switch openvswitch-datapath-dkms linux-headers-$kernel_version
1110
+        else
1111
+            ### FIXME(dtroyer): Find RPMs for OpenVSwitch
1112
+            echo "OpenVSwitch packages need to be located"
1113
+        fi
1120 1114
         # Set up integration bridge
1121 1115
         OVS_BRIDGE=${OVS_BRIDGE:-br-int}
1116
+        for PORT in `sudo ovs-vsctl --no-wait list-ports $OVS_BRIDGE`; do
1117
+            if [[ "$PORT" =~ tap* ]]; then echo `sudo ip link delete $PORT` > /dev/null; fi
1118
+            sudo ovs-vsctl --no-wait del-port $OVS_BRIDGE $PORT
1119
+        done
1122 1120
         sudo ovs-vsctl --no-wait -- --if-exists del-br $OVS_BRIDGE
1123 1121
         sudo ovs-vsctl --no-wait add-br $OVS_BRIDGE
1124 1122
         sudo ovs-vsctl --no-wait br-set-external-id $OVS_BRIDGE bridge-id br-int
1125
-
1126
-        # Start up the quantum <-> openvswitch agent
1127
-        QUANTUM_OVS_CONF_DIR=$QUANTUM_CONF_DIR/plugins/openvswitch
1128
-        mkdir -p $QUANTUM_OVS_CONF_DIR
1129
-        QUANTUM_OVS_CONFIG_FILE=$QUANTUM_OVS_CONF_DIR/ovs_quantum_plugin.ini
1130
-        if [[ -e $QUANTUM_DIR/etc/quantum/plugins/openvswitch/ovs_quantum_plugin.ini ]]; then
1131
-            sudo mv $QUANTUM_DIR/etc/quantum/plugins/openvswitch/ovs_quantum_plugin.ini $QUANTUM_OVS_CONFIG_FILE
1132
-        fi
1133
-        sudo sed -i -e "s/^sql_connection =.*$/sql_connection = mysql:\/\/$MYSQL_USER:$MYSQL_PASSWORD@$MYSQL_HOST\/ovs_quantum?charset=utf8/g" $QUANTUM_OVS_CONFIG_FILE
1134
-        screen_it q-agt "sleep 4; sudo python $QUANTUM_DIR/quantum/plugins/openvswitch/agent/ovs_quantum_agent.py $QUANTUM_OVS_CONFIG_FILE -v"
1123
+        sudo sed -i -e "s/.*local-ip = .*/local-ip = $HOST_IP/g" /$Q_PLUGIN_CONF_FILE
1124
+        AGENT_BINARY=$QUANTUM_DIR/quantum/plugins/openvswitch/agent/ovs_quantum_agent.py
1135 1125
     elif [[ "$Q_PLUGIN" = "linuxbridge" ]]; then
1136 1126
        # Start up the quantum <-> linuxbridge agent
1137 1127
        install_package bridge-utils
1138
-       sudo sed -i -e "s/^physical_interface = .*$/physical_interface = $QUANTUM_LB_PRIVATE_INTERFACE/g" $QUANTUM_LB_CONFIG_FILE
1139
-       if grep -Fxq "user = " $QUANTUM_LB_CONFIG_FILE
1140
-       then
1141
-           sudo sed -i -e "s/^connection = sqlite$/#connection = sqlite/g" $QUANTUM_LB_CONFIG_FILE
1142
-           sudo sed -i -e "s/^#connection = mysql$/connection = mysql/g" $QUANTUM_LB_CONFIG_FILE
1143
-           sudo sed -i -e "s/^user = .*$/user = $MYSQL_USER/g" $QUANTUM_LB_CONFIG_FILE
1144
-           sudo sed -i -e "s/^pass = .*$/pass = $MYSQL_PASSWORD/g" $QUANTUM_LB_CONFIG_FILE
1145
-           sudo sed -i -e "s/^host = .*$/host = $MYSQL_HOST/g" $QUANTUM_LB_CONFIG_FILE
1146
-       else
1147
-           sudo sed -i -e "s/^sql_connection =.*$/sql_connection = mysql:\/\/$MYSQL_USER:$MYSQL_PASSWORD@$MYSQL_HOST\/quantum_linux_bridge?charset=utf8/g" $QUANTUM_LB_CONFIG_FILE
1148
-       fi
1149
-
1150
-       screen_it q-agt "sleep 4; sudo python $QUANTUM_DIR/quantum/plugins/linuxbridge/agent/linuxbridge_quantum_agent.py $QUANTUM_LB_CONFIG_FILE -v"
1128
+        #set the default network interface
1129
+       QUANTUM_LB_PRIVATE_INTERFACE=${QUANTUM_LB_PRIVATE_INTERFACE:-$GUEST_INTERFACE_DEFAULT}
1130
+       sudo sed -i -e "s/^physical_interface = .*$/physical_interface = $QUANTUM_LB_PRIVATE_INTERFACE/g" /$Q_PLUGIN_CONF_FILE
1131
+       AGENT_BINARY=$QUANTUM_DIR/quantum/plugins/linuxbridge/agent/linuxbridge_quantum_agent.py
1151 1132
     fi
1133
+    # Start up the quantum agent
1134
+    screen_it q-agt "sudo python $AGENT_BINARY /$Q_PLUGIN_CONF_FILE -v"
1152 1135
 fi
1153 1136
 
1154 1137
 # Melange service
... ...
@@ -1278,6 +1261,21 @@ if is_service_enabled n-cpu; then
1278 1278
         fi
1279 1279
     fi
1280 1280
 
1281
+    QEMU_CONF=/etc/libvirt/qemu.conf
1282
+    if is_service_enabled quantum && [[ $Q_PLUGIN = "openvswitch" ]] && ! sudo grep -q '^cgroup_device_acl' $QEMU_CONF ; then
1283
+        # add /dev/net/tun to cgroup_device_acls, needed for type=ethernet interfaces
1284
+        sudo chmod 666 $QEMU_CONF
1285
+        sudo cat <<EOF >> /etc/libvirt/qemu.conf
1286
+cgroup_device_acl = [
1287
+    "/dev/null", "/dev/full", "/dev/zero",
1288
+    "/dev/random", "/dev/urandom",
1289
+    "/dev/ptmx", "/dev/kvm", "/dev/kqemu",
1290
+    "/dev/rtc", "/dev/hpet","/dev/net/tun",
1291
+]
1292
+EOF
1293
+        sudo chmod 644 $QEMU_CONF
1294
+    fi
1295
+
1281 1296
     if [[ "$os_PACKAGE" = "deb" ]]; then
1282 1297
         LIBVIRT_DAEMON=libvirt-bin
1283 1298
     else
... ...
@@ -1616,17 +1614,18 @@ if is_service_enabled quantum; then
1616 1616
         add_nova_opt "melange_host=$M_HOST"
1617 1617
         add_nova_opt "melange_port=$M_PORT"
1618 1618
     fi
1619
-    if is_service_enabled q-svc && [[ "$Q_PLUGIN" = "openvswitch" ]]; then
1620
-        add_nova_opt "libvirt_vif_type=ethernet"
1621
-        add_nova_opt "libvirt_vif_driver=nova.virt.libvirt.vif.LibvirtOpenVswitchDriver"
1622
-        add_nova_opt "linuxnet_interface_driver=nova.network.linux_net.LinuxOVSInterfaceDriver"
1623
-        add_nova_opt "quantum_use_dhcp=True"
1624
-    elif is_service_enabled q-svc && [[ "$Q_PLUGIN" = "linuxbridge" ]]; then
1625
-        add_nova_opt "libvirt_vif_type=ethernet"
1626
-        add_nova_opt "libvirt_vif_driver=nova.virt.libvirt.vif.QuantumLinuxBridgeVIFDriver"
1627
-        add_nova_opt "linuxnet_interface_driver=nova.network.linux_net.QuantumLinuxBridgeInterfaceDriver"
1628
-        add_nova_opt "quantum_use_dhcp=True"
1619
+
1620
+    if [[ "$Q_PLUGIN" = "openvswitch" ]]; then
1621
+        NOVA_VIF_DRIVER="nova.virt.libvirt.vif.LibvirtOpenVswitchDriver"
1622
+        LINUXNET_VIF_DRIVER="nova.network.linux_net.LinuxOVSInterfaceDriver"
1623
+    elif [[ "$Q_PLUGIN" = "linuxbridge" ]]; then
1624
+        NOVA_VIF_DRIVER="nova.virt.libvirt.vif.QuantumLinuxBridgeVIFDriver"
1625
+        LINUXNET_VIF_DRIVER="nova.network.linux_net.QuantumLinuxBridgeInterfaceDriver"
1629 1626
     fi
1627
+    add_nova_opt "libvirt_vif_type=ethernet"
1628
+    add_nova_opt "libvirt_vif_driver=$NOVA_VIF_DRIVER"
1629
+    add_nova_opt "linuxnet_interface_driver=$LINUXNET_VIF_DRIVER"
1630
+    add_nova_opt "quantum_use_dhcp=True"
1630 1631
 else
1631 1632
     add_nova_opt "network_manager=nova.network.manager.$NET_MAN"
1632 1633
 fi