This commit adds support for deploying glance as a wsgi script under
uwsgi. To get around limitations in the uwsgi protocol when using
python3 for chunked encoding we have to setup uwsgi in http mode on a
random port listening on localhost and use mod_proxy to forward the
incoming requests. The alternative approach of having apache buffer the
requests locally with the send_cl option with mod_proxy_uwsgi only
worked on python2 and also has the limitation that apache is buffering
the entire chunked object, which could be several gigabytes in size.
Depends-On: I089a22a4be4227a551c32442dba27c426f54c87d
Change-Id: Ie98fb7da5e8ecfa49cd680b88139cb7034d5f88f
| ... | ... |
@@ -260,12 +260,64 @@ function write_uwsgi_config {
|
| 260 | 260 |
else |
| 261 | 261 |
local apache_conf="" |
| 262 | 262 |
apache_conf=$(apache_site_config_for $name) |
| 263 |
- echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}/\" retry=0 " | sudo tee $apache_conf
|
|
| 263 |
+ echo "SetEnv proxy-sendcl 1" | sudo tee $apache_conf |
|
| 264 |
+ echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}/\" retry=0 " | sudo tee -a $apache_conf
|
|
| 264 | 265 |
enable_apache_site $name |
| 265 | 266 |
restart_apache_server |
| 266 | 267 |
fi |
| 267 | 268 |
} |
| 268 | 269 |
|
| 270 |
+# For services using chunked encoding, the only services known to use this |
|
| 271 |
+# currently are Glance and Swift, we need to use an http proxy instead of |
|
| 272 |
+# mod_proxy_uwsgi because the chunked encoding gets dropped. See: |
|
| 273 |
+# https://github.com/unbit/uwsgi/issues/1540 You can workaround this on python2 |
|
| 274 |
+# but that involves having apache buffer the request before sending it to |
|
| 275 |
+# uswgi. |
|
| 276 |
+function write_local_uwsgi_http_config {
|
|
| 277 |
+ local file=$1 |
|
| 278 |
+ local wsgi=$2 |
|
| 279 |
+ local url=$3 |
|
| 280 |
+ name=$(basename $wsgi) |
|
| 281 |
+ |
|
| 282 |
+ # create a home for the sockets; note don't use /tmp -- apache has |
|
| 283 |
+ # a private view of it on some platforms. |
|
| 284 |
+ |
|
| 285 |
+ # always cleanup given that we are using iniset here |
|
| 286 |
+ rm -rf $file |
|
| 287 |
+ iniset "$file" uwsgi wsgi-file "$wsgi" |
|
| 288 |
+ port=$(get_random_port) |
|
| 289 |
+ iniset "$file" uwsgi http "127.0.0.1:$port" |
|
| 290 |
+ iniset "$file" uwsgi processes $API_WORKERS |
|
| 291 |
+ # This is running standalone |
|
| 292 |
+ iniset "$file" uwsgi master true |
|
| 293 |
+ # Set die-on-term & exit-on-reload so that uwsgi shuts down |
|
| 294 |
+ iniset "$file" uwsgi die-on-term true |
|
| 295 |
+ iniset "$file" uwsgi exit-on-reload true |
|
| 296 |
+ iniset "$file" uwsgi enable-threads true |
|
| 297 |
+ iniset "$file" uwsgi plugins python |
|
| 298 |
+ # uwsgi recommends this to prevent thundering herd on accept. |
|
| 299 |
+ iniset "$file" uwsgi thunder-lock true |
|
| 300 |
+ # Override the default size for headers from the 4k default. |
|
| 301 |
+ iniset "$file" uwsgi buffer-size 65535 |
|
| 302 |
+ # Make sure the client doesn't try to re-use the connection. |
|
| 303 |
+ iniset "$file" uwsgi add-header "Connection: close" |
|
| 304 |
+ # This ensures that file descriptors aren't shared between processes. |
|
| 305 |
+ iniset "$file" uwsgi lazy-apps true |
|
| 306 |
+ iniset "$file" uwsgi chmod-socket 666 |
|
| 307 |
+ iniset "$file" uwsgi http-raw-body true |
|
| 308 |
+ iniset "$file" uwsgi http-chunked-input true |
|
| 309 |
+ iniset "$file" uwsgi http-auto-chunked true |
|
| 310 |
+ |
|
| 311 |
+ enable_apache_mod proxy |
|
| 312 |
+ enable_apache_mod proxy_http |
|
| 313 |
+ local apache_conf="" |
|
| 314 |
+ apache_conf=$(apache_site_config_for $name) |
|
| 315 |
+ echo "KeepAlive Off" | sudo tee $apache_conf |
|
| 316 |
+ echo "ProxyPass \"${url}\" \"http://127.0.0.1:$port\" retry=0 " | sudo tee -a $apache_conf
|
|
| 317 |
+ enable_apache_site $name |
|
| 318 |
+ restart_apache_server |
|
| 319 |
+} |
|
| 320 |
+ |
|
| 269 | 321 |
function remove_uwsgi_config {
|
| 270 | 322 |
local file=$1 |
| 271 | 323 |
local wsgi=$2 |
| ... | ... |
@@ -347,7 +347,7 @@ function configure_cinder {
|
| 347 | 347 |
|
| 348 | 348 |
iniset $CINDER_CONF DEFAULT osapi_volume_workers "$API_WORKERS" |
| 349 | 349 |
|
| 350 |
- iniset $CINDER_CONF DEFAULT glance_api_servers "${GLANCE_SERVICE_PROTOCOL}://${GLANCE_HOSTPORT}"
|
|
| 350 |
+ iniset $CINDER_CONF DEFAULT glance_api_servers "$GLANCE_URL" |
|
| 351 | 351 |
if is_service_enabled tls-proxy; then |
| 352 | 352 |
iniset $CINDER_CONF DEFAULT glance_protocol https |
| 353 | 353 |
iniset $CINDER_CONF DEFAULT glance_ca_certificates_file $SSL_BUNDLE_FILE |
| ... | ... |
@@ -71,6 +71,16 @@ GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$GLANCE_SERVICE_HOST:$GLANCE_SERVICE_PORT}
|
| 71 | 71 |
GLANCE_SERVICE_PROTOCOL=${GLANCE_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
|
| 72 | 72 |
GLANCE_REGISTRY_PORT=${GLANCE_REGISTRY_PORT:-9191}
|
| 73 | 73 |
GLANCE_REGISTRY_PORT_INT=${GLANCE_REGISTRY_PORT_INT:-19191}
|
| 74 |
+GLANCE_UWSGI=$GLANCE_BIN_DIR/glance-wsgi-api |
|
| 75 |
+GLANCE_UWSGI_CONF=$GLANCE_CONF_DIR/glance-uswgi.ini |
|
| 76 |
+# If wsgi mode is uwsgi run glance under uwsgi, else default to eventlet |
|
| 77 |
+# TODO(mtreinish): Remove the eventlet path here and in all the similar |
|
| 78 |
+# conditionals below after the Pike release |
|
| 79 |
+if [[ "$WSGI_MODE" == "uwsgi" ]]; then |
|
| 80 |
+ GLANCE_URL="$GLANCE_SERVICE_PROTOCOL://$GLANCE_SERVICE_HOST/image" |
|
| 81 |
+else |
|
| 82 |
+ GLANCE_URL="$GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT" |
|
| 83 |
+fi |
|
| 74 | 84 |
|
| 75 | 85 |
# Functions |
| 76 | 86 |
# --------- |
| ... | ... |
@@ -103,16 +113,13 @@ function configure_glance {
|
| 103 | 103 |
dburl=`database_connection_url glance` |
| 104 | 104 |
iniset $GLANCE_REGISTRY_CONF database connection $dburl |
| 105 | 105 |
iniset $GLANCE_REGISTRY_CONF DEFAULT use_syslog $SYSLOG |
| 106 |
- iniset $GLANCE_REGISTRY_CONF DEFAULT workers "$API_WORKERS" |
|
| 107 | 106 |
iniset $GLANCE_REGISTRY_CONF paste_deploy flavor keystone |
| 108 | 107 |
configure_auth_token_middleware $GLANCE_REGISTRY_CONF glance $GLANCE_AUTH_CACHE_DIR/registry |
| 109 | 108 |
iniset $GLANCE_REGISTRY_CONF oslo_messaging_notifications driver messagingv2 |
| 110 | 109 |
iniset_rpc_backend glance $GLANCE_REGISTRY_CONF |
| 111 | 110 |
iniset $GLANCE_REGISTRY_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT" |
| 112 | 111 |
|
| 113 |
- cp $GLANCE_DIR/etc/glance-api.conf $GLANCE_API_CONF |
|
| 114 | 112 |
iniset $GLANCE_API_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL |
| 115 |
- iniset $GLANCE_API_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS |
|
| 116 | 113 |
inicomment $GLANCE_API_CONF DEFAULT log_file |
| 117 | 114 |
iniset $GLANCE_API_CONF database connection $dburl |
| 118 | 115 |
iniset $GLANCE_API_CONF DEFAULT use_syslog $SYSLOG |
| ... | ... |
@@ -140,8 +147,6 @@ function configure_glance {
|
| 140 | 140 |
iniset $GLANCE_API_CONF glance_store filesystem_store_datadir $GLANCE_IMAGE_DIR/ |
| 141 | 141 |
iniset $GLANCE_API_CONF DEFAULT registry_host $GLANCE_SERVICE_HOST |
| 142 | 142 |
|
| 143 |
- iniset $GLANCE_API_CONF DEFAULT workers "$API_WORKERS" |
|
| 144 |
- |
|
| 145 | 143 |
# CORS feature support - to allow calls from Horizon by default |
| 146 | 144 |
if [ -n "$GLANCE_CORS_ALLOWED_ORIGIN" ]; then |
| 147 | 145 |
iniset $GLANCE_API_CONF cors allowed_origin "$GLANCE_CORS_ALLOWED_ORIGIN" |
| ... | ... |
@@ -198,7 +203,6 @@ function configure_glance {
|
| 198 | 198 |
setup_logging $GLANCE_REGISTRY_CONF |
| 199 | 199 |
|
| 200 | 200 |
cp -p $GLANCE_DIR/etc/glance-registry-paste.ini $GLANCE_REGISTRY_PASTE_INI |
| 201 |
- |
|
| 202 | 201 |
cp -p $GLANCE_DIR/etc/glance-api-paste.ini $GLANCE_API_PASTE_INI |
| 203 | 202 |
|
| 204 | 203 |
cp $GLANCE_DIR/etc/glance-cache.conf $GLANCE_CACHE_CONF |
| ... | ... |
@@ -231,6 +235,13 @@ function configure_glance {
|
| 231 | 231 |
iniset $GLANCE_API_CONF DEFAULT cinder_endpoint_template "https://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/%(project_id)s" |
| 232 | 232 |
iniset $GLANCE_CACHE_CONF DEFAULT cinder_endpoint_template "https://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/%(project_id)s" |
| 233 | 233 |
fi |
| 234 |
+ |
|
| 235 |
+ if [[ "$WSGI_MODE" == "uwsgi" ]]; then |
|
| 236 |
+ write_local_uwsgi_http_config "$GLANCE_UWSGI_CONF" "$GLANCE_UWSGI" "/image" |
|
| 237 |
+ else |
|
| 238 |
+ iniset $GLANCE_API_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS |
|
| 239 |
+ iniset $GLANCE_API_CONF DEFAULT workers "$API_WORKERS" |
|
| 240 |
+ fi |
|
| 234 | 241 |
} |
| 235 | 242 |
|
| 236 | 243 |
# create_glance_accounts() - Set up common required glance accounts |
| ... | ... |
@@ -255,7 +266,7 @@ function create_glance_accounts {
|
| 255 | 255 |
get_or_create_endpoint \ |
| 256 | 256 |
"image" \ |
| 257 | 257 |
"$REGION_NAME" \ |
| 258 |
- "$GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT" |
|
| 258 |
+ "$GLANCE_URL" |
|
| 259 | 259 |
|
| 260 | 260 |
# Note(frickler): Crude workaround for https://bugs.launchpad.net/glance-store/+bug/1620999 |
| 261 | 261 |
service_domain_id=$(get_or_create_domain $SERVICE_DOMAIN_NAME) |
| ... | ... |
@@ -320,15 +331,21 @@ function install_glance {
|
| 320 | 320 |
function start_glance {
|
| 321 | 321 |
local service_protocol=$GLANCE_SERVICE_PROTOCOL |
| 322 | 322 |
if is_service_enabled tls-proxy; then |
| 323 |
- start_tls_proxy glance-service '*' $GLANCE_SERVICE_PORT $GLANCE_SERVICE_HOST $GLANCE_SERVICE_PORT_INT |
|
| 323 |
+ if [[ "$WSGI_MODE" != "uwsgi" ]]; then |
|
| 324 |
+ start_tls_proxy glance-service '*' $GLANCE_SERVICE_PORT $GLANCE_SERVICE_HOST $GLANCE_SERVICE_PORT_INT |
|
| 325 |
+ fi |
|
| 324 | 326 |
start_tls_proxy glance-registry '*' $GLANCE_REGISTRY_PORT $GLANCE_SERVICE_HOST $GLANCE_REGISTRY_PORT_INT |
| 325 | 327 |
fi |
| 326 | 328 |
|
| 327 | 329 |
run_process g-reg "$GLANCE_BIN_DIR/glance-registry --config-file=$GLANCE_CONF_DIR/glance-registry.conf" |
| 328 |
- run_process g-api "$GLANCE_BIN_DIR/glance-api --config-file=$GLANCE_CONF_DIR/glance-api.conf" |
|
| 330 |
+ if [[ "$WSGI_MODE" == "uwsgi" ]]; then |
|
| 331 |
+ run_process g-api "$GLANCE_BIN_DIR/uwsgi --ini $GLANCE_UWSGI_CONF" |
|
| 332 |
+ else |
|
| 333 |
+ run_process g-api "$GLANCE_BIN_DIR/glance-api --config-file=$GLANCE_CONF_DIR/glance-api.conf" |
|
| 334 |
+ fi |
|
| 329 | 335 |
|
| 330 |
- echo "Waiting for g-api ($GLANCE_HOSTPORT) to start..." |
|
| 331 |
- if ! wait_for_service $SERVICE_TIMEOUT $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT; then |
|
| 336 |
+ echo "Waiting for g-api ($GLANCE_SERVICE_HOST) to start..." |
|
| 337 |
+ if ! wait_for_service $SERVICE_TIMEOUT $GLANCE_URL; then |
|
| 332 | 338 |
die $LINENO "g-api did not start" |
| 333 | 339 |
fi |
| 334 | 340 |
} |
| ... | ... |
@@ -574,7 +574,7 @@ function create_nova_conf {
|
| 574 | 574 |
# enable notifications, but it will allow them to function when enabled. |
| 575 | 575 |
iniset $NOVA_CONF oslo_messaging_notifications driver "messagingv2" |
| 576 | 576 |
iniset_rpc_backend nova $NOVA_CONF |
| 577 |
- iniset $NOVA_CONF glance api_servers "${GLANCE_SERVICE_PROTOCOL}://${GLANCE_HOSTPORT}"
|
|
| 577 |
+ iniset $NOVA_CONF glance api_servers "$GLANCE_URL" |
|
| 578 | 578 |
|
| 579 | 579 |
iniset $NOVA_CONF DEFAULT osapi_compute_workers "$API_WORKERS" |
| 580 | 580 |
iniset $NOVA_CONF DEFAULT metadata_workers "$API_WORKERS" |