Browse code

Use uwsgi for glance-api

This commit adds support for deploying glance as a wsgi script under
uwsgi. To get around limitations in the uwsgi protocol when using
python3 for chunked encoding we have to setup uwsgi in http mode on a
random port listening on localhost and use mod_proxy to forward the
incoming requests. The alternative approach of having apache buffer the
requests locally with the send_cl option with mod_proxy_uwsgi only
worked on python2 and also has the limitation that apache is buffering
the entire chunked object, which could be several gigabytes in size.

Depends-On: I089a22a4be4227a551c32442dba27c426f54c87d
Change-Id: Ie98fb7da5e8ecfa49cd680b88139cb7034d5f88f

Matthew Treinish authored on 2017/04/25 05:49:04
Showing 4 changed files
... ...
@@ -260,12 +260,64 @@ function write_uwsgi_config {
260 260
     else
261 261
         local apache_conf=""
262 262
         apache_conf=$(apache_site_config_for $name)
263
-        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}/\" retry=0 " | sudo tee $apache_conf
263
+        echo "SetEnv proxy-sendcl 1" | sudo tee $apache_conf
264
+        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}/\" retry=0 " | sudo tee -a $apache_conf
264 265
         enable_apache_site $name
265 266
         restart_apache_server
266 267
     fi
267 268
 }
268 269
 
270
+# For services using chunked encoding, the only services known to use this
271
+# currently are Glance and Swift, we need to use an http proxy instead of
272
+# mod_proxy_uwsgi because the chunked encoding gets dropped. See:
273
+# https://github.com/unbit/uwsgi/issues/1540 You can workaround this on python2
274
+# but that involves having apache buffer the request before sending it to
275
+# uswgi.
276
+function write_local_uwsgi_http_config {
277
+    local file=$1
278
+    local wsgi=$2
279
+    local url=$3
280
+    name=$(basename $wsgi)
281
+
282
+    # create a home for the sockets; note don't use /tmp -- apache has
283
+    # a private view of it on some platforms.
284
+
285
+    # always cleanup given that we are using iniset here
286
+    rm -rf $file
287
+    iniset "$file" uwsgi wsgi-file "$wsgi"
288
+    port=$(get_random_port)
289
+    iniset "$file" uwsgi http "127.0.0.1:$port"
290
+    iniset "$file" uwsgi processes $API_WORKERS
291
+    # This is running standalone
292
+    iniset "$file" uwsgi master true
293
+    # Set die-on-term & exit-on-reload so that uwsgi shuts down
294
+    iniset "$file" uwsgi die-on-term true
295
+    iniset "$file" uwsgi exit-on-reload true
296
+    iniset "$file" uwsgi enable-threads true
297
+    iniset "$file" uwsgi plugins python
298
+    # uwsgi recommends this to prevent thundering herd on accept.
299
+    iniset "$file" uwsgi thunder-lock true
300
+    # Override the default size for headers from the 4k default.
301
+    iniset "$file" uwsgi buffer-size 65535
302
+    # Make sure the client doesn't try to re-use the connection.
303
+    iniset "$file" uwsgi add-header "Connection: close"
304
+    # This ensures that file descriptors aren't shared between processes.
305
+    iniset "$file" uwsgi lazy-apps true
306
+    iniset "$file" uwsgi chmod-socket 666
307
+    iniset "$file" uwsgi http-raw-body true
308
+    iniset "$file" uwsgi http-chunked-input true
309
+    iniset "$file" uwsgi http-auto-chunked true
310
+
311
+    enable_apache_mod proxy
312
+    enable_apache_mod proxy_http
313
+    local apache_conf=""
314
+    apache_conf=$(apache_site_config_for $name)
315
+    echo "KeepAlive Off" | sudo tee $apache_conf
316
+    echo "ProxyPass \"${url}\" \"http://127.0.0.1:$port\" retry=0 " | sudo tee -a $apache_conf
317
+    enable_apache_site $name
318
+    restart_apache_server
319
+}
320
+
269 321
 function remove_uwsgi_config {
270 322
     local file=$1
271 323
     local wsgi=$2
... ...
@@ -347,7 +347,7 @@ function configure_cinder {
347 347
 
348 348
     iniset $CINDER_CONF DEFAULT osapi_volume_workers "$API_WORKERS"
349 349
 
350
-    iniset $CINDER_CONF DEFAULT glance_api_servers "${GLANCE_SERVICE_PROTOCOL}://${GLANCE_HOSTPORT}"
350
+    iniset $CINDER_CONF DEFAULT glance_api_servers "$GLANCE_URL"
351 351
     if is_service_enabled tls-proxy; then
352 352
         iniset $CINDER_CONF DEFAULT glance_protocol https
353 353
         iniset $CINDER_CONF DEFAULT glance_ca_certificates_file $SSL_BUNDLE_FILE
... ...
@@ -71,6 +71,16 @@ GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$GLANCE_SERVICE_HOST:$GLANCE_SERVICE_PORT}
71 71
 GLANCE_SERVICE_PROTOCOL=${GLANCE_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
72 72
 GLANCE_REGISTRY_PORT=${GLANCE_REGISTRY_PORT:-9191}
73 73
 GLANCE_REGISTRY_PORT_INT=${GLANCE_REGISTRY_PORT_INT:-19191}
74
+GLANCE_UWSGI=$GLANCE_BIN_DIR/glance-wsgi-api
75
+GLANCE_UWSGI_CONF=$GLANCE_CONF_DIR/glance-uswgi.ini
76
+# If wsgi mode is uwsgi run glance under uwsgi, else default to eventlet
77
+# TODO(mtreinish): Remove the eventlet path here and in all the similar
78
+# conditionals below after the Pike release
79
+if [[ "$WSGI_MODE" == "uwsgi" ]]; then
80
+    GLANCE_URL="$GLANCE_SERVICE_PROTOCOL://$GLANCE_SERVICE_HOST/image"
81
+else
82
+    GLANCE_URL="$GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT"
83
+fi
74 84
 
75 85
 # Functions
76 86
 # ---------
... ...
@@ -103,16 +113,13 @@ function configure_glance {
103 103
     dburl=`database_connection_url glance`
104 104
     iniset $GLANCE_REGISTRY_CONF database connection $dburl
105 105
     iniset $GLANCE_REGISTRY_CONF DEFAULT use_syslog $SYSLOG
106
-    iniset $GLANCE_REGISTRY_CONF DEFAULT workers "$API_WORKERS"
107 106
     iniset $GLANCE_REGISTRY_CONF paste_deploy flavor keystone
108 107
     configure_auth_token_middleware $GLANCE_REGISTRY_CONF glance $GLANCE_AUTH_CACHE_DIR/registry
109 108
     iniset $GLANCE_REGISTRY_CONF oslo_messaging_notifications driver messagingv2
110 109
     iniset_rpc_backend glance $GLANCE_REGISTRY_CONF
111 110
     iniset $GLANCE_REGISTRY_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"
112 111
 
113
-    cp $GLANCE_DIR/etc/glance-api.conf $GLANCE_API_CONF
114 112
     iniset $GLANCE_API_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
115
-    iniset $GLANCE_API_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS
116 113
     inicomment $GLANCE_API_CONF DEFAULT log_file
117 114
     iniset $GLANCE_API_CONF database connection $dburl
118 115
     iniset $GLANCE_API_CONF DEFAULT use_syslog $SYSLOG
... ...
@@ -140,8 +147,6 @@ function configure_glance {
140 140
     iniset $GLANCE_API_CONF glance_store filesystem_store_datadir $GLANCE_IMAGE_DIR/
141 141
     iniset $GLANCE_API_CONF DEFAULT registry_host $GLANCE_SERVICE_HOST
142 142
 
143
-    iniset $GLANCE_API_CONF DEFAULT workers "$API_WORKERS"
144
-
145 143
     # CORS feature support - to allow calls from Horizon by default
146 144
     if [ -n "$GLANCE_CORS_ALLOWED_ORIGIN" ]; then
147 145
         iniset $GLANCE_API_CONF cors allowed_origin "$GLANCE_CORS_ALLOWED_ORIGIN"
... ...
@@ -198,7 +203,6 @@ function configure_glance {
198 198
     setup_logging $GLANCE_REGISTRY_CONF
199 199
 
200 200
     cp -p $GLANCE_DIR/etc/glance-registry-paste.ini $GLANCE_REGISTRY_PASTE_INI
201
-
202 201
     cp -p $GLANCE_DIR/etc/glance-api-paste.ini $GLANCE_API_PASTE_INI
203 202
 
204 203
     cp $GLANCE_DIR/etc/glance-cache.conf $GLANCE_CACHE_CONF
... ...
@@ -231,6 +235,13 @@ function configure_glance {
231 231
         iniset $GLANCE_API_CONF DEFAULT cinder_endpoint_template "https://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/%(project_id)s"
232 232
         iniset $GLANCE_CACHE_CONF DEFAULT cinder_endpoint_template "https://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/%(project_id)s"
233 233
     fi
234
+
235
+    if [[ "$WSGI_MODE" == "uwsgi" ]]; then
236
+        write_local_uwsgi_http_config "$GLANCE_UWSGI_CONF" "$GLANCE_UWSGI" "/image"
237
+    else
238
+        iniset $GLANCE_API_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS
239
+        iniset $GLANCE_API_CONF DEFAULT workers "$API_WORKERS"
240
+    fi
234 241
 }
235 242
 
236 243
 # create_glance_accounts() - Set up common required glance accounts
... ...
@@ -255,7 +266,7 @@ function create_glance_accounts {
255 255
         get_or_create_endpoint \
256 256
             "image" \
257 257
             "$REGION_NAME" \
258
-            "$GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT"
258
+            "$GLANCE_URL"
259 259
 
260 260
         # Note(frickler): Crude workaround for https://bugs.launchpad.net/glance-store/+bug/1620999
261 261
         service_domain_id=$(get_or_create_domain $SERVICE_DOMAIN_NAME)
... ...
@@ -320,15 +331,21 @@ function install_glance {
320 320
 function start_glance {
321 321
     local service_protocol=$GLANCE_SERVICE_PROTOCOL
322 322
     if is_service_enabled tls-proxy; then
323
-        start_tls_proxy glance-service '*' $GLANCE_SERVICE_PORT $GLANCE_SERVICE_HOST $GLANCE_SERVICE_PORT_INT
323
+        if [[ "$WSGI_MODE" != "uwsgi" ]]; then
324
+            start_tls_proxy glance-service '*' $GLANCE_SERVICE_PORT $GLANCE_SERVICE_HOST $GLANCE_SERVICE_PORT_INT
325
+        fi
324 326
         start_tls_proxy glance-registry '*' $GLANCE_REGISTRY_PORT $GLANCE_SERVICE_HOST $GLANCE_REGISTRY_PORT_INT
325 327
     fi
326 328
 
327 329
     run_process g-reg "$GLANCE_BIN_DIR/glance-registry --config-file=$GLANCE_CONF_DIR/glance-registry.conf"
328
-    run_process g-api "$GLANCE_BIN_DIR/glance-api --config-file=$GLANCE_CONF_DIR/glance-api.conf"
330
+    if [[ "$WSGI_MODE" == "uwsgi" ]]; then
331
+        run_process g-api "$GLANCE_BIN_DIR/uwsgi --ini $GLANCE_UWSGI_CONF"
332
+    else
333
+        run_process g-api "$GLANCE_BIN_DIR/glance-api --config-file=$GLANCE_CONF_DIR/glance-api.conf"
334
+    fi
329 335
 
330
-    echo "Waiting for g-api ($GLANCE_HOSTPORT) to start..."
331
-    if ! wait_for_service $SERVICE_TIMEOUT $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT; then
336
+    echo "Waiting for g-api ($GLANCE_SERVICE_HOST) to start..."
337
+    if ! wait_for_service $SERVICE_TIMEOUT $GLANCE_URL; then
332 338
         die $LINENO "g-api did not start"
333 339
     fi
334 340
 }
... ...
@@ -574,7 +574,7 @@ function create_nova_conf {
574 574
     # enable notifications, but it will allow them to function when enabled.
575 575
     iniset $NOVA_CONF oslo_messaging_notifications driver "messagingv2"
576 576
     iniset_rpc_backend nova $NOVA_CONF
577
-    iniset $NOVA_CONF glance api_servers "${GLANCE_SERVICE_PROTOCOL}://${GLANCE_HOSTPORT}"
577
+    iniset $NOVA_CONF glance api_servers "$GLANCE_URL"
578 578
 
579 579
     iniset $NOVA_CONF DEFAULT osapi_compute_workers "$API_WORKERS"
580 580
     iniset $NOVA_CONF DEFAULT metadata_workers "$API_WORKERS"