#!/bin/bash
#
# lib/tls
# Functions to control the configuration and operation of the TLS proxy service
# !! source _before_ any services that use ``SERVICE_HOST``
#
# Dependencies:
#
# - ``functions`` file
# - ``DEST``, ``DATA_DIR`` must be defined
# - ``HOST_IP``, ``SERVICE_HOST``
# - ``KEYSTONE_TOKEN_FORMAT`` must be defined
# Entry points:
#
# - configure_CA
# - init_CA
# - configure_proxy
# - start_tls_proxy
# - stop_tls_proxy
# - cleanup_CA
# - make_root_CA
# - make_int_CA
# - make_cert ca-dir cert-name "common-name" ["alt-name" ...]
# - start_tls_proxy HOST_IP 5000 localhost 5000
# - ensure_certificates
# - is_ssl_enabled_service
# - enable_mod_ssl
# Defaults
# --------
if is_service_enabled tls-proxy; then
# TODO(dtroyer): revisit this below after the search for HOST_IP has been done
TLS_IP=${TLS_IP:-$(ipv6_unquote $SERVICE_HOST)}
fi
DEVSTACK_HOSTNAME=$(hostname -f)
DEVSTACK_CERT_NAME=devstack-cert
DEVSTACK_CERT=$DATA_DIR/$DEVSTACK_CERT_NAME.pem
# CA configuration
ROOT_CA_DIR=${ROOT_CA_DIR:-$DATA_DIR/CA/root-ca}
INT_CA_DIR=${INT_CA_DIR:-$DATA_DIR/CA/int-ca}
ORG_NAME="OpenStack"
ORG_UNIT_NAME="DevStack"
# Stud configuration
STUD_PROTO="--tls"
STUD_CIPHERS='TLSv1+HIGH:!DES:!aNULL:!eNULL:@STRENGTH'
# CA Functions
# ============
# There may be more than one, get specific
OPENSSL=${OPENSSL:-/usr/bin/openssl}
# Do primary CA configuration
function configure_CA {
# build common config file
# Verify ``TLS_IP`` is good
if [[ -n "$SERVICE_HOST" && "$(ipv6_unquote $SERVICE_HOST)" != "$TLS_IP" ]]; then
# auto-discover has changed the IP
TLS_IP=$(ipv6_unquote $SERVICE_HOST)
fi
}
# Creates a new CA directory structure
# create_CA_base ca-dir
function create_CA_base {
local ca_dir=$1
if [[ -d $ca_dir ]]; then
# Bail out it exists
return 0
fi
local i
for i in certs crl newcerts private; do
mkdir -p $ca_dir/$i
done
chmod 710 $ca_dir/private
echo "01" >$ca_dir/serial
cp /dev/null $ca_dir/index.txt
}
# Create a new CA configuration file
# create_CA_config ca-dir common-name
function create_CA_config {
local ca_dir=$1
local common_name=$2
echo "
[ ca ]
default_ca = CA_default
[ CA_default ]
dir = $ca_dir
policy = policy_match
database = \$dir/index.txt
serial = \$dir/serial
certs = \$dir/certs
crl_dir = \$dir/crl
new_certs_dir = \$dir/newcerts
certificate = \$dir/cacert.pem
private_key = \$dir/private/cacert.key
RANDFILE = \$dir/private/.rand
default_md = sha256
[ req ]
default_bits = 2048
default_md = sha256
prompt = no
distinguished_name = ca_distinguished_name
x509_extensions = ca_extensions
[ ca_distinguished_name ]
organizationName = $ORG_NAME
organizationalUnitName = $ORG_UNIT_NAME Certificate Authority
commonName = $common_name
[ policy_match ]
countryName = optional
stateOrProvinceName = optional
organizationName = match
organizationalUnitName = optional
commonName = supplied
[ ca_extensions ]
basicConstraints = critical,CA:true
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer
keyUsage = cRLSign, keyCertSign
" >$ca_dir/ca.conf
}
# Create a new signing configuration file
# create_signing_config ca-dir
function create_signing_config {
local ca_dir=$1
echo "
[ ca ]
default_ca = CA_default
[ CA_default ]
dir = $ca_dir
policy = policy_match
database = \$dir/index.txt
serial = \$dir/serial
certs = \$dir/certs
crl_dir = \$dir/crl
new_certs_dir = \$dir/newcerts
certificate = \$dir/cacert.pem
private_key = \$dir/private/cacert.key
RANDFILE = \$dir/private/.rand
default_md = default
[ req ]
default_bits = 1024
default_md = sha1
prompt = no
distinguished_name = req_distinguished_name
x509_extensions = req_extensions
[ req_distinguished_name ]
organizationName = $ORG_NAME
organizationalUnitName = $ORG_UNIT_NAME Server Farm
[ policy_match ]
countryName = optional
stateOrProvinceName = optional
organizationName = match
organizationalUnitName = optional
commonName = supplied
[ req_extensions ]
basicConstraints = CA:false
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer
keyUsage = digitalSignature, keyEncipherment, keyAgreement
extendedKeyUsage = serverAuth, clientAuth
subjectAltName = \$ENV::SUBJECT_ALT_NAME
" >$ca_dir/signing.conf
}
# Create root and intermediate CAs
# init_CA
function init_CA {
# Ensure CAs are built
make_root_CA $ROOT_CA_DIR
make_int_CA $INT_CA_DIR $ROOT_CA_DIR
# Create the CA bundle
cat $ROOT_CA_DIR/cacert.pem $INT_CA_DIR/cacert.pem >>$INT_CA_DIR/ca-chain.pem
cat $INT_CA_DIR/ca-chain.pem >> $SSL_BUNDLE_FILE
if is_fedora; then
sudo cp $INT_CA_DIR/ca-chain.pem /usr/share/pki/ca-trust-source/anchors/devstack-chain.pem
sudo update-ca-trust
elif is_suse; then
sudo cp $INT_CA_DIR/ca-chain.pem /usr/share/pki/trust/anchors/devstack-chain.pem
sudo update-ca-certificates
elif is_ubuntu; then
sudo cp $INT_CA_DIR/ca-chain.pem /usr/local/share/ca-certificates/devstack-int.crt
sudo cp $ROOT_CA_DIR/cacert.pem /usr/local/share/ca-certificates/devstack-root.crt
sudo update-ca-certificates
fi
}
# Create an initial server cert
# init_cert
function init_cert {
if [[ ! -r $DEVSTACK_CERT ]]; then
if [[ -n "$TLS_IP" ]]; then
TLS_IP="IP:$TLS_IP"
if [[ -n "$HOST_IPV6" ]]; then
TLS_IP="$TLS_IP,IP:$HOST_IPV6"
fi
fi
make_cert $INT_CA_DIR $DEVSTACK_CERT_NAME $DEVSTACK_HOSTNAME "$TLS_IP"
# Create a cert bundle
cat $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/cacert.pem >$DEVSTACK_CERT
fi
}
# make_cert creates and signs a new certificate with the given commonName and CA
# make_cert ca-dir cert-name "common-name" ["alt-name" ...]
function make_cert {
local ca_dir=$1
local cert_name=$2
local common_name=$3
local alt_names=$4
if [ "$common_name" != "$SERVICE_HOST" ]; then
if is_ipv4_address "$SERVICE_HOST" ; then
if [[ -z "$alt_names" ]]; then
alt_names="IP:$SERVICE_HOST"
else
alt_names="$alt_names,IP:$SERVICE_HOST"
fi
fi
fi
# Only generate the certificate if it doesn't exist yet on the disk
if [ ! -r "$ca_dir/$cert_name.crt" ]; then
# Generate a signing request
$OPENSSL req \
-sha1 \
-newkey rsa \
-nodes \
-keyout $ca_dir/private/$cert_name.key \
-out $ca_dir/$cert_name.csr \
-subj "/O=${ORG_NAME}/OU=${ORG_UNIT_NAME} Servers/CN=${common_name}"
if [[ -z "$alt_names" ]]; then
alt_names="DNS:${common_name}"
else
alt_names="DNS:${common_name},${alt_names}"
fi
# Sign the request valid for 1 year
SUBJECT_ALT_NAME="$alt_names" \
$OPENSSL ca -config $ca_dir/signing.conf \
-extensions req_extensions \
-days 365 \
-notext \
-in $ca_dir/$cert_name.csr \
-out $ca_dir/$cert_name.crt \
-subj "/O=${ORG_NAME}/OU=${ORG_UNIT_NAME} Servers/CN=${common_name}" \
-batch
fi
}
# Make an intermediate CA to sign everything else
# make_int_CA ca-dir signing-ca-dir
function make_int_CA {
local ca_dir=$1
local signing_ca_dir=$2
# Create the root CA
create_CA_base $ca_dir
create_CA_config $ca_dir 'Intermediate CA'
create_signing_config $ca_dir
if [ ! -r "$ca_dir/cacert.pem" ]; then
# Create a signing certificate request
$OPENSSL req -config $ca_dir/ca.conf \
-sha1 \
-newkey rsa \
-nodes \
-keyout $ca_dir/private/cacert.key \
-out $ca_dir/cacert.csr \
-outform PEM
# Sign the intermediate request valid for 1 year
$OPENSSL ca -config $signing_ca_dir/ca.conf \
-extensions ca_extensions \
-days 365 \
-notext \
-in $ca_dir/cacert.csr \
-out $ca_dir/cacert.pem \
-batch
fi
}
# Make a root CA to sign other CAs
# make_root_CA ca-dir
function make_root_CA {
local ca_dir=$1
# Create the root CA
create_CA_base $ca_dir
create_CA_config $ca_dir 'Root CA'
if [ ! -r "$ca_dir/cacert.pem" ]; then
# Create a self-signed certificate valid for 5 years
$OPENSSL req -config $ca_dir/ca.conf \
-x509 \
-nodes \
-newkey rsa \
-days 21360 \
-keyout $ca_dir/private/cacert.key \
-out $ca_dir/cacert.pem \
-outform PEM
fi
}
# Deploy the service cert & key to a service specific
# location
function deploy_int_cert {
local cert_target_file=$1
local key_target_file=$2
sudo cp "$INT_CA_DIR/$DEVSTACK_CERT_NAME.crt" "$cert_target_file"
sudo cp "$INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key" "$key_target_file"
}
# Deploy the intermediate CA cert bundle file to a service
# specific location
function deploy_int_CA {
local ca_target_file=$1
sudo cp "$INT_CA_DIR/ca-chain.pem" "$ca_target_file"
}
# If a non-system python-requests is installed then it will use the
# built-in CA certificate store rather than the distro-specific
# CA certificate store. Detect this and symlink to the correct
# one. If the value for the CA is not rooted in /etc then we know
# we need to change it.
function fix_system_ca_bundle_path {
if is_service_enabled tls-proxy; then
local capath
capath=$(python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
if [[ ! $capath == "" && ! $capath =~ ^/etc/.* && ! -L $capath ]]; then
if is_fedora; then
sudo rm -f $capath
sudo ln -s /etc/pki/tls/certs/ca-bundle.crt $capath
elif is_ubuntu; then
sudo rm -f $capath
sudo ln -s /etc/ssl/certs/ca-certificates.crt $capath
elif is_suse; then
sudo rm -f $capath
sudo ln -s /etc/ssl/ca-bundle.pem $capath
else
echo "Don't know how to set the CA bundle, expect the install to fail."
fi
fi
fi
}
# Only for compatibility, return if the tls-proxy is enabled
function is_ssl_enabled_service {
return is_service_enabled tls-proxy
}
# Certificate Input Configuration
# ===============================
# Ensure that the certificates for a service are in place. This function does
# not check that a service is SSL enabled, this should already have been
# completed.
#
# The function expects to find a certificate, key and CA certificate in the
# variables ``{service}_SSL_CERT``, ``{service}_SSL_KEY`` and ``{service}_SSL_CA``. For
# example for keystone this would be ``KEYSTONE_SSL_CERT``, ``KEYSTONE_SSL_KEY`` and
# ``KEYSTONE_SSL_CA``.
#
# If it does not find these certificates then the DevStack-issued server
# certificate, key and CA certificate will be associated with the service.
#
# If only some of the variables are provided then the function will quit.
function ensure_certificates {
local service=$1
local cert_var="${service}_SSL_CERT"
local key_var="${service}_SSL_KEY"
local ca_var="${service}_SSL_CA"
local cert=${!cert_var}
local key=${!key_var}
local ca=${!ca_var}
if [[ -z "$cert" && -z "$key" && -z "$ca" ]]; then
local cert="$INT_CA_DIR/$DEVSTACK_CERT_NAME.crt"
local key="$INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key"
local ca="$INT_CA_DIR/ca-chain.pem"
eval ${service}_SSL_CERT=\$cert
eval ${service}_SSL_KEY=\$key
eval ${service}_SSL_CA=\$ca
return # the CA certificate is already in the bundle
elif [[ -z "$cert" || -z "$key" || -z "$ca" ]]; then
die $LINENO "Missing either the ${cert_var} ${key_var} or ${ca_var}" \
"variable to enable SSL for ${service}"
fi
cat $ca >> $SSL_BUNDLE_FILE
}
# Enable the mod_ssl plugin in Apache
function enable_mod_ssl {
echo "Enabling mod_ssl"
if is_ubuntu; then
sudo a2enmod ssl
elif is_suse; then
sudo a2enmod ssl
sudo a2enflag SSL
elif is_fedora; then
# Fedora enables mod_ssl by default
:
fi
if ! sudo `which httpd || which apache2ctl` -M | grep -w -q ssl_module; then
die $LINENO "mod_ssl is not enabled in apache2/httpd, please check for it manually and run stack.sh again"
fi
}
# Proxy Functions
# ===============
function tune_apache_connections {
local tuning_file=$APACHE_SETTINGS_DIR/connection-tuning.conf
if ! [ -f $tuning_file ] ; then
sudo bash -c "cat > $tuning_file" << EOF
# worker MPM
# StartServers: initial number of server processes to start
# MinSpareThreads: minimum number of worker threads which are kept spare
# MaxSpareThreads: maximum number of worker threads which are kept spare
# ThreadLimit: ThreadsPerChild can be changed to this maximum value during a
# graceful restart. ThreadLimit can only be changed by stopping
# and starting Apache.
# ThreadsPerChild: constant number of worker threads in each server process
# MaxClients: maximum number of simultaneous client connections
# MaxRequestsPerChild: maximum number of requests a server process serves
#
# We want to be memory thrifty so tune down apache to allow 256 total
# connections. This should still be plenty for a dev env yet lighter than
# apache defaults.
<IfModule mpm_worker_module>
# Note that the next three conf values must be changed together.
# MaxClients = ServerLimit * ThreadsPerChild
ServerLimit 8
ThreadsPerChild 32
MaxClients 256
StartServers 2
MinSpareThreads 32
MaxSpareThreads 96
ThreadLimit 64
MaxRequestsPerChild 0
</IfModule>
<IfModule mpm_event_module>
# Note that the next three conf values must be changed together.
# MaxClients = ServerLimit * ThreadsPerChild
ServerLimit 8
ThreadsPerChild 32
MaxClients 256
StartServers 2
MinSpareThreads 32
MaxSpareThreads 96
ThreadLimit 64
MaxRequestsPerChild 0
</IfModule>
EOF
restart_apache_server
fi
}
# Starts the TLS proxy for the given IP/ports
# start_tls_proxy service-name front-host front-port back-host back-port
function start_tls_proxy {
local b_service="$1-tls-proxy"
local f_host=$2
local f_port=$3
local b_host=$4
local b_port=$5
# 8190 is the default apache size.
local f_header_size=${6:-8190}
tune_apache_connections
local config_file
config_file=$(apache_site_config_for $b_service)
local listen_string
# Default apache configs on ubuntu and centos listen on 80 and 443
# newer apache seems fine with duplicate listen directive but older
# apache does not so special case 80 and 443.
if [[ "$f_port" == "80" ]] || [[ "$f_port" == "443" ]]; then
listen_string=""
elif [[ "$f_host" == '*' ]] ; then
listen_string="Listen $f_port"
else
listen_string="Listen $f_host:$f_port"
fi
sudo bash -c "cat >$config_file" << EOF
$listen_string
<VirtualHost $f_host:$f_port>
SSLEngine On
SSLCertificateFile $DEVSTACK_CERT
# Disable KeepAlive to fix bug #1630664 a.k.a the
# ('Connection aborted.', BadStatusLine("''",)) error
KeepAlive Off
# This increase in allowed request header sizes is required
# for swift functional testing to work with tls enabled. It is 2 bytes
# larger than the apache default of 8190.
LimitRequestFieldSize $f_header_size
RequestHeader set X-Forwarded-Proto "https"
# Avoid races (at the cost of performance) to re-use a pooled connection
# where the connection is closed (bug 1807518).
SetEnv proxy-initial-not-pooled
<Location />
ProxyPass http://$b_host:$b_port/ retry=0 nocanon
ProxyPassReverse http://$b_host:$b_port/
</Location>
ErrorLog $APACHE_LOG_DIR/tls-proxy_error.log
ErrorLogFormat "%{cu}t [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] [frontend\ %A] %M% ,\ referer\ %{Referer}i"
LogLevel info
CustomLog $APACHE_LOG_DIR/tls-proxy_access.log "%{%Y-%m-%d}t %{%T}t.%{msec_frac}t [%l] %a \"%r\" %>s %b"
</VirtualHost>
EOF
if is_suse ; then
sudo a2enflag SSL
fi
for mod in headers ssl proxy proxy_http; do
enable_apache_mod $mod
done
enable_apache_site $b_service
restart_apache_server
}
# Follow TLS proxy
function follow_tls_proxy {
sudo touch /var/log/$APACHE_NAME/tls-proxy_error.log
tail_log tls-error /var/log/$APACHE_NAME/tls-proxy_error.log
sudo touch /var/log/$APACHE_NAME/tls-proxy_access.log
tail_log tls-proxy /var/log/$APACHE_NAME/tls-proxy_access.log
}
# Cleanup Functions
# =================
# Stops the apache service. This should be done only after all services
# using tls configuration are down.
function stop_tls_proxy {
stop_apache_server
# NOTE(jh): Removing all tls-proxy configs is a bit of a hack, but
# necessary so that we can restart after an unstack. A better
# solution would be to ensure that each service calling
# start_tls_proxy will call stop_tls_proxy with the same
# parameters on shutdown so we can use the disable_apache_site
# function and remove individual files there.
if is_ubuntu; then
sudo rm -f /etc/apache2/sites-enabled/*-tls-proxy.conf
else
for i in $APACHE_CONF_DIR/*-tls-proxy.conf; do
sudo mv $i $i.disabled
done
fi
}
# Clean up the CA files
# cleanup_CA
function cleanup_CA {
if is_fedora; then
sudo rm -f /usr/share/pki/ca-trust-source/anchors/devstack-chain.pem
sudo update-ca-trust
elif is_ubuntu; then
sudo rm -f /usr/local/share/ca-certificates/devstack-int.crt
sudo rm -f /usr/local/share/ca-certificates/devstack-root.crt
sudo update-ca-certificates
fi
rm -rf "$INT_CA_DIR" "$ROOT_CA_DIR" "$DEVSTACK_CERT"
}
# Tell emacs to use shell-script-mode
## Local variables:
## mode: shell-script
## End: