#!/bin/bash
 #

 # lib/cinder

 # Install and start **Cinder** volume service

 
 # Dependencies:

 #

 # - functions

 # - DEST, DATA_DIR, STACK_USER must be defined

 # - SERVICE_{TENANT_NAME|PASSWORD} must be defined
 # - ``KEYSTONE_TOKEN_FORMAT`` must be defined

 
 # stack.sh
 # ---------

 # - install_cinder
 # - configure_cinder
 # - init_cinder
 # - start_cinder
 # - stop_cinder
 # - cleanup_cinder

 # Save trace setting
 XTRACE=$(set +o | grep xtrace)
 set +o xtrace

 
 
 # Defaults
 # --------
 

 # set up default driver
 CINDER_DRIVER=${CINDER_DRIVER:-default}

 CINDER_PLUGINS=$TOP_DIR/lib/cinder_plugins

 CINDER_BACKENDS=$TOP_DIR/lib/cinder_backends

 
 # grab plugin config if specified via cinder_driver
 if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
     source $CINDER_PLUGINS/$CINDER_DRIVER
 fi

 # set up default directories

 GITDIR["python-cinderclient"]=$DEST/python-cinderclient

 CINDER_DIR=$DEST/cinder

 CINDER_STATE_PATH=${CINDER_STATE_PATH:=$DATA_DIR/cinder}

 CINDER_AUTH_CACHE_DIR=${CINDER_AUTH_CACHE_DIR:-/var/cache/cinder}
 

 CINDER_CONF_DIR=/etc/cinder
 CINDER_CONF=$CINDER_CONF_DIR/cinder.conf

 CINDER_API_PASTE_INI=$CINDER_CONF_DIR/api-paste.ini

 # Public facing bits

 if is_ssl_enabled_service "cinder" || is_service_enabled tls-proxy; then
     CINDER_SERVICE_PROTOCOL="https"
 fi

 CINDER_SERVICE_HOST=${CINDER_SERVICE_HOST:-$SERVICE_HOST}
 CINDER_SERVICE_PORT=${CINDER_SERVICE_PORT:-8776}
 CINDER_SERVICE_PORT_INT=${CINDER_SERVICE_PORT_INT:-18776}
 CINDER_SERVICE_PROTOCOL=${CINDER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 

 # Support entry points installation of console scripts
 if [[ -d $CINDER_DIR/bin ]]; then

     CINDER_BIN_DIR=$CINDER_DIR/bin
 else

     CINDER_BIN_DIR=$(get_python_exec_prefix)

 fi

 
 # Maintain this here for backward-compatibility with the old configuration
 # DEPRECATED: Use CINDER_ENABLED_BACKENDS instead

 # Support for multi lvm backend configuration (default is no support)

 CINDER_MULTI_LVM_BACKEND=$(trueorfalse False CINDER_MULTI_LVM_BACKEND)

 # Default backends
 # The backend format is type:name where type is one of the supported backend
 # types (lvm, nfs, etc) and name is the identifier used in the Cinder
 # configuration and for the volume type name.  Multiple backends are
 # comma-separated.
 if [[ $CINDER_MULTI_LVM_BACKEND == "False" ]]; then
     CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1}
 else
     CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1,lvm:lvmdriver-2}
 fi
 
 

 # Should cinder perform secure deletion of volumes?
 # Defaults to true, can be set to False to avoid this bug when testing:
 # https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1023755

 CINDER_SECURE_DELETE=$(trueorfalse True CINDER_SECURE_DELETE)

 # Cinder reports allocations back to the scheduler on periodic intervals
 # it turns out we can get an "out of space" issue when we run tests too
 # quickly just because cinder didn't realize we'd freed up resources.
 # Make this configurable so that devstack-gate/tempest can set it to
 # less than the 60 second default
 # https://bugs.launchpad.net/cinder/+bug/1180976
 CINDER_PERIODIC_INTERVAL=${CINDER_PERIODIC_INTERVAL:-60}
 

 # Tell Tempest this project is present
 TEMPEST_SERVICES+=,cinder
 

 # Source the enabled backends
 if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
     for be in ${CINDER_ENABLED_BACKENDS//,/ }; do

         be_type=${be%%:*}
         be_name=${be##*:}
         if [[ -r $CINDER_BACKENDS/${be_type} ]]; then
             source $CINDER_BACKENDS/${be_type}

         fi
     done
 fi
 

 # Change the default nova_catalog_info and nova_catalog_admin_info values in
 # cinder so that the service name cinder is searching for matches that set for
 # nova in keystone.
 CINDER_NOVA_CATALOG_INFO=${CINDER_NOVA_CATALOG_INFO:-compute:nova:publicURL}
 CINDER_NOVA_CATALOG_ADMIN_INFO=${CINDER_NOVA_CATALOG_ADMIN_INFO:-compute:nova:adminURL}
 

 # Functions
 # ---------

 
 # Test if any Cinder services are enabled
 # is_cinder_enabled
 function is_cinder_enabled {
     [[ ,${ENABLED_SERVICES} =~ ,"c-" ]] && return 0
     return 1
 }
 

 # cleanup_cinder() - Remove residual data files, anything left over from previous
 # runs that a clean run would need to clean up

 function cleanup_cinder {

     # ensure the volume group is cleared up because fails might
     # leave dead volumes in the group

     local targets=$(sudo tgtadm --op show --mode target)

     if [ $? -ne 0 ]; then
         # If tgt driver isn't running this won't work obviously
         # So check the response and restart if need be
         echo "tgtd seems to be in a bad state, restarting..."
         if is_ubuntu; then
             restart_service tgt
         else
             restart_service tgtd
         fi

         targets=$(sudo tgtadm --op show --mode target)

     fi
 

     if [[ -n "$targets" ]]; then
         local iqn_list=( $(grep --no-filename -r iqn $SCSI_PERSIST_DIR | sed 's/<target //' | sed 's/>//') )

         for i in "${iqn_list[@]}"; do
             echo removing iSCSI target: $i
             sudo tgt-admin --delete $i
         done
     fi
 
     if is_ubuntu; then
         stop_service tgt
     else
         stop_service tgtd
     fi
 

     if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then

         local be be_name be_type

         for be in ${CINDER_ENABLED_BACKENDS//,/ }; do

             be_type=${be%%:*}
             be_name=${be##*:}
             if type cleanup_cinder_backend_${be_type} >/dev/null 2>&1; then
                 cleanup_cinder_backend_${be_type} ${be_name}

             fi
         done

     fi

 }
 

 # configure_cinder_rootwrap() - configure Cinder's rootwrap

 function configure_cinder_rootwrap {

     # Set the paths of certain binaries

     local cinder_rootwrap=$(get_rootwrap_location cinder)

     # Deploy new rootwrap filters files (owned by root).
     # Wipe any existing rootwrap.d files first
     if [[ -d $CINDER_CONF_DIR/rootwrap.d ]]; then
         sudo rm -rf $CINDER_CONF_DIR/rootwrap.d

     fi

     # Deploy filters to /etc/cinder/rootwrap.d
     sudo mkdir -m 755 $CINDER_CONF_DIR/rootwrap.d
     sudo cp $CINDER_DIR/etc/cinder/rootwrap.d/*.filters $CINDER_CONF_DIR/rootwrap.d
     sudo chown -R root:root $CINDER_CONF_DIR/rootwrap.d
     sudo chmod 644 $CINDER_CONF_DIR/rootwrap.d/*
     # Set up rootwrap.conf, pointing to /etc/cinder/rootwrap.d
     sudo cp $CINDER_DIR/etc/cinder/rootwrap.conf $CINDER_CONF_DIR/
     sudo sed -e "s:^filters_path=.*$:filters_path=$CINDER_CONF_DIR/rootwrap.d:" -i $CINDER_CONF_DIR/rootwrap.conf
     sudo chown root:root $CINDER_CONF_DIR/rootwrap.conf
     sudo chmod 0644 $CINDER_CONF_DIR/rootwrap.conf
     # Specify rootwrap.conf as first parameter to rootwrap

     ROOTWRAP_CSUDOER_CMD="$cinder_rootwrap $CINDER_CONF_DIR/rootwrap.conf *"

 
     # Set up the rootwrap sudoers for cinder

     local tempfile=`mktemp`
     echo "$STACK_USER ALL=(root) NOPASSWD: $ROOTWRAP_CSUDOER_CMD" >$tempfile
     chmod 0440 $tempfile
     sudo chown root:root $tempfile
     sudo mv $tempfile /etc/sudoers.d/cinder-rootwrap

 }
 
 # configure_cinder() - Set config files, create data dirs, etc

 function configure_cinder {

     if [[ ! -d $CINDER_CONF_DIR ]]; then
         sudo mkdir -p $CINDER_CONF_DIR
     fi
     sudo chown $STACK_USER $CINDER_CONF_DIR
 
     cp -p $CINDER_DIR/etc/cinder/policy.json $CINDER_CONF_DIR
 

     rm -f $CINDER_CONF
 

     configure_cinder_rootwrap

     cp $CINDER_DIR/etc/cinder/api-paste.ini $CINDER_API_PASTE_INI

 
     inicomment $CINDER_API_PASTE_INI filter:authtoken auth_host
     inicomment $CINDER_API_PASTE_INI filter:authtoken auth_port
     inicomment $CINDER_API_PASTE_INI filter:authtoken auth_protocol

     inicomment $CINDER_API_PASTE_INI filter:authtoken cafile

     inicomment $CINDER_API_PASTE_INI filter:authtoken admin_tenant_name
     inicomment $CINDER_API_PASTE_INI filter:authtoken admin_user
     inicomment $CINDER_API_PASTE_INI filter:authtoken admin_password
     inicomment $CINDER_API_PASTE_INI filter:authtoken signing_dir

     configure_auth_token_middleware $CINDER_CONF cinder $CINDER_AUTH_CACHE_DIR

     iniset $CINDER_CONF DEFAULT nova_catalog_info $CINDER_NOVA_CATALOG_INFO
     iniset $CINDER_CONF DEFAULT nova_catalog_admin_info $CINDER_NOVA_CATALOG_ADMIN_INFO
 

     iniset $CINDER_CONF DEFAULT auth_strategy keystone

     iniset $CINDER_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL

     iniset $CINDER_CONF DEFAULT verbose True

     iniset $CINDER_CONF DEFAULT my_ip "$CINDER_SERVICE_HOST"

     iniset $CINDER_CONF DEFAULT iscsi_helper tgtadm

     iniset $CINDER_CONF DEFAULT sql_connection `database_connection_url cinder`

     iniset $CINDER_CONF DEFAULT api_paste_config $CINDER_API_PASTE_INI

     iniset $CINDER_CONF DEFAULT rootwrap_config "$CINDER_CONF_DIR/rootwrap.conf"

     iniset $CINDER_CONF DEFAULT osapi_volume_extension cinder.api.contrib.standard_extensions

     iniset $CINDER_CONF DEFAULT state_path $CINDER_STATE_PATH

     iniset $CINDER_CONF DEFAULT lock_path $CINDER_STATE_PATH

     iniset $CINDER_CONF DEFAULT periodic_interval $CINDER_PERIODIC_INTERVAL

     # NOTE(thingee): Cinder V1 API is deprecated and defaults to off as of
     # Juno. Keep it enabled so we can continue testing while it's still
     # supported.
     iniset $CINDER_CONF DEFAULT enable_v1_api true

     if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then

         local enabled_backends=""
         local default_name=""
         local be be_name be_type

         for be in ${CINDER_ENABLED_BACKENDS//,/ }; do

             be_type=${be%%:*}
             be_name=${be##*:}
             if type configure_cinder_backend_${be_type} >/dev/null 2>&1; then
                 configure_cinder_backend_${be_type} ${be_name}

             fi

             if [[ -z "$default_name" ]]; then
                 default_name=$be_name

             fi

             enabled_backends+=$be_name,

         done
         iniset $CINDER_CONF DEFAULT enabled_backends ${enabled_backends%,*}

         if [[ -n "$default_name" ]]; then
             iniset $CINDER_CONF DEFAULT default_volume_type ${default_name}

         fi
     fi
 

     if is_service_enabled swift; then

         iniset $CINDER_CONF DEFAULT backup_swift_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:8080/v1/AUTH_"

     fi
 

     if is_service_enabled ceilometer; then

         iniset $CINDER_CONF DEFAULT notification_driver "messaging"

     fi
 

     if is_service_enabled tls-proxy; then
         # Set the service port for a proxy to take the original
         iniset $CINDER_CONF DEFAULT osapi_volume_listen_port $CINDER_SERVICE_PORT_INT
     fi
 

     if [ "$SYSLOG" != "False" ]; then
         iniset $CINDER_CONF DEFAULT use_syslog True
     fi
 

     iniset_rpc_backend cinder $CINDER_CONF DEFAULT

     if [[ "$CINDER_SECURE_DELETE" == "False" ]]; then
         iniset $CINDER_CONF DEFAULT secure_delete False

         iniset $CINDER_CONF DEFAULT volume_clear none

     fi
 

     # Format logging

     if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then

         setup_colorized_logging $CINDER_CONF DEFAULT "project_id" "user_id"

     fi

     if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
         configure_cinder_driver

     fi

     if is_fedora && [[ $DISTRO =~ (rhel6) ]]; then

         # Cinder clones are slightly larger due to some extra
         # metadata.  RHEL6 will not allow auto-extending of LV's
         # without this, leading to clones giving hard-to-track disk
         # I/O errors.
         # see https://bugzilla.redhat.com/show_bug.cgi?id=975052
         sudo sed -i~ \
             -e 's/snapshot_autoextend_threshold =.*/snapshot_autoextend_threshold = 80/' \
             -e 's/snapshot_autoextend_percent =.*/snapshot_autoextend_percent = 20/' \
             /etc/lvm/lvm.conf
     fi
 

     iniset $CINDER_CONF DEFAULT osapi_volume_workers "$API_WORKERS"

 
     iniset $CINDER_CONF DEFAULT glance_api_servers "${GLANCE_SERVICE_PROTOCOL}://${GLANCE_HOSTPORT}"
     if is_ssl_enabled_service glance || is_service_enabled tls-proxy; then
         iniset $CINDER_CONF DEFAULT glance_protocol https

         iniset $CINDER_CONF DEFAULT glance_ca_certificates_file $SSL_BUNDLE_FILE

     fi
 
     # Register SSL certificates if provided
     if is_ssl_enabled_service cinder; then
         ensure_certificates CINDER
 
         iniset $CINDER_CONF DEFAULT ssl_cert_file "$CINDER_SSL_CERT"
         iniset $CINDER_CONF DEFAULT ssl_key_file "$CINDER_SSL_KEY"
     fi
 

 }
 

 # create_cinder_accounts() - Set up common required cinder accounts
 
 # Tenant               User       Roles
 # ------------------------------------------------------------------
 # service              cinder     admin        # if enabled
 
 # Migrated from keystone_data.sh

 function create_cinder_accounts {

     local service_tenant=$(openstack project list | awk "/ $SERVICE_TENANT_NAME / { print \$2 }")
     local admin_role=$(openstack role list | awk "/ admin / { print \$2 }")

 
     # Cinder
     if [[ "$ENABLED_SERVICES" =~ "c-api" ]]; then

         local cinder_user=$(get_or_create_user "cinder" \
             "$SERVICE_PASSWORD" $service_tenant)
         get_or_add_user_role $admin_role $cinder_user $service_tenant

         if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then

             local cinder_service=$(get_or_create_service "cinder" \

                 "volume" "Cinder Volume Service")

             get_or_create_endpoint $cinder_service "$REGION_NAME" \

                 "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/\$(tenant_id)s" \
                 "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/\$(tenant_id)s" \
                 "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/\$(tenant_id)s"
 

             local cinder_v2_service=$(get_or_create_service "cinderv2" \

                 "volumev2" "Cinder Volume Service V2")

             get_or_create_endpoint $cinder_v2_service "$REGION_NAME" \

                 "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v2/\$(tenant_id)s" \
                 "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v2/\$(tenant_id)s" \
                 "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v2/\$(tenant_id)s"

         fi
     fi
 }
 

 # create_cinder_cache_dir() - Part of the init_cinder() process

 function create_cinder_cache_dir {

     # Create cache dir
     sudo mkdir -p $CINDER_AUTH_CACHE_DIR
     sudo chown $STACK_USER $CINDER_AUTH_CACHE_DIR
     rm -f $CINDER_AUTH_CACHE_DIR/*
 }
 

 # init_cinder() - Initialize database and volume group

 # Uses global ``NOVA_ENABLED_APIS``

 function init_cinder {

     # Force nova volumes off
     NOVA_ENABLED_APIS=$(echo $NOVA_ENABLED_APIS | sed "s/osapi_volume,//")
 

     if is_service_enabled $DATABASE_BACKENDS; then

         # (Re)create cinder database

         recreate_database cinder utf8

         # Migrate cinder database

         $CINDER_BIN_DIR/cinder-manage db sync

     fi
 

     if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then

         local be be_name be_type

         for be in ${CINDER_ENABLED_BACKENDS//,/ }; do

             be_type=${be%%:*}
             be_name=${be##*:}
             if type init_cinder_backend_${be_type} >/dev/null 2>&1; then
                 init_cinder_backend_${be_type} ${be_name}

             fi

         done

     fi

     mkdir -p $CINDER_STATE_PATH/volumes

     create_cinder_cache_dir

 }
 
 # install_cinder() - Collect source and prepare

 function install_cinder {

     git_clone $CINDER_REPO $CINDER_DIR $CINDER_BRANCH

     setup_develop $CINDER_DIR

 }

 # install_cinderclient() - Collect source and prepare

 function install_cinderclient {

     if use_library_from_git "python-cinderclient"; then
         git_clone_by_name "python-cinderclient"
         setup_dev_lib "python-cinderclient"
         sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-cinderclient"]}/tools/,/etc/bash_completion.d/}cinder.bash_completion

     fi

 }
 

 # apply config.d approach for cinder volumes directory

 function _configure_tgt_for_config_d {

     if [[ ! -d /etc/tgt/stack.d/ ]]; then
         sudo ln -sf $CINDER_STATE_PATH/volumes /etc/tgt/stack.d
         echo "include /etc/tgt/stack.d/*" | sudo tee -a /etc/tgt/targets.conf

     fi
 }
 

 # start_cinder() - Start running processes, including screen

 function start_cinder {

     local service_port=$CINDER_SERVICE_PORT
     local service_protocol=$CINDER_SERVICE_PROTOCOL
     if is_service_enabled tls-proxy; then
         service_port=$CINDER_SERVICE_PORT_INT
         service_protocol="http"
     fi

     if is_service_enabled c-vol; then

         # Delete any old stack.conf
         sudo rm -f /etc/tgt/conf.d/stack.conf

         _configure_tgt_for_config_d

         if is_ubuntu; then

             sudo service tgt restart

         elif is_fedora; then

             if [[ $DISTRO =~ (rhel6) ]]; then
                 sudo /sbin/service tgtd restart
             else
                 # bypass redirection to systemctl during restart
                 sudo /sbin/service --skip-redirect tgtd restart
             fi

         elif is_suse; then
             restart_service tgtd
         else
             # note for other distros: unstack.sh also uses the tgt/tgtd service
             # name, and would need to be adjusted too
             exit_distro_not_supported "restarting tgt"

         fi

         # NOTE(gfidente): ensure tgtd is running in debug mode
         sudo tgtadm --mode system --op update --name debug --value on

     fi
 

     run_process c-api "$CINDER_BIN_DIR/cinder-api --config-file $CINDER_CONF"

     echo "Waiting for Cinder API to start..."

     if ! wait_for_service $SERVICE_TIMEOUT $service_protocol://$CINDER_SERVICE_HOST:$service_port; then

         die $LINENO "c-api did not start"
     fi
 

     run_process c-sch "$CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF"
     run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF"
     run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF"

 
     # NOTE(jdg): For cinder, startup order matters.  To ensure that repor_capabilities is received
     # by the scheduler start the cinder-volume service last (or restart it) after the scheduler
     # has started.  This is a quick fix for lp bug/1189595

 
     # Start proxies if enabled
     if is_service_enabled c-api && is_service_enabled tls-proxy; then
         start_tls_proxy '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_PORT_INT &
     fi

 }
 

 # stop_cinder() - Stop running processes

 function stop_cinder {

     # Kill the cinder screen windows

     local serv

     for serv in c-api c-bak c-sch c-vol; do

         stop_process $serv

     done

 
     if is_service_enabled c-vol; then

         if is_ubuntu; then
             stop_service tgt
         else
             stop_service tgtd
         fi

     fi
 }

 # create_volume_types() - Create Cinder's configured volume types
 function create_volume_types {
     # Create volume types
     if is_service_enabled c-api && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then

         local be be_name be_type

         for be in ${CINDER_ENABLED_BACKENDS//,/ }; do

             be_type=${be%%:*}
             be_name=${be##*:}
             # openstack volume type create --property volume_backend_name="${be_type}" ${be_name}
             cinder type-create ${be_name} && \
                 cinder type-key ${be_name} set volume_backend_name="${be_name}"

         done
     fi
 }
 
 # Compatibility for Grenade
 
 function create_cinder_volume_group {
     # During a transition period Grenade needs to have this function defined
     # It is effectively a no-op in the Grenade 'target' use case
     :
 }
 

 # Restore xtrace
 $XTRACE

 # Tell emacs to use shell-script-mode
 ## Local variables:
 ## mode: shell-script
 ## End: