Browse code
Build wheel cache for venvs
Building a bunch of virtual envs later is going to be tedious if we do not
pre-cache certain annoying-to-build packages.
* tools/build_wheels.sh: pre-build some wheels for annoying package installs
* list distro package dependencies in files/*/venv
* list packages to pre-build as wheels in files/venv-requirements.txt
* install database Python modules when setting up the database
Change-Id: Idff1ea69a5ca12ba56098e664dbf6924fe6a2e47
Dean Troyer authored on 2015/02/17 04:58:35Showing 13 changed files
- clean.sh
- files/debs/devlibs
- files/debs/postgresql
- files/rpms-suse/devlibs
- files/rpms-suse/postgresql
- files/rpms/devlibs
- files/rpms/postgresql
- files/venv-requirements.txt
- lib/databases/mysql
- lib/databases/postgresql
- stack.sh
- stackrc
- tools/build_wheels.sh
| ... | ... |
@@ -119,6 +119,10 @@ if [[ -n "$SCREEN_LOGDIR" ]] && [[ -d "$SCREEN_LOGDIR" ]]; then |
| 119 | 119 |
sudo rm -rf $SCREEN_LOGDIR |
| 120 | 120 |
fi |
| 121 | 121 |
|
| 122 |
+# Clean up venvs |
|
| 123 |
+DIRS_TO_CLEAN="$WHEELHOUSE" |
|
| 124 |
+rm -rf $DIRS_TO_CLEAN |
|
| 125 |
+ |
|
| 122 | 126 |
# Clean up files |
| 123 | 127 |
|
| 124 | 128 |
FILES_TO_CLEAN=".localrc.auto docs/files docs/html shocco/ stack-screenrc test*.conf* test.ini*" |
| ... | ... |
@@ -671,6 +671,21 @@ fi |
| 671 | 671 |
source $TOP_DIR/tools/fixup_stuff.sh |
| 672 | 672 |
|
| 673 | 673 |
|
| 674 |
+# Virtual Environment |
|
| 675 |
+# ------------------- |
|
| 676 |
+ |
|
| 677 |
+# Temporary hack for testing |
|
| 678 |
+# This belongs in d-g functions.sh setup_host() or devstack-vm-gate.sh |
|
| 679 |
+if [[ -d /var/cache/pip ]]; then |
|
| 680 |
+ sudo chown -R $STACK_USER:$STACK_USER /var/cache/pip |
|
| 681 |
+fi |
|
| 682 |
+ |
|
| 683 |
+# Pre-build some problematic wheels |
|
| 684 |
+if [[ ! -d ${WHEELHOUSE:-} ]]; then
|
|
| 685 |
+ source tools/build_wheels.sh |
|
| 686 |
+fi |
|
| 687 |
+ |
|
| 688 |
+ |
|
| 674 | 689 |
# Extras Pre-install |
| 675 | 690 |
# ------------------ |
| 676 | 691 |
|
| ... | ... |
@@ -112,6 +112,11 @@ elif [[ -f $RC_DIR/.localrc.auto ]]; then |
| 112 | 112 |
source $RC_DIR/.localrc.auto |
| 113 | 113 |
fi |
| 114 | 114 |
|
| 115 |
+# Configure wheel cache location |
|
| 116 |
+export WHEELHOUSE=${WHEELHOUSE:-$DEST/.wheelhouse}
|
|
| 117 |
+export PIP_WHEEL_DIR=${PIP_WHEEL_DIR:-$WHEELHOUSE}
|
|
| 118 |
+export PIP_FIND_LINKS=${PIP_FIND_LINKS:-file://$WHEELHOUSE}
|
|
| 119 |
+ |
|
| 115 | 120 |
# This can be used to turn database query logging on and off |
| 116 | 121 |
# (currently only implemented for MySQL backend) |
| 117 | 122 |
DATABASE_QUERY_LOGGING=$(trueorfalse True DATABASE_QUERY_LOGGING) |
| 118 | 123 |
new file mode 100755 |
| ... | ... |
@@ -0,0 +1,77 @@ |
| 0 |
+#!/usr/bin/env bash |
|
| 1 |
+# |
|
| 2 |
+# **tools/build_wheels.sh** - Build a cache of Python wheels |
|
| 3 |
+# |
|
| 4 |
+# build_wheels.sh [package [...]] |
|
| 5 |
+# |
|
| 6 |
+# System package prerequisites listed in files/*/devlibs will be installed |
|
| 7 |
+# |
|
| 8 |
+# Builds wheels for all virtual env requirements listed in |
|
| 9 |
+# ``venv-requirements.txt`` plus any supplied on the command line. |
|
| 10 |
+# |
|
| 11 |
+# Assumes ``tools/install_pip.sh`` has been run and a suitable pip/setuptools is available. |
|
| 12 |
+ |
|
| 13 |
+# If TOP_DIR is set we're being sourced rather than running stand-alone |
|
| 14 |
+# or in a sub-shell |
|
| 15 |
+if [[ -z "$TOP_DIR" ]]; then |
|
| 16 |
+ |
|
| 17 |
+ set -o errexit |
|
| 18 |
+ set -o nounset |
|
| 19 |
+ |
|
| 20 |
+ # Keep track of the devstack directory |
|
| 21 |
+ TOP_DIR=$(cd $(dirname "$0")/.. && pwd) |
|
| 22 |
+ FILES=$TOP_DIR/files |
|
| 23 |
+ |
|
| 24 |
+ # Import common functions |
|
| 25 |
+ source $TOP_DIR/functions |
|
| 26 |
+ |
|
| 27 |
+ GetDistro |
|
| 28 |
+ |
|
| 29 |
+ source $TOP_DIR/stackrc |
|
| 30 |
+ |
|
| 31 |
+ trap err_trap ERR |
|
| 32 |
+ |
|
| 33 |
+fi |
|
| 34 |
+ |
|
| 35 |
+# Get additional packages to build |
|
| 36 |
+MORE_PACKAGES="$@" |
|
| 37 |
+ |
|
| 38 |
+# Set a fall-back default, assume that since this script is being called |
|
| 39 |
+# package builds are to happen even if WHEELHOUSE is not configured |
|
| 40 |
+export WHEELHOUSE=${WHEELHOUSE:-.wheelhouse}
|
|
| 41 |
+ |
|
| 42 |
+# Exit on any errors so that errors don't compound |
|
| 43 |
+function err_trap {
|
|
| 44 |
+ local r=$? |
|
| 45 |
+ set +o xtrace |
|
| 46 |
+ |
|
| 47 |
+ rm -rf $TMP_VENV_PATH |
|
| 48 |
+ |
|
| 49 |
+ exit $r |
|
| 50 |
+} |
|
| 51 |
+ |
|
| 52 |
+# Get system prereqs |
|
| 53 |
+install_package $(get_packages devlibs) |
|
| 54 |
+ |
|
| 55 |
+# Get a modern ``virtualenv`` |
|
| 56 |
+pip_install virtualenv |
|
| 57 |
+ |
|
| 58 |
+# Prepare the workspace |
|
| 59 |
+TMP_VENV_PATH=$(mktemp -d tmp-venv-XXXX) |
|
| 60 |
+virtualenv $TMP_VENV_PATH |
|
| 61 |
+ |
|
| 62 |
+# Install modern pip and wheel |
|
| 63 |
+$TMP_VENV_PATH/bin/pip install -U pip wheel |
|
| 64 |
+ |
|
| 65 |
+# VENV_PACKAGES is a list of packages we want to pre-install |
|
| 66 |
+VENV_PACKAGE_FILE=$FILES/venv-requirements.txt |
|
| 67 |
+if [[ -r $VENV_PACKAGE_FILE ]]; then |
|
| 68 |
+ VENV_PACKAGES=$(grep -v '^#' $VENV_PACKAGE_FILE) |
|
| 69 |
+fi |
|
| 70 |
+ |
|
| 71 |
+for pkg in ${VENV_PACKAGES,/ } ${MORE_PACKAGES}; do
|
|
| 72 |
+ $TMP_VENV_PATH/bin/pip wheel $pkg |
|
| 73 |
+done |
|
| 74 |
+ |
|
| 75 |
+# Clean up wheel workspace |
|
| 76 |
+rm -rf $TMP_VENV_PATH |