Browse code

Enable optional Python 3 support

Add USE_PYTHON3 and PYTHON3_VERSION variables to allow services to use
python 3 if they indicate support in their python package metadata.

Tested in Heat here -> I837c2fba682ab430d50e9f43913f2fed20325a7a.
Project config change to add a dedicated job to Heat is here -> I0837e62d6ccc66397a5e409f0961edd4be31f467

Change-Id: I079e18b58b214bf8362945c253d6d894ca8b1a6b

Doug Hellmann authored on 2015/05/08 06:06:24
Showing 5 changed files
... ...
@@ -28,10 +28,13 @@ declare -A PROJECT_VENV
28 28
 # Get the path to the pip command.
29 29
 # get_pip_command
30 30
 function get_pip_command {
31
-    which pip || which pip-python
31
+    local version="$1"
32
+    # NOTE(dhellmann): I don't know if we actually get a pip3.4-python
33
+    # under any circumstances.
34
+    which pip${version} || which pip${version}-python
32 35
 
33 36
     if [ $? -ne 0 ]; then
34
-        die $LINENO "Unable to find pip; cannot continue"
37
+        die $LINENO "Unable to find pip${version}; cannot continue"
35 38
     fi
36 39
 }
37 40
 
... ...
@@ -66,6 +69,13 @@ function pip_install_gr {
66 66
     pip_install $clean_name
67 67
 }
68 68
 
69
+# Determine the python versions supported by a package
70
+function get_python_versions_for_package {
71
+    local name=$1
72
+    cd $name && python setup.py --classifiers \
73
+        | grep 'Language' | cut -f5 -d: | grep '\.' | tr '\n' ' '
74
+}
75
+
69 76
 # Wrapper for ``pip install`` to set cache and proxy environment variables
70 77
 # Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
71 78
 # ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``,
... ...
@@ -104,8 +114,22 @@ function pip_install {
104 104
             local sudo_pip="env"
105 105
         else
106 106
             local cmd_pip
107
-            cmd_pip=$(get_pip_command)
107
+            cmd_pip=$(get_pip_command $PYTHON2_VERSION)
108 108
             local sudo_pip="sudo -H"
109
+            if python3_enabled; then
110
+                # Look at the package classifiers to find the python
111
+                # versions supported, and if we find the version of
112
+                # python3 we've been told to use, use that instead of the
113
+                # default pip
114
+                local package_dir=${!#}
115
+                local python_versions
116
+                if [[ -d "$package_dir" ]]; then
117
+                    python_versions=$(get_python_versions_for_package $package_dir)
118
+                    if [[ $python_versions =~ $PYTHON3_VERSION ]]; then
119
+                        cmd_pip=$(get_pip_command $PYTHON3_VERSION)
120
+                    fi
121
+                fi
122
+            fi
109 123
         fi
110 124
     fi
111 125
 
... ...
@@ -113,6 +137,8 @@ function pip_install {
113 113
     # Always apply constraints
114 114
     cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
115 115
 
116
+    # FIXME(dhellmann): Need to force multiple versions of pip for
117
+    # packages like setuptools?
116 118
     local pip_version
117 119
     pip_version=$(python -c "import pip; \
118 120
                         print(pip.__version__.strip('.')[0])")
... ...
@@ -276,6 +302,21 @@ function setup_package {
276 276
     fi
277 277
 }
278 278
 
279
+# Report whether python 3 should be used
280
+function python3_enabled {
281
+    if [[ $USE_PYTHON3 == "True" ]]; then
282
+        return 0
283
+    else
284
+        return 1
285
+    fi
286
+}
287
+
288
+# Install python3 packages
289
+function install_python3 {
290
+    if is_ubuntu; then
291
+        apt_get install python3.4 python3.4-dev
292
+    fi
293
+}
279 294
 
280 295
 # Restore xtrace
281 296
 $INC_PY_TRACE
... ...
@@ -19,6 +19,7 @@
19 19
 function stack_install_service {
20 20
     local service=$1
21 21
     if type install_${service} >/dev/null 2>&1; then
22
+        # FIXME(dhellmann): Needs to be python3-aware at some point.
22 23
         if [[ ${USE_VENV} = True && -n ${PROJECT_VENV[$service]:-} ]]; then
23 24
             rm -rf ${PROJECT_VENV[$service]}
24 25
             source $TOP_DIR/tools/build_venv.sh ${PROJECT_VENV[$service]} ${ADDITIONAL_VENV_PACKAGES//,/ }
... ...
@@ -118,6 +118,17 @@ if [[ -r $RC_DIR/.localrc.password ]]; then
118 118
     source $RC_DIR/.localrc.password
119 119
 fi
120 120
 
121
+# Control whether Python 3 should be used.
122
+export USE_PYTHON3=${USE_PYTHON3:-False}
123
+
124
+# When Python 3 is supported by an application, adding the specific
125
+# version of Python 3 to this variable will install the app using that
126
+# version of the interpreter instead of 2.7.
127
+export PYTHON3_VERSION=${PYTHON3_VERSION:-3.4}
128
+
129
+# Just to be more explicit on the Python 2 version to use.
130
+export PYTHON2_VERSION=${PYTHON2_VERSION:-2.7}
131
+
121 132
 # allow local overrides of env variables, including repo config
122 133
 if [[ -f $RC_DIR/localrc ]]; then
123 134
     # Old-style user-supplied config
... ...
@@ -8,6 +8,7 @@
8 8
 
9 9
 # Assumptions:
10 10
 # - update pip to $INSTALL_PIP_VERSION
11
+# - if USE_PYTHON3=True, PYTHON3_VERSION refers to a version already installed
11 12
 
12 13
 set -o errexit
13 14
 set -o xtrace
... ...
@@ -31,6 +32,8 @@ GetDistro
31 31
 echo "Distro: $DISTRO"
32 32
 
33 33
 function get_versions {
34
+    # FIXME(dhellmann): Deal with multiple python versions here? This
35
+    # is just used for reporting, so maybe not?
34 36
     PIP=$(which pip 2>/dev/null || which pip-python 2>/dev/null || true)
35 37
     if [[ -n $PIP ]]; then
36 38
         PIP_VERSION=$($PIP --version | awk '{ print $2}')
... ...
@@ -75,6 +78,9 @@ function install_get_pip {
75 75
         touch $LOCAL_PIP.downloaded
76 76
     fi
77 77
     sudo -H -E python $LOCAL_PIP
78
+    if python3_enabled; then
79
+        sudo -H -E python${PYTHON3_VERSION} $LOCAL_PIP
80
+    fi
78 81
 }
79 82
 
80 83
 
... ...
@@ -114,6 +120,7 @@ get_versions
114 114
 # python in f23 depends on the python-pip package
115 115
 if ! { is_fedora && [[ $DISTRO == "f23" ]]; }; then
116 116
     uninstall_package python-pip
117
+    uninstall_package python3-pip
117 118
 fi
118 119
 
119 120
 install_get_pip
... ...
@@ -122,6 +129,7 @@ if [[ -n $PYPI_ALTERNATIVE_URL ]]; then
122 122
     configure_pypi_alternative_url
123 123
 fi
124 124
 
125
+set -x
125 126
 pip_install -U setuptools
126 127
 
127 128
 get_versions
... ...
@@ -81,6 +81,9 @@ if [[ -n "$SYSLOG" && "$SYSLOG" != "False" ]]; then
81 81
     fi
82 82
 fi
83 83
 
84
+if python3_enabled; then
85
+    install_python3
86
+fi
84 87
 
85 88
 # Mark end of run
86 89
 # ---------------