Browse code

allow config to manage python3 use explicitly

Add variables ENABLED_PYTHON3_PACKAGES and DISABLED_PYTHON3_PACKAGES to
work like ENABLED_SERVICES and DISABLED_SERVICES and to manage which
packages are installed using Python 3. Move the list of whitelisted
packages in pip_install to the default for ENABLED_PYTHON3_PACKAGES,
except swift which is not enabled by default for now.

Add enable_python3_package and disable_python3_package functions to make
editing the variables from local.conf easier.

Add python3_enabled_for and python3_disabled_for functions to check the
settings against packages being installed by pip.

Update pip_install to check if python3 is disabled for a service, then
see if it is explicitly enabled, and only then fall back to looking at
the classifiers in the packaging metadata.

Update pip_install messages to give more detail about why the choice
between python 2 and 3 is being made for a given package.

Change-Id: I69857d4e11f4767928614a3b637c894bcd03491f
Signed-off-by: Doug Hellmann <doug@doughellmann.com>

Doug Hellmann authored on 2017/01/10 06:24:24
Showing 3 changed files
... ...
@@ -97,6 +97,111 @@ function check_python3_support_for_package_remote {
97 97
     echo $classifier
98 98
 }
99 99
 
100
+# python3_enabled_for() checks if the service(s) specified as arguments are
101
+# enabled by the user in ``ENABLED_PYTHON3_PACKAGES``.
102
+#
103
+# Multiple services specified as arguments are ``OR``'ed together; the test
104
+# is a short-circuit boolean, i.e it returns on the first match.
105
+#
106
+# Uses global ``ENABLED_PYTHON3_PACKAGES``
107
+# python3_enabled_for dir [dir ...]
108
+function python3_enabled_for {
109
+    local xtrace
110
+    xtrace=$(set +o | grep xtrace)
111
+    set +o xtrace
112
+
113
+    local enabled=1
114
+    local dirs=$@
115
+    local dir
116
+    for dir in ${dirs}; do
117
+        [[ ,${ENABLED_PYTHON3_PACKAGES}, =~ ,${dir}, ]] && enabled=0
118
+    done
119
+
120
+    $xtrace
121
+    return $enabled
122
+}
123
+
124
+# python3_disabled_for() checks if the service(s) specified as arguments are
125
+# disabled by the user in ``DISABLED_PYTHON3_PACKAGES``.
126
+#
127
+# Multiple services specified as arguments are ``OR``'ed together; the test
128
+# is a short-circuit boolean, i.e it returns on the first match.
129
+#
130
+# Uses global ``DISABLED_PYTHON3_PACKAGES``
131
+# python3_disabled_for dir [dir ...]
132
+function python3_disabled_for {
133
+    local xtrace
134
+    xtrace=$(set +o | grep xtrace)
135
+    set +o xtrace
136
+
137
+    local enabled=1
138
+    local dirs=$@
139
+    local dir
140
+    for dir in ${dirs}; do
141
+        [[ ,${DISABLED_PYTHON3_PACKAGES}, =~ ,${dir}, ]] && enabled=0
142
+    done
143
+
144
+    $xtrace
145
+    return $enabled
146
+}
147
+
148
+# enable_python3_package() adds the repositories passed as argument to the
149
+# ``ENABLED_PYTHON3_PACKAGES`` list, if they are not already present.
150
+#
151
+# For example:
152
+#   enable_python3_package nova
153
+#
154
+# Uses global ``ENABLED_PYTHON3_PACKAGES``
155
+# enable_python3_package dir [dir ...]
156
+function enable_python3_package {
157
+    local xtrace
158
+    xtrace=$(set +o | grep xtrace)
159
+    set +o xtrace
160
+
161
+    local tmpsvcs="${ENABLED_PYTHON3_PACKAGES}"
162
+    local python3
163
+    for dir in $@; do
164
+        if [[ ,${DISABLED_PYTHON3_PACKAGES}, =~ ,${dir}, ]]; then
165
+            warn $LINENO "Attempt to enable_python3_package ${dir} when it has been disabled"
166
+            continue
167
+        fi
168
+        if ! python3_enabled_for $dir; then
169
+            tmpsvcs+=",$dir"
170
+        fi
171
+    done
172
+    ENABLED_PYTHON3_PACKAGES=$(_cleanup_service_list "$tmpsvcs")
173
+
174
+    $xtrace
175
+}
176
+
177
+# disable_python3_package() prepares the services passed as argument to be
178
+# removed from the ``ENABLED_PYTHON3_PACKAGES`` list, if they are present.
179
+#
180
+# For example:
181
+#   disable_python3_package swift
182
+#
183
+# Uses globals ``ENABLED_PYTHON3_PACKAGES`` and ``DISABLED_PYTHON3_PACKAGES``
184
+# disable_python3_package dir [dir ...]
185
+function disable_python3_package {
186
+    local xtrace
187
+    xtrace=$(set +o | grep xtrace)
188
+    set +o xtrace
189
+
190
+    local disabled_svcs="${DISABLED_PYTHON3_PACKAGES}"
191
+    local enabled_svcs=",${ENABLED_PYTHON3_PACKAGES},"
192
+    local dir
193
+    for dir in $@; do
194
+        disabled_svcs+=",$dir"
195
+        if python3_enabled_for $dir; then
196
+            enabled_svcs=${enabled_svcs//,$dir,/,}
197
+        fi
198
+    done
199
+    DISABLED_PYTHON3_PACKAGES=$(_cleanup_service_list "$disabled_svcs")
200
+    ENABLED_PYTHON3_PACKAGES=$(_cleanup_service_list "$enabled_svcs")
201
+
202
+    $xtrace
203
+}
204
+
100 205
 # Wrapper for ``pip install`` to set cache and proxy environment variables
101 206
 # Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
102 207
 # ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``,
... ...
@@ -149,16 +254,16 @@ function pip_install {
149 149
                 # support for python3 in progress, but don't claim support
150 150
                 # in their classifier
151 151
                 echo "Check python version for : $package_dir"
152
-                if [[ ${package_dir##*/} == "nova" || ${package_dir##*/} == "glance" || \
153
-                        ${package_dir##*/} == "cinder" || ${package_dir##*/} == "swift" || \
154
-                        ${package_dir##*/} == "uwsgi" ]]; then
155
-                    echo "Using $PYTHON3_VERSION version to install $package_dir"
152
+                if python3_disabled_for ${package_dir##*/}; then
153
+                    echo "Explicitly using $PYTHON2_VERSION version to install $package_dir based on DISABLED_PYTHON3_PACKAGES"
154
+                elif python3_enabled_for ${package_dir##*/}; then
155
+                    echo "Explicitly using $PYTHON3_VERSION version to install $package_dir based on ENABLED_PYTHON3_PACKAGES"
156 156
                     sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
157 157
                     cmd_pip=$(get_pip_command $PYTHON3_VERSION)
158 158
                 elif [[ -d "$package_dir" ]]; then
159 159
                     python_versions=$(get_python_versions_for_package $package_dir)
160 160
                     if [[ $python_versions =~ $PYTHON3_VERSION ]]; then
161
-                        echo "Using $PYTHON3_VERSION version to install $package_dir"
161
+                        echo "Automatically using $PYTHON3_VERSION version to install $package_dir based on classifiers"
162 162
                         sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
163 163
                         cmd_pip=$(get_pip_command $PYTHON3_VERSION)
164 164
                     else
... ...
@@ -167,7 +272,7 @@ function pip_install {
167 167
                         # a warning.
168 168
                         python3_classifier=$(check_python3_support_for_package_local $package_dir)
169 169
                         if [[ ! -z "$python3_classifier" ]]; then
170
-                            echo "Using $PYTHON3_VERSION version to install $package_dir"
170
+                            echo "Automatically using $PYTHON3_VERSION version to install $package_dir based on local package settings"
171 171
                             sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
172 172
                             cmd_pip=$(get_pip_command $PYTHON3_VERSION)
173 173
                         fi
... ...
@@ -177,7 +282,7 @@ function pip_install {
177 177
                     package=$(echo $package_dir | grep -o '^[.a-zA-Z0-9_-]*')
178 178
                     python3_classifier=$(check_python3_support_for_package_remote $package)
179 179
                     if [[ ! -z "$python3_classifier" ]]; then
180
-                        echo "Using $PYTHON3_VERSION version to install $package"
180
+                        echo "Automatically using $PYTHON3_VERSION version to install $package based on remote package settings"
181 181
                         sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
182 182
                         cmd_pip=$(get_pip_command $PYTHON3_VERSION)
183 183
                     fi
... ...
@@ -102,9 +102,19 @@ if [[ -r $RC_DIR/.localrc.password ]]; then
102 102
     source $RC_DIR/.localrc.password
103 103
 fi
104 104
 
105
-# Control whether Python 3 should be used.
105
+# Control whether Python 3 should be used at all.
106 106
 export USE_PYTHON3=$(trueorfalse False USE_PYTHON3)
107 107
 
108
+# Control whether Python 3 is enabled for specific services by the
109
+# base name of the directory from which they are installed. See
110
+# enable_python3_package to edit this variable and use_python3_for to
111
+# test membership.
112
+export ENABLED_PYTHON3_PACKAGES="nova,glance,cinder,uwsgi"
113
+
114
+# Explicitly list services not to run under Python 3. See
115
+# disable_python3_package to edit this variable.
116
+export DISABLED_PYTHON3_PACKAGES=""
117
+
108 118
 # When Python 3 is supported by an application, adding the specific
109 119
 # version of Python 3 to this variable will install the app using that
110 120
 # version of the interpreter instead of 2.7.
111 121
new file mode 100755
... ...
@@ -0,0 +1,30 @@
0
+#!/usr/bin/env bash
1
+
2
+# Tests for DevStack INI functions
3
+
4
+TOP=$(cd $(dirname "$0")/.. && pwd)
5
+
6
+source $TOP/functions-common
7
+source $TOP/inc/python
8
+
9
+source $TOP/tests/unittest.sh
10
+
11
+echo "Testing Python 3 functions"
12
+
13
+# Initialize variables manipulated by functions under test.
14
+export ENABLED_PYTHON3_PACKAGES=""
15
+export DISABLED_PYTHON3_PACKAGES=""
16
+
17
+assert_false "should not be enabled yet" python3_enabled_for testpackage1
18
+
19
+enable_python3_package testpackage1
20
+assert_equal "$ENABLED_PYTHON3_PACKAGES" "testpackage1"  "unexpected result"
21
+assert_true "should be enabled" python3_enabled_for testpackage1
22
+
23
+assert_false "should not be disabled yet" python3_disabled_for testpackage2
24
+
25
+disable_python3_package testpackage2
26
+assert_equal "$DISABLED_PYTHON3_PACKAGES" "testpackage2"  "unexpected result"
27
+assert_true "should be disabled" python3_disabled_for testpackage2
28
+
29
+report_results