Browse code

[stable-2.9] Explicitly use multiprocessing fork start method (#63581) (#63591)

* Explicitly use multiprocessing fork start method

* Remove unused import

* Remove unused import

* Fallback to just multiprocessing on py2
(cherry picked from commit 82ee341)

Co-authored-by: Matt Martz <matt@sivel.net>

Matt Martz authored on 2019/11/13 06:01:27
Showing 4 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,2 @@
0
+bugfixes:
1
+- TaskQueueManager - Explicitly set the mutliprocessing start method to ``fork`` to avoid issues with the default on macOS now being ``spawn``.
... ...
@@ -19,7 +19,6 @@
19 19
 from __future__ import (absolute_import, division, print_function)
20 20
 __metaclass__ = type
21 21
 
22
-import multiprocessing
23 22
 import os
24 23
 import sys
25 24
 import traceback
... ...
@@ -41,13 +40,14 @@ from ansible.executor.task_executor import TaskExecutor
41 41
 from ansible.executor.task_result import TaskResult
42 42
 from ansible.module_utils._text import to_text
43 43
 from ansible.utils.display import Display
44
+from ansible.utils.multiprocessing import context as multiprocessing_context
44 45
 
45 46
 __all__ = ['WorkerProcess']
46 47
 
47 48
 display = Display()
48 49
 
49 50
 
50
-class WorkerProcess(multiprocessing.Process):
51
+class WorkerProcess(multiprocessing_context.Process):
51 52
     '''
52 53
     The worker thread class, which uses TaskExecutor to run tasks
53 54
     read from a job queue and pushes results into a results queue
... ...
@@ -19,7 +19,6 @@
19 19
 from __future__ import (absolute_import, division, print_function)
20 20
 __metaclass__ = type
21 21
 
22
-import multiprocessing
23 22
 import os
24 23
 import tempfile
25 24
 
... ...
@@ -41,6 +40,7 @@ from ansible.utils.helpers import pct_to_int
41 41
 from ansible.vars.hostvars import HostVars
42 42
 from ansible.vars.reserved import warn_if_reserved
43 43
 from ansible.utils.display import Display
44
+from ansible.utils.multiprocessing import context as multiprocessing_context
44 45
 
45 46
 
46 47
 __all__ = ['TaskQueueManager']
... ...
@@ -97,7 +97,7 @@ class TaskQueueManager:
97 97
         self._unreachable_hosts = dict()
98 98
 
99 99
         try:
100
-            self._final_q = multiprocessing.Queue()
100
+            self._final_q = multiprocessing_context.Queue()
101 101
         except OSError as e:
102 102
             raise AnsibleError("Unable to use multiprocessing, this is normally caused by lack of access to /dev/shm: %s" % to_native(e))
103 103
 
104 104
new file mode 100644
... ...
@@ -0,0 +1,21 @@
0
+# Copyright (c) 2019 Matt Martz <matt@sivel.net>
1
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
2
+
3
+# Make coding more python3-ish
4
+from __future__ import (absolute_import, division, print_function)
5
+__metaclass__ = type
6
+
7
+import multiprocessing
8
+
9
+# Explicit multiprocessing context using the fork start method
10
+# This exists as a compat layer now that Python3.8 has changed the default
11
+# start method for macOS to ``spawn`` which is incompatible with our
12
+# code base currently
13
+#
14
+# This exists in utils to allow it to be easily imported into various places
15
+# without causing circular import or dependency problems
16
+try:
17
+    context = multiprocessing.get_context('fork')
18
+except AttributeError:
19
+    # Py2 has no context functionality, and only supports fork
20
+    context = multiprocessing