Browse code

Move ssh and local connection plugins from using raw select to selectors

At the moment, this change will use EPoll on Linux, KQueue on *BSDs,
etc, so it should alleviate problems with too many open file
descriptors.

* Bundle a copy of selectors2 so that we have the selectors API everywhere.
* Add licensing information to selectors2 file so it's clear what the
licensing terms and conditions are.
* Exclude the bundled copy of selectors2 from our boilerplate code-smell test
* Rewrite ssh_run tests to attempt to work around problem with mocking
select on shippable

Fixes #14143

Toshio Kuratomi authored on 2017/02/02 01:48:23
Showing 7 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,47 @@
0
+# (c) 2014, 2017 Toshio Kuratomi <tkuratomi@ansible.com>
1
+#
2
+# This file is part of Ansible
3
+#
4
+# Ansible is free software: you can redistribute it and/or modify
5
+# it under the terms of the GNU General Public License as published by
6
+# the Free Software Foundation, either version 3 of the License, or
7
+# (at your option) any later version.
8
+#
9
+# Ansible is distributed in the hope that it will be useful,
10
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
+# GNU General Public License for more details.
13
+#
14
+# You should have received a copy of the GNU General Public License
15
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
16
+
17
+# Make coding more python3-ish
18
+from __future__ import (absolute_import, division, print_function)
19
+__metaclass__ = type
20
+
21
+'''
22
+Compat selectors library.  Python-3.5 has this builtin.  The selectors2
23
+package exists on pypi to backport the functionality as far as python-2.6.
24
+'''
25
+# The following makes it easier for us to script updates of the bundled code
26
+_BUNDLED_METADATA = { "pypi_name": "selectors2", "version": "1.1.0" }
27
+
28
+import os.path
29
+import sys
30
+
31
+try:
32
+    # Python 3.4+
33
+    import selectors as _system_selectors
34
+except ImportError:
35
+    try:
36
+        # backport package installed in the system
37
+        import selectors2 as _system_selectors
38
+    except ImportError:
39
+        _system_selectors = None
40
+
41
+if _system_selectors:
42
+    selectors = _system_selectors
43
+else:
44
+    # Our bundled copy
45
+    from . import _selectors2 as selectors
46
+sys.modules['ansible.compat.selectors'] = selectors
0 47
new file mode 100644
... ...
@@ -0,0 +1,667 @@
0
+# This file is from the selectors2.py package.  It backports the PSF Licensed
1
+# selectors module from the Python-3.5 stdlib to older versions of Python.
2
+# The author, Seth Michael Larson, dual licenses his modifications under the
3
+# PSF License and MIT License:
4
+# https://github.com/SethMichaelLarson/selectors2#license
5
+#
6
+# Seth's copy of the MIT license is reproduced below
7
+#
8
+# MIT License
9
+#
10
+# Copyright (c) 2016 Seth Michael Larson
11
+#
12
+# Permission is hereby granted, free of charge, to any person obtaining a copy
13
+# of this software and associated documentation files (the "Software"), to deal
14
+# in the Software without restriction, including without limitation the rights
15
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
16
+# copies of the Software, and to permit persons to whom the Software is
17
+# furnished to do so, subject to the following conditions:
18
+#
19
+# The above copyright notice and this permission notice shall be included in all
20
+# copies or substantial portions of the Software.
21
+#
22
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
23
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
25
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
27
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
28
+# SOFTWARE.
29
+
30
+
31
+# Backport of selectors.py from Python 3.5+ to support Python < 3.4
32
+# Also has the behavior specified in PEP 475 which is to retry syscalls
33
+# in the case of an EINTR error. This module is required because selectors34
34
+# does not follow this behavior and instead returns that no dile descriptor
35
+# events have occurred rather than retry the syscall. The decision to drop
36
+# support for select.devpoll is made to maintain 100% test coverage.
37
+
38
+import errno
39
+import math
40
+import select
41
+import socket
42
+import sys
43
+import time
44
+from collections import namedtuple, Mapping
45
+
46
+try:
47
+    monotonic = time.monotonic
48
+except (AttributeError, ImportError):  # Python 3.3<
49
+    monotonic = time.time
50
+
51
+__author__ = 'Seth Michael Larson'
52
+__email__ = 'sethmichaellarson@protonmail.com'
53
+__version__ = '1.1.0'
54
+__license__ = 'MIT'
55
+
56
+__all__ = [
57
+    'EVENT_READ',
58
+    'EVENT_WRITE',
59
+    'SelectorError',
60
+    'SelectorKey',
61
+    'DefaultSelector'
62
+]
63
+
64
+EVENT_READ = (1 << 0)
65
+EVENT_WRITE = (1 << 1)
66
+
67
+HAS_SELECT = True  # Variable that shows whether the platform has a selector.
68
+_SYSCALL_SENTINEL = object()  # Sentinel in case a system call returns None.
69
+
70
+
71
+class SelectorError(Exception):
72
+    def __init__(self, errcode):
73
+        super(SelectorError, self).__init__()
74
+        self.errno = errcode
75
+
76
+    def __repr__(self):
77
+        return "<SelectorError errno={0}>".format(self.errno)
78
+
79
+    def __str__(self):
80
+        return self.__repr__()
81
+
82
+
83
+def _fileobj_to_fd(fileobj):
84
+    """ Return a file descriptor from a file object. If
85
+    given an integer will simply return that integer back. """
86
+    if isinstance(fileobj, int):
87
+        fd = fileobj
88
+    else:
89
+        try:
90
+            fd = int(fileobj.fileno())
91
+        except (AttributeError, TypeError, ValueError):
92
+            raise ValueError("Invalid file object: {0!r}".format(fileobj))
93
+    if fd < 0:
94
+        raise ValueError("Invalid file descriptor: {0}".format(fd))
95
+    return fd
96
+
97
+# Python 3.5 uses a more direct route to wrap system calls to increase speed.
98
+if sys.version_info >= (3, 5):
99
+    def _syscall_wrapper(func, _, *args, **kwargs):
100
+        """ This is the short-circuit version of the below logic
101
+        because in Python 3.5+ all selectors restart system calls. """
102
+        try:
103
+            return func(*args, **kwargs)
104
+        except (OSError, IOError, select.error) as e:
105
+            errcode = None
106
+            if hasattr(e, "errno"):
107
+                errcode = e.errno
108
+            elif hasattr(e, "args"):
109
+                errcode = e.args[0]
110
+            raise SelectorError(errcode)
111
+else:
112
+    def _syscall_wrapper(func, recalc_timeout, *args, **kwargs):
113
+        """ Wrapper function for syscalls that could fail due to EINTR.
114
+        All functions should be retried if there is time left in the timeout
115
+        in accordance with PEP 475. """
116
+        timeout = kwargs.get("timeout", None)
117
+        if timeout is None:
118
+            expires = None
119
+            recalc_timeout = False
120
+        else:
121
+            timeout = float(timeout)
122
+            if timeout < 0.0:  # Timeout less than 0 treated as no timeout.
123
+                expires = None
124
+            else:
125
+                expires = monotonic() + timeout
126
+
127
+        args = list(args)
128
+        if recalc_timeout and "timeout" not in kwargs:
129
+            raise ValueError(
130
+                "Timeout must be in args or kwargs to be recalculated")
131
+
132
+        result = _SYSCALL_SENTINEL
133
+        while result is _SYSCALL_SENTINEL:
134
+            try:
135
+                result = func(*args, **kwargs)
136
+            # OSError is thrown by select.select
137
+            # IOError is thrown by select.epoll.poll
138
+            # select.error is thrown by select.poll.poll
139
+            # Aren't we thankful for Python 3.x rework for exceptions?
140
+            except (OSError, IOError, select.error) as e:
141
+                # select.error wasn't a subclass of OSError in the past.
142
+                errcode = None
143
+                if hasattr(e, "errno"):
144
+                    errcode = e.errno
145
+                elif hasattr(e, "args"):
146
+                    errcode = e.args[0]
147
+
148
+                # Also test for the Windows equivalent of EINTR.
149
+                is_interrupt = (errcode == errno.EINTR or (hasattr(errno, "WSAEINTR") and
150
+                                                           errcode == errno.WSAEINTR))
151
+
152
+                if is_interrupt:
153
+                    if expires is not None:
154
+                        current_time = monotonic()
155
+                        if current_time > expires:
156
+                            raise OSError(errno=errno.ETIMEDOUT)
157
+                        if recalc_timeout:
158
+                            if "timeout" in kwargs:
159
+                                kwargs["timeout"] = expires - current_time
160
+                    continue
161
+                if errcode:
162
+                    raise SelectorError(errcode)
163
+                else:
164
+                    raise
165
+        return result
166
+
167
+
168
+SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data'])
169
+
170
+
171
+class _SelectorMapping(Mapping):
172
+    """ Mapping of file objects to selector keys """
173
+
174
+    def __init__(self, selector):
175
+        self._selector = selector
176
+
177
+    def __len__(self):
178
+        return len(self._selector._fd_to_key)
179
+
180
+    def __getitem__(self, fileobj):
181
+        try:
182
+            fd = self._selector._fileobj_lookup(fileobj)
183
+            return self._selector._fd_to_key[fd]
184
+        except KeyError:
185
+            raise KeyError("{0!r} is not registered.".format(fileobj))
186
+
187
+    def __iter__(self):
188
+        return iter(self._selector._fd_to_key)
189
+
190
+
191
+class BaseSelector(object):
192
+    """ Abstract Selector class
193
+
194
+    A selector supports registering file objects to be monitored
195
+    for specific I/O events.
196
+
197
+    A file object is a file descriptor or any object with a
198
+    `fileno()` method. An arbitrary object can be attached to the
199
+    file object which can be used for example to store context info,
200
+    a callback, etc.
201
+
202
+    A selector can use various implementations (select(), poll(), epoll(),
203
+    and kqueue()) depending on the platform. The 'DefaultSelector' class uses
204
+    the most efficient implementation for the current platform.
205
+    """
206
+    def __init__(self):
207
+        # Maps file descriptors to keys.
208
+        self._fd_to_key = {}
209
+
210
+        # Read-only mapping returned by get_map()
211
+        self._map = _SelectorMapping(self)
212
+
213
+    def _fileobj_lookup(self, fileobj):
214
+        """ Return a file descriptor from a file object.
215
+        This wraps _fileobj_to_fd() to do an exhaustive
216
+        search in case the object is invalid but we still
217
+        have it in our map. Used by unregister() so we can
218
+        unregister an object that was previously registered
219
+        even if it is closed. It is also used by _SelectorMapping
220
+        """
221
+        try:
222
+            return _fileobj_to_fd(fileobj)
223
+        except ValueError:
224
+
225
+            # Search through all our mapped keys.
226
+            for key in self._fd_to_key.values():
227
+                if key.fileobj is fileobj:
228
+                    return key.fd
229
+
230
+            # Raise ValueError after all.
231
+            raise
232
+
233
+    def register(self, fileobj, events, data=None):
234
+        """ Register a file object for a set of events to monitor. """
235
+        if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)):
236
+            raise ValueError("Invalid events: {0!r}".format(events))
237
+
238
+        key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data)
239
+
240
+        if key.fd in self._fd_to_key:
241
+            raise KeyError("{0!r} (FD {1}) is already registered"
242
+                           .format(fileobj, key.fd))
243
+
244
+        self._fd_to_key[key.fd] = key
245
+        return key
246
+
247
+    def unregister(self, fileobj):
248
+        """ Unregister a file object from being monitored. """
249
+        try:
250
+            key = self._fd_to_key.pop(self._fileobj_lookup(fileobj))
251
+        except KeyError:
252
+            raise KeyError("{0!r} is not registered".format(fileobj))
253
+
254
+        # Getting the fileno of a closed socket on Windows errors with EBADF.
255
+        except socket.error as err:
256
+            if err.errno != errno.EBADF:
257
+                raise
258
+            else:
259
+                for key in self._fd_to_key.values():
260
+                    if key.fileobj is fileobj:
261
+                        self._fd_to_key.pop(key.fd)
262
+                        break
263
+                else:
264
+                    raise KeyError("{0!r} is not registered".format(fileobj))
265
+        return key
266
+
267
+    def modify(self, fileobj, events, data=None):
268
+        """ Change a registered file object monitored events and data. """
269
+        # NOTE: Some subclasses optimize this operation even further.
270
+        try:
271
+            key = self._fd_to_key[self._fileobj_lookup(fileobj)]
272
+        except KeyError:
273
+            raise KeyError("{0!r} is not registered".format(fileobj))
274
+
275
+        if events != key.events:
276
+            self.unregister(fileobj)
277
+            key = self.register(fileobj, events, data)
278
+
279
+        elif data != key.data:
280
+            # Use a shortcut to update the data.
281
+            key = key._replace(data=data)
282
+            self._fd_to_key[key.fd] = key
283
+
284
+        return key
285
+
286
+    def select(self, timeout=None):
287
+        """ Perform the actual selection until some monitored file objects
288
+        are ready or the timeout expires. """
289
+        raise NotImplementedError()
290
+
291
+    def close(self):
292
+        """ Close the selector. This must be called to ensure that all
293
+        underlying resources are freed. """
294
+        self._fd_to_key.clear()
295
+        self._map = None
296
+
297
+    def get_key(self, fileobj):
298
+        """ Return the key associated with a registered file object. """
299
+        mapping = self.get_map()
300
+        if mapping is None:
301
+            raise RuntimeError("Selector is closed")
302
+        try:
303
+            return mapping[fileobj]
304
+        except KeyError:
305
+            raise KeyError("{0!r} is not registered".format(fileobj))
306
+
307
+    def get_map(self):
308
+        """ Return a mapping of file objects to selector keys """
309
+        return self._map
310
+
311
+    def _key_from_fd(self, fd):
312
+        """ Return the key associated to a given file descriptor
313
+         Return None if it is not found. """
314
+        try:
315
+            return self._fd_to_key[fd]
316
+        except KeyError:
317
+            return None
318
+
319
+    def __enter__(self):
320
+        return self
321
+
322
+    def __exit__(self, *args):
323
+        self.close()
324
+
325
+# Almost all platforms have select.select()
326
+if hasattr(select, "select"):
327
+    class SelectSelector(BaseSelector):
328
+        """ Select-based selector. """
329
+        def __init__(self):
330
+            super(SelectSelector, self).__init__()
331
+            self._readers = set()
332
+            self._writers = set()
333
+
334
+        def register(self, fileobj, events, data=None):
335
+            key = super(SelectSelector, self).register(fileobj, events, data)
336
+            if events & EVENT_READ:
337
+                self._readers.add(key.fd)
338
+            if events & EVENT_WRITE:
339
+                self._writers.add(key.fd)
340
+            return key
341
+
342
+        def unregister(self, fileobj):
343
+            key = super(SelectSelector, self).unregister(fileobj)
344
+            self._readers.discard(key.fd)
345
+            self._writers.discard(key.fd)
346
+            return key
347
+
348
+        def _select(self, r, w, timeout=None):
349
+            """ Wrapper for select.select because timeout is a positional arg """
350
+            return select.select(r, w, [], timeout)
351
+
352
+        def select(self, timeout=None):
353
+            # Selecting on empty lists on Windows errors out.
354
+            if not len(self._readers) and not len(self._writers):
355
+                return []
356
+
357
+            timeout = None if timeout is None else max(timeout, 0.0)
358
+            ready = []
359
+            r, w, _ = _syscall_wrapper(self._select, True, self._readers,
360
+                                       self._writers, timeout)
361
+            r = set(r)
362
+            w = set(w)
363
+            for fd in r | w:
364
+                events = 0
365
+                if fd in r:
366
+                    events |= EVENT_READ
367
+                if fd in w:
368
+                    events |= EVENT_WRITE
369
+
370
+                key = self._key_from_fd(fd)
371
+                if key:
372
+                    ready.append((key, events & key.events))
373
+            return ready
374
+
375
+    __all__.append('SelectSelector')
376
+
377
+
378
+if hasattr(select, "poll"):
379
+    class PollSelector(BaseSelector):
380
+        """ Poll-based selector """
381
+        def __init__(self):
382
+            super(PollSelector, self).__init__()
383
+            self._poll = select.poll()
384
+
385
+        def register(self, fileobj, events, data=None):
386
+            key = super(PollSelector, self).register(fileobj, events, data)
387
+            event_mask = 0
388
+            if events & EVENT_READ:
389
+                event_mask |= select.POLLIN
390
+            if events & EVENT_WRITE:
391
+                event_mask |= select.POLLOUT
392
+            self._poll.register(key.fd, event_mask)
393
+            return key
394
+
395
+        def unregister(self, fileobj):
396
+            key = super(PollSelector, self).unregister(fileobj)
397
+            self._poll.unregister(key.fd)
398
+            return key
399
+
400
+        def _wrap_poll(self, timeout=None):
401
+            """ Wrapper function for select.poll.poll() so that
402
+            _syscall_wrapper can work with only seconds. """
403
+            if timeout is not None:
404
+                if timeout <= 0:
405
+                    timeout = 0
406
+                else:
407
+                    # select.poll.poll() has a resolution of 1 millisecond,
408
+                    # round away from zero to wait *at least* timeout seconds.
409
+                    timeout = math.ceil(timeout * 1e3)
410
+
411
+            result = self._poll.poll(timeout)
412
+            return result
413
+
414
+        def select(self, timeout=None):
415
+            ready = []
416
+            fd_events = _syscall_wrapper(self._wrap_poll, True, timeout=timeout)
417
+            for fd, event_mask in fd_events:
418
+                events = 0
419
+                if event_mask & ~select.POLLIN:
420
+                    events |= EVENT_WRITE
421
+                if event_mask & ~select.POLLOUT:
422
+                    events |= EVENT_READ
423
+
424
+                key = self._key_from_fd(fd)
425
+                if key:
426
+                    ready.append((key, events & key.events))
427
+
428
+            return ready
429
+
430
+    __all__.append('PollSelector')
431
+
432
+if hasattr(select, "epoll"):
433
+    class EpollSelector(BaseSelector):
434
+        """ Epoll-based selector """
435
+        def __init__(self):
436
+            super(EpollSelector, self).__init__()
437
+            self._epoll = select.epoll()
438
+
439
+        def fileno(self):
440
+            return self._epoll.fileno()
441
+
442
+        def register(self, fileobj, events, data=None):
443
+            key = super(EpollSelector, self).register(fileobj, events, data)
444
+            events_mask = 0
445
+            if events & EVENT_READ:
446
+                events_mask |= select.EPOLLIN
447
+            if events & EVENT_WRITE:
448
+                events_mask |= select.EPOLLOUT
449
+            _syscall_wrapper(self._epoll.register, False, key.fd, events_mask)
450
+            return key
451
+
452
+        def unregister(self, fileobj):
453
+            key = super(EpollSelector, self).unregister(fileobj)
454
+            try:
455
+                _syscall_wrapper(self._epoll.unregister, False, key.fd)
456
+            except SelectorError:
457
+                # This can occur when the fd was closed since registry.
458
+                pass
459
+            return key
460
+
461
+        def select(self, timeout=None):
462
+            if timeout is not None:
463
+                if timeout <= 0:
464
+                    timeout = 0.0
465
+                else:
466
+                    # select.epoll.poll() has a resolution of 1 millisecond
467
+                    # but luckily takes seconds so we don't need a wrapper
468
+                    # like PollSelector. Just for better rounding.
469
+                    timeout = math.ceil(timeout * 1e3) * 1e-3
470
+                timeout = float(timeout)
471
+            else:
472
+                timeout = -1.0  # epoll.poll() must have a float.
473
+
474
+            # We always want at least 1 to ensure that select can be called
475
+            # with no file descriptors registered. Otherwise will fail.
476
+            max_events = max(len(self._fd_to_key), 1)
477
+
478
+            ready = []
479
+            fd_events = _syscall_wrapper(self._epoll.poll, True,
480
+                                         timeout=timeout,
481
+                                         maxevents=max_events)
482
+            for fd, event_mask in fd_events:
483
+                events = 0
484
+                if event_mask & ~select.EPOLLIN:
485
+                    events |= EVENT_WRITE
486
+                if event_mask & ~select.EPOLLOUT:
487
+                    events |= EVENT_READ
488
+
489
+                key = self._key_from_fd(fd)
490
+                if key:
491
+                    ready.append((key, events & key.events))
492
+            return ready
493
+
494
+        def close(self):
495
+            self._epoll.close()
496
+            super(EpollSelector, self).close()
497
+
498
+    __all__.append('EpollSelector')
499
+
500
+
501
+if hasattr(select, "devpoll"):
502
+    class DevpollSelector(BaseSelector):
503
+        """Solaris /dev/poll selector."""
504
+
505
+        def __init__(self):
506
+            super(DevpollSelector, self).__init__()
507
+            self._devpoll = select.devpoll()
508
+
509
+        def fileno(self):
510
+            return self._devpoll.fileno()
511
+
512
+        def register(self, fileobj, events, data=None):
513
+            key = super(DevpollSelector, self).register(fileobj, events, data)
514
+            poll_events = 0
515
+            if events & EVENT_READ:
516
+                poll_events |= select.POLLIN
517
+            if events & EVENT_WRITE:
518
+                poll_events |= select.POLLOUT
519
+            self._devpoll.register(key.fd, poll_events)
520
+            return key
521
+
522
+        def unregister(self, fileobj):
523
+            key = super(DevpollSelector, self).unregister(fileobj)
524
+            self._devpoll.unregister(key.fd)
525
+            return key
526
+
527
+        def _wrap_poll(self, timeout=None):
528
+            """ Wrapper function for select.poll.poll() so that
529
+            _syscall_wrapper can work with only seconds. """
530
+            if timeout is not None:
531
+                if timeout <= 0:
532
+                    timeout = 0
533
+                else:
534
+                    # select.devpoll.poll() has a resolution of 1 millisecond,
535
+                    # round away from zero to wait *at least* timeout seconds.
536
+                    timeout = math.ceil(timeout * 1e3)
537
+
538
+            result = self._devpoll.poll(timeout)
539
+            return result
540
+
541
+        def select(self, timeout=None):
542
+            ready = []
543
+            fd_events = _syscall_wrapper(self._wrap_poll, True, timeout=timeout)
544
+            for fd, event_mask in fd_events:
545
+                events = 0
546
+                if event_mask & ~select.POLLIN:
547
+                    events |= EVENT_WRITE
548
+                if event_mask & ~select.POLLOUT:
549
+                    events |= EVENT_READ
550
+
551
+                key = self._key_from_fd(fd)
552
+                if key:
553
+                    ready.append((key, events & key.events))
554
+
555
+            return ready
556
+
557
+        def close(self):
558
+            self._devpoll.close()
559
+            super(DevpollSelector, self).close()
560
+
561
+    __all__.append('DevpollSelector')
562
+
563
+
564
+if hasattr(select, "kqueue"):
565
+    class KqueueSelector(BaseSelector):
566
+        """ Kqueue / Kevent-based selector """
567
+        def __init__(self):
568
+            super(KqueueSelector, self).__init__()
569
+            self._kqueue = select.kqueue()
570
+
571
+        def fileno(self):
572
+            return self._kqueue.fileno()
573
+
574
+        def register(self, fileobj, events, data=None):
575
+            key = super(KqueueSelector, self).register(fileobj, events, data)
576
+            if events & EVENT_READ:
577
+                kevent = select.kevent(key.fd,
578
+                                       select.KQ_FILTER_READ,
579
+                                       select.KQ_EV_ADD)
580
+
581
+                _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
582
+
583
+            if events & EVENT_WRITE:
584
+                kevent = select.kevent(key.fd,
585
+                                       select.KQ_FILTER_WRITE,
586
+                                       select.KQ_EV_ADD)
587
+
588
+                _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
589
+
590
+            return key
591
+
592
+        def unregister(self, fileobj):
593
+            key = super(KqueueSelector, self).unregister(fileobj)
594
+            if key.events & EVENT_READ:
595
+                kevent = select.kevent(key.fd,
596
+                                       select.KQ_FILTER_READ,
597
+                                       select.KQ_EV_DELETE)
598
+                try:
599
+                    _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
600
+                except SelectorError:
601
+                    pass
602
+            if key.events & EVENT_WRITE:
603
+                kevent = select.kevent(key.fd,
604
+                                       select.KQ_FILTER_WRITE,
605
+                                       select.KQ_EV_DELETE)
606
+                try:
607
+                    _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
608
+                except SelectorError:
609
+                    pass
610
+
611
+            return key
612
+
613
+        def select(self, timeout=None):
614
+            if timeout is not None:
615
+                timeout = max(timeout, 0)
616
+
617
+            max_events = len(self._fd_to_key) * 2
618
+            ready_fds = {}
619
+
620
+            kevent_list = _syscall_wrapper(self._kqueue.control, True,
621
+                                           None, max_events, timeout)
622
+
623
+            for kevent in kevent_list:
624
+                fd = kevent.ident
625
+                event_mask = kevent.filter
626
+                events = 0
627
+                if event_mask == select.KQ_FILTER_READ:
628
+                    events |= EVENT_READ
629
+                if event_mask == select.KQ_FILTER_WRITE:
630
+                    events |= EVENT_WRITE
631
+
632
+                key = self._key_from_fd(fd)
633
+                if key:
634
+                    if key.fd not in ready_fds:
635
+                        ready_fds[key.fd] = (key, events & key.events)
636
+                    else:
637
+                        old_events = ready_fds[key.fd][1]
638
+                        ready_fds[key.fd] = (key, (events | old_events) & key.events)
639
+
640
+            return list(ready_fds.values())
641
+
642
+        def close(self):
643
+            self._kqueue.close()
644
+            super(KqueueSelector, self).close()
645
+
646
+    __all__.append('KqueueSelector')
647
+
648
+
649
+# Choose the best implementation, roughly:
650
+# kqueue == epoll == devpoll > poll > select.
651
+# select() also can't accept a FD > FD_SETSIZE (usually around 1024)
652
+if 'KqueueSelector' in globals():  # Platform-specific: Mac OS and BSD
653
+    DefaultSelector = KqueueSelector
654
+if 'DevpollSelector' in globals():
655
+    DefaultSelector = DevpollSelector
656
+elif 'EpollSelector' in globals():  # Platform-specific: Linux
657
+    DefaultSelector = EpollSelector
658
+elif 'PollSelector' in globals():  # Platform-specific: Linux
659
+    DefaultSelector = PollSelector
660
+elif 'SelectSelector' in globals():  # Platform-specific: Windows
661
+    DefaultSelector = SelectSelector
662
+else:  # Platform-specific: AppEngine
663
+    def no_selector(_):
664
+        raise ValueError("Platform does not have a selector")
665
+    DefaultSelector = no_selector
666
+    HAS_SELECT = False
... ...
@@ -1,5 +1,5 @@
1 1
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
2
-# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com>
2
+# (c) 2015, 2017 Toshio Kuratomi <tkuratomi@ansible.com>
3 3
 #
4 4
 # This file is part of Ansible
5 5
 #
... ...
@@ -19,16 +19,14 @@ from __future__ import (absolute_import, division, print_function)
19 19
 __metaclass__ = type
20 20
 
21 21
 import os
22
-import select
23 22
 import shutil
24 23
 import subprocess
25 24
 import fcntl
26 25
 import getpass
27 26
 
28
-from ansible.compat.six import text_type, binary_type
29
-
30 27
 import ansible.constants as C
31
-
28
+from ansible.compat import selectors
29
+from ansible.compat.six import text_type, binary_type
32 30
 from ansible.errors import AnsibleError, AnsibleFileNotFound
33 31
 from ansible.module_utils._text import to_bytes, to_native
34 32
 from ansible.plugins.connection import ConnectionBase
... ...
@@ -90,21 +88,31 @@ class Connection(ConnectionBase):
90 90
         if self._play_context.prompt and sudoable:
91 91
             fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
92 92
             fcntl.fcntl(p.stderr, fcntl.F_SETFL, fcntl.fcntl(p.stderr, fcntl.F_GETFL) | os.O_NONBLOCK)
93
+            selector = selectors.DefaultSelector()
94
+            selector.register(p.stdout, selectors.EVENT_READ)
95
+            selector.register(p.stderr, selectors.EVENT_READ)
96
+
93 97
             become_output = b''
94
-            while not self.check_become_success(become_output) and not self.check_password_prompt(become_output):
95
-
96
-                rfd, wfd, efd = select.select([p.stdout, p.stderr], [], [p.stdout, p.stderr], self._play_context.timeout)
97
-                if p.stdout in rfd:
98
-                    chunk = p.stdout.read()
99
-                elif p.stderr in rfd:
100
-                    chunk = p.stderr.read()
101
-                else:
102
-                    stdout, stderr = p.communicate()
103
-                    raise AnsibleError('timeout waiting for privilege escalation password prompt:\n' + to_native(become_output))
104
-                if not chunk:
105
-                    stdout, stderr = p.communicate()
106
-                    raise AnsibleError('privilege output closed while waiting for password prompt:\n' + to_native(become_output))
107
-                become_output += chunk
98
+            try:
99
+                while not self.check_become_success(become_output) and not self.check_password_prompt(become_output):
100
+                    events = selector.select(self._play_context.timeout)
101
+                    if not events:
102
+                        stdout, stderr = p.communicate()
103
+                        raise AnsibleError('timeout waiting for privilege escalation password prompt:\n' + to_native(become_output))
104
+
105
+                    for key, event in events:
106
+                        if key.fileobj == p.stdout:
107
+                            chunk = p.stdout.read()
108
+                        elif key.fileobj == p.stderr:
109
+                            chunk = p.stderr.read()
110
+
111
+                    if not chunk:
112
+                        stdout, stderr = p.communicate()
113
+                        raise AnsibleError('privilege output closed while waiting for password prompt:\n' + to_native(become_output))
114
+                    become_output += chunk
115
+            finally:
116
+                selector.close()
117
+
108 118
             if not self.check_become_success(become_output):
109 119
                 p.stdin.write(to_bytes(self._play_context.become_pass, errors='surrogate_or_strict') + b'\n')
110 120
             fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK)
... ...
@@ -1,5 +1,6 @@
1 1
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
2 2
 # Copyright 2015 Abhijit Menon-Sen <ams@2ndQuadrant.com>
3
+# Copyright 2017 Toshio Kuratomi <tkuratomi@ansible.com>
3 4
 #
4 5
 # This file is part of Ansible
5 6
 #
... ...
@@ -24,11 +25,11 @@ import fcntl
24 24
 import hashlib
25 25
 import os
26 26
 import pty
27
-import select
28 27
 import subprocess
29 28
 import time
30 29
 
31 30
 from ansible import constants as C
31
+from ansible.compat import selectors
32 32
 from ansible.compat.six import PY3, text_type, binary_type
33 33
 from ansible.compat.six.moves import shlex_quote
34 34
 from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
... ...
@@ -443,148 +444,158 @@ class Connection(ConnectionBase):
443 443
         # they will race each other when we can't connect, and the connect
444 444
         # timeout usually fails
445 445
         timeout = 2 + self._play_context.timeout
446
-        rpipes = [p.stdout, p.stderr]
447
-        for fd in rpipes:
446
+        for fd in (p.stdout, p.stderr):
448 447
             fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
449 448
 
450
-        # If we can send initial data without waiting for anything, we do so
451
-        # before we call select.
449
+        ### TODO: bcoca would like to use SelectSelector() when open
450
+        # filehandles is low, then switch to more efficient ones when higher.
451
+        # select is faster when filehandles is low.
452
+        selector = selectors.DefaultSelector()
453
+        selector.register(p.stdout, selectors.EVENT_READ)
454
+        selector.register(p.stderr, selectors.EVENT_READ)
452 455
 
456
+        # If we can send initial data without waiting for anything, we do so
457
+        # before we start polling
453 458
         if states[state] == 'ready_to_send' and in_data:
454 459
             self._send_initial_data(stdin, in_data)
455 460
             state += 1
456 461
 
457
-        while True:
458
-            rfd, wfd, efd = select.select(rpipes, [], [], timeout)
462
+        try:
463
+            while True:
464
+                events = selector.select(timeout)
465
+
466
+                # We pay attention to timeouts only while negotiating a prompt.
467
+
468
+                if not events:
469
+                    # We timed out
470
+                    if state <= states.index('awaiting_escalation'):
471
+                        # If the process has already exited, then it's not really a
472
+                        # timeout; we'll let the normal error handling deal with it.
473
+                        if p.poll() is not None:
474
+                            break
475
+                        self._terminate_process(p)
476
+                        raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, to_native(b_stdout)))
477
+
478
+                # Read whatever output is available on stdout and stderr, and stop
479
+                # listening to the pipe if it's been closed.
480
+
481
+                for key, event in events:
482
+                    if key.fileobj == p.stdout:
483
+                        b_chunk = p.stdout.read()
484
+                        if b_chunk == b'':
485
+                            # stdout has been closed, stop watching it
486
+                            selector.unregister(p.stdout)
487
+                            # When ssh has ControlMaster (+ControlPath/Persist) enabled, the
488
+                            # first connection goes into the background and we never see EOF
489
+                            # on stderr. If we see EOF on stdout, lower the select timeout
490
+                            # to reduce the time wasted selecting on stderr if we observe
491
+                            # that the process has not yet existed after this EOF. Otherwise
492
+                            # we may spend a long timeout period waiting for an EOF that is
493
+                            # not going to arrive until the persisted connection closes.
494
+                            timeout = 1
495
+                        b_tmp_stdout += b_chunk
496
+                        display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk)))
497
+                    elif key.fileobj == p.stderr:
498
+                        b_chunk = p.stderr.read()
499
+                        if b_chunk == b'':
500
+                            # stderr has been closed, stop watching it
501
+                            selector.unregister(p.stderr)
502
+                        b_tmp_stderr += b_chunk
503
+                        display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk)))
504
+
505
+                # We examine the output line-by-line until we have negotiated any
506
+                # privilege escalation prompt and subsequent success/error message.
507
+                # Afterwards, we can accumulate output without looking at it.
508
+
509
+                if state < states.index('ready_to_send'):
510
+                    if b_tmp_stdout:
511
+                        b_output, b_unprocessed = self._examine_output('stdout', states[state], b_tmp_stdout, sudoable)
512
+                        b_stdout += b_output
513
+                        b_tmp_stdout = b_unprocessed
514
+
515
+                    if b_tmp_stderr:
516
+                        b_output, b_unprocessed = self._examine_output('stderr', states[state], b_tmp_stderr, sudoable)
517
+                        b_stderr += b_output
518
+                        b_tmp_stderr = b_unprocessed
519
+                else:
520
+                    b_stdout += b_tmp_stdout
521
+                    b_stderr += b_tmp_stderr
522
+                    b_tmp_stdout = b_tmp_stderr = b''
523
+
524
+                # If we see a privilege escalation prompt, we send the password.
525
+                # (If we're expecting a prompt but the escalation succeeds, we
526
+                # didn't need the password and can carry on regardless.)
527
+
528
+                if states[state] == 'awaiting_prompt':
529
+                    if self._flags['become_prompt']:
530
+                        display.debug('Sending become_pass in response to prompt')
531
+                        stdin.write(to_bytes(self._play_context.become_pass) + b'\n')
532
+                        self._flags['become_prompt'] = False
533
+                        state += 1
534
+                    elif self._flags['become_success']:
535
+                        state += 1
536
+
537
+                # We've requested escalation (with or without a password), now we
538
+                # wait for an error message or a successful escalation.
539
+
540
+                if states[state] == 'awaiting_escalation':
541
+                    if self._flags['become_success']:
542
+                        display.debug('Escalation succeeded')
543
+                        self._flags['become_success'] = False
544
+                        state += 1
545
+                    elif self._flags['become_error']:
546
+                        display.debug('Escalation failed')
547
+                        self._terminate_process(p)
548
+                        self._flags['become_error'] = False
549
+                        raise AnsibleError('Incorrect %s password' % self._play_context.become_method)
550
+                    elif self._flags['become_nopasswd_error']:
551
+                        display.debug('Escalation requires password')
552
+                        self._terminate_process(p)
553
+                        self._flags['become_nopasswd_error'] = False
554
+                        raise AnsibleError('Missing %s password' % self._play_context.become_method)
555
+                    elif self._flags['become_prompt']:
556
+                        # This shouldn't happen, because we should see the "Sorry,
557
+                        # try again" message first.
558
+                        display.debug('Escalation prompt repeated')
559
+                        self._terminate_process(p)
560
+                        self._flags['become_prompt'] = False
561
+                        raise AnsibleError('Incorrect %s password' % self._play_context.become_method)
562
+
563
+                # Once we're sure that the privilege escalation prompt, if any, has
564
+                # been dealt with, we can send any initial data and start waiting
565
+                # for output.
566
+
567
+                if states[state] == 'ready_to_send':
568
+                    if in_data:
569
+                        self._send_initial_data(stdin, in_data)
570
+                    state += 1
459 571
 
460
-            # We pay attention to timeouts only while negotiating a prompt.
572
+                # Now we're awaiting_exit: has the child process exited? If it has,
573
+                # and we've read all available output from it, we're done.
461 574
 
462
-            if not rfd:
463
-                if state <= states.index('awaiting_escalation'):
464
-                    # If the process has already exited, then it's not really a
465
-                    # timeout; we'll let the normal error handling deal with it.
466
-                    if p.poll() is not None:
575
+                if p.poll() is not None:
576
+                    if not selector.get_map() or not events:
467 577
                         break
468
-                    self._terminate_process(p)
469
-                    raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, to_native(b_stdout)))
470
-
471
-            # Read whatever output is available on stdout and stderr, and stop
472
-            # listening to the pipe if it's been closed.
473
-
474
-            if p.stdout in rfd:
475
-                b_chunk = p.stdout.read()
476
-                if b_chunk == b'':
477
-                    rpipes.remove(p.stdout)
478
-                    # When ssh has ControlMaster (+ControlPath/Persist) enabled, the
479
-                    # first connection goes into the background and we never see EOF
480
-                    # on stderr. If we see EOF on stdout, lower the select timeout
481
-                    # to reduce the time wasted selecting on stderr if we observe
482
-                    # that the process has not yet existed after this EOF. Otherwise
483
-                    # we may spend a long timeout period waiting for an EOF that is
484
-                    # not going to arrive until the persisted connection closes.
485
-                    timeout = 1
486
-                b_tmp_stdout += b_chunk
487
-                display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk)))
488
-
489
-            if p.stderr in rfd:
490
-                b_chunk = p.stderr.read()
491
-                if b_chunk == b'':
492
-                    rpipes.remove(p.stderr)
493
-                b_tmp_stderr += b_chunk
494
-                display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk)))
495
-
496
-            # We examine the output line-by-line until we have negotiated any
497
-            # privilege escalation prompt and subsequent success/error message.
498
-            # Afterwards, we can accumulate output without looking at it.
499
-
500
-            if state < states.index('ready_to_send'):
501
-                if b_tmp_stdout:
502
-                    b_output, b_unprocessed = self._examine_output('stdout', states[state], b_tmp_stdout, sudoable)
503
-                    b_stdout += b_output
504
-                    b_tmp_stdout = b_unprocessed
505
-
506
-                if b_tmp_stderr:
507
-                    b_output, b_unprocessed = self._examine_output('stderr', states[state], b_tmp_stderr, sudoable)
508
-                    b_stderr += b_output
509
-                    b_tmp_stderr = b_unprocessed
510
-            else:
511
-                b_stdout += b_tmp_stdout
512
-                b_stderr += b_tmp_stderr
513
-                b_tmp_stdout = b_tmp_stderr = b''
514
-
515
-            # If we see a privilege escalation prompt, we send the password.
516
-            # (If we're expecting a prompt but the escalation succeeds, we
517
-            # didn't need the password and can carry on regardless.)
518
-
519
-            if states[state] == 'awaiting_prompt':
520
-                if self._flags['become_prompt']:
521
-                    display.debug('Sending become_pass in response to prompt')
522
-                    stdin.write(to_bytes(self._play_context.become_pass) + b'\n')
523
-                    self._flags['become_prompt'] = False
524
-                    state += 1
525
-                elif self._flags['become_success']:
526
-                    state += 1
578
+                    # We should not see further writes to the stdout/stderr file
579
+                    # descriptors after the process has closed, set the select
580
+                    # timeout to gather any last writes we may have missed.
581
+                    timeout = 0
582
+                    continue
527 583
 
528
-            # We've requested escalation (with or without a password), now we
529
-            # wait for an error message or a successful escalation.
584
+                # If the process has not yet exited, but we've already read EOF from
585
+                # its stdout and stderr (and thus no longer watching any file
586
+                # descriptors), we can just wait for it to exit.
530 587
 
531
-            if states[state] == 'awaiting_escalation':
532
-                if self._flags['become_success']:
533
-                    display.debug('Escalation succeeded')
534
-                    self._flags['become_success'] = False
535
-                    state += 1
536
-                elif self._flags['become_error']:
537
-                    display.debug('Escalation failed')
538
-                    self._terminate_process(p)
539
-                    self._flags['become_error'] = False
540
-                    raise AnsibleError('Incorrect %s password' % self._play_context.become_method)
541
-                elif self._flags['become_nopasswd_error']:
542
-                    display.debug('Escalation requires password')
543
-                    self._terminate_process(p)
544
-                    self._flags['become_nopasswd_error'] = False
545
-                    raise AnsibleError('Missing %s password' % self._play_context.become_method)
546
-                elif self._flags['become_prompt']:
547
-                    # This shouldn't happen, because we should see the "Sorry,
548
-                    # try again" message first.
549
-                    display.debug('Escalation prompt repeated')
550
-                    self._terminate_process(p)
551
-                    self._flags['become_prompt'] = False
552
-                    raise AnsibleError('Incorrect %s password' % self._play_context.become_method)
553
-
554
-            # Once we're sure that the privilege escalation prompt, if any, has
555
-            # been dealt with, we can send any initial data and start waiting
556
-            # for output.
557
-
558
-            if states[state] == 'ready_to_send':
559
-                if in_data:
560
-                    self._send_initial_data(stdin, in_data)
561
-                state += 1
562
-
563
-            # Now we're awaiting_exit: has the child process exited? If it has,
564
-            # and we've read all available output from it, we're done.
565
-
566
-            if p.poll() is not None:
567
-                if not rpipes or not rfd:
588
+                elif not selector.get_map():
589
+                    p.wait()
568 590
                     break
569
-                # We should not see further writes to the stdout/stderr file
570
-                # descriptors after the process has closed, set the select
571
-                # timeout to gather any last writes we may have missed.
572
-                timeout = 0
573
-                continue
574
-
575
-            # If the process has not yet exited, but we've already read EOF from
576
-            # its stdout and stderr (and thus removed both from rpipes), we can
577
-            # just wait for it to exit.
578
-
579
-            elif not rpipes:
580
-                p.wait()
581
-                break
582
-
583
-            # Otherwise there may still be outstanding data to read.
584 591
 
585
-        # close stdin after process is terminated and stdout/stderr are read
586
-        # completely (see also issue #848)
587
-        stdin.close()
592
+                # Otherwise there may still be outstanding data to read.
593
+        finally:
594
+            selector.close()
595
+            # close stdin after process is terminated and stdout/stderr are read
596
+            # completely (see also issue #848)
597
+            stdin.close()
588 598
 
589 599
         if C.HOST_KEY_CHECKING:
590 600
             if cmd[0] == b"sshpass" and p.returncode == 6:
... ...
@@ -18,8 +18,8 @@ setup(name='ansible',
18 18
       author_email='info@ansible.com',
19 19
       url='http://ansible.com/',
20 20
       license='GPLv3',
21
-      # Ansible will also make use of a system copy of python-six if installed but use a
22
-      # Bundled copy if it's not.
21
+      # Ansible will also make use of a system copy of python-six and
22
+      # python-selectors2 if installed but use a Bundled copy if it's not.
23 23
       install_requires=['paramiko', 'jinja2', "PyYAML", 'setuptools', 'pycrypto >= 2.6'],
24 24
       package_dir={ '': 'lib' },
25 25
       packages=find_packages('lib'),
... ...
@@ -7,6 +7,7 @@ metaclass2=$(find ./lib/ansible -path ./lib/ansible/modules -prune \
7 7
         -o -path ./lib/ansible/modules/__init__.py \
8 8
         -o -path ./lib/ansible/module_utils -prune \
9 9
         -o -path ./lib/ansible/compat/six/_six.py -prune \
10
+        -o -path ./lib/ansible/compat/selectors/_selectors2.py -prune \
10 11
         -o -path ./lib/ansible/utils/module_docs_fragments -prune \
11 12
         -o -name '*.py' -exec grep -HL '__metaclass__ = type' '{}' '+')
12 13
 
... ...
@@ -14,6 +15,7 @@ future2=$(find ./lib/ansible -path ./lib/ansible/modules -prune \
14 14
         -o -path ./lib/ansible/modules/__init__.py \
15 15
         -o -path ./lib/ansible/module_utils -prune \
16 16
         -o -path ./lib/ansible/compat/six/_six.py -prune \
17
+        -o -path ./lib/ansible/compat/selectors/_selectors2.py -prune \
17 18
         -o -path ./lib/ansible/utils/module_docs_fragments -prune \
18 19
         -o -name '*.py' -exec grep -HL 'from __future__ import (absolute_import, division, print_function)' '{}' '+')
19 20
 
... ...
@@ -23,10 +23,13 @@ __metaclass__ = type
23 23
 
24 24
 from io import StringIO
25 25
 
26
+import pytest
27
+
26 28
 from ansible.compat.tests import unittest
27 29
 from ansible.compat.tests.mock import patch, MagicMock
28 30
 
29 31
 from ansible import constants as C
32
+from ansible.compat.selectors import SelectorKey, EVENT_READ
30 33
 from ansible.compat.six.moves import shlex_quote
31 34
 from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
32 35
 from ansible.playbook.play_context import PlayContext
... ...
@@ -83,82 +86,6 @@ class TestConnectionBaseClass(unittest.TestCase):
83 83
         res, stdout, stderr = conn._exec_command('ssh')
84 84
         res, stdout, stderr = conn._exec_command('ssh', 'this is some data')
85 85
 
86
-    @patch('select.select')
87
-    @patch('fcntl.fcntl')
88
-    @patch('os.write')
89
-    @patch('os.close')
90
-    @patch('pty.openpty')
91
-    @patch('subprocess.Popen')
92
-    def test_plugins_connection_ssh__run(self, mock_Popen, mock_openpty, mock_osclose, mock_oswrite, mock_fcntl, mock_select):
93
-        pc = PlayContext()
94
-        new_stdin = StringIO()
95
-
96
-        conn = ssh.Connection(pc, new_stdin)
97
-        conn._send_initial_data = MagicMock()
98
-        conn._examine_output = MagicMock()
99
-        conn._terminate_process = MagicMock()
100
-        conn.sshpass_pipe = [MagicMock(), MagicMock()]
101
-
102
-        mock_popen_res = MagicMock()
103
-        mock_popen_res.poll   = MagicMock()
104
-        mock_popen_res.wait   = MagicMock()
105
-        mock_popen_res.stdin  = MagicMock()
106
-        mock_popen_res.stdin.fileno.return_value = 1000
107
-        mock_popen_res.stdout = MagicMock()
108
-        mock_popen_res.stdout.fileno.return_value = 1001
109
-        mock_popen_res.stderr = MagicMock()
110
-        mock_popen_res.stderr.fileno.return_value = 1002
111
-        mock_popen_res.return_code = 0
112
-        mock_Popen.return_value = mock_popen_res
113
-
114
-        def _mock_select(rlist, wlist, elist, timeout=None):
115
-            rvals = []
116
-            if mock_popen_res.stdin in rlist:
117
-                rvals.append(mock_popen_res.stdin)
118
-            if mock_popen_res.stderr in rlist:
119
-                rvals.append(mock_popen_res.stderr)
120
-            return (rvals, [], [])
121
-
122
-        mock_select.side_effect = _mock_select
123
-
124
-        mock_popen_res.stdout.read.side_effect = [b"some data", b""]
125
-        mock_popen_res.stderr.read.side_effect = [b""]
126
-        conn._run("ssh", "this is input data")
127
-
128
-        # test with a password set to trigger the sshpass write
129
-        pc.password = '12345'
130
-        mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
131
-        mock_popen_res.stderr.read.side_effect = [b""]
132
-        conn._run(["ssh", "is", "a", "cmd"], "this is more data")
133
-
134
-        # test with password prompting enabled
135
-        pc.password = None
136
-        pc.prompt = True
137
-        mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
138
-        mock_popen_res.stderr.read.side_effect = [b""]
139
-        conn._run("ssh", "this is input data")
140
-
141
-        # test with some become settings
142
-        pc.prompt = False
143
-        pc.become = True
144
-        pc.success_key = 'BECOME-SUCCESS-abcdefg'
145
-        mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
146
-        mock_popen_res.stderr.read.side_effect = [b""]
147
-        conn._run("ssh", "this is input data")
148
-
149
-        # simulate no data input
150
-        mock_openpty.return_value = (98, 99)
151
-        mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
152
-        mock_popen_res.stderr.read.side_effect = [b""]
153
-        conn._run("ssh", "")
154
-
155
-        # simulate no data input but Popen using new pty's fails
156
-        mock_Popen.return_value = None
157
-        mock_Popen.side_effect = [OSError(), mock_popen_res]
158
-        mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
159
-        mock_popen_res.stderr.read.side_effect = [b""]
160
-        conn._run("ssh", "")
161
-
162 86
     def test_plugins_connection_ssh__examine_output(self):
163 87
         pc = PlayContext()
164 88
         new_stdin = StringIO()
... ...
@@ -341,7 +268,6 @@ class TestConnectionBaseClass(unittest.TestCase):
341 341
         conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
342 342
         conn._run.assert_called_with('some command to run', expected_in_data, checkrc=False)
343 343
 
344
-
345 344
         # test that a non-zero rc raises an error
346 345
         conn._run.return_value = (1, 'stdout', 'some errors')
347 346
         self.assertRaises(AnsibleError, conn.put_file, '/path/to/bad/file', '/remote/path/to/file')
... ...
@@ -398,3 +324,215 @@ class TestConnectionBaseClass(unittest.TestCase):
398 398
         # test that a non-zero rc raises an error
399 399
         conn._run.return_value = (1, 'stdout', 'some errors')
400 400
         self.assertRaises(AnsibleError, conn.fetch_file, '/path/to/bad/file', '/remote/path/to/file')
401
+
402
+
403
+class MockSelector(object):
404
+    def __init__(self):
405
+        self.files_watched = 0
406
+        self.register = MagicMock(side_effect=self._register)
407
+        self.unregister = MagicMock(side_effect=self._unregister)
408
+        self.close = MagicMock()
409
+        self.get_map = MagicMock(side_effect=self._get_map)
410
+        self.select = MagicMock()
411
+
412
+    def _register(self, *args, **kwargs):
413
+        self.files_watched += 1
414
+
415
+    def _unregister(self, *args, **kwargs):
416
+        self.files_watched -= 1
417
+
418
+    def _get_map(self, *args, **kwargs):
419
+        return self.files_watched
420
+
421
+
422
+@pytest.fixture
423
+def mock_run_env(request, mocker):
424
+    pc = PlayContext()
425
+    new_stdin = StringIO()
426
+
427
+    conn = ssh.Connection(pc, new_stdin)
428
+    conn._send_initial_data = MagicMock()
429
+    conn._examine_output = MagicMock()
430
+    conn._terminate_process = MagicMock()
431
+    conn.sshpass_pipe = [MagicMock(), MagicMock()]
432
+
433
+    request.cls.pc = pc
434
+    request.cls.conn = conn
435
+
436
+    mock_popen_res = MagicMock()
437
+    mock_popen_res.poll = MagicMock()
438
+    mock_popen_res.wait = MagicMock()
439
+    mock_popen_res.stdin = MagicMock()
440
+    mock_popen_res.stdin.fileno.return_value = 1000
441
+    mock_popen_res.stdout = MagicMock()
442
+    mock_popen_res.stdout.fileno.return_value = 1001
443
+    mock_popen_res.stderr = MagicMock()
444
+    mock_popen_res.stderr.fileno.return_value = 1002
445
+    mock_popen_res.returncode = 0
446
+    request.cls.mock_popen_res = mock_popen_res
447
+
448
+    mock_popen = mocker.patch('subprocess.Popen', return_value=mock_popen_res)
449
+    request.cls.mock_popen = mock_popen
450
+
451
+    request.cls.mock_selector = MockSelector()
452
+    mocker.patch('ansible.compat.selectors.DefaultSelector', lambda: request.cls.mock_selector)
453
+
454
+    request.cls.mock_openpty = mocker.patch('pty.openpty')
455
+
456
+    mocker.patch('fcntl.fcntl')
457
+    mocker.patch('os.write')
458
+    mocker.patch('os.close')
459
+
460
+
461
+@pytest.mark.usefixtures('mock_run_env')
462
+class TestSSHConnectionRun(object):
463
+    # FIXME:
464
+    # These tests are little more than a smoketest.  Need to enhance them
465
+    # a bit to check that they're calling the relevant functions and making
466
+    # complete coverage of the code paths
467
+    def test_no_escalation(self):
468
+        self.mock_popen_res.stdout.read.side_effect = [b"my_stdout\n", b"second_line"]
469
+        self.mock_popen_res.stderr.read.side_effect = [b"my_stderr"]
470
+        self.mock_selector.select.side_effect = [
471
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
472
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
473
+            [(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ)],
474
+            []]
475
+        self.mock_selector.get_map.side_effect = lambda: True
476
+
477
+        return_code, b_stdout, b_stderr = self.conn._run("ssh", "this is input data")
478
+        assert return_code == 0
479
+        assert b_stdout == b'my_stdout\nsecond_line'
480
+        assert b_stderr == b'my_stderr'
481
+        assert self.mock_selector.register.called is True
482
+        assert self.mock_selector.register.call_count == 2
483
+        assert self.conn._send_initial_data.called is True
484
+        assert self.conn._send_initial_data.call_count == 1
485
+        assert self.conn._send_initial_data.call_args[0][1] == 'this is input data'
486
+
487
+    def test_with_password(self):
488
+        # test with a password set to trigger the sshpass write
489
+        self.pc.password = '12345'
490
+        self.mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
491
+        self.mock_popen_res.stderr.read.side_effect = [b""]
492
+        self.mock_selector.select.side_effect = [
493
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
494
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
495
+            [(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ)],
496
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
497
+            []]
498
+        self.mock_selector.get_map.side_effect = lambda: True
499
+
500
+        return_code, b_stdout, b_stderr = self.conn._run(["ssh", "is", "a", "cmd"], "this is more data")
501
+        assert return_code == 0
502
+        assert b_stdout == b'some data'
503
+        assert b_stderr == b''
504
+        assert self.mock_selector.register.called is True
505
+        assert self.mock_selector.register.call_count == 2
506
+        assert self.conn._send_initial_data.called is True
507
+        assert self.conn._send_initial_data.call_count == 1
508
+        assert self.conn._send_initial_data.call_args[0][1] == 'this is more data'
509
+
510
+    def _password_with_prompt_examine_output(self, sourice, state, b_chunk, sudoable):
511
+        if state == 'awaiting_prompt':
512
+            self.conn._flags['become_prompt'] = True
513
+        elif state == 'awaiting_escalation':
514
+            self.conn._flags['become_success'] = True
515
+        return (b'', b'')
516
+
517
+    def test_pasword_with_prompt(self):
518
+        # test with password prompting enabled
519
+        self.pc.password = None
520
+        self.pc.prompt = b'Password:'
521
+        self.conn._examine_output.side_effect = self._password_with_prompt_examine_output
522
+        self.mock_popen_res.stdout.read.side_effect = [b"Password:", b"Success", b""]
523
+        self.mock_popen_res.stderr.read.side_effect = [b""]
524
+        self.mock_selector.select.side_effect = [
525
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
526
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
527
+            [(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ),
528
+             (SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
529
+            []]
530
+        self.mock_selector.get_map.side_effect = lambda: True
531
+
532
+        return_code, b_stdout, b_stderr = self.conn._run("ssh", "this is input data")
533
+        assert return_code == 0
534
+        assert b_stdout == b''
535
+        assert b_stderr == b''
536
+        assert self.mock_selector.register.called is True
537
+        assert self.mock_selector.register.call_count == 2
538
+        assert self.conn._send_initial_data.called is True
539
+        assert self.conn._send_initial_data.call_count == 1
540
+        assert self.conn._send_initial_data.call_args[0][1] == 'this is input data'
541
+
542
+    def test_pasword_with_become(self):
543
+        # test with some become settings
544
+        self.pc.prompt = b'Password:'
545
+        self.pc.become = True
546
+        self.pc.success_key = 'BECOME-SUCCESS-abcdefg'
547
+        self.conn._examine_output.side_effect = self._password_with_prompt_examine_output
548
+        self.mock_popen_res.stdout.read.side_effect = [b"Password:", b"BECOME-SUCCESS-abcdefg", b"abc"]
549
+        self.mock_popen_res.stderr.read.side_effect = [b"123"]
550
+        self.mock_selector.select.side_effect = [
551
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
552
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
553
+            [(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ)],
554
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
555
+            []]
556
+        self.mock_selector.get_map.side_effect = lambda: True
557
+
558
+        return_code, b_stdout, b_stderr = self.conn._run("ssh", "this is input data")
559
+        assert return_code == 0
560
+        assert b_stdout == b'abc'
561
+        assert b_stderr == b'123'
562
+        assert self.mock_selector.register.called is True
563
+        assert self.mock_selector.register.call_count == 2
564
+        assert self.conn._send_initial_data.called is True
565
+        assert self.conn._send_initial_data.call_count == 1
566
+        assert self.conn._send_initial_data.call_args[0][1] == 'this is input data'
567
+
568
+    def test_pasword_without_data(self):
569
+        # simulate no data input
570
+        self.mock_openpty.return_value = (98, 99)
571
+        self.mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
572
+        self.mock_popen_res.stderr.read.side_effect = [b""]
573
+        self.mock_selector.select.side_effect = [
574
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
575
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
576
+            [(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ)],
577
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
578
+            []]
579
+        self.mock_selector.get_map.side_effect = lambda: True
580
+
581
+        return_code, b_stdout, b_stderr = self.conn._run("ssh", "")
582
+        assert return_code == 0
583
+        assert b_stdout == b'some data'
584
+        assert b_stderr == b''
585
+        assert self.mock_selector.register.called is True
586
+        assert self.mock_selector.register.call_count == 2
587
+        assert self.conn._send_initial_data.called is False
588
+
589
+    def test_pasword_without_data(self):
590
+        # simulate no data input but Popen using new pty's fails
591
+        self.mock_popen.return_value = None
592
+        self.mock_popen.side_effect = [OSError(), self.mock_popen_res]
593
+
594
+        # simulate no data input
595
+        self.mock_openpty.return_value = (98, 99)
596
+        self.mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
597
+        self.mock_popen_res.stderr.read.side_effect = [b""]
598
+        self.mock_selector.select.side_effect = [
599
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
600
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
601
+            [(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ)],
602
+            [(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
603
+            []]
604
+        self.mock_selector.get_map.side_effect = lambda: True
605
+
606
+        return_code, b_stdout, b_stderr = self.conn._run("ssh", "")
607
+        assert return_code == 0
608
+        assert b_stdout == b'some data'
609
+        assert b_stderr == b''
610
+        assert self.mock_selector.register.called is True
611
+        assert self.mock_selector.register.call_count == 2
612
+        assert self.conn._send_initial_data.called is False