Browse code

Bug fixes and cleanup for ansible-test. (#45991)

* Remove unused imports.
* Clean up ConfigParser usage in ansible-test.
* Fix bare except statements in ansible-test.
* Miscellaneous cleanup from PyCharm inspections.
* Enable pylint no-self-use for ansible-test.
* Remove obsolete pylint ignores for Python 3.7.
* Fix shellcheck issuers under newer shellcheck.
* Use newer path for ansible-test.
* Fix issues in code-smell tests.

Matt Clay authored on 2018/09/22 03:38:22
Showing 24 changed files
... ...
@@ -122,7 +122,7 @@ ifneq ($(REPOTAG),)
122 122
 endif
123 123
 
124 124
 # ansible-test parameters
125
-ANSIBLE_TEST ?= test/runner/ansible-test
125
+ANSIBLE_TEST ?= bin/ansible-test
126 126
 TEST_FLAGS ?=
127 127
 
128 128
 # ansible-test units parameters (make test / make test-py3)
... ...
@@ -25,7 +25,6 @@ NOTE: Running ansible-test with the --tox option or inside a virtual environment
25 25
 
26 26
 from __future__ import absolute_import, print_function
27 27
 
28
-import errno
29 28
 import json
30 29
 import os
31 30
 import sys
... ...
@@ -203,10 +202,10 @@ def find_executable(executable):
203 203
     :rtype: str
204 204
     """
205 205
     self = os.path.abspath(__file__)
206
-    path = os.environ.get('PATH', os.defpath)
206
+    path = os.environ.get('PATH', os.path.defpath)
207 207
     seen_dirs = set()
208 208
 
209
-    for path_dir in path.split(os.pathsep):
209
+    for path_dir in path.split(os.path.pathsep):
210 210
         if path_dir in seen_dirs:
211 211
             continue
212 212
 
... ...
@@ -25,8 +25,8 @@ def ansible_environment(args, color=True):
25 25
 
26 26
     ansible_path = os.path.join(os.getcwd(), 'bin')
27 27
 
28
-    if not path.startswith(ansible_path + os.pathsep):
29
-        path = ansible_path + os.pathsep + path
28
+    if not path.startswith(ansible_path + os.path.pathsep):
29
+        path = ansible_path + os.path.pathsep + path
30 30
 
31 31
     if isinstance(args, IntegrationConfig):
32 32
         ansible_config = 'test/integration/%s.cfg' % args.command
... ...
@@ -28,13 +28,6 @@ from lib.docker_util import (
28 28
     get_docker_container_id,
29 29
 )
30 30
 
31
-try:
32
-    # noinspection PyPep8Naming
33
-    import ConfigParser as configparser
34
-except ImportError:
35
-    # noinspection PyUnresolvedReferences
36
-    import configparser
37
-
38 31
 
39 32
 class ACMEProvider(CloudProvider):
40 33
     """ACME plugin. Sets up cloud resources for tests."""
... ...
@@ -17,6 +17,7 @@ from lib.util import (
17 17
     display,
18 18
     SubprocessError,
19 19
     is_shippable,
20
+    ConfigParser,
20 21
 )
21 22
 
22 23
 from lib.http import (
... ...
@@ -34,13 +35,6 @@ from lib.docker_util import (
34 34
     get_docker_container_id,
35 35
 )
36 36
 
37
-try:
38
-    # noinspection PyPep8Naming
39
-    import ConfigParser as configparser
40
-except ImportError:
41
-    # noinspection PyUnresolvedReferences
42
-    import configparser
43
-
44 37
 
45 38
 class CsCloudProvider(CloudProvider):
46 39
     """CloudStack cloud provider plugin. Sets up cloud resources before delegation."""
... ...
@@ -119,7 +113,7 @@ class CsCloudProvider(CloudProvider):
119 119
 
120 120
     def _setup_static(self):
121 121
         """Configure CloudStack tests for use with static configuration."""
122
-        parser = configparser.RawConfigParser()
122
+        parser = ConfigParser()
123 123
         parser.read(self.config_static_path)
124 124
 
125 125
         self.endpoint = parser.get('cloudstack', 'endpoint')
... ...
@@ -211,7 +205,7 @@ class CsCloudProvider(CloudProvider):
211 211
             containers = bridge['Containers']
212 212
             container = [containers[container] for container in containers if containers[container]['Name'] == self.DOCKER_SIMULATOR_NAME][0]
213 213
             return re.sub(r'/[0-9]+$', '', container['IPv4Address'])
214
-        except:
214
+        except Exception:
215 215
             display.error('Failed to process the following docker network inspect output:\n%s' %
216 216
                           json.dumps(networks, indent=4, sort_keys=True))
217 217
             raise
... ...
@@ -6,9 +6,7 @@ from __future__ import absolute_import, print_function
6 6
 import os
7 7
 
8 8
 from lib.util import (
9
-    ApplicationError,
10 9
     display,
11
-    is_shippable,
12 10
 )
13 11
 
14 12
 from lib.cloud import (
... ...
@@ -16,9 +14,6 @@ from lib.cloud import (
16 16
     CloudEnvironment,
17 17
 )
18 18
 
19
-from lib.core_ci import (
20
-    AnsibleCoreCI, )
21
-
22 19
 
23 20
 class GcpCloudProvider(CloudProvider):
24 21
     """GCP cloud provider plugin. Sets up cloud resources before delegation."""
... ...
@@ -1,17 +1,12 @@
1 1
 """OpenNebula plugin for integration tests."""
2 2
 
3
-import os
4
-
5 3
 from lib.cloud import (
6 4
     CloudProvider,
7 5
     CloudEnvironment
8 6
 )
9 7
 
10 8
 from lib.util import (
11
-    find_executable,
12
-    ApplicationError,
13 9
     display,
14
-    is_shippable,
15 10
 )
16 11
 
17 12
 
... ...
@@ -4,20 +4,13 @@ from __future__ import absolute_import, print_function
4 4
 import os
5 5
 import time
6 6
 
7
-try:
8
-    # noinspection PyPep8Naming
9
-    import ConfigParser as configparser
10
-except ImportError:
11
-    # noinspection PyUnresolvedReferences
12
-    import configparser
13
-
14 7
 from lib.util import (
15 8
     display,
16 9
     ApplicationError,
17 10
     is_shippable,
18 11
     run_command,
19
-    generate_password,
20 12
     SubprocessError,
13
+    ConfigParser,
21 14
 )
22 15
 
23 16
 from lib.cloud import (
... ...
@@ -27,15 +20,6 @@ from lib.cloud import (
27 27
 
28 28
 from lib.core_ci import (
29 29
     AnsibleCoreCI,
30
-    InstanceConnection,
31
-)
32
-
33
-from lib.manage_ci import (
34
-    ManagePosixCI,
35
-)
36
-
37
-from lib.http import (
38
-    HttpClient,
39 30
 )
40 31
 
41 32
 
... ...
@@ -219,7 +203,7 @@ class TowerConfig(object):
219 219
         :type path: str
220 220
         :rtype: TowerConfig
221 221
         """
222
-        parser = configparser.RawConfigParser()
222
+        parser = ConfigParser()
223 223
         parser.read(path)
224 224
 
225 225
         keys = (
... ...
@@ -21,13 +21,6 @@ from lib.docker_util import (
21 21
     get_docker_container_id,
22 22
 )
23 23
 
24
-try:
25
-    # noinspection PyPep8Naming
26
-    import ConfigParser as configparser
27
-except ImportError:
28
-    # noinspection PyUnresolvedReferences
29
-    import configparser
30
-
31 24
 
32 25
 class VcenterProvider(CloudProvider):
33 26
     """VMware vcenter/esx plugin. Sets up cloud resources for tests."""
... ...
@@ -172,8 +172,8 @@ def docker_inspect(args, container_id):
172 172
     except SubprocessError as ex:
173 173
         try:
174 174
             return json.loads(ex.stdout)
175
-        except:
176
-            raise ex  # pylint: disable=locally-disabled, raising-bad-type
175
+        except Exception:
176
+            raise ex
177 177
 
178 178
 
179 179
 def docker_network_disconnect(args, container_id, network):
... ...
@@ -200,8 +200,8 @@ def docker_network_inspect(args, network):
200 200
     except SubprocessError as ex:
201 201
         try:
202 202
             return json.loads(ex.stdout)
203
-        except:
204
-            raise ex  # pylint: disable=locally-disabled, raising-bad-type
203
+        except Exception:
204
+            raise ex
205 205
 
206 206
 
207 207
 def docker_exec(args, container_id, cmd, options=None, capture=False, stdin=None, stdout=None):
... ...
@@ -541,6 +541,7 @@ def command_windows_integration(args):
541 541
                 instance.result.stop()
542 542
 
543 543
 
544
+# noinspection PyUnusedLocal
544 545
 def windows_init(args, internal_targets):  # pylint: disable=locally-disabled, unused-argument
545 546
     """
546 547
     :type args: WindowsIntegrationConfig
... ...
@@ -2,7 +2,6 @@
2 2
 from __future__ import absolute_import, print_function
3 3
 
4 4
 import os
5
-import re
6 5
 
7 6
 from lib.sanity import (
8 7
     SanityMultipleVersion,
... ...
@@ -2,7 +2,6 @@
2 2
 from __future__ import absolute_import, print_function
3 3
 
4 4
 import os
5
-import re
6 5
 
7 6
 from lib.sanity import (
8 7
     SanityMultipleVersion,
... ...
@@ -6,11 +6,6 @@ import json
6 6
 import os
7 7
 import datetime
8 8
 
9
-try:
10
-    import ConfigParser as configparser
11
-except ImportError:
12
-    import configparser
13
-
14 9
 from lib.sanity import (
15 10
     SanitySingleVersion,
16 11
     SanityMessage,
... ...
@@ -23,8 +18,8 @@ from lib.util import (
23 23
     SubprocessError,
24 24
     run_command,
25 25
     display,
26
-    find_executable,
27 26
     read_lines_without_comments,
27
+    ConfigParser,
28 28
 )
29 29
 
30 30
 from lib.executor import (
... ...
@@ -245,7 +240,7 @@ class PylintTest(SanitySingleVersion):
245 245
         if not os.path.exists(rcfile):
246 246
             rcfile = 'test/sanity/pylint/config/default'
247 247
 
248
-        parser = configparser.SafeConfigParser()
248
+        parser = ConfigParser()
249 249
         parser.read(rcfile)
250 250
 
251 251
         if parser.has_section('ansible-test'):
... ...
@@ -268,7 +263,7 @@ class PylintTest(SanitySingleVersion):
268 268
         ] + paths
269 269
 
270 270
         env = ansible_environment(args)
271
-        env['PYTHONPATH'] += '%s%s' % (os.pathsep, self.plugin_dir)
271
+        env['PYTHONPATH'] += '%s%s' % (os.path.pathsep, self.plugin_dir)
272 272
 
273 273
         if paths:
274 274
             try:
... ...
@@ -16,7 +16,6 @@ from lib.util import (
16 16
     run_command,
17 17
     parse_to_list_of_dict,
18 18
     display,
19
-    find_executable,
20 19
     read_lines_without_comments,
21 20
 )
22 21
 
... ...
@@ -62,7 +62,8 @@ class YamllintTest(SanitySingleVersion):
62 62
 
63 63
         return SanitySuccess(self.name)
64 64
 
65
-    def test_paths(self, args, paths):
65
+    @staticmethod
66
+    def test_paths(args, paths):
66 67
         """
67 68
         :type args: SanityConfig
68 69
         :type paths: list[str]
... ...
@@ -30,7 +30,7 @@ class WrappedThread(threading.Thread):
30 30
         Run action and capture results or exception.
31 31
         Do not override. Do not call directly. Executed by the start() method.
32 32
         """
33
-        # noinspection PyBroadException
33
+        # noinspection PyBroadException, PyPep8
34 34
         try:
35 35
             self._result.put((self.action(), None))
36 36
         except:  # pylint: disable=locally-disabled, bare-except
... ...
@@ -5,7 +5,6 @@ from __future__ import absolute_import, print_function
5 5
 import atexit
6 6
 import contextlib
7 7
 import errno
8
-import filecmp
9 8
 import fcntl
10 9
 import inspect
11 10
 import json
... ...
@@ -32,6 +31,13 @@ except ImportError:
32 32
     from abc import ABCMeta
33 33
     ABC = ABCMeta('ABC', (), {})
34 34
 
35
+try:
36
+    # noinspection PyCompatibility
37
+    from ConfigParser import SafeConfigParser as ConfigParser
38
+except ImportError:
39
+    # noinspection PyCompatibility
40
+    from configparser import ConfigParser
41
+
35 42
 DOCKER_COMPLETION = {}
36 43
 
37 44
 coverage_path = ''  # pylint: disable=locally-disabled, invalid-name
... ...
@@ -117,10 +123,10 @@ def find_executable(executable, cwd=None, path=None, required=True):
117 117
             match = executable
118 118
     else:
119 119
         if path is None:
120
-            path = os.environ.get('PATH', os.defpath)
120
+            path = os.environ.get('PATH', os.path.defpath)
121 121
 
122 122
         if path:
123
-            path_dirs = path.split(os.pathsep)
123
+            path_dirs = path.split(os.path.pathsep)
124 124
             seen_dirs = set()
125 125
 
126 126
             for path_dir in path_dirs:
... ...
@@ -197,7 +203,7 @@ def intercept_command(args, cmd, target_name, capture=False, env=None, data=None
197 197
     coverage_file = os.path.abspath(os.path.join(inject_path, '..', 'output', '%s=%s=%s=%s=coverage' % (
198 198
         args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version)))
199 199
 
200
-    env['PATH'] = inject_path + os.pathsep + env['PATH']
200
+    env['PATH'] = inject_path + os.path.pathsep + env['PATH']
201 201
     env['ANSIBLE_TEST_PYTHON_VERSION'] = version
202 202
     env['ANSIBLE_TEST_PYTHON_INTERPRETER'] = interpreter
203 203
 
... ...
@@ -388,7 +394,7 @@ def common_environment():
388 388
     """Common environment used for executing all programs."""
389 389
     env = dict(
390 390
         LC_ALL='en_US.UTF-8',
391
-        PATH=os.environ.get('PATH', os.defpath),
391
+        PATH=os.environ.get('PATH', os.path.defpath),
392 392
     )
393 393
 
394 394
     required = (
... ...
@@ -17,6 +17,7 @@ if [ ! -f /usr/bin/virtualenv ] && [ -f /usr/bin/virtualenv-3 ]; then
17 17
 fi
18 18
 
19 19
 # Improve prompts on remote host for interactive use.
20
+# shellcheck disable=SC1117
20 21
 cat << EOF > ~/.bashrc
21 22
 alias ls='ls --color=auto'
22 23
 export PS1='\[\e]0;\u@\h: \w\a\]\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
... ...
@@ -76,6 +76,7 @@ if [ ! -f "${HOME}/.ssh/id_rsa.pub" ]; then
76 76
 fi
77 77
 
78 78
 # Improve prompts on remote host for interactive use.
79
+# shellcheck disable=SC1117
79 80
 cat << EOF > ~/.bashrc
80 81
 alias ls='ls -G'
81 82
 export PS1='\[\e]0;\u@\h: \w\a\]\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
... ...
@@ -1,7 +1,6 @@
1 1
 #!/usr/bin/env python
2 2
 
3 3
 import os
4
-import re
5 4
 import sys
6 5
 
7 6
 
... ...
@@ -1,7 +1,6 @@
1 1
 [MESSAGES CONTROL]
2 2
 
3 3
 disable=
4
-    no-self-use,
5 4
     too-few-public-methods,
6 5
     too-many-arguments,
7 6
     too-many-branches,
... ...
@@ -63,48 +63,3 @@ lib/ansible/modules/storage/infinidat/infini_vol.py ansible-format-automatic-spe
63 63
 lib/ansible/modules/storage/purestorage/purefa_host.py ansible-format-automatic-specification
64 64
 lib/ansible/modules/storage/purestorage/purefa_pg.py ansible-format-automatic-specification
65 65
 lib/ansible/modules/system/firewalld.py ansible-format-automatic-specification
66
-test/runner/injector/importer.py missing-docstring 3.7
67
-test/runner/injector/injector.py missing-docstring 3.7
68
-test/runner/lib/ansible_util.py missing-docstring 3.7
69
-test/runner/lib/changes.py missing-docstring 3.7
70
-test/runner/lib/classification.py missing-docstring 3.7
71
-test/runner/lib/cloud/__init__.py missing-docstring 3.7
72
-test/runner/lib/cloud/aws.py missing-docstring 3.7
73
-test/runner/lib/cloud/azure.py missing-docstring 3.7
74
-test/runner/lib/cloud/cs.py missing-docstring 3.7
75
-test/runner/lib/cloud/vcenter.py missing-docstring 3.7
76
-test/runner/lib/config.py missing-docstring 3.7
77
-test/runner/lib/core_ci.py missing-docstring 3.7
78
-test/runner/lib/cover.py missing-docstring 3.7
79
-test/runner/lib/delegation.py missing-docstring 3.7
80
-test/runner/lib/delegation.py redefined-variable-type 2.7
81
-test/runner/lib/diff.py missing-docstring 3.7
82
-test/runner/lib/docker_util.py missing-docstring 3.7
83
-test/runner/lib/executor.py missing-docstring 3.7
84
-test/runner/lib/git.py missing-docstring 3.7
85
-test/runner/lib/http.py missing-docstring 3.7
86
-test/runner/lib/import_analysis.py missing-docstring 3.7
87
-test/runner/lib/manage_ci.py missing-docstring 3.7
88
-test/runner/lib/metadata.py missing-docstring 3.7
89
-test/runner/lib/powershell_import_analysis.py missing-docstring 3.7
90
-test/runner/lib/pytar.py missing-docstring 3.7
91
-test/runner/lib/sanity/__init__.py missing-docstring 3.7
92
-test/runner/lib/sanity/ansible_doc.py missing-docstring 3.7
93
-test/runner/lib/sanity/compile.py missing-docstring 3.7
94
-test/runner/lib/sanity/import.py missing-docstring 3.7
95
-test/runner/lib/sanity/pep8.py missing-docstring 3.7
96
-test/runner/lib/sanity/pslint.py missing-docstring 3.7
97
-test/runner/lib/sanity/pylint.py missing-docstring 3.7
98
-test/runner/lib/sanity/rstcheck.py missing-docstring 3.7
99
-test/runner/lib/sanity/sanity_docs.py missing-docstring 3.7
100
-test/runner/lib/sanity/shellcheck.py missing-docstring 3.7
101
-test/runner/lib/sanity/validate_modules.py missing-docstring 3.7
102
-test/runner/lib/sanity/yamllint.py missing-docstring 3.7
103
-test/runner/lib/target.py missing-docstring 3.7
104
-test/runner/lib/test.py missing-docstring 3.7
105
-test/runner/lib/thread.py missing-docstring 3.7
106
-test/runner/lib/util.py missing-docstring 3.7
107
-test/runner/retry.py missing-docstring 3.7
108
-test/runner/shippable.py missing-docstring 3.7
109
-test/runner/units/test_diff.py missing-docstring 3.7
110
-test/sanity/import/importer.py missing-docstring 3.7
... ...
@@ -23,10 +23,10 @@ if [ -d /home/shippable/cache/ ]; then
23 23
     ls -la /home/shippable/cache/
24 24
 fi
25 25
 
26
-which python
26
+command -v python
27 27
 python -V
28 28
 
29
-which pip
29
+command -v pip
30 30
 pip --version
31 31
 pip list --disable-pip-version-check
32 32