diff --git a/test/runner/lib/cloud/tower.py b/test/runner/lib/cloud/tower.py index 29e147f9e9..ac64d725f7 100644 --- a/test/runner/lib/cloud/tower.py +++ b/test/runner/lib/cloud/tower.py @@ -15,7 +15,6 @@ from lib.util import ( display, ApplicationError, is_shippable, - find_pip, run_command, generate_password, SubprocessError, @@ -151,8 +150,7 @@ class TowerCloudEnvironment(CloudEnvironment): display.info('Installing Tower CLI version: %s' % tower_cli_version) - pip = find_pip(version=self.args.python_version) - cmd = [pip, 'install', '--disable-pip-version-check', 'ansible-tower-cli==%s' % tower_cli_version] + cmd = self.args.pip_command + ['install', '--disable-pip-version-check', 'ansible-tower-cli==%s' % tower_cli_version] run_command(self.args, cmd) diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py index eb6b5f665e..b1f23954fe 100644 --- a/test/runner/lib/config.py +++ b/test/runner/lib/config.py @@ -9,6 +9,8 @@ from lib.util import ( CommonConfig, is_shippable, docker_qualify_image, + find_python, + generate_pip_command, ) from lib.metadata import ( @@ -67,6 +69,20 @@ class EnvironmentConfig(CommonConfig): if self.delegate: self.requirements = True + @property + def python_executable(self): + """ + :rtype: str + """ + return find_python(self.python_version) + + @property + def pip_command(self): + """ + :rtype: list[str] + """ + return generate_pip_command(self.python_executable) + class TestConfig(EnvironmentConfig): """Configuration common to all test commands.""" diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index 2f2b86973c..119b0f889d 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -45,7 +45,6 @@ from lib.util import ( make_dirs, is_shippable, is_binary_file, - find_pip, find_executable, raw_command, get_coverage_path, @@ -151,7 +150,7 @@ def install_command_requirements(args): if args.junit: packages.append('junit-xml') - pip = find_pip(version=args.python_version) + pip = args.pip_command commands = [generate_pip_install(pip, args.command, packages=packages)] @@ -183,7 +182,7 @@ def install_command_requirements(args): def run_pip_commands(args, pip, commands, detect_pip_changes=False): """ :type args: EnvironmentConfig - :type pip: str + :type pip: list[str] :type commands: list[list[str]] :type detect_pip_changes: bool :rtype: list[list[str]] @@ -210,7 +209,7 @@ def run_pip_commands(args, pip, commands, detect_pip_changes=False): # AttributeError: 'Requirement' object has no attribute 'project_name' # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258 # Upgrading pip works around the issue. - run_command(args, [pip, 'install', '--upgrade', 'pip']) + run_command(args, pip + ['install', '--upgrade', 'pip']) run_command(args, cmd) after_list = pip_list(args, pip) if detect_pip_changes else None @@ -224,10 +223,10 @@ def run_pip_commands(args, pip, commands, detect_pip_changes=False): def pip_list(args, pip): """ :type args: EnvironmentConfig - :type pip: str + :type pip: list[str] :rtype: str """ - stdout, _ = run_command(args, [pip, 'list'], capture=True) + stdout, _ = run_command(args, pip + ['list'], capture=True) return stdout @@ -238,12 +237,12 @@ def generate_egg_info(args): if os.path.isdir('lib/ansible.egg-info'): return - run_command(args, ['python%s' % args.python_version, 'setup.py', 'egg_info'], capture=args.verbosity < 3) + run_command(args, [args.python_executable, 'setup.py', 'egg_info'], capture=args.verbosity < 3) def generate_pip_install(pip, command, packages=None): """ - :type pip: str + :type pip: list[str] :type command: str :type packages: list[str] | None :rtype: list[str] | None @@ -262,7 +261,7 @@ def generate_pip_install(pip, command, packages=None): if not options: return None - return [pip, 'install', '--disable-pip-version-check', '-c', constraints] + options + return pip + ['install', '--disable-pip-version-check', '-c', constraints] + options def command_shell(args): diff --git a/test/runner/lib/sanity/__init__.py b/test/runner/lib/sanity/__init__.py index a773e917fb..2517239b3b 100644 --- a/test/runner/lib/sanity/__init__.py +++ b/test/runner/lib/sanity/__init__.py @@ -221,7 +221,11 @@ class SanityCodeSmellTest(SanityTest): :type targets: SanityTargets :rtype: SanityResult """ - cmd = [self.path] + if self.path.endswith('.py'): + cmd = [args.python_executable, self.path] + else: + cmd = [self.path] + env = ansible_environment(args, color=False) pattern = None diff --git a/test/runner/lib/sanity/compile.py b/test/runner/lib/sanity/compile.py index e754448057..aa0f5fdcc6 100644 --- a/test/runner/lib/sanity/compile.py +++ b/test/runner/lib/sanity/compile.py @@ -16,6 +16,7 @@ from lib.util import ( SubprocessError, run_command, display, + find_python, ) from lib.config import ( @@ -50,7 +51,7 @@ class CompileTest(SanityMultipleVersion): if not paths: return SanitySkipped(self.name, python_version=python_version) - cmd = ['python%s' % python_version, 'test/sanity/compile/compile.py'] + cmd = [find_python(python_version), 'test/sanity/compile/compile.py'] data = '\n'.join(paths) diff --git a/test/runner/lib/sanity/import.py b/test/runner/lib/sanity/import.py index b6c7ede108..edccce9b54 100644 --- a/test/runner/lib/sanity/import.py +++ b/test/runner/lib/sanity/import.py @@ -18,6 +18,7 @@ from lib.util import ( intercept_command, remove_tree, display, + find_python, ) from lib.ansible_util import ( @@ -66,7 +67,7 @@ class ImportTest(SanityMultipleVersion): remove_tree(virtual_environment_path) - cmd = ['virtualenv', virtual_environment_path, '--python', 'python%s' % python_version, '--no-setuptools', '--no-wheel'] + cmd = ['virtualenv', virtual_environment_path, '--python', find_python(python_version), '--no-setuptools', '--no-wheel'] if not args.coverage: cmd.append('--no-pip') @@ -84,8 +85,8 @@ class ImportTest(SanityMultipleVersion): # make sure coverage is available in the virtual environment if needed if args.coverage: - run_command(args, generate_pip_install('pip', 'sanity.import', packages=['setuptools']), env=env) - run_command(args, generate_pip_install('pip', 'sanity.import', packages=['coverage']), env=env) + run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['setuptools']), env=env) + run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['coverage']), env=env) run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'setuptools'], env=env) run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'pip'], env=env) diff --git a/test/runner/lib/sanity/pep8.py b/test/runner/lib/sanity/pep8.py index 9544fc20ca..4ebfe939c0 100644 --- a/test/runner/lib/sanity/pep8.py +++ b/test/runner/lib/sanity/pep8.py @@ -15,7 +15,6 @@ from lib.util import ( SubprocessError, display, run_command, - find_executable, ) from lib.config import ( @@ -56,8 +55,8 @@ class Pep8Test(SanitySingleVersion): paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths_set) cmd = [ - 'python%s' % args.python_version, - find_executable('pycodestyle'), + args.python_executable, + '-m', 'pycodestyle', '--max-line-length', '160', '--config', '/dev/null', '--ignore', ','.join(sorted(current_ignore)), diff --git a/test/runner/lib/sanity/pylint.py b/test/runner/lib/sanity/pylint.py index e41af68e8c..ead962fbf8 100644 --- a/test/runner/lib/sanity/pylint.py +++ b/test/runner/lib/sanity/pylint.py @@ -252,8 +252,8 @@ class PylintTest(SanitySingleVersion): load_plugins = set(self.plugin_names) - disable_plugins cmd = [ - 'python%s' % args.python_version, - find_executable('pylint'), + args.python_executable, + '-m', 'pylint', '--jobs', '0', '--reports', 'n', '--max-line-length', '160', diff --git a/test/runner/lib/sanity/rstcheck.py b/test/runner/lib/sanity/rstcheck.py index b1e1d9ceb2..c3113fa3e4 100644 --- a/test/runner/lib/sanity/rstcheck.py +++ b/test/runner/lib/sanity/rstcheck.py @@ -15,6 +15,7 @@ from lib.util import ( SubprocessError, run_command, parse_to_dict, + display, find_executable, ) @@ -22,6 +23,10 @@ from lib.config import ( SanityConfig, ) +UNSUPPORTED_PYTHON_VERSIONS = ( + '2.6', +) + class RstcheckTest(SanitySingleVersion): """Sanity test using rstcheck.""" @@ -31,6 +36,10 @@ class RstcheckTest(SanitySingleVersion): :type targets: SanityTargets :rtype: SanityResult """ + if args.python_version in UNSUPPORTED_PYTHON_VERSIONS: + display.warning('Skipping rstcheck on unsupported Python version %s.' % args.python_version) + return SanitySkipped(self.name) + with open('test/sanity/rstcheck/ignore-substitutions.txt', 'r') as ignore_fd: ignore_substitutions = sorted(set(ignore_fd.read().splitlines())) @@ -40,8 +49,8 @@ class RstcheckTest(SanitySingleVersion): return SanitySkipped(self.name) cmd = [ - 'python%s' % args.python_version, - find_executable('rstcheck'), + args.python_executable, + '-m', 'rstcheck', '--report', 'warning', '--ignore-substitutions', ','.join(ignore_substitutions), ] + paths diff --git a/test/runner/lib/sanity/validate_modules.py b/test/runner/lib/sanity/validate_modules.py index 40c0758c15..1dcbd64a72 100644 --- a/test/runner/lib/sanity/validate_modules.py +++ b/test/runner/lib/sanity/validate_modules.py @@ -57,7 +57,7 @@ class ValidateModulesTest(SanitySingleVersion): return SanitySkipped(self.name) cmd = [ - 'python%s' % args.python_version, + args.python_executable, 'test/sanity/validate-modules/validate-modules', '--format', 'json', '--arg-spec', diff --git a/test/runner/lib/sanity/yamllint.py b/test/runner/lib/sanity/yamllint.py index 9aed5b94ad..8f3fa275af 100644 --- a/test/runner/lib/sanity/yamllint.py +++ b/test/runner/lib/sanity/yamllint.py @@ -65,7 +65,7 @@ class YamllintTest(SanitySingleVersion): :rtype: list[SanityMessage] """ cmd = [ - 'python%s' % args.python_version, + args.python_executable, 'test/sanity/yamllint/yamllinter.py', ] diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py index c303d7dba6..a12aa334bc 100644 --- a/test/runner/lib/util.py +++ b/test/runner/lib/util.py @@ -63,51 +63,6 @@ def remove_file(path): os.remove(path) -def find_pip(path=None, version=None): - """ - :type path: str | None - :type version: str | None - :rtype: str - """ - if version: - version_info = version.split('.') - python_bin = find_executable('python%s' % version, path=path) - else: - version_info = sys.version_info - python_bin = sys.executable - - choices = ( - 'pip%s' % '.'.join(str(i) for i in version_info[:2]), - 'pip%s' % version_info[0], - 'pip', - ) - - pip = None - - for choice in choices: - pip = find_executable(choice, required=False, path=path) - - if pip: - break - - if not pip: - raise ApplicationError('Required program not found: %s' % ', '.join(choices)) - - with open(pip) as pip_fd: - shebang = pip_fd.readline().strip() - - if not shebang.startswith('#!') or ' ' in shebang: - raise ApplicationError('Unexpected shebang in "%s": %s' % (pip, shebang)) - - our_python = os.path.realpath(python_bin) - pip_python = os.path.realpath(shebang[2:]) - - if our_python != pip_python and not filecmp.cmp(our_python, pip_python, False): - raise ApplicationError('Current interpreter "%s" does not match "%s" interpreter "%s".' % (our_python, pip, pip_python)) - - return pip - - def find_executable(executable, cwd=None, path=None, required=True): """ :type executable: str @@ -160,6 +115,30 @@ def find_executable(executable, cwd=None, path=None, required=True): return match +def find_python(version, path=None): + """ + :type version: str + :type path: str | None + :rtype: str + """ + version_info = tuple(int(n) for n in version.split('.')) + + if not path and version_info == sys.version_info[:len(version_info)]: + python_bin = sys.executable + else: + python_bin = find_executable('python%s' % version, path=path) + + return python_bin + + +def generate_pip_command(python): + """ + :type python: str + :rtype: list[str] + """ + return [python, '-m', 'pip.__main__'] + + def intercept_command(args, cmd, target_name, capture=False, env=None, data=None, cwd=None, python_version=None, path=None): """ :type args: TestConfig @@ -180,7 +159,7 @@ def intercept_command(args, cmd, target_name, capture=False, env=None, data=None inject_path = get_coverage_path(args) config_path = os.path.join(inject_path, 'injector.json') version = python_version or args.python_version - interpreter = find_executable('python%s' % version, path=path) + interpreter = find_python(version, path) coverage_file = os.path.abspath(os.path.join(inject_path, '..', 'output', '%s=%s=%s=%s=coverage' % ( args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version))) diff --git a/test/runner/requirements/sanity.txt b/test/runner/requirements/sanity.txt index 34d6e07f47..3fe5ae6dc2 100644 --- a/test/runner/requirements/sanity.txt +++ b/test/runner/requirements/sanity.txt @@ -5,7 +5,7 @@ paramiko pycodestyle pylint ; python_version >= '2.7' # pylint 1.5.3+ is required for non-buggy JSON output, but 1.4+ requires python 2.7+ pytest -rstcheck +rstcheck ; python_version >= '2.7' # rstcheck requires python 2.7+ sphinx virtualenv voluptuous diff --git a/test/runner/test.py b/test/runner/test.py index b26e0b7dcb..51d290aba8 100755 --- a/test/runner/test.py +++ b/test/runner/test.py @@ -12,8 +12,8 @@ from lib.util import ( ApplicationError, display, raw_command, - find_pip, get_docker_completion, + generate_pip_command, ) from lib.delegation import ( @@ -112,7 +112,7 @@ def parse_args(): except ImportError: if '--requirements' not in sys.argv: raise - raw_command(generate_pip_install(find_pip(), 'ansible-test')) + raw_command(generate_pip_install(generate_pip_command(sys.executable), 'ansible-test')) import argparse try: