mirror of
https://github.com/ansible-collections/community.general.git
synced 2025-04-27 12:51:25 -07:00
Move ssh and local connection plugins from using raw select to selectors
At the moment, this change will use EPoll on Linux, KQueue on *BSDs, etc, so it should alleviate problems with too many open file descriptors. * Bundle a copy of selectors2 so that we have the selectors API everywhere. * Add licensing information to selectors2 file so it's clear what the licensing terms and conditions are. * Exclude the bundled copy of selectors2 from our boilerplate code-smell test * Rewrite ssh_run tests to attempt to work around problem with mocking select on shippable Fixes #14143
This commit is contained in:
Toshio Kuratomi
parent
2c70450e23
commit
d1a6b07fe1
7 changed files with 1100 additions and 227 deletions
|
|
@ -23,10 +23,13 @@ __metaclass__ = type
|
|||
|
||||
from io import StringIO
|
||||
|
||||
import pytest
|
||||
|
||||
from ansible.compat.tests import unittest
|
||||
from ansible.compat.tests.mock import patch, MagicMock
|
||||
|
||||
from ansible import constants as C
|
||||
from ansible.compat.selectors import SelectorKey, EVENT_READ
|
||||
from ansible.compat.six.moves import shlex_quote
|
||||
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
|
||||
from ansible.playbook.play_context import PlayContext
|
||||
|
|
@ -83,82 +86,6 @@ class TestConnectionBaseClass(unittest.TestCase):
|
|||
res, stdout, stderr = conn._exec_command('ssh')
|
||||
res, stdout, stderr = conn._exec_command('ssh', 'this is some data')
|
||||
|
||||
@patch('select.select')
|
||||
@patch('fcntl.fcntl')
|
||||
@patch('os.write')
|
||||
@patch('os.close')
|
||||
@patch('pty.openpty')
|
||||
@patch('subprocess.Popen')
|
||||
def test_plugins_connection_ssh__run(self, mock_Popen, mock_openpty, mock_osclose, mock_oswrite, mock_fcntl, mock_select):
|
||||
pc = PlayContext()
|
||||
new_stdin = StringIO()
|
||||
|
||||
conn = ssh.Connection(pc, new_stdin)
|
||||
conn._send_initial_data = MagicMock()
|
||||
conn._examine_output = MagicMock()
|
||||
conn._terminate_process = MagicMock()
|
||||
conn.sshpass_pipe = [MagicMock(), MagicMock()]
|
||||
|
||||
mock_popen_res = MagicMock()
|
||||
mock_popen_res.poll = MagicMock()
|
||||
mock_popen_res.wait = MagicMock()
|
||||
mock_popen_res.stdin = MagicMock()
|
||||
mock_popen_res.stdin.fileno.return_value = 1000
|
||||
mock_popen_res.stdout = MagicMock()
|
||||
mock_popen_res.stdout.fileno.return_value = 1001
|
||||
mock_popen_res.stderr = MagicMock()
|
||||
mock_popen_res.stderr.fileno.return_value = 1002
|
||||
mock_popen_res.return_code = 0
|
||||
mock_Popen.return_value = mock_popen_res
|
||||
|
||||
def _mock_select(rlist, wlist, elist, timeout=None):
|
||||
rvals = []
|
||||
if mock_popen_res.stdin in rlist:
|
||||
rvals.append(mock_popen_res.stdin)
|
||||
if mock_popen_res.stderr in rlist:
|
||||
rvals.append(mock_popen_res.stderr)
|
||||
return (rvals, [], [])
|
||||
|
||||
mock_select.side_effect = _mock_select
|
||||
|
||||
mock_popen_res.stdout.read.side_effect = [b"some data", b""]
|
||||
mock_popen_res.stderr.read.side_effect = [b""]
|
||||
conn._run("ssh", "this is input data")
|
||||
|
||||
# test with a password set to trigger the sshpass write
|
||||
pc.password = '12345'
|
||||
mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
|
||||
mock_popen_res.stderr.read.side_effect = [b""]
|
||||
conn._run(["ssh", "is", "a", "cmd"], "this is more data")
|
||||
|
||||
# test with password prompting enabled
|
||||
pc.password = None
|
||||
pc.prompt = True
|
||||
mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
|
||||
mock_popen_res.stderr.read.side_effect = [b""]
|
||||
conn._run("ssh", "this is input data")
|
||||
|
||||
# test with some become settings
|
||||
pc.prompt = False
|
||||
pc.become = True
|
||||
pc.success_key = 'BECOME-SUCCESS-abcdefg'
|
||||
mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
|
||||
mock_popen_res.stderr.read.side_effect = [b""]
|
||||
conn._run("ssh", "this is input data")
|
||||
|
||||
# simulate no data input
|
||||
mock_openpty.return_value = (98, 99)
|
||||
mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
|
||||
mock_popen_res.stderr.read.side_effect = [b""]
|
||||
conn._run("ssh", "")
|
||||
|
||||
# simulate no data input but Popen using new pty's fails
|
||||
mock_Popen.return_value = None
|
||||
mock_Popen.side_effect = [OSError(), mock_popen_res]
|
||||
mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
|
||||
mock_popen_res.stderr.read.side_effect = [b""]
|
||||
conn._run("ssh", "")
|
||||
|
||||
def test_plugins_connection_ssh__examine_output(self):
|
||||
pc = PlayContext()
|
||||
new_stdin = StringIO()
|
||||
|
|
@ -341,7 +268,6 @@ class TestConnectionBaseClass(unittest.TestCase):
|
|||
conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
|
||||
conn._run.assert_called_with('some command to run', expected_in_data, checkrc=False)
|
||||
|
||||
|
||||
# test that a non-zero rc raises an error
|
||||
conn._run.return_value = (1, 'stdout', 'some errors')
|
||||
self.assertRaises(AnsibleError, conn.put_file, '/path/to/bad/file', '/remote/path/to/file')
|
||||
|
|
@ -398,3 +324,215 @@ class TestConnectionBaseClass(unittest.TestCase):
|
|||
# test that a non-zero rc raises an error
|
||||
conn._run.return_value = (1, 'stdout', 'some errors')
|
||||
self.assertRaises(AnsibleError, conn.fetch_file, '/path/to/bad/file', '/remote/path/to/file')
|
||||
|
||||
|
||||
class MockSelector(object):
|
||||
def __init__(self):
|
||||
self.files_watched = 0
|
||||
self.register = MagicMock(side_effect=self._register)
|
||||
self.unregister = MagicMock(side_effect=self._unregister)
|
||||
self.close = MagicMock()
|
||||
self.get_map = MagicMock(side_effect=self._get_map)
|
||||
self.select = MagicMock()
|
||||
|
||||
def _register(self, *args, **kwargs):
|
||||
self.files_watched += 1
|
||||
|
||||
def _unregister(self, *args, **kwargs):
|
||||
self.files_watched -= 1
|
||||
|
||||
def _get_map(self, *args, **kwargs):
|
||||
return self.files_watched
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_run_env(request, mocker):
|
||||
pc = PlayContext()
|
||||
new_stdin = StringIO()
|
||||
|
||||
conn = ssh.Connection(pc, new_stdin)
|
||||
conn._send_initial_data = MagicMock()
|
||||
conn._examine_output = MagicMock()
|
||||
conn._terminate_process = MagicMock()
|
||||
conn.sshpass_pipe = [MagicMock(), MagicMock()]
|
||||
|
||||
request.cls.pc = pc
|
||||
request.cls.conn = conn
|
||||
|
||||
mock_popen_res = MagicMock()
|
||||
mock_popen_res.poll = MagicMock()
|
||||
mock_popen_res.wait = MagicMock()
|
||||
mock_popen_res.stdin = MagicMock()
|
||||
mock_popen_res.stdin.fileno.return_value = 1000
|
||||
mock_popen_res.stdout = MagicMock()
|
||||
mock_popen_res.stdout.fileno.return_value = 1001
|
||||
mock_popen_res.stderr = MagicMock()
|
||||
mock_popen_res.stderr.fileno.return_value = 1002
|
||||
mock_popen_res.returncode = 0
|
||||
request.cls.mock_popen_res = mock_popen_res
|
||||
|
||||
mock_popen = mocker.patch('subprocess.Popen', return_value=mock_popen_res)
|
||||
request.cls.mock_popen = mock_popen
|
||||
|
||||
request.cls.mock_selector = MockSelector()
|
||||
mocker.patch('ansible.compat.selectors.DefaultSelector', lambda: request.cls.mock_selector)
|
||||
|
||||
request.cls.mock_openpty = mocker.patch('pty.openpty')
|
||||
|
||||
mocker.patch('fcntl.fcntl')
|
||||
mocker.patch('os.write')
|
||||
mocker.patch('os.close')
|
||||
|
||||
|
||||
@pytest.mark.usefixtures('mock_run_env')
|
||||
class TestSSHConnectionRun(object):
|
||||
# FIXME:
|
||||
# These tests are little more than a smoketest. Need to enhance them
|
||||
# a bit to check that they're calling the relevant functions and making
|
||||
# complete coverage of the code paths
|
||||
def test_no_escalation(self):
|
||||
self.mock_popen_res.stdout.read.side_effect = [b"my_stdout\n", b"second_line"]
|
||||
self.mock_popen_res.stderr.read.side_effect = [b"my_stderr"]
|
||||
self.mock_selector.select.side_effect = [
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ)],
|
||||
[]]
|
||||
self.mock_selector.get_map.side_effect = lambda: True
|
||||
|
||||
return_code, b_stdout, b_stderr = self.conn._run("ssh", "this is input data")
|
||||
assert return_code == 0
|
||||
assert b_stdout == b'my_stdout\nsecond_line'
|
||||
assert b_stderr == b'my_stderr'
|
||||
assert self.mock_selector.register.called is True
|
||||
assert self.mock_selector.register.call_count == 2
|
||||
assert self.conn._send_initial_data.called is True
|
||||
assert self.conn._send_initial_data.call_count == 1
|
||||
assert self.conn._send_initial_data.call_args[0][1] == 'this is input data'
|
||||
|
||||
def test_with_password(self):
|
||||
# test with a password set to trigger the sshpass write
|
||||
self.pc.password = '12345'
|
||||
self.mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
|
||||
self.mock_popen_res.stderr.read.side_effect = [b""]
|
||||
self.mock_selector.select.side_effect = [
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[]]
|
||||
self.mock_selector.get_map.side_effect = lambda: True
|
||||
|
||||
return_code, b_stdout, b_stderr = self.conn._run(["ssh", "is", "a", "cmd"], "this is more data")
|
||||
assert return_code == 0
|
||||
assert b_stdout == b'some data'
|
||||
assert b_stderr == b''
|
||||
assert self.mock_selector.register.called is True
|
||||
assert self.mock_selector.register.call_count == 2
|
||||
assert self.conn._send_initial_data.called is True
|
||||
assert self.conn._send_initial_data.call_count == 1
|
||||
assert self.conn._send_initial_data.call_args[0][1] == 'this is more data'
|
||||
|
||||
def _password_with_prompt_examine_output(self, sourice, state, b_chunk, sudoable):
|
||||
if state == 'awaiting_prompt':
|
||||
self.conn._flags['become_prompt'] = True
|
||||
elif state == 'awaiting_escalation':
|
||||
self.conn._flags['become_success'] = True
|
||||
return (b'', b'')
|
||||
|
||||
def test_pasword_with_prompt(self):
|
||||
# test with password prompting enabled
|
||||
self.pc.password = None
|
||||
self.pc.prompt = b'Password:'
|
||||
self.conn._examine_output.side_effect = self._password_with_prompt_examine_output
|
||||
self.mock_popen_res.stdout.read.side_effect = [b"Password:", b"Success", b""]
|
||||
self.mock_popen_res.stderr.read.side_effect = [b""]
|
||||
self.mock_selector.select.side_effect = [
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ),
|
||||
(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[]]
|
||||
self.mock_selector.get_map.side_effect = lambda: True
|
||||
|
||||
return_code, b_stdout, b_stderr = self.conn._run("ssh", "this is input data")
|
||||
assert return_code == 0
|
||||
assert b_stdout == b''
|
||||
assert b_stderr == b''
|
||||
assert self.mock_selector.register.called is True
|
||||
assert self.mock_selector.register.call_count == 2
|
||||
assert self.conn._send_initial_data.called is True
|
||||
assert self.conn._send_initial_data.call_count == 1
|
||||
assert self.conn._send_initial_data.call_args[0][1] == 'this is input data'
|
||||
|
||||
def test_pasword_with_become(self):
|
||||
# test with some become settings
|
||||
self.pc.prompt = b'Password:'
|
||||
self.pc.become = True
|
||||
self.pc.success_key = 'BECOME-SUCCESS-abcdefg'
|
||||
self.conn._examine_output.side_effect = self._password_with_prompt_examine_output
|
||||
self.mock_popen_res.stdout.read.side_effect = [b"Password:", b"BECOME-SUCCESS-abcdefg", b"abc"]
|
||||
self.mock_popen_res.stderr.read.side_effect = [b"123"]
|
||||
self.mock_selector.select.side_effect = [
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[]]
|
||||
self.mock_selector.get_map.side_effect = lambda: True
|
||||
|
||||
return_code, b_stdout, b_stderr = self.conn._run("ssh", "this is input data")
|
||||
assert return_code == 0
|
||||
assert b_stdout == b'abc'
|
||||
assert b_stderr == b'123'
|
||||
assert self.mock_selector.register.called is True
|
||||
assert self.mock_selector.register.call_count == 2
|
||||
assert self.conn._send_initial_data.called is True
|
||||
assert self.conn._send_initial_data.call_count == 1
|
||||
assert self.conn._send_initial_data.call_args[0][1] == 'this is input data'
|
||||
|
||||
def test_pasword_without_data(self):
|
||||
# simulate no data input
|
||||
self.mock_openpty.return_value = (98, 99)
|
||||
self.mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
|
||||
self.mock_popen_res.stderr.read.side_effect = [b""]
|
||||
self.mock_selector.select.side_effect = [
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[]]
|
||||
self.mock_selector.get_map.side_effect = lambda: True
|
||||
|
||||
return_code, b_stdout, b_stderr = self.conn._run("ssh", "")
|
||||
assert return_code == 0
|
||||
assert b_stdout == b'some data'
|
||||
assert b_stderr == b''
|
||||
assert self.mock_selector.register.called is True
|
||||
assert self.mock_selector.register.call_count == 2
|
||||
assert self.conn._send_initial_data.called is False
|
||||
|
||||
def test_pasword_without_data(self):
|
||||
# simulate no data input but Popen using new pty's fails
|
||||
self.mock_popen.return_value = None
|
||||
self.mock_popen.side_effect = [OSError(), self.mock_popen_res]
|
||||
|
||||
# simulate no data input
|
||||
self.mock_openpty.return_value = (98, 99)
|
||||
self.mock_popen_res.stdout.read.side_effect = [b"some data", b"", b""]
|
||||
self.mock_popen_res.stderr.read.side_effect = [b""]
|
||||
self.mock_selector.select.side_effect = [
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stderr, 1002, [EVENT_READ], None), EVENT_READ)],
|
||||
[(SelectorKey(self.mock_popen_res.stdout, 1001, [EVENT_READ], None), EVENT_READ)],
|
||||
[]]
|
||||
self.mock_selector.get_map.side_effect = lambda: True
|
||||
|
||||
return_code, b_stdout, b_stderr = self.conn._run("ssh", "")
|
||||
assert return_code == 0
|
||||
assert b_stdout == b'some data'
|
||||
assert b_stderr == b''
|
||||
assert self.mock_selector.register.called is True
|
||||
assert self.mock_selector.register.call_count == 2
|
||||
assert self.conn._send_initial_data.called is False
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue