Support unix socket with urls/uri (#43560)

* First pass at allowing unix socket with urls/uri. See #42341

* Only insert handler as needed

* Fix and add tests

* Add HTTPS functionality for unix sockets

* Additional test fixes

* Create context manager for monkey patching HTTPConnection.connect, de-dupe code, raise better errors

* doc

* Add a few more tests

* Fix __call__

* Remove unused import

* Patch HTTPConnection.connect with the functionality we want, instead of duplicating code and disabling

* Fix var name

* Remove unused var

* Add changelog fragment

* Update uri docs

* Fix rebase indentation issue

changelogs/fragments/uri-unix-socket.yml

@@ -0,0 +1,2 @@
+minor_changes:
+- uri/urls - Support unix domain sockets (https://github.com/ansible/ansible/pull/43560)

lib/ansible/module_utils/urls.py

@@ -42,6 +42,8 @@ import sys
 import tempfile
 import traceback
 
+from contextlib import contextmanager
+
 try:
     import httplib
 except ImportError:
@@ -362,10 +364,11 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler
         in place of HTTPSHandler
         '''
 
-        def __init__(self, client_cert=None, client_key=None, **kwargs):
+        def __init__(self, client_cert=None, client_key=None, unix_socket=None, **kwargs):
             urllib_request.HTTPSHandler.__init__(self, **kwargs)
             self.client_cert = client_cert
             self.client_key = client_key
+            self._unix_socket = unix_socket
 
         def https_open(self, req):
             return self.do_open(self._build_https_connection, req)
@@ -379,9 +382,69 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler
                 kwargs['context'] = self._context
             except AttributeError:
                 pass
+            if self._unix_socket:
+                return UnixHTTPSConnection(self._unix_socket)(host, **kwargs)
             return httplib.HTTPSConnection(host, **kwargs)
 
 
+@contextmanager
+def unix_socket_patch_httpconnection_connect():
+    '''Monkey patch ``httplib.HTTPConnection.connect`` to be ``UnixHTTPConnection.connect``
+    so that when calling ``super(UnixHTTPSConnection, self).connect()`` we get the
+    correct behavior of creating self.sock for the unix socket
+    '''
+    _connect = httplib.HTTPConnection.connect
+    httplib.HTTPConnection.connect = UnixHTTPConnection.connect
+    yield
+    httplib.HTTPConnection.connect = _connect
+
+
+class UnixHTTPSConnection(httplib.HTTPSConnection):
+    def __init__(self, unix_socket):
+        self._unix_socket = unix_socket
+
+    def connect(self):
+        # This method exists simply to ensure we monkeypatch
+        # httplib.HTTPConnection.connect to call UnixHTTPConnection.connect
+        with unix_socket_patch_httpconnection_connect():
+            super(UnixHTTPSConnection, self).connect()
+
+    def __call__(self, *args, **kwargs):
+        httplib.HTTPSConnection.__init__(self, *args, **kwargs)
+        return self
+
+
+class UnixHTTPConnection(httplib.HTTPConnection):
+    '''Handles http requests to a unix socket file'''
+
+    def __init__(self, unix_socket):
+        self._unix_socket = unix_socket
+
+    def connect(self):
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        try:
+            self.sock.connect(self._unix_socket)
+        except OSError as e:
+            raise OSError('Invalid Socket File (%s): %s' % (self._unix_socket, e))
+        if self.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
+            self.sock.settimeout(self.timeout)
+
+    def __call__(self, *args, **kwargs):
+        httplib.HTTPConnection.__init__(self, *args, **kwargs)
+        return self
+
+
+class UnixHTTPHandler(urllib_request.HTTPHandler):
+    '''Handler for Unix urls'''
+
+    def __init__(self, unix_socket, **kwargs):
+        urllib_request.HTTPHandler.__init__(self, **kwargs)
+        self._unix_socket = unix_socket
+
+    def http_open(self, req):
+        return self.do_open(UnixHTTPConnection(self._unix_socket), req)
+
+
 class ParseResultDottedDict(dict):
     '''
     A dict that acts similarly to the ParseResult named tuple from urllib
@@ -854,7 +917,7 @@ def rfc2822_date_string(timetuple, zone='-0000'):
 class Request:
     def __init__(self, headers=None, use_proxy=True, force=False, timeout=10, validate_certs=True,
                  url_username=None, url_password=None, http_agent=None, force_basic_auth=False,
-                 follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None):
+                 follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None, unix_socket=None):
         """This class works somewhat similarly to the ``Session`` class of from requests
         by defining a cookiejar that an be used across requests as well as cascaded defaults that
         can apply to repeated requests
@@ -875,7 +938,7 @@ class Request:
 
         self.headers = headers or {}
         if not isinstance(self.headers, dict):
-            raise ValueError("headers must be a dict")
+            raise ValueError("headers must be a dict: %r" % self.headers)
         self.use_proxy = use_proxy
         self.force = force
         self.timeout = timeout
@@ -887,6 +950,7 @@ class Request:
         self.follow_redirects = follow_redirects
         self.client_cert = client_cert
         self.client_key = client_key
+        self.unix_socket = unix_socket
         if isinstance(cookies, cookiejar.CookieJar):
             self.cookies = cookies
         else:
@@ -901,7 +965,8 @@ class Request:
              force=None, last_mod_time=None, timeout=None, validate_certs=None,
              url_username=None, url_password=None, http_agent=None,
              force_basic_auth=None, follow_redirects=None,
-             client_cert=None, client_key=None, cookies=None, use_gssapi=False):
+             client_cert=None, client_key=None, cookies=None, use_gssapi=False,
+             unix_socket=None):
         """
         Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
 
@@ -936,6 +1001,8 @@ class Request:
         :kwarg cookies: (optional) CookieJar object to send with the
             request
         :kwarg use_gssapi: (optional) Use GSSAPI handler of requests.
+        :kwarg unix_socket: (optional) String of file system path to unix socket file to use when establishing
+        connection to the provided url
         :returns: HTTPResponse
         """
 
@@ -959,8 +1026,13 @@ class Request:
         client_cert = self._fallback(client_cert, self.client_cert)
         client_key = self._fallback(client_key, self.client_key)
         cookies = self._fallback(cookies, self.cookies)
+        unix_socket = self._fallback(unix_socket, self.unix_socket)
 
         handlers = []
+
+        if unix_socket:
+            handlers.append(UnixHTTPHandler(unix_socket))
+
         ssl_handler = maybe_add_ssl_handler(url, validate_certs)
         if ssl_handler:
             handlers.append(ssl_handler)
@@ -1033,10 +1105,12 @@ class Request:
             context.check_hostname = False
             handlers.append(HTTPSClientAuthHandler(client_cert=client_cert,
                                                    client_key=client_key,
-                                                   context=context))
+                                                   context=context,
-        elif client_cert:
+                                                   unix_socket=unix_socket))
+        elif client_cert or unix_socket:
             handlers.append(HTTPSClientAuthHandler(client_cert=client_cert,
-                                                   client_key=client_key))
+                                                   client_key=client_key,
+                                                   unix_socket=unix_socket))
 
         # pre-2.6 versions of python cannot use the custom https
         # handler, since the socket class is lacking create_connection.
@@ -1164,7 +1238,7 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
              url_username=None, url_password=None, http_agent=None,
              force_basic_auth=False, follow_redirects='urllib2',
              client_cert=None, client_key=None, cookies=None,
-             use_gssapi=False):
+             use_gssapi=False, unix_socket=None):
     '''
     Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
 
@@ -1176,7 +1250,7 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
                           url_username=url_username, url_password=url_password, http_agent=http_agent,
                           force_basic_auth=force_basic_auth, follow_redirects=follow_redirects,
                           client_cert=client_cert, client_key=client_key, cookies=cookies,
-                          use_gssapi=use_gssapi)
+                          use_gssapi=use_gssapi, unix_socket=unix_socket)
 
 
 #
@@ -1212,7 +1286,7 @@ def url_argument_spec():
 
 def fetch_url(module, url, data=None, headers=None, method=None,
               use_proxy=True, force=False, last_mod_time=None, timeout=10,
-              use_gssapi=False):
+              use_gssapi=False, unix_socket=None):
     """Sends a request via HTTP(S) or FTP (needs the module as parameter)
 
     :arg module: The AnsibleModule (used to get username, password etc. (s.b.).
@@ -1226,6 +1300,8 @@ def fetch_url(module, url, data=None, headers=None, method=None,
     :kwarg last_mod_time: Default: None
     :kwarg int timeout:   Default: 10
     :kwarg boolean use_gssapi:   Default: False
+    :kwarg unix_socket: (optional) String of file system path to unix socket file to use when establishing
+    connection to the provided url
 
     :returns: A tuple of (**response**, **info**). Use ``response.read()`` to read the data.
         The **info** contains the 'status' and other meta data. When a HttpError (status > 400)
@@ -1275,7 +1351,8 @@ def fetch_url(module, url, data=None, headers=None, method=None,
                      validate_certs=validate_certs, url_username=username,
                      url_password=password, http_agent=http_agent, force_basic_auth=force_basic_auth,
                      follow_redirects=follow_redirects, client_cert=client_cert,
-                     client_key=client_key, cookies=cookies, use_gssapi=use_gssapi)
+                     client_key=client_key, cookies=cookies, use_gssapi=use_gssapi,
+                     unix_socket=unix_socket)
         # Lowercase keys, to conform to py2 behavior, so that py3 and py2 are predictable
         info.update(dict((k.lower(), v) for k, v in r.info().items()))
 

lib/ansible/modules/net_tools/basics/uri.py

@@ -170,6 +170,10 @@ options:
       - If C(no), it will not use a proxy, even if one is defined in an environment variable on the target hosts.
     type: bool
     default: yes
+  unix_socket:
+    description:
+    - Path to Unix domain socket to use for connection
+    version_added: '2.8'
 notes:
   - The dependency on httplib2 was removed in Ansible 2.1.
   - The module returns all the HTTP headers in lower-case.
@@ -460,7 +464,7 @@ def uri(module, url, dest, body, body_format, method, headers, socket_timeout):
             _, redir_info = fetch_url(module, url, data=body,
                                       headers=headers,
                                       method=method,
-                                      timeout=socket_timeout)
+                                      timeout=socket_timeout, unix_socket=module.params['unix_socket'])
             # if we are redirected, update the url with the location header,
             # and update dest with the new url filename
             if redir_info['status'] in (301, 302, 303, 307):
@@ -475,7 +479,8 @@ def uri(module, url, dest, body, body_format, method, headers, socket_timeout):
         module.params['follow_redirects'] = follow_redirects
 
     resp, info = fetch_url(module, url, data=data, headers=headers,
-                           method=method, timeout=socket_timeout, **kwargs)
+                           method=method, timeout=socket_timeout, unix_socket=module.params['unix_socket'],
+                           **kwargs)
 
     try:
         content = resp.read()
@@ -515,6 +520,7 @@ def main():
         status_code=dict(type='list', default=[200]),
         timeout=dict(type='int', default=30),
         headers=dict(type='dict', default={}),
+        unix_socket=dict(type='path'),
     )
 
     module = AnsibleModule(

test/units/module_utils/urls/test_Request.py

@@ -8,7 +8,8 @@ __metaclass__ = type
 import datetime
 import os
 
-from ansible.module_utils.urls import Request, open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod
+from ansible.module_utils.urls import (Request, open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod,
+                                       UnixHTTPHandler, UnixHTTPSConnection, httplib)
 from ansible.module_utils.urls import SSLValidationHandler, HTTPSClientAuthHandler, RedirectHandlerFactory
 
 import pytest
@@ -45,6 +46,7 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
         client_cert='/tmp/client.pem',
         client_key='/tmp/client.key',
         cookies=cookies,
+        unix_socket='/foo/bar/baz.sock',
     )
     fallback_mock = mocker.spy(request, '_fallback')
 
@@ -63,10 +65,11 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
         call(None, '/tmp/client.pem'),  # client_cert
         call(None, '/tmp/client.key'),  # client_key
         call(None, cookies),  # cookies
+        call(None, '/foo/bar/baz.sock'),  # unix_socket
     ]
     fallback_mock.assert_has_calls(calls)
 
-    assert fallback_mock.call_count == 12  # All but headers use fallback
+    assert fallback_mock.call_count == 13  # All but headers use fallback
 
     args = urlopen_mock.call_args[0]
     assert args[1] is None  # data, this is handled in the Request not urlopen
@@ -125,6 +128,39 @@ def test_Request_open_http(urlopen_mock, install_opener_mock):
     assert len(found_handlers) == 0
 
 
+def test_Request_open_unix_socket(urlopen_mock, install_opener_mock):
+    r = Request().open('GET', 'http://ansible.com/', unix_socket='/foo/bar/baz.sock')
+    args = urlopen_mock.call_args[0]
+
+    opener = install_opener_mock.call_args[0][0]
+    handlers = opener.handlers
+
+    found_handlers = []
+    for handler in handlers:
+        if isinstance(handler, UnixHTTPHandler):
+            found_handlers.append(handler)
+
+    assert len(found_handlers) == 1
+
+
+def test_Request_open_https_unix_socket(urlopen_mock, install_opener_mock):
+    r = Request().open('GET', 'https://ansible.com/', unix_socket='/foo/bar/baz.sock')
+    args = urlopen_mock.call_args[0]
+
+    opener = install_opener_mock.call_args[0][0]
+    handlers = opener.handlers
+
+    found_handlers = []
+    for handler in handlers:
+        if isinstance(handler, HTTPSClientAuthHandler):
+            found_handlers.append(handler)
+
+    assert len(found_handlers) == 1
+
+    inst = found_handlers[0]._build_https_connection('foo')
+    assert isinstance(inst, UnixHTTPSConnection)
+
+
 def test_Request_open_ftp(urlopen_mock, install_opener_mock, mocker):
     mocker.patch('ansible.module_utils.urls.ParseResultDottedDict.as_list', side_effect=AssertionError)
 
@@ -270,6 +306,10 @@ def test_Request_open_no_validate_certs(urlopen_mock, install_opener_mock):
             break
 
     assert ssl_handler is not None
+
+    inst = ssl_handler._build_https_connection('foo')
+    assert isinstance(inst, httplib.HTTPSConnection)
+
     context = ssl_handler._context
     assert context.protocol == ssl.PROTOCOL_SSLv23
     if ssl.OP_NO_SSLv2:
@@ -399,4 +439,5 @@ def test_open_url(urlopen_mock, install_opener_mock, mocker):
                                      force=False, last_mod_time=None, timeout=10, validate_certs=True,
                                      url_username=None, url_password=None, http_agent=None,
                                      force_basic_auth=False, follow_redirects='urllib2',
-                                     client_cert=None, client_key=None, cookies=None, use_gssapi=False)
+                                     client_cert=None, client_key=None, cookies=None, use_gssapi=False,
+                                     unix_socket=None)

test/units/module_utils/urls/test_fetch_url.py

@@ -67,7 +67,7 @@ def test_fetch_url(open_url_mock, fake_ansible_module):
     open_url_mock.assert_called_once_with('http://ansible.com/', client_cert=None, client_key=None, cookies=kwargs['cookies'], data=None,
                                           follow_redirects='urllib2', force=False, force_basic_auth='', headers=None,
                                           http_agent='ansible-httpget', last_mod_time=None, method=None, timeout=10, url_password='', url_username='',
-                                          use_proxy=True, validate_certs=True, use_gssapi=False)
+                                          use_proxy=True, validate_certs=True, use_gssapi=False, unix_socket=None)
 
 
 def test_fetch_url_params(open_url_mock, fake_ansible_module):
@@ -89,7 +89,7 @@ def test_fetch_url_params(open_url_mock, fake_ansible_module):
     open_url_mock.assert_called_once_with('http://ansible.com/', client_cert='client.pem', client_key='client.key', cookies=kwargs['cookies'], data=None,
                                           follow_redirects='all', force=False, force_basic_auth=True, headers=None,
                                           http_agent='ansible-test', last_mod_time=None, method=None, timeout=10, url_password='passwd', url_username='user',
-                                          use_proxy=True, validate_certs=False, use_gssapi=False)
+                                          use_proxy=True, validate_certs=False, use_gssapi=False, unix_socket=None)
 
 
 def test_fetch_url_cookies(mocker, fake_ansible_module):

test/units/module_utils/urls/test_urls.py

@@ -89,3 +89,11 @@ def test_ParseResultDottedDict():
     assert parts[0] == dotted_parts.scheme
 
     assert dotted_parts.as_list() == list(parts)
+
+
+def test_unix_socket_patch_httpconnection_connect(mocker):
+    unix_conn = mocker.patch.object(urls.UnixHTTPConnection, 'connect')
+    conn = urls.httplib.HTTPConnection('ansible.com')
+    with urls.unix_socket_patch_httpconnection_connect():
+        conn.connect()
+    assert unix_conn.call_count == 1