Fix url lookup for python 3 (#17295)

* Use six instead of urllib2, for python 3 compat

* Open the certificate file using binary mode

On python3, os.write requires 'bytes'. Also avoid
using a too broad exception, since the issue was hard
to spot due to it.

* Do not add the header User-agent if not set

Python3 module do raise a exception if a header is
not a string-like object, and the default value is None.

lib/ansible/plugins/lookup/url.py

@@ -17,12 +17,12 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import urllib2
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 from ansible.module_utils.urls import open_url, ConnectionError, SSLValidationError
 from ansible.utils.unicode import to_unicode
+from ansible.compat.six.moves.urllib.error import HTTPError, URLError
 
 try:
     from __main__ import display
@@ -42,9 +42,9 @@ class LookupModule(LookupBase):
             display.vvvv("url lookup connecting to %s" % term)
             try:
                 response = open_url(term, validate_certs=validate_certs)
-            except urllib2.HTTPError as e:
+            except HTTPError as e:
                 raise AnsibleError("Received HTTP error for %s : %s" % (term, str(e)))
-            except urllib2.URLError as e:
+            except URLError as e:
                 raise AnsibleError("Failed lookup url for %s : %s" % (term, str(e)))
             except SSLValidationError as e:
                 raise AnsibleError("Error validating the server's certificate for %s: %s" % (term, str(e)))