Make ini parsing slightly more robust

Prior to this commit, the ini parser would fail if the inventory was
not 100% utf-8.  This commit makes this slightly more robust by
omitting full line comments from that requirement.

Fixes #17593

lib/ansible/inventory/dir.py

@@ -45,11 +45,10 @@ def get_file_parser(hostsfile, groups, loader):
     parser = None
 
     try:
-        inv_file = open(hostsfile)
-        first_line = inv_file.readlines()[0]
-        inv_file.close()
-        if first_line.startswith('#!'):
-            shebang_present = True
+        with open(hostsfile, 'rb') as inv_file:
+            initial_chars = inv_file.read(2)
+            if initial_chars.startswith(b'#!'):
+                shebang_present = True
     except:
         pass
 

lib/ansible/inventory/ini.py

@@ -40,7 +40,6 @@ class InventoryParser(object):
     """
 
     def __init__(self, loader, groups, filename=C.DEFAULT_HOST_LIST):
-        self._loader = loader
         self.filename = filename
 
         # Start with an empty host list and whatever groups we're passed in
@@ -53,12 +52,17 @@ class InventoryParser(object):
         # Read in the hosts, groups, and variables defined in the
         # inventory file.
 
-        if loader:
-            (data, private) = loader._get_file_contents(filename)
-        else:
-            with open(filename) as fh:
-                data = to_text(fh.read())
-        data = data.split('\n')
+        with open(filename, 'rb') as fh:
+            data = fh.read()
+            try:
+                # Faster to do to_text once on a long string than many
+                # times on smaller strings
+                data = to_text(data, errors='surrogate_or_strict')
+                data = [line for line in data.splitlines() if not (line.startswith(u';') or line.startswith(u'#'))]
+            except UnicodeError:
+                # Skip comment lines here to avoid potential undecodable
+                # errors in comments: https://github.com/ansible/ansible/issues/17593
+                data = [to_text(line, errors='surrogate_or_strict') for line in data.splitlines() if not (line.startswith(b';') or line.startswith(b'#'))]
 
         self._parse(data)
 
@@ -88,8 +92,8 @@ class InventoryParser(object):
 
             line = line.strip()
 
-            # Skip empty lines and comments
-            if line == '' or line.startswith(";") or line.startswith("#"):
+            # Skip empty lines
+            if not line:
                 continue
 
             # Is this a [section] header? That tells us what group we're parsing