Add a yaml constructor for unicode strings:

* Changes AnsibleConstructor so that only unicode strings are returned
  (no str type)
* Tracks line, column numbers for strings
* Adds unittests for AnsibleLoader (generic for all the yaml parsing)

v2/ansible/parsing/yaml/composer.py

@@ -20,17 +20,27 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from yaml.composer import Composer
-from yaml.nodes import MappingNode
+from yaml.nodes import MappingNode, ScalarNode
 
 class AnsibleComposer(Composer):
     def __init__(self):
         self.__mapping_starts = []
         super(Composer, self).__init__()
+
     def compose_node(self, parent, index):
         # the line number where the previous token has ended (plus empty lines)
         node = Composer.compose_node(self, parent, index)
-        if isinstance(node, MappingNode):
+        if isinstance(node, ScalarNode):
+            # Scalars are pretty easy -- assume they start on the current
+            # token's line (what about multiline strings?  Perhaps we also
+            # need to use previous token ended
             node.__datasource__ = self.name
+            node.__line__ = self.line + 1
+            node.__column__ = self.column + 1
+        elif isinstance(node, MappingNode):
+            node.__datasource__ = self.name
+
+            # Need extra help to know where the mapping starts
             try:
                 (cur_line, cur_column) = self.__mapping_starts.pop()
             except:
@@ -38,7 +48,9 @@ class AnsibleComposer(Composer):
                 cur_column = None
             node.__line__   = cur_line
             node.__column__ = cur_column
+
         return node
+
     def compose_mapping_node(self, anchor):
         # the column here will point at the position in the file immediately
         # after the first key is found, which could be a space or a newline.

v2/ansible/parsing/yaml/constructor.py

@@ -20,7 +20,8 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from yaml.constructor import Constructor
-from ansible.parsing.yaml.objects import AnsibleMapping
+from ansible.utils.unicode import to_unicode
+from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleUnicode
 
 class AnsibleConstructor(Constructor):
     def __init__(self, file_name=None):
@@ -52,6 +53,22 @@ class AnsibleConstructor(Constructor):
 
         return ret
 
+    def construct_yaml_str(self, node):
+        # Override the default string handling function
+        # to always return unicode objects
+        value = self.construct_scalar(node)
+        value = to_unicode(value)
+        data = AnsibleUnicode(self.construct_scalar(node))
+
+        data._line_number = node.__line__
+        data._column_number = node.__column__
+        if self._ansible_file_name:
+            data._data_source = self._ansible_file_name
+        else:
+            data._data_source = node.__datasource__
+
+        return data
+
 AnsibleConstructor.add_constructor(
     u'tag:yaml.org,2002:map',
     AnsibleConstructor.construct_yaml_map)
@@ -60,3 +77,11 @@ AnsibleConstructor.add_constructor(
     u'tag:yaml.org,2002:python/dict',
     AnsibleConstructor.construct_yaml_map)
 
+AnsibleConstructor.add_constructor(
+    u'tag:yaml.org,2002:str',
+    AnsibleConstructor.construct_yaml_str)
+
+AnsibleConstructor.add_constructor(
+    u'tag:yaml.org,2002:python/unicode',
+    AnsibleConstructor.construct_yaml_str)
+

v2/ansible/parsing/yaml/objects.py

@@ -50,3 +50,6 @@ class AnsibleMapping(AnsibleBaseYAMLObject, dict):
     ''' sub class for dictionaries '''
     pass
 
+class AnsibleUnicode(AnsibleBaseYAMLObject, unicode):
+    ''' sub class for unicode objects '''
+    pass