Implement vault encrypted yaml variables. (#16274)

Make !vault-encrypted create a AnsibleVaultUnicode
yaml object that can be used as a regular string object.

This allows a playbook to include a encrypted vault
blob for the value of a yaml variable. A 'secret_password'
variable can have it's value encrypted instead of having
to vault encrypt an entire vars file.

Add __ENCRYPTED__ to the vault yaml types so
template.Template can treat it similar
to __UNSAFE__ flags.

vault.VaultLib api changes:
    - Split VaultLib.encrypt to encrypt and encrypt_bytestring

    - VaultLib.encrypt() previously accepted the plaintext data
      as either a byte string or a unicode string.
      Doing the right thing based on the input type would fail
      on py3 if given a arg of type 'bytes'. To simplify the
      API, vaultlib.encrypt() now assumes input plaintext is a
      py2 unicode or py3 str. It will encode to utf-8 then call
      the new encrypt_bytestring(). The new methods are less
      ambiguous.

    - moved VaultLib.is_encrypted logic to vault module scope
      and split to is_encrypted() and is_encrypted_file().

Add a test/unit/mock/yaml_helper.py
It has some helpers for testing parsing/yaml

Integration tests added as roles test_vault and test_vault_embedded

test/units/parsing/yaml/test_loader.py

@@ -26,20 +26,26 @@ from six import text_type, binary_type
 from collections import Sequence, Set, Mapping
 
 from ansible.compat.tests import unittest
-from ansible.compat.tests.mock import patch
 
+from ansible import errors
 from ansible.parsing.yaml.loader import AnsibleLoader
+from ansible.parsing import vault
+from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+from ansible.parsing.yaml.dumper import AnsibleDumper
+from ansible.utils.unicode import to_bytes
+
+from units.mock.yaml_helper import YamlTestUtils
 
 try:
     from _yaml import ParserError
 except ImportError:
     from yaml.parser import ParserError
 
-
 class NameStringIO(StringIO):
     """In py2.6, StringIO doesn't let you set name because a baseclass has it
     as readonly property"""
     name = None
+
     def __init__(self, *args, **kwargs):
         super(NameStringIO, self).__init__(*args, **kwargs)
 
@@ -159,6 +165,124 @@ class TestAnsibleLoaderBasic(unittest.TestCase):
         self.assertEqual(data[0][u'baz'].ansible_pos, ('myfile.yml', 2, 9))
 
 
+class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils):
+    def setUp(self):
+        self.vault_password = "hunter42"
+        self.vault = vault.VaultLib(self.vault_password)
+
+    def test_wrong_password(self):
+        plaintext = u"Ansible"
+        bob_password = "this is a different password"
+
+        bobs_vault = vault.VaultLib(bob_password)
+
+        ciphertext = bobs_vault.encrypt(plaintext)
+
+        try:
+            self.vault.decrypt(ciphertext)
+        except Exception as e:
+            self.assertIsInstance(e, errors.AnsibleError)
+            self.assertEqual(e.message, 'Decryption failed')
+
+    def _encrypt_plaintext(self, plaintext):
+        # Construct a yaml repr of a vault by hand
+        vaulted_var_bytes = self.vault.encrypt(plaintext)
+
+        # add yaml tag
+        vaulted_var = vaulted_var_bytes.decode()
+        lines = vaulted_var.splitlines()
+        lines2 = []
+        for line in lines:
+            lines2.append('        %s' % line)
+
+        vaulted_var = '\n'.join(lines2)
+        tagged_vaulted_var = u"""!vault-encrypted |\n%s""" % vaulted_var
+        return tagged_vaulted_var
+
+    def _build_stream(self, yaml_text):
+        stream = NameStringIO(yaml_text)
+        stream.name = 'my.yml'
+        return stream
+
+    def _loader(self, stream):
+        return AnsibleLoader(stream, vault_password=self.vault_password)
+
+    def _load_yaml(self, yaml_text, password):
+        stream = self._build_stream(yaml_text)
+        loader = self._loader(stream)
+
+        data_from_yaml = loader.get_single_data()
+
+        return data_from_yaml
+
+    def test_dump_load_cycle(self):
+        avu = AnsibleVaultEncryptedUnicode.from_plaintext('The plaintext for test_dump_load_cycle.', vault=self.vault)
+        self._dump_load_cycle(avu)
+
+    def test_embedded_vault_from_dump(self):
+        avu = AnsibleVaultEncryptedUnicode.from_plaintext('setec astronomy', vault=self.vault)
+        blip = {'stuff1': [{'a dict key': 24},
+                           {'shhh-ssh-secrets': avu,
+                            'nothing to see here': 'move along'}],
+                'another key': 24.1}
+
+        blip = ['some string', 'another string', avu]
+        stream = NameStringIO()
+
+        self._dump_stream(blip, stream, dumper=AnsibleDumper)
+
+        print(stream.getvalue())
+        stream.seek(0)
+
+        stream.seek(0)
+
+        loader = self._loader(stream)
+
+        data_from_yaml = loader.get_data()
+        stream2 = NameStringIO(u'')
+        # verify we can dump the object again
+        self._dump_stream(data_from_yaml, stream2, dumper=AnsibleDumper)
+
+    def test_embedded_vault(self):
+        plaintext_var = u"""This is the plaintext string."""
+        tagged_vaulted_var = self._encrypt_plaintext(plaintext_var)
+        another_vaulted_var = self._encrypt_plaintext(plaintext_var)
+
+        different_var = u"""A different string that is not the same as the first one."""
+        different_vaulted_var = self._encrypt_plaintext(different_var)
+
+        yaml_text = u"""---\nwebster: daniel\noed: oxford\nthe_secret: %s\nanother_secret: %s\ndifferent_secret: %s""" % (tagged_vaulted_var, another_vaulted_var, different_vaulted_var)
+
+        data_from_yaml = self._load_yaml(yaml_text, self.vault_password)
+        vault_string = data_from_yaml['the_secret']
+
+        self.assertEquals(plaintext_var, data_from_yaml['the_secret'])
+
+        test_dict = {}
+        test_dict[vault_string] = 'did this work?'
+
+        self.assertEquals(vault_string.data, vault_string)
+
+        # This looks weird and useless, but the object in question has a custom __eq__
+        self.assertEquals(vault_string, vault_string)
+
+        another_vault_string = data_from_yaml['another_secret']
+        different_vault_string = data_from_yaml['different_secret']
+
+        self.assertEquals(vault_string, another_vault_string)
+        self.assertNotEquals(vault_string, different_vault_string)
+
+        # More testing of __eq__/__ne__
+        self.assertTrue('some string' != vault_string)
+        self.assertNotEquals('some string', vault_string)
+
+        # Note this is a compare of the str/unicode of these, they are diferent types
+        # so we want to test self == other, and other == self etc
+        self.assertEquals(plaintext_var, vault_string)
+        self.assertEquals(vault_string, plaintext_var)
+        self.assertFalse(plaintext_var != vault_string)
+        self.assertFalse(vault_string != plaintext_var)
+
 class TestAnsibleLoaderPlay(unittest.TestCase):
 
     def setUp(self):
@@ -242,7 +366,7 @@ class TestAnsibleLoaderPlay(unittest.TestCase):
 
     def check_vars(self):
         # Numbers don't have line/col information yet
-        #self.assertEqual(self.data[0][u'vars'][u'number'].ansible_pos, (self.play_filename, 4, 21))
+        # self.assertEqual(self.data[0][u'vars'][u'number'].ansible_pos, (self.play_filename, 4, 21))
 
         self.assertEqual(self.data[0][u'vars'][u'string'].ansible_pos, (self.play_filename, 5, 29))
         self.assertEqual(self.data[0][u'vars'][u'utf8_string'].ansible_pos, (self.play_filename, 6, 34))
@@ -255,8 +379,8 @@ class TestAnsibleLoaderPlay(unittest.TestCase):
         self.assertEqual(self.data[0][u'vars'][u'list'][0].ansible_pos, (self.play_filename, 11, 25))
         self.assertEqual(self.data[0][u'vars'][u'list'][1].ansible_pos, (self.play_filename, 12, 25))
         # Numbers don't have line/col info yet
-        #self.assertEqual(self.data[0][u'vars'][u'list'][2].ansible_pos, (self.play_filename, 13, 25))
-        #self.assertEqual(self.data[0][u'vars'][u'list'][3].ansible_pos, (self.play_filename, 14, 25))
+        # self.assertEqual(self.data[0][u'vars'][u'list'][2].ansible_pos, (self.play_filename, 13, 25))
+        # self.assertEqual(self.data[0][u'vars'][u'list'][3].ansible_pos, (self.play_filename, 14, 25))
 
     def check_tasks(self):
         #