Support multiple vault passwords (#22756)

Fixes #13243

** Add --vault-id to name/identify multiple vault passwords

Use --vault-id to indicate id and path/type

 --vault-id=prompt  # prompt for default vault id password
 --vault-id=myorg@prompt  # prompt for a vault_id named 'myorg'
 --vault-id=a_password_file  # load ./a_password_file for default id
 --vault-id=myorg@a_password_file # load file for 'myorg' vault id

vault_id's are created implicitly for existing --vault-password-file
and --ask-vault-pass options.

Vault ids are just for UX purposes and bookkeeping. Only the vault
payload and the password bytestring is needed to decrypt a
vault blob.

Replace passing password around everywhere with
a VaultSecrets object.

If we specify a vault_id, mention that in password prompts

Specifying multiple -vault-password-files will
now try each until one works

** Rev vault format in a backwards compatible way

The 1.2 vault format adds the vault_id to the header line
of the vault text. This is backwards compatible with older
versions of ansible. Old versions will just ignore it and
treat it as the default (and only) vault id.

Note: only 2.4+ supports multiple vault passwords, so while
earlier ansible versions can read the vault-1.2 format, it
does not make them magically support multiple vault passwords.

use 1.1 format for 'default' vault_id

Vaulted items that need to include a vault_id will be
written in 1.2 format.

If we set a new DEFAULT_VAULT_IDENTITY, then the default will
use version 1.2

vault will only use a vault_id if one is specified. So if none
is specified and C.DEFAULT_VAULT_IDENTITY is 'default'
we use the old format.

** Changes/refactors needed to implement multiple vault passwords

raise exceptions on decrypt fail, check vault id early

split out parsing the vault plaintext envelope (with the
sha/original plaintext) to _split_plaintext_envelope()

some cli fixups for specifying multiple paths in
the unfrack_paths optparse callback

fix py3 dict.keys() 'dict_keys object is not indexable' error

pluralize cli.options.vault_password_file -> vault_password_files
pluralize cli.options.new_vault_password_file -> new_vault_password_files
pluralize cli.options.vault_id -> cli.options.vault_ids

** Add a config option (vault_id_match) to force vault id matching.

With 'vault_id_match=True' and an ansible
vault that provides a vault_id, then decryption will require
that a matching vault_id is required. (via
--vault-id=my_vault_id@password_file, for ex).

In other words, if the config option is true, then only
the vault secrets with matching vault ids are candidates for
decrypting a vault. If option is false (the default), then
all of the provided vault secrets will be selected.

If a user doesn't want all vault secrets to be tried to
decrypt any vault content, they can enable this option.

Note: The vault id used for the match is not encrypted or
cryptographically signed. It is just a label/id/nickname used
for referencing a specific vault secret.

test/units/parsing/test_dataloader.py

@@ -26,6 +26,8 @@ from ansible.compat.tests.mock import patch, mock_open
 from ansible.errors import AnsibleParserError, yaml_strings
 from ansible.module_utils._text import to_text
 from ansible.module_utils.six import PY3
+
+from units.mock.vault_helper import TextVaultSecret
 from ansible.parsing.dataloader import DataLoader
 
 from units.mock.path import mock_unfrackpath_noop
@@ -118,7 +120,8 @@ class TestDataLoaderWithVault(unittest.TestCase):
 
     def setUp(self):
         self._loader = DataLoader()
-        self._loader.set_vault_password('ansible')
+        vault_secrets = [('default', TextVaultSecret('ansible'))]
+        self._loader.set_vault_secrets(vault_secrets)
 
     def tearDown(self):
         pass

test/units/parsing/vault/test_vault.py

@@ -24,6 +24,7 @@ __metaclass__ = type
 import binascii
 import io
 import os
+import tempfile
 
 from binascii import hexlify
 import pytest
@@ -33,9 +34,141 @@ from ansible.compat.tests import unittest
 from ansible import errors
 from ansible.module_utils import six
 from ansible.module_utils._text import to_bytes, to_text
-from ansible.parsing.vault import VaultLib
 from ansible.parsing import vault
 
+from units.mock.loader import DictDataLoader
+from units.mock.vault_helper import TextVaultSecret
+
+
+class TestVaultSecret(unittest.TestCase):
+    def test(self):
+        secret = vault.VaultSecret()
+        secret.load()
+        self.assertIsNone(secret._bytes)
+
+    def test_bytes(self):
+        some_text = u'私はガラスを食べられます。それは私を傷つけません。'
+        _bytes = to_bytes(some_text)
+        secret = vault.VaultSecret(_bytes)
+        secret.load()
+        self.assertEqual(secret.bytes, _bytes)
+
+
+class TestPromptVaultSecret(unittest.TestCase):
+    def test_empty_prompt_formats(self):
+        secret = vault.PromptVaultSecret(vault_id='test_id', prompt_formats=[])
+        secret.load()
+        self.assertIsNone(secret._bytes)
+
+
+class TestFileVaultSecret(unittest.TestCase):
+    def test(self):
+        secret = vault.FileVaultSecret()
+        self.assertIsNone(secret._bytes)
+        self.assertIsNone(secret._text)
+
+    def test_repr_empty(self):
+        secret = vault.FileVaultSecret()
+        self.assertEqual(repr(secret), "FileVaultSecret()")
+
+    def test_repr(self):
+        tmp_file = tempfile.NamedTemporaryFile(delete=False)
+        fake_loader = DictDataLoader({tmp_file.name: 'sdfadf'})
+
+        secret = vault.FileVaultSecret(loader=fake_loader, filename=tmp_file.name)
+        filename = tmp_file.name
+        tmp_file.close()
+        self.assertEqual(repr(secret), "FileVaultSecret(filename='%s')" % filename)
+
+    def test_empty_bytes(self):
+        secret = vault.FileVaultSecret()
+        self.assertIsNone(secret.bytes)
+
+    def test_file(self):
+        password = 'some password'
+
+        tmp_file = tempfile.NamedTemporaryFile(delete=False)
+        tmp_file.write(to_bytes(password))
+        tmp_file.close()
+
+        fake_loader = DictDataLoader({tmp_file.name: 'sdfadf'})
+
+        secret = vault.FileVaultSecret(loader=fake_loader, filename=tmp_file.name)
+        secret.load()
+
+        os.unlink(tmp_file.name)
+
+        self.assertEqual(secret.bytes, to_bytes(password))
+
+    def test_file_not_a_directory(self):
+        filename = '/dev/null/foobar'
+        fake_loader = DictDataLoader({filename: 'sdfadf'})
+
+        secret = vault.FileVaultSecret(loader=fake_loader, filename=filename)
+        self.assertRaisesRegexp(errors.AnsibleError,
+                                '.*Could not read vault password file.*/dev/null/foobar.*Not a directory',
+                                secret.load)
+
+    def test_file_not_found(self):
+        tmp_file = tempfile.NamedTemporaryFile()
+        filename = tmp_file.name
+        tmp_file.close()
+
+        fake_loader = DictDataLoader({filename: 'sdfadf'})
+
+        secret = vault.FileVaultSecret(loader=fake_loader, filename=filename)
+        self.assertRaisesRegexp(errors.AnsibleError,
+                                '.*Could not read vault password file.*%s.*' % filename,
+                                secret.load)
+
+
+class TestScriptVaultSecret(unittest.TestCase):
+    def test(self):
+        secret = vault.ScriptVaultSecret()
+        self.assertIsNone(secret._bytes)
+        self.assertIsNone(secret._text)
+
+
+class TestGetFileVaultSecret(unittest.TestCase):
+    def test_file(self):
+        password = 'some password'
+
+        tmp_file = tempfile.NamedTemporaryFile(delete=False)
+        tmp_file.write(to_bytes(password))
+        tmp_file.close()
+
+        fake_loader = DictDataLoader({tmp_file.name: 'sdfadf'})
+
+        secret = vault.get_file_vault_secret(filename=tmp_file.name, loader=fake_loader)
+        secret.load()
+
+        os.unlink(tmp_file.name)
+
+        self.assertEqual(secret.bytes, to_bytes(password))
+
+    def test_file_not_a_directory(self):
+        filename = '/dev/null/foobar'
+        fake_loader = DictDataLoader({filename: 'sdfadf'})
+
+        self.assertRaisesRegexp(errors.AnsibleError,
+                                '.*The vault password file %s was not found.*' % filename,
+                                vault.get_file_vault_secret,
+                                filename=filename,
+                                loader=fake_loader)
+
+    def test_file_not_found(self):
+        tmp_file = tempfile.NamedTemporaryFile()
+        filename = tmp_file.name
+        tmp_file.close()
+
+        fake_loader = DictDataLoader({filename: 'sdfadf'})
+
+        self.assertRaisesRegexp(errors.AnsibleError,
+                                '.*The vault password file %s was not found.*' % filename,
+                                vault.get_file_vault_secret,
+                                filename=filename,
+                                loader=fake_loader)
+
 
 class TestVaultIsEncrypted(unittest.TestCase):
     def test_bytes_not_encrypted(self):
@@ -251,11 +384,59 @@ class TestVaultCipherAes256PyCrypto(TestVaultCipherAes256):
         super(TestVaultCipherAes256PyCrypto, self).tearDown()
 
 
+class TestMatchSecrets(unittest.TestCase):
+    def test_empty_tuple(self):
+        secrets = [tuple()]
+        vault_ids = ['vault_id_1']
+        self.assertRaises(ValueError,
+                          vault.match_secrets,
+                          secrets, vault_ids)
+
+    def test_empty_secrets(self):
+        matches = vault.match_secrets([], ['vault_id_1'])
+        self.assertEqual(matches, [])
+
+    def test_single_match(self):
+        secret = TextVaultSecret('password')
+        matches = vault.match_secrets([('default', secret)], ['default'])
+        self.assertEquals(matches, [('default', secret)])
+
+    def test_no_matches(self):
+        secret = TextVaultSecret('password')
+        matches = vault.match_secrets([('default', secret)], ['not_default'])
+        self.assertEquals(matches, [])
+
+    def test_multiple_matches(self):
+        secrets = [('vault_id1', TextVaultSecret('password1')),
+                   ('vault_id2', TextVaultSecret('password2')),
+                   ('vault_id1', TextVaultSecret('password3')),
+                   ('vault_id4', TextVaultSecret('password4'))]
+        vault_ids = ['vault_id1', 'vault_id4']
+        matches = vault.match_secrets(secrets, vault_ids)
+
+        self.assertEqual(len(matches), 3)
+        expected = [('vault_id1', TextVaultSecret('password1')),
+                    ('vault_id1', TextVaultSecret('password3')),
+                    ('vault_id4', TextVaultSecret('password4'))]
+        self.assertEqual([x for x, y in matches],
+                         [a for a, b in expected])
+
+
 @pytest.mark.skipif(not vault.HAS_CRYPTOGRAPHY,
                     reason="Skipping cryptography tests because cryptography is not installed")
 class TestVaultLib(unittest.TestCase):
     def setUp(self):
-        self.v = VaultLib('test-vault-password')
+        self.vault_password = "test-vault-password"
+        text_secret = TextVaultSecret(self.vault_password)
+        self.vault_secrets = [('default', text_secret),
+                              ('test_id', text_secret)]
+        self.v = vault.VaultLib(self.vault_secrets)
+
+    def _vault_secrets(self, vault_id, secret):
+        return [(vault_id, secret)]
+
+    def _vault_secrets_from_password(self, vault_id, password):
+        return [(vault_id, TextVaultSecret(password))]
 
     def test_encrypt(self):
         plaintext = u'Some text to encrypt in a café'
@@ -266,6 +447,15 @@ class TestVaultLib(unittest.TestCase):
         b_header = b'$ANSIBLE_VAULT;1.1;AES256\n'
         self.assertEqual(b_vaulttext[:len(b_header)], b_header)
 
+    def test_encrypt_vault_id(self):
+        plaintext = u'Some text to encrypt in a café'
+        b_vaulttext = self.v.encrypt(plaintext, vault_id='test_id')
+
+        self.assertIsInstance(b_vaulttext, six.binary_type)
+
+        b_header = b'$ANSIBLE_VAULT;1.2;AES256;test_id\n'
+        self.assertEqual(b_vaulttext[:len(b_header)], b_header)
+
     def test_encrypt_bytes(self):
 
         plaintext = to_bytes(u'Some text to encrypt in a café')
@@ -276,38 +466,60 @@ class TestVaultLib(unittest.TestCase):
         b_header = b'$ANSIBLE_VAULT;1.1;AES256\n'
         self.assertEqual(b_vaulttext[:len(b_header)], b_header)
 
+    def test_encrypt_no_secret_empty_secrets(self):
+        vault_secrets = []
+        v = vault.VaultLib(vault_secrets)
+
+        plaintext = u'Some text to encrypt in a café'
+        self.assertRaisesRegexp(vault.AnsibleVaultError,
+                                '.*A vault password must be specified to encrypt data.*',
+                                v.encrypt,
+                                plaintext)
+
     def test_is_encrypted(self):
         self.assertFalse(self.v.is_encrypted(b"foobar"), msg="encryption check on plaintext yielded false positive")
         b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
         self.assertTrue(self.v.is_encrypted(b_data), msg="encryption check on headered text failed")
 
-    def test_format_output(self):
-        self.v.cipher_name = "TEST"
+    def test_format_vaulttext_envelope(self):
+        cipher_name = "TEST"
         b_ciphertext = b"ansible"
-        b_vaulttext = self.v._format_output(b_ciphertext)
+        b_vaulttext = vault.format_vaulttext_envelope(b_ciphertext,
+                                                      cipher_name,
+                                                      version=self.v.b_version,
+                                                      vault_id='default')
         b_lines = b_vaulttext.split(b'\n')
         self.assertGreater(len(b_lines), 1, msg="failed to properly add header")
 
         b_header = b_lines[0]
-        self.assertTrue(b_header.endswith(b';TEST'), msg="header does not end with cipher name")
+        # self.assertTrue(b_header.endswith(b';TEST'), msg="header does not end with cipher name")
 
         b_header_parts = b_header.split(b';')
-        self.assertEqual(len(b_header_parts), 3, msg="header has the wrong number of parts")
+        self.assertEqual(len(b_header_parts), 4, msg="header has the wrong number of parts")
         self.assertEqual(b_header_parts[0], b'$ANSIBLE_VAULT', msg="header does not start with $ANSIBLE_VAULT")
         self.assertEqual(b_header_parts[1], self.v.b_version, msg="header version is incorrect")
         self.assertEqual(b_header_parts[2], b'TEST', msg="header does not end with cipher name")
 
-    def test_split_header(self):
+        # And just to verify, lets parse the results and compare
+        b_ciphertext2, b_version2, cipher_name2, vault_id2 = \
+            vault.parse_vaulttext_envelope(b_vaulttext)
+        self.assertEqual(b_ciphertext, b_ciphertext2)
+        self.assertEqual(self.v.b_version, b_version2)
+        self.assertEqual(cipher_name, cipher_name2)
+        self.assertEqual('default', vault_id2)
+
+    def test_parse_vaulttext_envelope(self):
         b_vaulttext = b"$ANSIBLE_VAULT;9.9;TEST\nansible"
-        b_ciphertext = self.v._split_header(b_vaulttext)
+        b_ciphertext, b_version, cipher_name, vault_id = vault.parse_vaulttext_envelope(b_vaulttext)
         b_lines = b_ciphertext.split(b'\n')
         self.assertEqual(b_lines[0], b"ansible", msg="Payload was not properly split from the header")
-        self.assertEqual(self.v.cipher_name, u'TEST', msg="cipher name was not properly set")
-        self.assertEqual(self.v.b_version, b"9.9", msg="version was not properly set")
+        self.assertEqual(cipher_name, u'TEST', msg="cipher name was not properly set")
+        self.assertEqual(b_version, b"9.9", msg="version was not properly set")
 
     def test_encrypt_decrypt_aes(self):
         self.v.cipher_name = u'AES'
-        self.v.b_password = b'ansible'
+        vault_secrets = self._vault_secrets_from_password('default', 'ansible')
+        self.v.secrets = vault_secrets
         # AES encryption code has been removed, so this is old output for
         # AES-encrypted 'foobar' with password 'ansible'.
         b_vaulttext = b'''$ANSIBLE_VAULT;1.1;AES
@@ -326,6 +538,67 @@ fe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e
         self.assertNotEqual(b_vaulttext, b"foobar", msg="encryption failed")
         self.assertEqual(b_plaintext, b"foobar", msg="decryption failed")
 
+    def test_encrypt_decrypt_aes256_none_secrets(self):
+        vault_secrets = self._vault_secrets_from_password('default', 'ansible')
+        v = vault.VaultLib(vault_secrets)
+
+        plaintext = u"foobar"
+        b_vaulttext = v.encrypt(plaintext)
+
+        # VaultLib will default to empty {} if secrets is None
+        v_none = vault.VaultLib(None)
+        # so set secrets None explicitly
+        v_none.secrets = None
+        self.assertRaisesRegexp(vault.AnsibleVaultError,
+                                '.*A vault password must be specified to decrypt data.*',
+                                v_none.decrypt,
+                                b_vaulttext)
+
+    def test_encrypt_decrypt_aes256_empty_secrets(self):
+        vault_secrets = self._vault_secrets_from_password('default', 'ansible')
+        v = vault.VaultLib(vault_secrets)
+
+        plaintext = u"foobar"
+        b_vaulttext = v.encrypt(plaintext)
+
+        vault_secrets_empty = []
+        v_none = vault.VaultLib(vault_secrets_empty)
+
+        self.assertRaisesRegexp(vault.AnsibleVaultError,
+                                '.*Attempting to decrypt but no vault secrets found.*',
+                                v_none.decrypt,
+                                b_vaulttext)
+
+    def test_encrypt_decrypt_aes256_multiple_secrets_all_wrong(self):
+        plaintext = u'Some text to encrypt in a café'
+        b_vaulttext = self.v.encrypt(plaintext)
+
+        vault_secrets = [('default', TextVaultSecret('another-wrong-password')),
+                         ('wrong-password', TextVaultSecret('wrong-password'))]
+
+        v_multi = vault.VaultLib(vault_secrets)
+        self.assertRaisesRegexp(errors.AnsibleError,
+                                '.*Decryption failed.*',
+                                v_multi.decrypt,
+                                b_vaulttext,
+                                filename='/dev/null/fake/filename')
+
+    def test_encrypt_decrypt_aes256_multiple_secrets_one_valid(self):
+        plaintext = u'Some text to encrypt in a café'
+        b_vaulttext = self.v.encrypt(plaintext)
+
+        correct_secret = TextVaultSecret(self.vault_password)
+        wrong_secret = TextVaultSecret('wrong-password')
+
+        vault_secrets = [('default', wrong_secret),
+                         ('corect_secret', correct_secret),
+                         ('wrong_secret', wrong_secret)]
+
+        v_multi = vault.VaultLib(vault_secrets)
+        b_plaintext = v_multi.decrypt(b_vaulttext)
+        self.assertNotEqual(b_vaulttext, to_bytes(plaintext), msg="encryption failed")
+        self.assertEqual(b_plaintext, to_bytes(plaintext), msg="decryption failed")
+
     def test_encrypt_decrypt_aes256_existing_vault(self):
         self.v.cipher_name = u'AES256'
         b_orig_plaintext = b"Setec Astronomy"
@@ -380,6 +653,27 @@ fe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e
         # assert we throw an error
         self.v.decrypt(b_invalid_ciphertext)
 
+    def test_decrypt_non_default_1_2(self):
+        b_expected_plaintext = to_bytes('foo bar\n')
+        vaulttext = '''$ANSIBLE_VAULT;1.2;AES256;ansible_devel
+65616435333934613466373335363332373764363365633035303466643439313864663837393234
+3330656363343637313962633731333237313636633534630a386264363438363362326132363239
+39363166646664346264383934393935653933316263333838386362633534326664646166663736
+6462303664383765650a356637643633366663643566353036303162386237336233393065393164
+6264'''
+
+        vault_secrets = self._vault_secrets_from_password('default', 'ansible')
+        v = vault.VaultLib(vault_secrets)
+
+        b_vaulttext = to_bytes(vaulttext)
+
+        b_plaintext = v.decrypt(b_vaulttext)
+        self.assertEqual(b_expected_plaintext, b_plaintext)
+
+        b_ciphertext, b_version, cipher_name, vault_id = vault.parse_vaulttext_envelope(b_vaulttext)
+        self.assertEqual('ansible_devel', vault_id)
+        self.assertEqual(b'1.2', b_version)
+
     def test_encrypt_encrypted(self):
         self.v.cipher_name = u'AES'
         b_vaulttext = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")

test/units/parsing/vault/test_vault_editor.py

@@ -30,8 +30,11 @@ from ansible.compat.tests.mock import patch
 
 from ansible import errors
 from ansible.parsing import vault
+from ansible.parsing.vault import VaultLib, VaultEditor, match_encrypt_secret
+
 from ansible.module_utils._text import to_bytes, to_text
 
+from units.mock.vault_helper import TextVaultSecret
 
 v10_data = """$ANSIBLE_VAULT;1.0;AES
 53616c7465645f5fd0026926a2d415a28a2622116273fbc90e377225c12a347e1daf4456d36a77f9
@@ -52,6 +55,14 @@ class TestVaultEditor(unittest.TestCase):
 
     def setUp(self):
         self._test_dir = None
+        self.vault_password = "test-vault-password"
+        vault_secret = TextVaultSecret(self.vault_password)
+        self.vault_secrets = [('vault_secret', vault_secret),
+                              ('default', vault_secret)]
+
+    @property
+    def vault_secret(self):
+        return match_encrypt_secret(self.vault_secrets)[1]
 
     def tearDown(self):
         if self._test_dir:
@@ -59,6 +70,11 @@ class TestVaultEditor(unittest.TestCase):
             # shutil.rmtree(self._test_dir)
         self._test_dir = None
 
+    def _secrets(self, password):
+        vault_secret = TextVaultSecret(password)
+        vault_secrets = [('default', vault_secret)]
+        return vault_secrets
+
     def test_methods_exist(self):
         v = vault.VaultEditor(None)
         slots = ['create_file',
@@ -83,6 +99,11 @@ class TestVaultEditor(unittest.TestCase):
         opened_file.close()
         return file_path
 
+    def _vault_editor(self, vault_secrets=None):
+        if vault_secrets is None:
+            vault_secrets = self._secrets(self.vault_password)
+        return VaultEditor(VaultLib(vault_secrets))
+
     @patch('ansible.parsing.vault.call')
     def test_edit_file_helper_empty_target(self, mock_sp_call):
         self._test_dir = self._create_test_dir()
@@ -91,9 +112,9 @@ class TestVaultEditor(unittest.TestCase):
         src_file_path = self._create_file(self._test_dir, 'src_file', content=src_contents)
 
         mock_sp_call.side_effect = self._faux_command
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
 
-        b_ciphertext = ve._edit_file_helper(src_file_path)
+        b_ciphertext = ve._edit_file_helper(src_file_path, self.vault_secret)
 
         self.assertNotEqual(src_contents, b_ciphertext)
 
@@ -107,12 +128,13 @@ class TestVaultEditor(unittest.TestCase):
         error_txt = 'calling editor raised an exception'
         mock_sp_call.side_effect = errors.AnsibleError(error_txt)
 
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
 
         self.assertRaisesRegexp(errors.AnsibleError,
                                 error_txt,
                                 ve._edit_file_helper,
-                                src_file_path)
+                                src_file_path,
+                                self.vault_secret)
 
     @patch('ansible.parsing.vault.call')
     def test_edit_file_helper_symlink_target(self, mock_sp_call):
@@ -126,9 +148,9 @@ class TestVaultEditor(unittest.TestCase):
         os.symlink(src_file_path, src_file_link_path)
 
         mock_sp_call.side_effect = self._faux_command
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
 
-        b_ciphertext = ve._edit_file_helper(src_file_link_path)
+        b_ciphertext = ve._edit_file_helper(src_file_link_path, self.vault_secret)
 
         self.assertNotEqual(src_file_contents, b_ciphertext,
                             'b_ciphertext should be encrypted and not equal to src_contents')
@@ -160,9 +182,9 @@ class TestVaultEditor(unittest.TestCase):
             self._faux_editor(editor_args, src_file_contents)
 
         mock_sp_call.side_effect = faux_editor
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
 
-        ve._edit_file_helper(src_file_path, existing_data=src_file_contents)
+        ve._edit_file_helper(src_file_path, self.vault_secret, existing_data=src_file_contents)
 
         new_target_file = open(src_file_path, 'rb')
         new_target_file_contents = new_target_file.read()
@@ -193,13 +215,17 @@ class TestVaultEditor(unittest.TestCase):
         src_file_contents = to_bytes("some info in a file\nyup.")
         src_file_path = self._create_file(self._test_dir, 'src_file', content=src_file_contents)
 
-        ve = vault.VaultEditor('password')
-        ve.encrypt_file(src_file_path)
+        ve = self._vault_editor()
+        ve.encrypt_file(src_file_path, self.vault_secret)
 
+        # FIXME: update to just set self._secrets or just a new vault secret id
         new_password = 'password2:electricbugaloo'
-        ve.rekey_file(src_file_path, new_password)
+        new_vault_secret = TextVaultSecret(new_password)
+        new_vault_secrets = [('default', new_vault_secret)]
+        ve.rekey_file(src_file_path, vault.match_encrypt_secret(new_vault_secrets)[1])
 
-        new_ve = vault.VaultEditor(new_password)
+        # FIXME: can just update self._secrets here
+        new_ve = vault.VaultEditor(VaultLib(new_vault_secrets))
         self._assert_file_is_encrypted(new_ve, src_file_path, src_file_contents)
 
     def test_rekey_file_no_new_password(self):
@@ -208,8 +234,8 @@ class TestVaultEditor(unittest.TestCase):
         src_file_contents = to_bytes("some info in a file\nyup.")
         src_file_path = self._create_file(self._test_dir, 'src_file', content=src_file_contents)
 
-        ve = vault.VaultEditor('password')
-        ve.encrypt_file(src_file_path)
+        ve = self._vault_editor()
+        ve.encrypt_file(src_file_path, self.vault_secret)
 
         self.assertRaisesRegexp(errors.AnsibleError,
                                 'The value for the new_password to rekey',
@@ -223,7 +249,7 @@ class TestVaultEditor(unittest.TestCase):
         src_file_contents = to_bytes("some info in a file\nyup.")
         src_file_path = self._create_file(self._test_dir, 'src_file', content=src_file_contents)
 
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
 
         new_password = 'password2:electricbugaloo'
         self.assertRaisesRegexp(errors.AnsibleError,
@@ -237,11 +263,11 @@ class TestVaultEditor(unittest.TestCase):
         src_file_contents = to_bytes("some info in a file\nyup.")
         src_file_path = self._create_file(self._test_dir, 'src_file', content=src_file_contents)
 
-        ve = vault.VaultEditor('password')
-        ve.encrypt_file(src_file_path)
+        ve = self._vault_editor()
+        ve.encrypt_file(src_file_path, self.vault_secret)
 
         res = ve.plaintext(src_file_path)
-        self.assertEquals(src_file_contents, res)
+        self.assertEqual(src_file_contents, res)
 
     def test_plaintext_not_encrypted(self):
         self._test_dir = self._create_test_dir()
@@ -249,7 +275,7 @@ class TestVaultEditor(unittest.TestCase):
         src_file_contents = to_bytes("some info in a file\nyup.")
         src_file_path = self._create_file(self._test_dir, 'src_file', content=src_file_contents)
 
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
         self.assertRaisesRegexp(errors.AnsibleError,
                                 'input is not vault encrypted data',
                                 ve.plaintext,
@@ -260,8 +286,8 @@ class TestVaultEditor(unittest.TestCase):
         src_file_contents = to_bytes("some info in a file\nyup.")
         src_file_path = self._create_file(self._test_dir, 'src_file', content=src_file_contents)
 
-        ve = vault.VaultEditor('password')
-        ve.encrypt_file(src_file_path)
+        ve = self._vault_editor()
+        ve.encrypt_file(src_file_path, self.vault_secret)
 
         self._assert_file_is_encrypted(ve, src_file_path, src_file_contents)
 
@@ -274,8 +300,8 @@ class TestVaultEditor(unittest.TestCase):
         src_file_link_path = os.path.join(self._test_dir, 'a_link_to_dest_file')
         os.symlink(src_file_path, src_file_link_path)
 
-        ve = vault.VaultEditor('password')
-        ve.encrypt_file(src_file_link_path)
+        ve = self._vault_editor()
+        ve.encrypt_file(src_file_link_path, self.vault_secret)
 
         self._assert_file_is_encrypted(ve, src_file_path, src_file_contents)
         self._assert_file_is_encrypted(ve, src_file_link_path, src_file_contents)
@@ -296,9 +322,9 @@ class TestVaultEditor(unittest.TestCase):
 
         mock_sp_call.side_effect = faux_editor
 
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
 
-        ve.encrypt_file(src_file_path)
+        ve.encrypt_file(src_file_path, self.vault_secret)
         ve.edit_file(src_file_path)
 
         new_src_file = open(src_file_path, 'rb')
@@ -308,7 +334,6 @@ class TestVaultEditor(unittest.TestCase):
         self.assertEqual(src_file_plaintext, new_src_contents)
 
         new_stat = os.stat(src_file_path)
-        print(new_stat)
 
     @patch('ansible.parsing.vault.call')
     def test_edit_file_symlink(self, mock_sp_call):
@@ -324,9 +349,9 @@ class TestVaultEditor(unittest.TestCase):
 
         mock_sp_call.side_effect = faux_editor
 
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
 
-        ve.encrypt_file(src_file_path)
+        ve.encrypt_file(src_file_path, self.vault_secret)
 
         src_file_link_path = os.path.join(self._test_dir, 'a_link_to_dest_file')
 
@@ -360,7 +385,7 @@ class TestVaultEditor(unittest.TestCase):
 
         mock_sp_call.side_effect = faux_editor
 
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
         self.assertRaisesRegexp(errors.AnsibleError,
                                 'input is not vault encrypted data',
                                 ve.edit_file,
@@ -371,18 +396,19 @@ class TestVaultEditor(unittest.TestCase):
         src_contents = to_bytes("some info in a file\nyup.")
         src_file_path = self._create_file(self._test_dir, 'src_file', content=src_contents)
 
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
         self.assertRaisesRegexp(errors.AnsibleError,
                                 'please use .edit. instead',
                                 ve.create_file,
-                                src_file_path)
+                                src_file_path,
+                                self.vault_secret)
 
     def test_decrypt_file_exception(self):
         self._test_dir = self._create_test_dir()
         src_contents = to_bytes("some info in a file\nyup.")
         src_file_path = self._create_file(self._test_dir, 'src_file', content=src_contents)
 
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
         self.assertRaisesRegexp(errors.AnsibleError,
                                 'input is not vault encrypted data',
                                 ve.decrypt_file,
@@ -398,8 +424,9 @@ class TestVaultEditor(unittest.TestCase):
         tmp_file = tempfile.NamedTemporaryFile()
         os.unlink(tmp_file.name)
 
-        ve = vault.VaultEditor("ansible")
-        ve.create_file(tmp_file.name)
+        _secrets = self._secrets('ansible')
+        ve = self._vault_editor(_secrets)
+        ve.create_file(tmp_file.name, vault.match_encrypt_secret(_secrets)[1])
 
         self.assertTrue(os.path.exists(tmp_file.name))
 
@@ -409,7 +436,7 @@ class TestVaultEditor(unittest.TestCase):
         with v10_file as f:
             f.write(to_bytes(v10_data))
 
-        ve = vault.VaultEditor("ansible")
+        ve = self._vault_editor(self._secrets("ansible"))
 
         # make sure the password functions for the cipher
         error_hit = False
@@ -417,6 +444,7 @@ class TestVaultEditor(unittest.TestCase):
             ve.decrypt_file(v10_file.name)
         except errors.AnsibleError:
             error_hit = True
+            raise
 
         # verify decrypted content
         f = open(v10_file.name, "rb")
@@ -426,7 +454,7 @@ class TestVaultEditor(unittest.TestCase):
         os.unlink(v10_file.name)
 
         assert error_hit is False, "error decrypting 1.0 file"
-        self.assertEquals(fdata.strip(), "foo")
+        self.assertEqual(fdata.strip(), "foo")
         assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()
 
     def test_decrypt_1_1(self):
@@ -434,13 +462,14 @@ class TestVaultEditor(unittest.TestCase):
         with v11_file as f:
             f.write(to_bytes(v11_data))
 
-        ve = vault.VaultEditor("ansible")
+        ve = self._vault_editor(self._secrets("ansible"))
 
         # make sure the password functions for the cipher
         error_hit = False
         try:
             ve.decrypt_file(v11_file.name)
         except errors.AnsibleError:
+            raise
             error_hit = True
 
         # verify decrypted content
@@ -450,21 +479,23 @@ class TestVaultEditor(unittest.TestCase):
 
         os.unlink(v11_file.name)
 
-        assert error_hit is False, "error decrypting 1.0 file"
-        assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()
+        assert error_hit is False, "error decrypting 1.1 file"
+        assert fdata.strip() == "foo", "incorrect decryption of 1.1 file: %s" % fdata.strip()
 
     def test_rekey_migration(self):
         v10_file = tempfile.NamedTemporaryFile(delete=False)
         with v10_file as f:
             f.write(to_bytes(v10_data))
 
-        ve = vault.VaultEditor("ansible")
+        ve = self._vault_editor(self._secrets("ansible"))
 
         # make sure the password functions for the cipher
         error_hit = False
+        new_secrets = self._secrets("ansible2")
         try:
-            ve.rekey_file(v10_file.name, 'ansible2')
+            ve.rekey_file(v10_file.name, vault.match_encrypt_secret(new_secrets)[1])
         except errors.AnsibleError:
+            raise
             error_hit = True
 
         # verify decrypted content
@@ -475,30 +506,31 @@ class TestVaultEditor(unittest.TestCase):
         assert error_hit is False, "error rekeying 1.0 file to 1.1"
 
         # ensure filedata can be decrypted, is 1.1 and is AES256
-        vl = vault.VaultLib("ansible2")
+        vl = VaultLib(new_secrets)
         dec_data = None
         error_hit = False
         try:
             dec_data = vl.decrypt(fdata)
         except errors.AnsibleError:
+            raise
             error_hit = True
 
         os.unlink(v10_file.name)
 
-        assert vl.cipher_name == "AES256", "wrong cipher name set after rekey: %s" % vl.cipher_name
+        self.assertIn(b'AES256', fdata, 'AES256 was not found in vault file %s' % to_text(fdata))
         assert error_hit is False, "error decrypting migrated 1.0 file"
         assert dec_data.strip() == b"foo", "incorrect decryption of rekeyed/migrated file: %s" % dec_data
 
     def test_real_path_dash(self):
         filename = '-'
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
 
         res = ve._real_path(filename)
         self.assertEqual(res, '-')
 
     def test_real_path_dev_null(self):
         filename = '/dev/null'
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
 
         res = ve._real_path(filename)
         self.assertEqual(res, '/dev/null')
@@ -510,7 +542,7 @@ class TestVaultEditor(unittest.TestCase):
 
         os.symlink(file_path, file_link_path)
 
-        ve = vault.VaultEditor('password')
+        ve = self._vault_editor()
 
         res = ve._real_path(file_link_path)
         self.assertEqual(res, file_path)

test/units/parsing/yaml/test_dumper.py

@@ -19,12 +19,6 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import io
-import yaml
-
-try:
-    from _yaml import ParserError
-except ImportError:
-    from yaml.parser import ParserError
 
 from ansible.compat.tests import unittest
 from ansible.parsing import vault
@@ -32,12 +26,15 @@ from ansible.parsing.yaml import dumper, objects
 from ansible.parsing.yaml.loader import AnsibleLoader
 
 from units.mock.yaml_helper import YamlTestUtils
+from units.mock.vault_helper import TextVaultSecret
 
 
 class TestAnsibleDumper(unittest.TestCase, YamlTestUtils):
     def setUp(self):
         self.vault_password = "hunter42"
-        self.good_vault = vault.VaultLib(self.vault_password)
+        vault_secret = TextVaultSecret(self.vault_password)
+        self.vault_secrets = [('vault_secret', vault_secret)]
+        self.good_vault = vault.VaultLib(self.vault_secrets)
         self.vault = self.good_vault
         self.stream = self._build_stream()
         self.dumper = dumper.AnsibleDumper
@@ -48,11 +45,12 @@ class TestAnsibleDumper(unittest.TestCase, YamlTestUtils):
         return stream
 
     def _loader(self, stream):
-        return AnsibleLoader(stream, vault_password=self.vault_password)
+        return AnsibleLoader(stream, vault_secrets=self.vault.secrets)
 
     def test(self):
         plaintext = 'This is a string we are going to encrypt.'
-        avu = objects.AnsibleVaultEncryptedUnicode.from_plaintext(plaintext, vault=self.vault)
+        avu = objects.AnsibleVaultEncryptedUnicode.from_plaintext(plaintext, vault=self.vault,
+                                                                  secret=vault.match_secrets(self.vault_secrets, ['vault_secret'])[0][1])
 
         yaml_out = self._dump_string(avu, dumper=self.dumper)
         stream = self._build_stream(yaml_out)
@@ -60,4 +58,4 @@ class TestAnsibleDumper(unittest.TestCase, YamlTestUtils):
 
         data_from_yaml = loader.get_single_data()
 
-        self.assertEquals(plaintext, data_from_yaml.data)
+        self.assertEqual(plaintext, data_from_yaml.data)

test/units/parsing/yaml/test_loader.py

@@ -34,6 +34,7 @@ from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
 from ansible.parsing.yaml.dumper import AnsibleDumper
 
 from units.mock.yaml_helper import YamlTestUtils
+from units.mock.vault_helper import TextVaultSecret
 
 try:
     from _yaml import ParserError
@@ -176,25 +177,35 @@ class TestAnsibleLoaderBasic(unittest.TestCase):
 class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils):
     def setUp(self):
         self.vault_password = "hunter42"
-        self.vault = vault.VaultLib(self.vault_password)
+        vault_secret = TextVaultSecret(self.vault_password)
+        self.vault_secrets = [('vault_secret', vault_secret),
+                              ('default', vault_secret)]
+        self.vault = vault.VaultLib(self.vault_secrets)
+
+    @property
+    def vault_secret(self):
+        return vault.match_encrypt_secret(self.vault_secrets)[1]
 
     def test_wrong_password(self):
         plaintext = u"Ansible"
         bob_password = "this is a different password"
 
-        bobs_vault = vault.VaultLib(bob_password)
+        bobs_secret = TextVaultSecret(bob_password)
+        bobs_secrets = [('default', bobs_secret)]
 
-        ciphertext = bobs_vault.encrypt(plaintext)
+        bobs_vault = vault.VaultLib(bobs_secrets)
+
+        ciphertext = bobs_vault.encrypt(plaintext, vault.match_encrypt_secret(bobs_secrets)[1])
 
         try:
             self.vault.decrypt(ciphertext)
         except Exception as e:
             self.assertIsInstance(e, errors.AnsibleError)
-            self.assertEqual(e.message, 'Decryption failed')
+            self.assertEqual(e.message, 'Decryption failed (no vault secrets would found that could decrypt)')
 
     def _encrypt_plaintext(self, plaintext):
         # Construct a yaml repr of a vault by hand
-        vaulted_var_bytes = self.vault.encrypt(plaintext)
+        vaulted_var_bytes = self.vault.encrypt(plaintext, self.vault_secret)
 
         # add yaml tag
         vaulted_var = vaulted_var_bytes.decode()
@@ -213,7 +224,7 @@ class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils):
         return stream
 
     def _loader(self, stream):
-        return AnsibleLoader(stream, vault_password=self.vault_password)
+        return AnsibleLoader(stream, vault_secrets=self.vault.secrets)
 
     def _load_yaml(self, yaml_text, password):
         stream = self._build_stream(yaml_text)
@@ -224,11 +235,11 @@ class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils):
         return data_from_yaml
 
     def test_dump_load_cycle(self):
-        avu = AnsibleVaultEncryptedUnicode.from_plaintext('The plaintext for test_dump_load_cycle.', vault=self.vault)
+        avu = AnsibleVaultEncryptedUnicode.from_plaintext('The plaintext for test_dump_load_cycle.', self.vault, self.vault_secret)
         self._dump_load_cycle(avu)
 
     def test_embedded_vault_from_dump(self):
-        avu = AnsibleVaultEncryptedUnicode.from_plaintext('setec astronomy', vault=self.vault)
+        avu = AnsibleVaultEncryptedUnicode.from_plaintext('setec astronomy', self.vault, self.vault_secret)
         blip = {'stuff1': [{'a dict key': 24},
                            {'shhh-ssh-secrets': avu,
                             'nothing to see here': 'move along'}],
@@ -239,7 +250,6 @@ class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils):
 
         self._dump_stream(blip, stream, dumper=AnsibleDumper)
 
-        print(stream.getvalue())
         stream.seek(0)
 
         stream.seek(0)
@@ -247,6 +257,7 @@ class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils):
         loader = self._loader(stream)
 
         data_from_yaml = loader.get_data()
+
         stream2 = NameStringIO(u'')
         # verify we can dump the object again
         self._dump_stream(data_from_yaml, stream2, dumper=AnsibleDumper)
@@ -266,20 +277,20 @@ class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils):
         data_from_yaml = self._load_yaml(yaml_text, self.vault_password)
         vault_string = data_from_yaml['the_secret']
 
-        self.assertEquals(plaintext_var, data_from_yaml['the_secret'])
+        self.assertEqual(plaintext_var, data_from_yaml['the_secret'])
 
         test_dict = {}
         test_dict[vault_string] = 'did this work?'
 
-        self.assertEquals(vault_string.data, vault_string)
+        self.assertEqual(vault_string.data, vault_string)
 
         # This looks weird and useless, but the object in question has a custom __eq__
-        self.assertEquals(vault_string, vault_string)
+        self.assertEqual(vault_string, vault_string)
 
         another_vault_string = data_from_yaml['another_secret']
         different_vault_string = data_from_yaml['different_secret']
 
-        self.assertEquals(vault_string, another_vault_string)
+        self.assertEqual(vault_string, another_vault_string)
         self.assertNotEquals(vault_string, different_vault_string)
 
         # More testing of __eq__/__ne__
@@ -288,8 +299,8 @@ class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils):
 
         # Note this is a compare of the str/unicode of these, they are different types
         # so we want to test self == other, and other == self etc
-        self.assertEquals(plaintext_var, vault_string)
-        self.assertEquals(vault_string, plaintext_var)
+        self.assertEqual(plaintext_var, vault_string)
+        self.assertEqual(vault_string, plaintext_var)
         self.assertFalse(plaintext_var != vault_string)
         self.assertFalse(vault_string != plaintext_var)
 

test/units/parsing/yaml/test_objects.py

@@ -21,6 +21,7 @@ __metaclass__ = type
 
 from ansible.compat.tests import unittest
 
+from ansible.errors import AnsibleError
 
 from ansible.parsing import vault
 from ansible.parsing.yaml.loader import AnsibleLoader
@@ -29,6 +30,7 @@ from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.parsing.yaml import objects
 
 from units.mock.yaml_helper import YamlTestUtils
+from units.mock.vault_helper import TextVaultSecret
 
 
 class TestAnsibleVaultUnicodeNoVault(unittest.TestCase, YamlTestUtils):
@@ -68,16 +70,22 @@ class TestAnsibleVaultUnicodeNoVault(unittest.TestCase, YamlTestUtils):
 
 class TestAnsibleVaultEncryptedUnicode(unittest.TestCase, YamlTestUtils):
     def setUp(self):
-        self.vault_password = "hunter42"
-        self.good_vault = vault.VaultLib(self.vault_password)
+        self.good_vault_password = "hunter42"
+        good_vault_secret = TextVaultSecret(self.good_vault_password)
+        self.good_vault_secrets = [('good_vault_password', good_vault_secret)]
+        self.good_vault = vault.VaultLib(self.good_vault_secrets)
 
+        # TODO: make this use two vault secret identities instead of two vaultSecrets
         self.wrong_vault_password = 'not-hunter42'
-        self.wrong_vault = vault.VaultLib(self.wrong_vault_password)
+        wrong_vault_secret = TextVaultSecret(self.wrong_vault_password)
+        self.wrong_vault_secrets = [('wrong_vault_password', wrong_vault_secret)]
+        self.wrong_vault = vault.VaultLib(self.wrong_vault_secrets)
 
         self.vault = self.good_vault
+        self.vault_secrets = self.good_vault_secrets
 
     def _loader(self, stream):
-        return AnsibleLoader(stream, vault_password=self.vault_password)
+        return AnsibleLoader(stream, vault_secrets=self.vault_secrets)
 
     def test_dump_load_cycle(self):
         aveu = self._from_plaintext('the test string for TestAnsibleVaultEncryptedUnicode.test_dump_load_cycle')
@@ -86,12 +94,13 @@ class TestAnsibleVaultEncryptedUnicode(unittest.TestCase, YamlTestUtils):
     def assert_values(self, avu, seq):
         self.assertIsInstance(avu, objects.AnsibleVaultEncryptedUnicode)
 
-        self.assertEquals(avu, seq)
+        self.assertEqual(avu, seq)
         self.assertTrue(avu.vault is self.vault)
         self.assertIsInstance(avu.vault, vault.VaultLib)
 
     def _from_plaintext(self, seq):
-        return objects.AnsibleVaultEncryptedUnicode.from_plaintext(seq, vault=self.vault)
+        id_secret = vault.match_encrypt_secret(self.good_vault_secrets)
+        return objects.AnsibleVaultEncryptedUnicode.from_plaintext(seq, vault=self.vault, secret=id_secret[1])
 
     def _from_ciphertext(self, ciphertext):
         avu = objects.AnsibleVaultEncryptedUnicode(ciphertext)
@@ -126,7 +135,7 @@ class TestAnsibleVaultEncryptedUnicode(unittest.TestCase, YamlTestUtils):
         avu = self._from_plaintext(seq)
         b_avu = avu.encode('utf-8', 'strict')
         self.assertIsInstance(avu, objects.AnsibleVaultEncryptedUnicode)
-        self.assertEquals(b_avu, seq.encode('utf-8', 'strict'))
+        self.assertEqual(b_avu, seq.encode('utf-8', 'strict'))
         self.assertTrue(avu.vault is self.vault)
         self.assertIsInstance(avu.vault, vault.VaultLib)
 
@@ -135,4 +144,8 @@ class TestAnsibleVaultEncryptedUnicode(unittest.TestCase, YamlTestUtils):
         seq = ''
         self.vault = self.wrong_vault
         avu = self._from_plaintext(seq)
-        self.assert_values(avu, seq)
+
+        def compare(avu, seq):
+            return avu == seq
+
+        self.assertRaises(AnsibleError, compare, avu, seq)