Support multiple vault passwords (#22756)

Fixes #13243

** Add --vault-id to name/identify multiple vault passwords

Use --vault-id to indicate id and path/type

 --vault-id=prompt  # prompt for default vault id password
 --vault-id=myorg@prompt  # prompt for a vault_id named 'myorg'
 --vault-id=a_password_file  # load ./a_password_file for default id
 --vault-id=myorg@a_password_file # load file for 'myorg' vault id

vault_id's are created implicitly for existing --vault-password-file
and --ask-vault-pass options.

Vault ids are just for UX purposes and bookkeeping. Only the vault
payload and the password bytestring is needed to decrypt a
vault blob.

Replace passing password around everywhere with
a VaultSecrets object.

If we specify a vault_id, mention that in password prompts

Specifying multiple -vault-password-files will
now try each until one works

** Rev vault format in a backwards compatible way

The 1.2 vault format adds the vault_id to the header line
of the vault text. This is backwards compatible with older
versions of ansible. Old versions will just ignore it and
treat it as the default (and only) vault id.

Note: only 2.4+ supports multiple vault passwords, so while
earlier ansible versions can read the vault-1.2 format, it
does not make them magically support multiple vault passwords.

use 1.1 format for 'default' vault_id

Vaulted items that need to include a vault_id will be
written in 1.2 format.

If we set a new DEFAULT_VAULT_IDENTITY, then the default will
use version 1.2

vault will only use a vault_id if one is specified. So if none
is specified and C.DEFAULT_VAULT_IDENTITY is 'default'
we use the old format.

** Changes/refactors needed to implement multiple vault passwords

raise exceptions on decrypt fail, check vault id early

split out parsing the vault plaintext envelope (with the
sha/original plaintext) to _split_plaintext_envelope()

some cli fixups for specifying multiple paths in
the unfrack_paths optparse callback

fix py3 dict.keys() 'dict_keys object is not indexable' error

pluralize cli.options.vault_password_file -> vault_password_files
pluralize cli.options.new_vault_password_file -> new_vault_password_files
pluralize cli.options.vault_id -> cli.options.vault_ids

** Add a config option (vault_id_match) to force vault id matching.

With 'vault_id_match=True' and an ansible
vault that provides a vault_id, then decryption will require
that a matching vault_id is required. (via
--vault-id=my_vault_id@password_file, for ex).

In other words, if the config option is true, then only
the vault secrets with matching vault ids are candidates for
decrypting a vault. If option is false (the default), then
all of the provided vault secrets will be selected.

If a user doesn't want all vault secrets to be tried to
decrypt any vault content, they can enable this option.

Note: The vault id used for the match is not encrypted or
cryptographically signed. It is just a label/id/nickname used
for referencing a specific vault secret.

lib/ansible/parsing/yaml/constructor.py

@@ -23,9 +23,10 @@ from yaml.constructor import SafeConstructor, ConstructorError
 from yaml.nodes import MappingNode
 
 from ansible.module_utils._text import to_bytes
-from ansible.parsing.vault import VaultLib
-from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode, AnsibleVaultEncryptedUnicode
+from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode
+from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
 from ansible.utils.unsafe_proxy import wrap_var
+from ansible.parsing.vault import VaultLib, parse_vaulttext_envelope
 
 
 try:
@@ -36,12 +37,12 @@ except ImportError:
 
 
 class AnsibleConstructor(SafeConstructor):
-    def __init__(self, file_name=None, b_vault_password=None):
-        self._b_vault_password = b_vault_password
+    def __init__(self, file_name=None, vault_secrets=None):
         self._ansible_file_name = file_name
         super(AnsibleConstructor, self).__init__()
         self._vaults = {}
-        self._vaults['default'] = VaultLib(b_password=self._b_vault_password)
+        self.vault_secrets = vault_secrets or []
+        self._vaults['default'] = VaultLib(secrets=self.vault_secrets)
 
     def construct_yaml_map(self, node):
         data = AnsibleMapping()
@@ -96,17 +97,16 @@ class AnsibleConstructor(SafeConstructor):
 
     def construct_vault_encrypted_unicode(self, node):
         value = self.construct_scalar(node)
-        ciphertext_data = to_bytes(value)
-
-        if self._b_vault_password is None:
+        b_ciphertext_data = to_bytes(value)
+        # could pass in a key id here to choose the vault to associate with
+        # TODO/FIXME: plugin vault selector
+        vault = self._vaults['default']
+        if vault.secrets is None:
             raise ConstructorError(context=None, context_mark=None,
                                    problem="found !vault but no vault password provided",
                                    problem_mark=node.start_mark,
                                    note=None)
-
-        # could pass in a key id here to choose the vault to associate with
-        vault = self._vaults['default']
-        ret = AnsibleVaultEncryptedUnicode(ciphertext_data)
+        ret = AnsibleVaultEncryptedUnicode(b_ciphertext_data)
         ret.vault = vault
         return ret
 

lib/ansible/parsing/yaml/loader.py

@@ -32,9 +32,9 @@ from ansible.parsing.yaml.constructor import AnsibleConstructor
 if HAVE_PYYAML_C:
 
     class AnsibleLoader(CParser, AnsibleConstructor, Resolver):
-        def __init__(self, stream, file_name=None, vault_password=None):
+        def __init__(self, stream, file_name=None, vault_secrets=None):
             CParser.__init__(self, stream)
-            AnsibleConstructor.__init__(self, file_name=file_name, b_vault_password=vault_password)
+            AnsibleConstructor.__init__(self, file_name=file_name, vault_secrets=vault_secrets)
             Resolver.__init__(self)
 else:
     from yaml.composer import Composer
@@ -43,10 +43,10 @@ else:
     from yaml.parser import Parser
 
     class AnsibleLoader(Reader, Scanner, Parser, Composer, AnsibleConstructor, Resolver):
-        def __init__(self, stream, file_name=None, vault_password=None):
+        def __init__(self, stream, file_name=None, vault_secrets=None):
             Reader.__init__(self, stream)
             Scanner.__init__(self)
             Parser.__init__(self)
             Composer.__init__(self)
-            AnsibleConstructor.__init__(self, file_name=file_name, b_vault_password=vault_password)
+            AnsibleConstructor.__init__(self, file_name=file_name, vault_secrets=vault_secrets)
             Resolver.__init__(self)

lib/ansible/parsing/yaml/objects.py

@@ -76,11 +76,11 @@ class AnsibleVaultEncryptedUnicode(yaml.YAMLObject, AnsibleBaseYAMLObject):
     yaml_tag = u'!vault'
 
     @classmethod
-    def from_plaintext(cls, seq, vault):
+    def from_plaintext(cls, seq, vault, secret):
         if not vault:
             raise vault.AnsibleVaultError('Error creating AnsibleVaultEncryptedUnicode, invalid vault (%s) provided' % vault)
 
-        ciphertext = vault.encrypt(seq)
+        ciphertext = vault.encrypt(seq, secret)
         avu = cls(ciphertext)
         avu.vault = vault
         return avu