Terraform init -upgrade flag (#4455)

* Adds optional `-upgrade` flag to terraform init.

This allows Terraform to install provider dependencies into an existing project when the provider constraints change.

* fix transposed documentation keys

* Add integration tests for terraform init

* Revert to validate_certs: yes for general public testing

* skip integration tests on irrelevant platforms

* skip legacy Python versions from CI tests

* add changelog fragment

* Update plugins/modules/cloud/misc/terraform.py

Adds version_added metadata to the new module option.

Co-authored-by: Felix Fontein <felix@fontein.de>

* Change terraform_arch constant to Ansible fact mapping

* correct var typo, clarify task purpose

* Squashed some logic bugs, added override for local Terraform

If `existing_terraform_path` is provided, the playbook will not download Terraform or check its version.

I also tested this on a local system with Terraform installed, and squashed some bugs related to using of an
existing binary.

* revert to previous test behavior for TF install

* readability cleanup

* Update plugins/modules/cloud/misc/terraform.py

Co-authored-by: Felix Fontein <felix@fontein.de>

tests/integration/targets/terraform/tasks/main.yml

@@ -0,0 +1,70 @@
+---
+
+
+# This block checks and registers Terraform version of the binary found in path.
+
+- name: Check for existing Terraform in path
+  block:
+  - name: Check if terraform is present in path
+    command: "command -v terraform"
+    register: terraform_binary_path
+    ignore_errors: true
+
+  - name: Check Terraform version
+    command: terraform version
+    register: terraform_version_output
+    when: terraform_binary_path.rc == 0
+
+  - name: Set terraform version
+    set_fact:
+      terraform_version_installed: "{{ terraform_version_output.stdout | regex_search('(?!Terraform.*v)([0-9]+\\.[0-9]+\\.[0-9]+)') }}"
+    when: terraform_version_output.changed
+
+# This block handles the tasks of installing the Terraform binary. This happens if there is no existing
+# terraform in $PATH OR version does not match `terraform_version`.
+
+- name: Execute Terraform install tasks
+  block:
+
+  - name: Install Terraform
+    debug:
+      msg: "Installing terraform {{ terraform_version }}, found: {{ terraform_version_installed | default('no terraform binary found') }}."
+
+  - name: Ensure unzip is present
+    ansible.builtin.package:
+      name: unzip
+      state: present
+
+  - name: Install Terraform binary
+    unarchive:
+      src: "{{ terraform_url }}"
+      dest: "{{ remote_tmp_dir }}"
+      mode: 0755
+      remote_src: yes
+      validate_certs: "{{ validate_certs }}"
+
+  when: terraform_version_installed is not defined or terraform_version_installed != terraform_version
+
+# This sets `terraform_binary_path` to coalesced output of first non-empty string in this order:
+# path from the 'Check if terraform is present in path' task, and lastly, the fallback path.
+
+- name: Set path to terraform binary
+  set_fact:
+    terraform_binary_path: "{{ terraform_binary_path.stdout or remote_tmp_dir ~ '/terraform' }}"
+
+- name: Create terraform project directory
+  file:
+    path: "{{ terraform_project_dir }}/{{ item['name'] }}"
+    state: directory
+    mode: 0755
+  loop: "{{ terraform_provider_versions }}"
+  loop_control:
+    index_var: provider_index
+
+- name: Loop over provider upgrade test tasks
+  include_tasks: test_provider_upgrade.yml
+  vars:
+    tf_provider: "{{ terraform_provider_versions[provider_index] }}"
+  loop: "{{ terraform_provider_versions }}"
+  loop_control:
+    index_var: provider_index

tests/integration/targets/terraform/tasks/test_provider_upgrade.yml

@@ -0,0 +1,23 @@
+---
+
+- name: Output terraform provider test project
+  ansible.builtin.template:
+    src: templates/provider_test/main.tf.j2
+    dest: "{{ terraform_project_dir }}/{{ tf_provider['name'] }}/main.tf"
+    force: yes
+  register: terraform_provider_hcl
+
+# The purpose of this task is to init terraform multiple times with different provider module
+# versions, so that we can verify that provider upgrades during init work as intended.
+
+- name: Init Terraform configuration with pinned provider version
+  community.general.terraform:
+    project_path: "{{ terraform_provider_hcl.dest | dirname }}"
+    binary_path: "{{ terraform_binary_path }}"
+    force_init: yes
+    provider_upgrade: "{{ terraform_provider_upgrade }}"
+    state: present
+  register: terraform_init_result
+
+- assert:
+    that: terraform_init_result is not failed