Terraform init -upgrade flag (#4455)

* Adds optional `-upgrade` flag to terraform init.

This allows Terraform to install provider dependencies into an existing project when the provider constraints change.

* fix transposed documentation keys

* Add integration tests for terraform init

* Revert to validate_certs: yes for general public testing

* skip integration tests on irrelevant platforms

* skip legacy Python versions from CI tests

* add changelog fragment

* Update plugins/modules/cloud/misc/terraform.py

Adds version_added metadata to the new module option.

Co-authored-by: Felix Fontein <felix@fontein.de>

* Change terraform_arch constant to Ansible fact mapping

* correct var typo, clarify task purpose

* Squashed some logic bugs, added override for local Terraform

If `existing_terraform_path` is provided, the playbook will not download Terraform or check its version.

I also tested this on a local system with Terraform installed, and squashed some bugs related to using of an
existing binary.

* revert to previous test behavior for TF install

* readability cleanup

* Update plugins/modules/cloud/misc/terraform.py

Co-authored-by: Felix Fontein <felix@fontein.de>

tests/integration/targets/terraform/.gitignore

@@ -0,0 +1,4 @@
+**/.terraform/*
+*.tfstate
+*.tfstate.*
+.terraform.lock.hcl

tests/integration/targets/terraform/aliases

@@ -0,0 +1,7 @@
+shippable/posix/group1
+skip/windows
+skip/aix
+skip/osx
+skip/macos
+skip/freebsd
+skip/python2

tests/integration/targets/terraform/meta/main.yml

@@ -0,0 +1,3 @@
+dependencies:
+  - setup_pkg_mgr
+  - setup_remote_tmp_dir

tests/integration/targets/terraform/tasks/main.yml

@@ -0,0 +1,70 @@
+---
+
+
+# This block checks and registers Terraform version of the binary found in path.
+
+- name: Check for existing Terraform in path
+  block:
+  - name: Check if terraform is present in path
+    command: "command -v terraform"
+    register: terraform_binary_path
+    ignore_errors: true
+
+  - name: Check Terraform version
+    command: terraform version
+    register: terraform_version_output
+    when: terraform_binary_path.rc == 0
+
+  - name: Set terraform version
+    set_fact:
+      terraform_version_installed: "{{ terraform_version_output.stdout | regex_search('(?!Terraform.*v)([0-9]+\\.[0-9]+\\.[0-9]+)') }}"
+    when: terraform_version_output.changed
+
+# This block handles the tasks of installing the Terraform binary. This happens if there is no existing
+# terraform in $PATH OR version does not match `terraform_version`.
+
+- name: Execute Terraform install tasks
+  block:
+
+  - name: Install Terraform
+    debug:
+      msg: "Installing terraform {{ terraform_version }}, found: {{ terraform_version_installed | default('no terraform binary found') }}."
+
+  - name: Ensure unzip is present
+    ansible.builtin.package:
+      name: unzip
+      state: present
+
+  - name: Install Terraform binary
+    unarchive:
+      src: "{{ terraform_url }}"
+      dest: "{{ remote_tmp_dir }}"
+      mode: 0755
+      remote_src: yes
+      validate_certs: "{{ validate_certs }}"
+
+  when: terraform_version_installed is not defined or terraform_version_installed != terraform_version
+
+# This sets `terraform_binary_path` to coalesced output of first non-empty string in this order:
+# path from the 'Check if terraform is present in path' task, and lastly, the fallback path.
+
+- name: Set path to terraform binary
+  set_fact:
+    terraform_binary_path: "{{ terraform_binary_path.stdout or remote_tmp_dir ~ '/terraform' }}"
+
+- name: Create terraform project directory
+  file:
+    path: "{{ terraform_project_dir }}/{{ item['name'] }}"
+    state: directory
+    mode: 0755
+  loop: "{{ terraform_provider_versions }}"
+  loop_control:
+    index_var: provider_index
+
+- name: Loop over provider upgrade test tasks
+  include_tasks: test_provider_upgrade.yml
+  vars:
+    tf_provider: "{{ terraform_provider_versions[provider_index] }}"
+  loop: "{{ terraform_provider_versions }}"
+  loop_control:
+    index_var: provider_index

tests/integration/targets/terraform/tasks/test_provider_upgrade.yml

@@ -0,0 +1,23 @@
+---
+
+- name: Output terraform provider test project
+  ansible.builtin.template:
+    src: templates/provider_test/main.tf.j2
+    dest: "{{ terraform_project_dir }}/{{ tf_provider['name'] }}/main.tf"
+    force: yes
+  register: terraform_provider_hcl
+
+# The purpose of this task is to init terraform multiple times with different provider module
+# versions, so that we can verify that provider upgrades during init work as intended.
+
+- name: Init Terraform configuration with pinned provider version
+  community.general.terraform:
+    project_path: "{{ terraform_provider_hcl.dest | dirname }}"
+    binary_path: "{{ terraform_binary_path }}"
+    force_init: yes
+    provider_upgrade: "{{ terraform_provider_upgrade }}"
+    state: present
+  register: terraform_init_result
+
+- assert:
+    that: terraform_init_result is not failed

tests/integration/targets/terraform/templates/provider_test/main.tf.j2

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    {{ tf_provider['name'] }} = {
+      source  = "{{ tf_provider['source'] }}"
+      version = "{{ tf_provider['version'] }}"
+    }
+  }
+}

tests/integration/targets/terraform/vars/main.yml

@@ -0,0 +1,37 @@
+---
+
+# Terraform version that will be downloaded
+terraform_version: 1.1.7
+
+# Architecture of the downloaded Terraform release (needs to match target testing platform)
+
+terraform_arch: "{{ ansible_system | lower }}_{{terraform_arch_map[ansible_architecture] }}"
+
+# URL of where the Terraform binary will be downloaded from
+terraform_url: "https://releases.hashicorp.com/terraform/{{ terraform_version }}/terraform_{{ terraform_version }}_{{ terraform_arch }}.zip"
+
+# Controls whether the unarchive task will validate TLS certs of the Terraform binary host
+validate_certs: yes
+
+# Directory where Terraform tests will be created
+terraform_project_dir: "{{ remote_tmp_dir }}/tf_provider_test"
+
+# Controls whether terraform init will use the `-upgrade` flag
+terraform_provider_upgrade: yes
+
+# list of dicts containing Terraform providers that will be tested
+# The null provider is a good candidate, as it's small and has no external dependencies
+terraform_provider_versions: 
+  - name: "null" 
+    source: "hashicorp/null"
+    version: ">=2.0.0, < 3.0.0"
+  - name: "null" 
+    source: "hashicorp/null"
+    version: ">=3.0.0"
+
+# mapping between values returned from ansible_architecture and arch names used by golang builds of Terraform
+# see https://www.terraform.io/downloads
+
+terraform_arch_map:
+  x86_64: amd64
+  arm64: arm64