Fix inventory plugin cache + add tests (#38229)

* Fix setting the cache when refresh_cache or --flush-cache are used

* Use jsonify function that handles datetime objects in jsonfile cache plugin

* Don't access self._options directly

* Add initial integration tests for aws_ec2 inventory plugin

* Add CI alias

* Fix and add a few more unit tests

* Add integration tests for constructed

* Fix typo

* Use inventory config templates

* Collect all instances that are not terminated by default

* Create separate playbook for setting up the VPC, subnet, security group, and finding an image for the host

Create a separate playbook for removing the resources

* Allow easier grouping by region and add an example

* use a unified json encode/decode that can handle unsafe and vault

test/units/plugins/inventory/test_aws_ec2.py

@@ -28,7 +28,7 @@ import datetime
 boto3 = pytest.importorskip('boto3')
 botocore = pytest.importorskip('botocore')
 
-from ansible.errors import AnsibleError
+from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.plugins.inventory.aws_ec2 import InventoryModule
 from ansible.plugins.inventory.aws_ec2 import instance_data_filter_to_boto_attr
 
@@ -111,62 +111,101 @@ instances = {
 }
 
 
-def test_compile_values():
-    inv = InventoryModule()
+@pytest.fixture(scope="module")
+def inventory():
+    return InventoryModule()
+
+
+def test_compile_values(inventory):
     found_value = instances['Instances'][0]
     chain_of_keys = instance_data_filter_to_boto_attr['instance.group-id']
     for attr in chain_of_keys:
-        found_value = inv._compile_values(found_value, attr)
+        found_value = inventory._compile_values(found_value, attr)
     assert found_value == "sg-12345678"
 
 
-def test_get_boto_attr_chain():
-    inv = InventoryModule()
+def test_get_boto_attr_chain(inventory):
     instance = instances['Instances'][0]
-    assert inv._get_boto_attr_chain('network-interface.addresses.private-ip-address', instance) == "098.76.54.321"
+    assert inventory._get_boto_attr_chain('network-interface.addresses.private-ip-address', instance) == "098.76.54.321"
 
 
-def test_boto3_conn():
-    inv = InventoryModule()
-    inv._options = {"boto_profile": "first_precedence",
-                    "aws_access_key_id": "test_access_key",
-                    "aws_secret_access_key": "test_secret_key",
-                    "aws_security_token": "test_security_token"}
-    inv._set_credentials()
+def test_boto3_conn(inventory):
+    inventory._options = {"boto_profile": "first_precedence",
+                          "aws_access_key_id": "test_access_key",
+                          "aws_secret_access_key": "test_secret_key",
+                          "aws_security_token": "test_security_token"}
+    inventory._set_credentials()
     with pytest.raises(AnsibleError) as error_message:
-        for connection, region in inv._boto3_conn(regions=['us-east-1']):
+        for connection, region in inventory._boto3_conn(regions=['us-east-1']):
             assert error_message == "Insufficient credentials found."
 
 
-def test_get_hostname_default():
-    inv = InventoryModule()
+def test_get_hostname_default(inventory):
     instance = instances['Instances'][0]
-    assert inv._get_hostname(instance, hostnames=None) == "ec2-12-345-67-890.compute-1.amazonaws.com"
+    assert inventory._get_hostname(instance, hostnames=None) == "ec2-12-345-67-890.compute-1.amazonaws.com"
 
 
-def test_get_hostname():
+def test_get_hostname(inventory):
     hostnames = ['ip-address', 'dns-name']
-    inv = InventoryModule()
     instance = instances['Instances'][0]
-    assert inv._get_hostname(instance, hostnames) == "12.345.67.890"
+    assert inventory._get_hostname(instance, hostnames) == "12.345.67.890"
 
 
-def test_set_credentials(monkeypatch):
-    inv = InventoryModule()
-    inv._options = {'aws_access_key_id': 'test_access_key',
-                    'aws_secret_access_key': 'test_secret_key',
-                    'aws_security_token': 'test_security_token',
-                    'boto_profile': 'test_profile'}
-    inv._set_credentials()
+def test_set_credentials(inventory):
+    inventory._options = {'aws_access_key_id': 'test_access_key',
+                          'aws_secret_access_key': 'test_secret_key',
+                          'aws_security_token': 'test_security_token',
+                          'boto_profile': 'test_profile'}
+    inventory._set_credentials()
 
-    assert inv.boto_profile == "test_profile"
-    assert inv.aws_access_key_id == "test_access_key"
-    assert inv.aws_secret_access_key == "test_secret_key"
-    assert inv.aws_security_token == "test_security_token"
+    assert inventory.boto_profile == "test_profile"
+    assert inventory.aws_access_key_id == "test_access_key"
+    assert inventory.aws_secret_access_key == "test_secret_key"
+    assert inventory.aws_security_token == "test_security_token"
 
 
-def test_insufficient_credentials(monkeypatch):
-    inv = InventoryModule()
+def test_insufficient_credentials(inventory):
+    inventory._options = {
+        'aws_access_key_id': None,
+        'aws_secret_access_key': None,
+        'aws_security_token': None,
+        'boto_profile': None
+    }
     with pytest.raises(AnsibleError) as error_message:
-        inv._set_credentials()
+        inventory._set_credentials()
         assert "Insufficient boto credentials found" in error_message
+
+
+def test_validate_option(inventory):
+    assert ['us-east-1'] == inventory._validate_option('regions', list, 'us-east-1')
+    assert ['us-east-1'] == inventory._validate_option('regions', list, ['us-east-1'])
+
+
+def test_illegal_option(inventory):
+    bad_filters = [{'tag:Environment': 'dev'}]
+    with pytest.raises(AnsibleParserError) as error_message:
+        inventory._validate_option('filters', dict, bad_filters)
+        assert "The option filters ([{'tag:Environment': 'dev'}]) must be a <class 'dict'>" == error_message
+
+
+def test_empty_config_query_options(inventory):
+    regions, filters, hostnames, strict_permissions = inventory._get_query_options({})
+    assert regions == filters == hostnames == []
+    assert strict_permissions is True
+
+
+def test_conig_query_options(inventory):
+    regions, filters, hostnames, strict_permissions = inventory._get_query_options(
+        {'regions': ['us-east-1', 'us-east-2'],
+         'filters': {'tag:Environment': ['dev', 'prod']},
+         'hostnames': 'ip-address',
+         'strict_permissions': False}
+    )
+    assert regions == ['us-east-1', 'us-east-2']
+    assert filters == [{'Name': 'tag:Environment', 'Values': ['dev', 'prod']}]
+    assert hostnames == ['ip-address']
+    assert strict_permissions is False
+
+
+def test_verify_file_bad_config(inventory):
+    assert inventory.verify_file('not_aws_config.yml') is False