Continue running network tests after failure (#26407)

Previously tests would stop running on the first failure.
With this PR we move onto running the tests for the next module. This
allows DCI runs to have a fuller picture of what's broken.

This only effects manual and DCI runs, it doesn't effect Shippable runs
as this the modules are tested directly.

test/integration/eos.yaml

@@ -7,14 +7,86 @@
     limit_to: "*"
     debug: false
 
-  roles:
-    - { role: eos_banner, when: "limit_to in ['*', 'eos_banner']" }
-    - { role: eos_command, when: "limit_to in ['*', 'eos_command']" }
-    - { role: eos_config, when: "limit_to in ['*', 'eos_config']" }
-    - { role: eos_template, when: "limit_to in ['*', 'eos_template']" }
-    - { role: eos_facts, when: "limit_to in ['*', 'eos_facts']" }
-    - { role: eos_eapi, debug: yes, when: "limit_to in ['*', 'eos_eapi']" }
-    - { role: eos_system, debug: yes, when: "limit_to in ['*', 'eos_system']" }
-    - { role: eos_user, when: "limit_to in ['*', 'eos_user']" }
-    - { role: eos_vlan, debug: yes, when: "limit_to in ['*', 'eos_vlan']" }
-    - { role: eos_vrf, debug: yes, when: "limit_to in ['*', 'eos_vrf']" }
+# Run the tests within blocks allows the the next module to be tested if the previous one fails.
+# This is done to allow https://github.com/ansible/dci-partner-ansible/ to run the full set of tests.
+
+
+  tasks:
+    - set_fact:
+        test_failed: false
+    - block:
+      - include_role:
+          name: eos_banner
+        when: "limit_to in ['*', 'eos_banner']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_command
+        when: "limit_to in ['*', 'eos_command']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_config
+        when: "limit_to in ['*', 'eos_config']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_template
+        when: "limit_to in ['*', 'eos_template']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_facts
+        when: "limit_to in ['*', 'eos_facts']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_eapi
+        when: "limit_to in ['*', 'eos_eapi']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_system
+        when: "limit_to in ['*', 'eos_system']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_user
+        when: "limit_to in ['*', 'eos_user']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_vlan
+        when: "limit_to in ['*', 'eos_vlan']"
+      rescue:
+        - set_fact: test_failed=true
+
+    - block:
+      - include_role:
+          name: eos_vrf
+        when: "limit_to in ['*', 'eos_vrf']"
+      rescue:
+        - set_fact: test_failed=true
+
+
+###########
+    - name: Has any previous test failed?
+      fail:
+        msg: "One or more tests failed, check log for details"
+      when: test_failed