[stable-1] AZP Bootstrap (#1505)

* AZP Bootstrap

(cherry picked from commit 33126b7267)

* AZP: Correct Cloud jobs

(cherry picked from commit 2fea31b292377e17ba5447dcb79c4b753a6fad58)

* Fix AZP CI (#1508)

* Fix 2.9/2.10 cloud

* Fix splunk callback tests.

* ansible_virtualization_type on AZP can be one of container/containerd instead of docker for dockerized tests.

* Disable nomad tests.

* Work around AZP bugs.

(cherry picked from commit dd55c3c3bb)

* Run tests on all groups.

* Reduce 2.9 coverage to decrease large number of jobs.

* Try to fix test.

* Revert "Try to fix test."

This reverts commit 23f51451c6.

* Other target selection for 2.9.

Co-authored-by: Felix Fontein <felix@fontein.de>

tests/integration/targets/filesystem/tasks/main.yml

@@ -33,7 +33,7 @@
     - 'not (item.0.key == "btrfs" and (ansible_distribution == "Ubuntu" and ansible_distribution_release == "trusty"))'
     - 'not (item.0.key == "btrfs" and (ansible_facts.os_family == "RedHat" and ansible_facts.distribution_major_version is version("8", ">=")))'
     - 'not (item.0.key == "lvm" and ansible_system == "FreeBSD")' # LVM not available on FreeBSD
-    - 'not (item.0.key == "lvm" and ansible_virtualization_type == "docker")' # Tests use losetup which can not be used inside unprivileged container
+    - 'not (item.0.key == "lvm" and ansible_virtualization_type in ["docker", "container", "containerd"])' # Tests use losetup which can not be used inside unprivileged container
     - 'not (item.0.key == "ocfs2" and ansible_os_family != "Debian")' # ocfs2 only available on Debian based distributions
     - 'not (item.0.key == "f2fs" and ansible_system == "FreeBSD")'
     # f2fs-tools package not available with RHEL/CentOS

tests/integration/targets/nomad/aliases

@@ -4,3 +4,4 @@ destructive
 skip/aix
 skip/centos6
 skip/freebsd
+disabled  # FIXME

tests/utils/shippable/shippable.sh

@@ -14,13 +14,16 @@ function join {
     echo "$*";
 }
 
+# Ensure we can write other collections to this dir
+sudo chown "$(whoami)" "${PWD}/../../"
+
 test="$(join / "${args[@]:1}")"
 
 docker images ansible/ansible
 docker images quay.io/ansible/*
 docker ps
 
-for container in $(docker ps --format '{{.Image}} {{.ID}}' | grep -v '^drydock/' | sed 's/^.* //'); do
+for container in $(docker ps --format '{{.Image}} {{.ID}}' | grep -v -e '^drydock/' -e '^quay.io/ansible/azure-pipelines-test-container:' | sed 's/^.* //'); do
     docker rm -f "${container}" || true  # ignore errors
 done
 
@@ -59,12 +62,16 @@ else
     retry pip install "https://github.com/ansible/ansible/archive/stable-${ansible_version}.tar.gz" --disable-pip-version-check
 fi
 
-export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible"
-SHIPPABLE_RESULT_DIR="$(pwd)/shippable"
-TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/general"
-mkdir -p "${TEST_DIR}"
-cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}"
-cd "${TEST_DIR}"
+if [ "${SHIPPABLE_BUILD_ID:-}" ]; then
+    export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible"
+    SHIPPABLE_RESULT_DIR="$(pwd)/shippable"
+    TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/general"
+    mkdir -p "${TEST_DIR}"
+    cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}"
+    cd "${TEST_DIR}"
+else
+    export ANSIBLE_COLLECTIONS_PATHS="${PWD}/../../../"
+fi
 
 # START: HACK install dependencies
 retry ansible-galaxy -vvv collection install ansible.netcommon
@@ -202,7 +209,7 @@ function cleanup
     fi
 }
 
-trap cleanup EXIT
+if [ "${SHIPPABLE_BUILD_ID:-}" ]; then trap cleanup EXIT; fi
 
 if [[ "${COVERAGE:-}" == "--coverage" ]]; then
     timeout=60
@@ -212,5 +219,5 @@ fi
 
 ansible-test env --dump --show --timeout "${timeout}" --color -v
 
-"tests/utils/shippable/check_matrix.py"
+if [ "${SHIPPABLE_BUILD_ID:-}" ]; then "tests/utils/shippable/check_matrix.py"; fi
 "tests/utils/shippable/${script}.sh" "${test}"