Overhaul ansible-test sanity implementation. (#22177)

- Tests are run to completion instead of stopping on first failure.
- Test results are now parsed instead of passing through to the console.
- Test results can be saved in junit xml format.
- Test results will show up on the Shippable "Tests" result tab.
- Added an experimental --lint option for easier integration with other tools.
- Code smell tests are now usable with the --list-tests, --test and --skip-test options.
- Code split out from executor.py into sanity.py.
- Rename download-logs to download.py and add support for test and coverage results.
- Miscellaneous improvements.

test/utils/shippable/download.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python
+# PYTHON_ARGCOMPLETE_OK
+
+# (c) 2016 Red Hat, Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import print_function
+
+import json
+import os
+import re
+import requests
+
+from argparse import ArgumentParser
+
+try:
+    import argcomplete
+except ImportError:
+    argcomplete = None
+
+
+def main():
+    api_key = get_api_key()
+
+    parser = ArgumentParser(description='Download results from a Shippable run.')
+
+    parser.add_argument('run_id',
+                        help='shippable run id.')
+
+    parser.add_argument('-v', '--verbose',
+                        dest='verbose',
+                        action='store_true',
+                        help='show what is being downloaded')
+
+    parser.add_argument('-t', '--test',
+                        dest='test',
+                        action='store_true',
+                        help='show what would be downloaded without downloading')
+
+    parser.add_argument('--key',
+                        dest='api_key',
+                        default=api_key,
+                        required=api_key is None,
+                        help='api key for accessing Shippable')
+
+    parser.add_argument('--console-logs',
+                        action='store_true',
+                        help='download console logs')
+
+    parser.add_argument('--test-results',
+                        action='store_true',
+                        help='download test results')
+
+    parser.add_argument('--coverage-results',
+                        action='store_true',
+                        help='download code coverage results')
+
+    parser.add_argument('--all',
+                        action='store_true',
+                        help='download everything')
+
+    parser.add_argument('--job-number',
+                        action='append',
+                        type=int,
+                        help='limit downloads to the given job number')
+
+    if argcomplete:
+        argcomplete.autocomplete(parser)
+
+    args = parser.parse_args()
+
+    if args.all:
+        args.console_logs = True
+        args.test_results = True
+        args.coverage_results = True
+
+    if not args.console_logs and not args.test_results and not args.coverage_results:
+        parser.error('At least one download option is required: --console-logs, --test-results, --coverage-results')
+
+    headers = dict(
+        Authorization='apiToken %s' % args.api_key,
+    )
+
+    response = requests.get('https://api.shippable.com/jobs?runIds=%s' % args.run_id, headers=headers)
+
+    if response.status_code != 200:
+        raise Exception(response.content)
+
+    body = response.json()
+    output_dir = args.run_id
+
+    if not args.test:
+        if not os.path.exists(output_dir):
+            os.mkdir(output_dir)
+
+    for j in body:
+        job_id = j['id']
+        job_number = j['jobNumber']
+
+        if args.job_number and job_number not in args.job_number:
+            continue
+
+        if args.console_logs:
+            path = os.path.join(output_dir, '%s-console.log' % job_number)
+            url = 'https://api.shippable.com/jobs/%s/consoles?download=true' % job_id
+            download(args, headers, path, url)
+
+        if args.test_results:
+            path = os.path.join(output_dir, '%s-test.json' % job_number)
+            url = 'https://api.shippable.com/jobs/%s/jobTestReports' % job_id
+            download(args, headers, path, url)
+            extract_contents(args, path, os.path.join(output_dir, '%s-test' % job_number))
+
+        if args.coverage_results:
+            path = os.path.join(output_dir, '%s-coverage.json' % job_number)
+            url = 'https://api.shippable.com/jobs/%s/jobCoverageReports' % job_id
+            download(args, headers, path, url)
+            extract_contents(args, path, os.path.join(output_dir, '%s-coverage' % job_number))
+
+
+def extract_contents(args, path, output_dir):
+    if not args.test:
+        with open(path, 'r') as json_fd:
+            items = json.load(json_fd)
+
+            for item in items:
+                contents = item['contents']
+                path = output_dir + '/' + re.sub('^/*', '', item['path'])
+
+                directory = os.path.dirname(path)
+
+                if not os.path.exists(directory):
+                    os.makedirs(directory)
+
+                if args.verbose:
+                    print(path)
+
+                if not os.path.exists(path):
+                    with open(path, 'w') as output_fd:
+                        output_fd.write(contents)
+
+
+def download(args, headers, path, url):
+    if args.verbose or args.test:
+        print(path)
+
+    if os.path.exists(path):
+        return
+
+    if not args.test:
+        response = requests.get(url, headers=headers)
+
+        if response.status_code != 200:
+            raise Exception(response.content)
+
+        content = response.content
+
+        with open(path, 'w') as f:
+            f.write(content)
+
+
+def get_api_key():
+    path = os.path.join(os.environ['HOME'], '.shippable.key')
+
+    try:
+        with open(path, 'r') as f:
+            return f.read().strip()
+    except IOError:
+        return None
+
+
+if __name__ == '__main__':
+    main()