import base64 import datetime import itertools import json from xml.sax.saxutils import quoteattr as quo import lit.Test def by_suite_and_test_path(test): # Suite names are not necessarily unique. Include object identity in sort # key to avoid mixing tests of different suites. return (test.suite.name, id(test.suite), test.path_in_suite) class JsonReport(object): def __init__(self, output_file): self.output_file = output_file def write_results(self, tests, elapsed): unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED} tests = [t for t in tests if t.result.code not in unexecuted_codes] # Construct the data we will write. data = {} # Encode the current lit version as a schema version. data["__version__"] = lit.__versioninfo__ data["elapsed"] = elapsed # FIXME: Record some information on the lit configuration used? # FIXME: Record information from the individual test suites? # Encode the tests. data["tests"] = tests_data = [] for test in tests: test_data = { "name": test.getFullName(), "code": test.result.code.name, "output": test.result.output, "elapsed": test.result.elapsed, } # Add test metrics, if present. if test.result.metrics: test_data["metrics"] = metrics_data = {} for key, value in test.result.metrics.items(): metrics_data[key] = value.todata() # Report micro-tests separately, if present if test.result.microResults: for key, micro_test in test.result.microResults.items(): # Expand parent test name with micro test name parent_name = test.getFullName() micro_full_name = parent_name + ":" + key micro_test_data = { "name": micro_full_name, "code": micro_test.code.name, "output": micro_test.output, "elapsed": micro_test.elapsed, } if micro_test.metrics: micro_test_data["metrics"] = micro_metrics_data = {} for key, value in micro_test.metrics.items(): micro_metrics_data[key] = value.todata() tests_data.append(micro_test_data) tests_data.append(test_data) with open(self.output_file, "w") as file: json.dump(data, file, indent=2, sort_keys=True) file.write("\n") _invalid_xml_chars_dict = { c: None for c in range(32) if chr(c) not in ("\t", "\n", "\r") } def remove_invalid_xml_chars(s): # According to the XML 1.0 spec, control characters other than # \t,\r, and \n are not permitted anywhere in the document # (https://www.w3.org/TR/xml/#charsets) and therefore this function # removes them to produce a valid XML document. # # Note: In XML 1.1 only \0 is illegal (https://www.w3.org/TR/xml11/#charsets) # but lit currently produces XML 1.0 output. return s.translate(_invalid_xml_chars_dict) class XunitReport(object): def __init__(self, output_file): self.output_file = output_file self.skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED} def write_results(self, tests, elapsed): tests.sort(key=by_suite_and_test_path) tests_by_suite = itertools.groupby(tests, lambda t: t.suite) with open(self.output_file, "w") as file: file.write('\n') file.write('\n'.format(time=elapsed)) for suite, test_iter in tests_by_suite: self._write_testsuite(file, suite, list(test_iter)) file.write("\n") def _write_testsuite(self, file, suite, tests): skipped = sum(1 for t in tests if t.result.code in self.skipped_codes) failures = sum(1 for t in tests if t.isFailure()) name = suite.config.name.replace(".", "-") file.write( f'\n' ) for test in tests: self._write_test(file, test, name) file.write("\n") def _write_test(self, file, test, suite_name): path = "/".join(test.path_in_suite[:-1]).replace(".", "_") class_name = f"{suite_name}.{path or suite_name}" name = test.path_in_suite[-1] time = test.result.elapsed or 0.0 file.write( f'\n ", "]]]]>") if isinstance(output, bytes): output = output.decode("utf-8", "ignore") # Failing test output sometimes contains control characters like # \x1b (e.g. if there was some -fcolor-diagnostics output) which are # not allowed inside XML files. # This causes problems with CI systems: for example, the Jenkins # JUnit XML will throw an exception when ecountering those # characters and similar problems also occur with GitLab CI. output = remove_invalid_xml_chars(output) file.write(output) file.write("]]>\n\n") elif test.result.code in self.skipped_codes: reason = self._get_skip_reason(test) file.write(f">\n \n\n") else: file.write("/>\n") def _get_skip_reason(self, test): code = test.result.code if code == lit.Test.EXCLUDED: return "Test not selected (--filter, --max-tests)" if code == lit.Test.SKIPPED: return "User interrupt" assert code == lit.Test.UNSUPPORTED features = test.getMissingRequiredFeatures() if features: return "Missing required feature(s): " + ", ".join(features) return "Unsupported configuration" def gen_resultdb_test_entry( test_name, start_time, elapsed_time, test_output, result_code, is_expected ): test_data = { "testId": test_name, "start_time": datetime.datetime.fromtimestamp(start_time).isoformat() + "Z", "duration": "%.9fs" % elapsed_time, "summary_html": '

', "artifacts": { "artifact-content-in-request": { "contents": base64.b64encode(test_output.encode("utf-8")).decode( "utf-8" ), }, }, "expected": is_expected, } if ( result_code == lit.Test.PASS or result_code == lit.Test.XPASS or result_code == lit.Test.FLAKYPASS ): test_data["status"] = "PASS" elif result_code == lit.Test.FAIL or result_code == lit.Test.XFAIL: test_data["status"] = "FAIL" elif ( result_code == lit.Test.UNSUPPORTED or result_code == lit.Test.SKIPPED or result_code == lit.Test.EXCLUDED ): test_data["status"] = "SKIP" elif result_code == lit.Test.UNRESOLVED or result_code == lit.Test.TIMEOUT: test_data["status"] = "ABORT" return test_data class ResultDBReport(object): def __init__(self, output_file): self.output_file = output_file def write_results(self, tests, elapsed): unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED} tests = [t for t in tests if t.result.code not in unexecuted_codes] data = {} data["__version__"] = lit.__versioninfo__ data["elapsed"] = elapsed # Encode the tests. data["tests"] = tests_data = [] for test in tests: tests_data.append( gen_resultdb_test_entry( test_name=test.getFullName(), start_time=test.result.start, elapsed_time=test.result.elapsed, test_output=test.result.output, result_code=test.result.code, is_expected=not test.result.code.isFailure, ) ) if test.result.microResults: for key, micro_test in test.result.microResults.items(): # Expand parent test name with micro test name parent_name = test.getFullName() micro_full_name = parent_name + ":" + key + "microres" tests_data.append( gen_resultdb_test_entry( test_name=micro_full_name, start_time=micro_test.start if micro_test.start else test.result.start, elapsed_time=micro_test.elapsed if micro_test.elapsed else test.result.elapsed, test_output=micro_test.output, result_code=micro_test.code, is_expected=not micro_test.code.isFailure, ) ) with open(self.output_file, "w") as file: json.dump(data, file, indent=2, sort_keys=True) file.write("\n") class TimeTraceReport(object): def __init__(self, output_file): self.output_file = output_file self.skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED} def write_results(self, tests, elapsed): # Find when first test started so we can make start times relative. first_start_time = min([t.result.start for t in tests]) events = [ self._get_test_event(x, first_start_time) for x in tests if x.result.code not in self.skipped_codes ] json_data = {"traceEvents": events} with open(self.output_file, "w") as time_trace_file: json.dump(json_data, time_trace_file, indent=2, sort_keys=True) def _get_test_event(self, test, first_start_time): test_name = test.getFullName() elapsed_time = test.result.elapsed or 0.0 start_time = test.result.start - first_start_time if test.result.start else 0.0 pid = test.result.pid or 0 return { "pid": pid, "tid": 1, "ph": "X", "ts": int(start_time * 1000000.0), "dur": int(elapsed_time * 1000000.0), "name": test_name, }