377 lines
11 KiB
Python
377 lines
11 KiB
Python
import argparse
|
|
import enum
|
|
import os
|
|
import shlex
|
|
import sys
|
|
|
|
import lit.reports
|
|
import lit.util
|
|
|
|
|
|
@enum.unique
|
|
class TestOrder(enum.Enum):
|
|
LEXICAL = "lexical"
|
|
RANDOM = "random"
|
|
SMART = "smart"
|
|
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser(prog="lit", fromfile_prefix_chars="@")
|
|
parser.add_argument(
|
|
"test_paths",
|
|
nargs="+",
|
|
metavar="TEST_PATH",
|
|
help="File or path to include in the test suite",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--version", action="version", version="%(prog)s " + lit.__version__
|
|
)
|
|
|
|
parser.add_argument(
|
|
"-j",
|
|
"--threads",
|
|
"--workers",
|
|
dest="workers",
|
|
metavar="N",
|
|
help="Number of workers used for testing",
|
|
type=_positive_int,
|
|
default=os.getenv("LIT_MAX_WORKERS", lit.util.usable_core_count()),
|
|
)
|
|
parser.add_argument(
|
|
"--config-prefix",
|
|
dest="configPrefix",
|
|
metavar="NAME",
|
|
help="Prefix for 'lit' config files",
|
|
)
|
|
parser.add_argument(
|
|
"-D",
|
|
"--param",
|
|
dest="user_params",
|
|
metavar="NAME=VAL",
|
|
help="Add 'NAME' = 'VAL' to the user defined parameters",
|
|
action="append",
|
|
default=[],
|
|
)
|
|
|
|
format_group = parser.add_argument_group("Output Format")
|
|
# FIXME: I find these names very confusing, although I like the
|
|
# functionality.
|
|
format_group.add_argument(
|
|
"-q", "--quiet", help="Suppress no error output", action="store_true"
|
|
)
|
|
format_group.add_argument(
|
|
"-s",
|
|
"--succinct",
|
|
help="Reduce amount of output."
|
|
" Additionally, show a progress bar,"
|
|
" unless --no-progress-bar is specified.",
|
|
action="store_true",
|
|
)
|
|
format_group.add_argument(
|
|
"-v",
|
|
"--verbose",
|
|
dest="showOutput",
|
|
help="For failed tests, show all output. For example, each command is"
|
|
" printed before it is executed, so the last printed command is the one"
|
|
" that failed.",
|
|
action="store_true",
|
|
)
|
|
format_group.add_argument(
|
|
"-vv",
|
|
"--echo-all-commands",
|
|
dest="showOutput",
|
|
help="Deprecated alias for -v.",
|
|
action="store_true",
|
|
)
|
|
format_group.add_argument(
|
|
"-a",
|
|
"--show-all",
|
|
dest="showAllOutput",
|
|
help="Enable -v, but for all tests not just failed tests.",
|
|
action="store_true",
|
|
)
|
|
format_group.add_argument(
|
|
"-o",
|
|
"--output",
|
|
type=lit.reports.JsonReport,
|
|
help="Write test results to the provided path",
|
|
metavar="PATH",
|
|
)
|
|
format_group.add_argument(
|
|
"--no-progress-bar",
|
|
dest="useProgressBar",
|
|
help="Do not use curses based progress bar",
|
|
action="store_false",
|
|
)
|
|
|
|
# Note: this does not generate flags for user-defined result codes.
|
|
success_codes = [c for c in lit.Test.ResultCode.all_codes() if not c.isFailure]
|
|
for code in success_codes:
|
|
format_group.add_argument(
|
|
"--show-{}".format(code.name.lower()),
|
|
dest="shown_codes",
|
|
help="Show {} tests ({})".format(code.label.lower(), code.name),
|
|
action="append_const",
|
|
const=code,
|
|
default=[],
|
|
)
|
|
|
|
execution_group = parser.add_argument_group("Test Execution")
|
|
execution_group.add_argument(
|
|
"--gtest-sharding",
|
|
help="Enable sharding for GoogleTest format",
|
|
action="store_true",
|
|
default=True,
|
|
)
|
|
execution_group.add_argument(
|
|
"--no-gtest-sharding",
|
|
dest="gtest_sharding",
|
|
help="Disable sharding for GoogleTest format",
|
|
action="store_false",
|
|
)
|
|
execution_group.add_argument(
|
|
"--path",
|
|
help="Additional paths to add to testing environment",
|
|
action="append",
|
|
default=[],
|
|
type=os.path.abspath,
|
|
)
|
|
execution_group.add_argument(
|
|
"--vg", dest="useValgrind", help="Run tests under valgrind", action="store_true"
|
|
)
|
|
execution_group.add_argument(
|
|
"--vg-leak",
|
|
dest="valgrindLeakCheck",
|
|
help="Check for memory leaks under valgrind",
|
|
action="store_true",
|
|
)
|
|
execution_group.add_argument(
|
|
"--vg-arg",
|
|
dest="valgrindArgs",
|
|
metavar="ARG",
|
|
help="Specify an extra argument for valgrind",
|
|
action="append",
|
|
default=[],
|
|
)
|
|
execution_group.add_argument(
|
|
"--time-tests",
|
|
help="Track elapsed wall time for each test",
|
|
action="store_true",
|
|
)
|
|
execution_group.add_argument(
|
|
"--no-execute",
|
|
dest="noExecute",
|
|
help="Don't execute any tests (assume PASS)",
|
|
action="store_true",
|
|
)
|
|
execution_group.add_argument(
|
|
"--xunit-xml-output",
|
|
type=lit.reports.XunitReport,
|
|
help="Write XUnit-compatible XML test reports to the specified file",
|
|
)
|
|
execution_group.add_argument(
|
|
"--resultdb-output",
|
|
type=lit.reports.ResultDBReport,
|
|
help="Write LuCI ResuldDB compatible JSON to the specified file",
|
|
)
|
|
execution_group.add_argument(
|
|
"--time-trace-output",
|
|
type=lit.reports.TimeTraceReport,
|
|
help="Write Chrome tracing compatible JSON to the specified file",
|
|
)
|
|
execution_group.add_argument(
|
|
"--timeout",
|
|
dest="maxIndividualTestTime",
|
|
help="Maximum time to spend running a single test (in seconds). "
|
|
"0 means no time limit. [Default: 0]",
|
|
type=_non_negative_int,
|
|
)
|
|
execution_group.add_argument(
|
|
"--max-failures",
|
|
help="Stop execution after the given number of failures.",
|
|
type=_positive_int,
|
|
)
|
|
execution_group.add_argument(
|
|
"--allow-empty-runs",
|
|
help="Do not fail the run if all tests are filtered out",
|
|
action="store_true",
|
|
)
|
|
execution_group.add_argument(
|
|
"--per-test-coverage",
|
|
dest="per_test_coverage",
|
|
action="store_true",
|
|
help="Enable individual test case coverage",
|
|
)
|
|
execution_group.add_argument(
|
|
"--ignore-fail",
|
|
dest="ignoreFail",
|
|
action="store_true",
|
|
help="Exit with status zero even if some tests fail",
|
|
)
|
|
|
|
selection_group = parser.add_argument_group("Test Selection")
|
|
selection_group.add_argument(
|
|
"--max-tests",
|
|
metavar="N",
|
|
help="Maximum number of tests to run",
|
|
type=_positive_int,
|
|
)
|
|
selection_group.add_argument(
|
|
"--max-time",
|
|
dest="timeout",
|
|
metavar="N",
|
|
help="Maximum time to spend testing (in seconds)",
|
|
type=_positive_int,
|
|
)
|
|
selection_group.add_argument(
|
|
"--order",
|
|
choices=[x.value for x in TestOrder],
|
|
default=TestOrder.SMART,
|
|
help="Test order to use (default: smart)",
|
|
)
|
|
selection_group.add_argument(
|
|
"--shuffle",
|
|
dest="order",
|
|
help="Run tests in random order (DEPRECATED: use --order=random)",
|
|
action="store_const",
|
|
const=TestOrder.RANDOM,
|
|
)
|
|
selection_group.add_argument(
|
|
"-i",
|
|
"--incremental",
|
|
help="Run failed tests first (DEPRECATED: use --order=smart)",
|
|
action="store_true",
|
|
)
|
|
selection_group.add_argument(
|
|
"--filter",
|
|
metavar="REGEX",
|
|
type=_case_insensitive_regex,
|
|
help="Only run tests with paths matching the given regular expression",
|
|
default=os.environ.get("LIT_FILTER", ".*"),
|
|
)
|
|
selection_group.add_argument(
|
|
"--filter-out",
|
|
metavar="REGEX",
|
|
type=_case_insensitive_regex,
|
|
help="Filter out tests with paths matching the given regular expression",
|
|
default=os.environ.get("LIT_FILTER_OUT", "^$"),
|
|
)
|
|
selection_group.add_argument(
|
|
"--xfail",
|
|
metavar="LIST",
|
|
type=_semicolon_list,
|
|
help="XFAIL tests with paths in the semicolon separated list",
|
|
default=os.environ.get("LIT_XFAIL", ""),
|
|
)
|
|
selection_group.add_argument(
|
|
"--xfail-not",
|
|
metavar="LIST",
|
|
type=_semicolon_list,
|
|
help="do not XFAIL tests with paths in the semicolon separated list",
|
|
default=os.environ.get("LIT_XFAIL_NOT", ""),
|
|
)
|
|
selection_group.add_argument(
|
|
"--num-shards",
|
|
dest="numShards",
|
|
metavar="M",
|
|
help="Split testsuite into M pieces and only run one",
|
|
type=_positive_int,
|
|
default=os.environ.get("LIT_NUM_SHARDS"),
|
|
)
|
|
selection_group.add_argument(
|
|
"--run-shard",
|
|
dest="runShard",
|
|
metavar="N",
|
|
help="Run shard #N of the testsuite",
|
|
type=_positive_int,
|
|
default=os.environ.get("LIT_RUN_SHARD"),
|
|
)
|
|
|
|
debug_group = parser.add_argument_group("Debug and Experimental Options")
|
|
debug_group.add_argument(
|
|
"--debug", help="Enable debugging (for 'lit' development)", action="store_true"
|
|
)
|
|
debug_group.add_argument(
|
|
"--show-suites",
|
|
help="Show discovered test suites and exit",
|
|
action="store_true",
|
|
)
|
|
debug_group.add_argument(
|
|
"--show-tests", help="Show all discovered tests and exit", action="store_true"
|
|
)
|
|
debug_group.add_argument(
|
|
"--show-used-features",
|
|
help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit",
|
|
action="store_true",
|
|
)
|
|
|
|
# LIT is special: environment variables override command line arguments.
|
|
env_args = shlex.split(os.environ.get("LIT_OPTS", ""))
|
|
args = sys.argv[1:] + env_args
|
|
opts = parser.parse_args(args)
|
|
|
|
# Validate command line options
|
|
if opts.incremental:
|
|
print(
|
|
"WARNING: --incremental is deprecated. Failing tests now always run first."
|
|
)
|
|
|
|
if opts.numShards or opts.runShard:
|
|
if not opts.numShards or not opts.runShard:
|
|
parser.error("--num-shards and --run-shard must be used together")
|
|
if opts.runShard > opts.numShards:
|
|
parser.error("--run-shard must be between 1 and --num-shards (inclusive)")
|
|
opts.shard = (opts.runShard, opts.numShards)
|
|
else:
|
|
opts.shard = None
|
|
|
|
opts.reports = filter(
|
|
None,
|
|
[
|
|
opts.output,
|
|
opts.xunit_xml_output,
|
|
opts.resultdb_output,
|
|
opts.time_trace_output,
|
|
],
|
|
)
|
|
|
|
return opts
|
|
|
|
|
|
def _positive_int(arg):
|
|
return _int(arg, "positive", lambda i: i > 0)
|
|
|
|
|
|
def _non_negative_int(arg):
|
|
return _int(arg, "non-negative", lambda i: i >= 0)
|
|
|
|
|
|
def _int(arg, kind, pred):
|
|
desc = "requires {} integer, but found '{}'"
|
|
try:
|
|
i = int(arg)
|
|
except ValueError:
|
|
raise _error(desc, kind, arg)
|
|
if not pred(i):
|
|
raise _error(desc, kind, arg)
|
|
return i
|
|
|
|
|
|
def _case_insensitive_regex(arg):
|
|
import re
|
|
|
|
try:
|
|
return re.compile(arg, re.IGNORECASE)
|
|
except re.error as reason:
|
|
raise _error("invalid regular expression: '{}', {}", arg, reason)
|
|
|
|
|
|
def _semicolon_list(arg):
|
|
return arg.split(";")
|
|
|
|
|
|
def _error(desc, *args):
|
|
msg = desc.format(*args)
|
|
return argparse.ArgumentTypeError(msg)
|