diff --git a/Project.toml b/Project.toml index 3bcc5b7..abac725 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ParallelTestRunner" uuid = "d3525ed8-44d0-4b2c-a655-542cee43accc" authors = ["Valentin Churavy "] -version = "2.6.3" +version = "2.7.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index 1e27ae6..e94d53f 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -913,6 +913,17 @@ runtests(MyPackage, args; testsuite) Workers are automatically recycled when they exceed memory limits to prevent out-of-memory issues during long test runs. The memory limit is set based on system architecture. + +## Failure Handling + +With `recycle_on_failure = true`, a worker is recycled after any test that did not pass, so +a test that corrupts process-wide state (e.g. wedges a GPU driver) cannot poison subsequent +tests on the same worker. + +With `retries = N` (default 0), tests that did not pass are re-run up to `N` times after +the main run completes — sequentially, on a single fresh worker, with all other workers +stopped — so tests that failed due to resource pressure from concurrent workers get an +otherwise-idle system. Only the final attempt of each test is reported. """ function runtests(mod::Module, args::ParsedArgs; testsuite::Dict{String,Expr} = find_tests(pwd()), @@ -922,7 +933,8 @@ function runtests(mod::Module, args::ParsedArgs; exename = nothing, exeflags = nothing, env = Vector{Pair{String, String}}(), - stdout = Base.stdout, stderr = Base.stderr, max_worker_rss = get_max_worker_rss()) + stdout = Base.stdout, stderr = Base.stderr, max_worker_rss = get_max_worker_rss(), + recycle_on_failure::Bool = false, retries::Integer = 0) # # set-up @@ -962,6 +974,8 @@ function runtests(mod::Module, args::ParsedArgs; stdout, stderr, max_worker_rss, + recycle_on_failure, + retries, ) end @@ -981,6 +995,8 @@ function _runtests(mod::Module, args::ParsedArgs; stdout = Base.stdout, stderr = Base.stderr, max_worker_rss = get_max_worker_rss(), + recycle_on_failure::Bool = false, + retries::Integer = 0, ) # determine parallelism @@ -1190,6 +1206,7 @@ function _runtests(mod::Module, args::ParsedArgs; tests_to_start = Threads.Atomic{Int}(length(tests)) next_test = Threads.Atomic{Int}(1) + interrupted = false try @sync for _ in 1:length(tests) push!(worker_tasks, Threads.@spawn begin @@ -1261,6 +1278,11 @@ function _runtests(mod::Module, args::ParsedArgs; # the worker has reached the max-rss limit, recycle it # so future tests start with a smaller working set Malt.stop(wrkr) + elseif recycle_on_failure && anynonpass(result[]) + # a failing test may have left the worker in a bad state + # (e.g. a wedged GPU driver whose every later allocation + # fails); recycle it so future tests get a fresh process + Malt.stop(wrkr) end else # One of Malt.TerminatedWorkerException, Malt.RemoteException, or ErrorException @@ -1299,6 +1321,7 @@ function _runtests(mod::Module, args::ParsedArgs; end) end catch err + interrupted = true if !(err isa InterruptException) println(io_ctx.stderr, "\nCaught an error, stopping...") end @@ -1336,6 +1359,55 @@ function _runtests(mod::Module, args::ParsedArgs; end end + # retry failed tests, if requested: sequentially, on a single fresh worker, with every + # other worker gone — tests that failed due to resource pressure (e.g. GPU memory + # oversubscription from concurrent workers) reliably pass on an otherwise-idle system. + # only the retried result is reported; persistent failures fail again and are reported + # exactly once. + if retries > 0 && !interrupted && args.quickfail === nothing + local retry_wrkr = nothing + for round in 1:retries + retryable = [r.test for r in results.value + if r.result isa Exception || anynonpass(r.result[])] + isempty(retryable) && break + println(io_ctx.stdout) + printstyled(io_ctx.stdout, + "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n"; + color = :yellow) + for test in retryable + if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr) + retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename, + exeflags, env) + end + test_t0 = time() + result = try + Malt.remote_eval_wait(Main, retry_wrkr.w, :(import ParallelTestRunner)) + Malt.remote_call_fetch(invokelatest, retry_wrkr.w, runtest, + RecordType, testsuite[test], test, + init_code, test_t0, custom_args) + catch ex + isa(ex, InterruptException) && rethrow() + ex + end + test_t1 = time() + output = @lock retry_wrkr.io String(take!(retry_wrkr.io[])) + filter!(r -> r.test != test, results.value) + push!(results.value, (; test, result, output, test_t0, test_t1)) + if result isa AbstractTestRecord && !anynonpass(result[]) + printstyled(io_ctx.stdout, " $test passed on retry\n"; color = :green) + else + printstyled(io_ctx.stdout, " $test failed again\n"; color = :red) + # don't let a failure contaminate the next retry + Malt.stop(retry_wrkr) + retry_wrkr = nothing + end + end + end + if retry_wrkr !== nothing && Malt.isrunning(retry_wrkr) + Malt.stop(retry_wrkr) + end + end + # print the output generated by each testset # (`@sync` above joined all writers, so `results` is quiescent from here on) for (testname, result, output, _start, _stop) in results.value