Parallel Testing (#2388)

* Initial support for running tests in parallel * Include Distributed as dependency * Add test_args for parallelisation [skip ci] * Better code availability to workers - Fixes test/Rings - Moves dependencies to where they are needed * Some dependancy fixes (and indentation fixes) [skip ci] * Skip timing tests if parallel and many OMP threads * Review: Remove unneeded dependencies * Review: better readability of if clause * Review: Remove redundant `using` * Review: fix indentation, remove comment for Doctest * Review: get rid of try-catch * Review: use get() for env vars * Review: comments from @fingolfin * Review: don't redefine constant * Review: use nworkers() instead of length(workers()) * Remove test which was accidentally added back when resolving merge conflict.
oscar-system · Jul 5, 2023 · c5fef18 · c5fef18
1 parent 425131c
commit c5fef18
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 44 deletions.
diff --git a/test/PolyhedralGeometry/timing.jl b/test/PolyhedralGeometry/timing.jl
@@ -4,7 +4,11 @@
     # to avoid unnecessary failures we skip the timing tests
     haskey(ENV, "JULIA_PKGEVAL") && return
 
-    using Oscar
+    # If running multiple workers, and many OMP threads, CPU oversubscription may occur
+    # which fails the timing tests. In this situation, skip instead.
+    if isdefined(Main, :Distributed) && nworkers() > 1
+        get(ENV, "OMP_NUM_THREADS", "") == "1" || return
+    end
 
     # macos on github actions is very slow
     factor = Sys.isapple() && haskey(ENV,"GITHUB_ACTIONS") ? 5.0 : 1.0

diff --git a/test/Project.toml b/test/Project.toml
@@ -1,5 +1,6 @@
 [deps]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

diff --git a/test/Rings/mpoly-localizations.jl b/test/Rings/mpoly-localizations.jl
@@ -1,3 +1,6 @@
+import Oscar.Nemo.AbstractAlgebra
+include(joinpath(pathof(AbstractAlgebra), "..", "..", "test", "Rings-conformance-tests.jl"))
+
 const rng = Oscar.get_seeded_rng()
 
 @testset "mpoly-localizations" begin

diff --git a/test/Rings/slpolys.jl b/test/Rings/slpolys.jl
@@ -1,5 +1,7 @@
 using Oscar: SLPolynomialRing
 
+const SLP = Oscar.StraightLinePrograms
+
 replstr(c) = sprint((io, x) -> show(io, "text/plain", x), c)
 
 @testset "LazyPolyRing" begin

diff --git a/test/StraightLinePrograms/runtests.jl b/test/StraightLinePrograms/runtests.jl
@@ -2,7 +2,7 @@
 module SLPTest
 using ..Test
 using ..Oscar
-using ..Oscar: SLP
+const SLP = Oscar.StraightLinePrograms
 
 include("setup.jl")
 include("straightline.jl")

diff --git a/test/runtests.jl b/test/runtests.jl
@@ -1,31 +1,52 @@
 using Oscar
 using Test
+using Distributed
 
 import Random
 
+numprocs_str = get(ENV, "NUMPROCS", "1")
+
+if !isempty(ARGS)
+  jargs = [arg for arg in ARGS if startswith(arg, "-j")]
+  if !isempty(jargs)
+    numprocs_str = split(jargs[end], "-j")[end]
+  end
+end
+
+const numprocs = parse(Int, numprocs_str)
+
+if numprocs >= 2
+  println("Adding worker processes")
+  addprocs(numprocs)
+end
+
 if haskey(ENV, "JULIA_PKGEVAL") ||
-    get(ENV, "CI", "") == "true" ||
-    haskey(ENV, "OSCAR_RANDOM_SEED")
+  get(ENV, "CI", "") == "true" ||
+  haskey(ENV, "OSCAR_RANDOM_SEED")
   seed = parse(UInt32, get(ENV, "OSCAR_RANDOM_SEED", "42"))
   @info string(@__FILE__)*" -- fixed SEED $seed"
 else
   seed = rand(UInt32)
   @info string(@__FILE__)*" -- SEED $seed"
 end
-Oscar.set_seed!(seed)
 
-import Oscar.Nemo.AbstractAlgebra
-include(joinpath(pathof(AbstractAlgebra), "..", "..", "test", "Rings-conformance-tests.jl"))
+@everywhere using Test
+@everywhere using Oscar
+@everywhere Oscar.set_seed!($seed)
+
+# hotfix, otherwise StraightLinePrograms returns something which then leads to an error
+module SLPTest
+end
 
 # some helpers
-import Printf
-import PrettyTables
+@everywhere import Printf
+@everywhere import PrettyTables
 
 # the current code for extracting the compile times does not work on earlier
 # julia version
-const compiletimes = @static VERSION >= v"1.9.0-DEV" ? true : false
+@everywhere const compiletimes = @static VERSION >= v"1.9.0-DEV" ? true : false
 
-const stats_dict = Dict{String,NamedTuple}()
+@everywhere const stats_dict = Dict{String,NamedTuple}()
 
 function print_stats(io::IO; fmt=PrettyTables.tf_unicode, max=50)
   sorted = sort(collect(stats_dict), by=x->x[2].time, rev=true)
@@ -44,9 +65,9 @@ end
 
 # we only want to print stats for files that run tests and not those that just
 # include other files
-const innermost = Ref(true)
+@everywhere const innermost = Ref(true)
 # redefine include to print and collect some extra stats
-function include(str::String)
+@everywhere function include(str::String)
   innermost[] = true
   # we pass the identity to avoid recursing into this function again
   @static if compiletimes
@@ -80,58 +101,69 @@ end
   Base.cumulative_compile_timing(true)
 end
 
-# Used in both Rings/slpolys.jl and StraightLinePrograms/runtests.jl
-const SLP = Oscar.StraightLinePrograms
+println("Making test list")
 
-include("Aqua.jl")
+const testlist = [
+  "Aqua.jl",
 
-include("printing.jl")
+  "printing.jl",
 
-include("PolyhedralGeometry/runtests.jl")
-include("Combinatorics/runtests.jl")
+  "PolyhedralGeometry/runtests.jl",
+  "Combinatorics/runtests.jl",
 
-include("GAP/runtests.jl")
-include("Groups/runtests.jl")
+  "GAP/runtests.jl",
+  "Groups/runtests.jl",
 
-include("Rings/runtests.jl")
+  "Rings/runtests.jl",
 
-include("NumberTheory/nmbthy.jl")
-include("NumberTheory/galthy.jl")
+  "NumberTheory/nmbthy.jl",
+  "NumberTheory/galthy.jl",
 
 # Will automatically include all experimental packages following our
 # guidelines.
-include("../experimental/runtests.jl")
 
-include("Experimental/gmodule.jl")
-include("Experimental/ModStdQt.jl")
-include("Experimental/ModStdNF.jl")
-include("Experimental/MatrixGroups.jl")
-include("Experimental/ExteriorAlgebra.jl")
+  "../experimental/runtests.jl",
+
+  "Experimental/gmodule.jl",
+  "Experimental/ModStdQt.jl",
+  "Experimental/ModStdNF.jl",
+  "Experimental/MatrixGroups.jl",
+  "Experimental/ExteriorAlgebra.jl",
+
+  "Rings/ReesAlgebra.jl",
 
-include("Rings/ReesAlgebra.jl")
+  "Modules/runtests.jl",
 
-include("Modules/runtests.jl")
+  "InvariantTheory/runtests.jl",
 
-include("InvariantTheory/runtests.jl")
+  "AlgebraicGeometry/Schemes/runtests.jl",
+  "AlgebraicGeometry/ToricVarieties/runtests.jl",
+  "AlgebraicGeometry/Surfaces/K3Auto.jl",
 
-include("AlgebraicGeometry/Schemes/runtests.jl")
-include("AlgebraicGeometry/ToricVarieties/runtests.jl")
-include("AlgebraicGeometry/Surfaces/K3Auto.jl")
+  "TropicalGeometry/runtests.jl",
 
-include("TropicalGeometry/runtests.jl")
+  "Serialization/runtests.jl",
 
-include("Serialization/runtests.jl")
+  "StraightLinePrograms/runtests.jl"
+]
 
-include("StraightLinePrograms/runtests.jl")
+# if many workers, distribute tasks across them
+# otherwise, is essentially a serial loop
+pmap(include, testlist)
 
 @static if compiletimes
   Base.cumulative_compile_timing(false);
 end
 
-if haskey(ENV, "GITHUB_STEP_SUMMARY") && compiletimes
-  open(ENV["GITHUB_STEP_SUMMARY"], "a") do io
-    print_stats(io, fmt=PrettyTables.tf_markdown)
+#currently, print_stats will fail when running tests with external workers
+#TODO: potentially rewrite include as well as print_stats 
+#      to comply with parallel decisions
+if numprocs == 1
+  if haskey(ENV, "GITHUB_STEP_SUMMARY") && compiletimes
+    open(ENV["GITHUB_STEP_SUMMARY"], "a") do io
+      print_stats(io, fmt=PrettyTables.tf_markdown)
+    end
+  else
+    print_stats(stdout; max=10)
   end
-else
-  print_stats(stdout; max=10)
 end