From d461440c973e7525f959987a2c444913026a6c60 Mon Sep 17 00:00:00 2001 From: JamesWrigley Date: Mon, 2 Dec 2024 17:04:10 +0100 Subject: [PATCH 1/2] Move all the test dependencies into test/Project.toml --- Project.toml | 9 --------- test/Project.toml | 2 ++ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/Project.toml b/Project.toml index fd7508cd..27b7e9cb 100644 --- a/Project.toml +++ b/Project.toml @@ -64,12 +64,3 @@ StatsBase = "0.28, 0.29, 0.30, 0.31, 0.32, 0.33, 0.34" TaskLocalValues = "0.1" TimespanLogging = "0.1" julia = "1.9" - -[extras] -Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" -DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" -GraphViz = "f526b714-d49f-11e8-06ff-31ed36ee7ee0" -JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" -Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" diff --git a/test/Project.toml b/test/Project.toml index 97b58166..64e46aa8 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,6 +2,7 @@ ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" +Dagger = "d58978e5-989f-55fb-8d15-ea34adc7bf54" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" @@ -13,6 +14,7 @@ MemPool = "f9f48841-c794-520a-933b-121f7ba6ed94" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +Preferences = "21216c6a-2e73-6563-6e65-726566657250" PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" From 85602cd79e900da5675adf4d2fe51a465c97f0ea Mon Sep 17 00:00:00 2001 From: JamesWrigley Date: Sat, 30 Nov 2024 18:41:50 +0100 Subject: [PATCH 2/2] Add support for DistributedNext --- Project.toml | 6 +++++- lib/TimespanLogging/Project.toml | 4 ++++ lib/TimespanLogging/src/core.jl | 20 +++++++++++++++++++- src/Dagger.jl | 28 ++++++++++++++++++++++++++-- src/array/darray.jl | 2 +- src/array/sort.jl | 4 ++-- src/sch/Sch.jl | 8 +++++++- test/Project.toml | 1 - test/preferences.jl | 21 +++++++++++++++++++++ test/runtests.jl | 1 + 10 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 test/preferences.jl diff --git a/Project.toml b/Project.toml index 27b7e9cb..f25a2084 100644 --- a/Project.toml +++ b/Project.toml @@ -6,12 +6,14 @@ version = "0.18.13" Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +DistributedNext = "fab6aee4-877b-4bac-a744-3eca44acbb6f" Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" MemPool = "f9f48841-c794-520a-933b-121f7ba6ed94" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +Preferences = "21216c6a-2e73-6563-6e65-726566657250" Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Requires = "ae029012-a4dd-5104-9daa-d747884805df" @@ -47,15 +49,17 @@ Adapt = "4.0.4" Colors = "0.12, 0.13" DataFrames = "1" DataStructures = "0.18" +DistributedNext = "1.0.0" Distributions = "0.25" GraphViz = "0.2" Graphs = "1" JSON3 = "1" MacroTools = "0.5" -MemPool = "0.4.6" +MemPool = "0.4.11" OnlineStats = "1" Plots = "1" PrecompileTools = "1.2" +Preferences = "1.4.3" PythonCall = "0.9" Requires = "1" ScopedValues = "1.1" diff --git a/lib/TimespanLogging/Project.toml b/lib/TimespanLogging/Project.toml index 784466c9..95053cd0 100644 --- a/lib/TimespanLogging/Project.toml +++ b/lib/TimespanLogging/Project.toml @@ -5,7 +5,11 @@ version = "0.1.0" [deps] Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +DistributedNext = "fab6aee4-877b-4bac-a744-3eca44acbb6f" +Preferences = "21216c6a-2e73-6563-6e65-726566657250" Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" [compat] +DistributedNext = "1.0.0" +Preferences = "1.4.3" julia = "1" diff --git a/lib/TimespanLogging/src/core.jl b/lib/TimespanLogging/src/core.jl index 12276c1c..81534567 100644 --- a/lib/TimespanLogging/src/core.jl +++ b/lib/TimespanLogging/src/core.jl @@ -1,4 +1,10 @@ -using Distributed +import Preferences: @load_preference, @set_preferences! +if @load_preference("distributed-package") == "DistributedNext" + using DistributedNext +else + using Distributed +end + import Profile import Base.gc_num @@ -16,6 +22,18 @@ ProfilerResult(samples, lineinfo, tasks::Vector{Task}) = ProfilerResult(samples, lineinfo, tasks::Nothing) = ProfilerResult(samples, lineinfo, map(Base.pointer_from_objref, UInt[])) +""" + set_distributed_package!(value[="Distributed|DistributedNext"]) + +Set a [preference](https://github.com/JuliaPackaging/Preferences.jl) for using +either the Distributed.jl stdlib or DistributedNext.jl. You will need to restart +Julia after setting a new preference. +""" +function set_distributed_package!(value) + @set_preferences!("distributed-package" => value) + @info "TimespanLogging.jl preference has been set, restart your Julia session for this change to take effect!" +end + """ Timespan diff --git a/src/Dagger.jl b/src/Dagger.jl index b478ece0..8bc2c24a 100644 --- a/src/Dagger.jl +++ b/src/Dagger.jl @@ -8,8 +8,6 @@ import MemPool import MemPool: DRef, FileRef, poolget, poolset import Base: collect, reduce -import Distributed -import Distributed: Future, RemoteChannel, myid, workers, nworkers, procs, remotecall, remotecall_wait, remotecall_fetch import LinearAlgebra import LinearAlgebra: Adjoint, BLAS, Diagonal, Bidiagonal, Tridiagonal, LAPACK, LowerTriangular, PosDefException, Transpose, UpperTriangular, UnitLowerTriangular, UnitUpperTriangular, diagind, ishermitian, issymmetric @@ -33,6 +31,17 @@ import TimespanLogging: timespan_start, timespan_finish import Adapt +# Preferences +import Preferences: @load_preference, @set_preferences! + +if @load_preference("distributed-package") == "DistributedNext" + import DistributedNext + import DistributedNext: Future, RemoteChannel, myid, workers, nworkers, procs, remotecall, remotecall_wait, remotecall_fetch, check_same_host +else + import Distributed + import Distributed: Future, RemoteChannel, myid, workers, nworkers, procs, remotecall, remotecall_wait, remotecall_fetch, check_same_host +end + include("lib/util.jl") include("utils/dagdebug.jl") @@ -96,6 +105,21 @@ include("utils/logging-events.jl") include("utils/logging.jl") include("utils/viz.jl") +""" + set_distributed_package!(value[="Distributed|DistributedNext"]) + +Set a [preference](https://github.com/JuliaPackaging/Preferences.jl) for using +either the Distributed.jl stdlib or DistributedNext.jl. You will need to restart +Julia after setting a new preference. +""" +function set_distributed_package!(value) + MemPool.set_distributed_package!(value) + TimespanLogging.set_distributed_package!(value) + + @set_preferences!("distributed-package" => value) + @info "Dagger.jl preference has been set, restart your Julia session for this change to take effect!" +end + # Precompilation import PrecompileTools: @compile_workload include("precompile.jl") diff --git a/src/array/darray.jl b/src/array/darray.jl index ca1aebff..11feb53c 100644 --- a/src/array/darray.jl +++ b/src/array/darray.jl @@ -434,7 +434,7 @@ end function Distribute(data::AbstractArray{T,N}) where {T,N} nprocs = sum(w->length(Dagger.get_processors(OSProc(w))), - Distributed.procs()) + procs()) p = Blocks(ntuple(i->max(cld(size(data, i), nprocs), 1), N)) return Distribute(partition(p, domain(data)), p, data) end diff --git a/src/array/sort.jl b/src/array/sort.jl index 3040e11d..2993d45e 100644 --- a/src/array/sort.jl +++ b/src/array/sort.jl @@ -144,8 +144,8 @@ const use_shared_array = Ref(true) function _promote_array(x::AbstractArray{T}, y::AbstractArray{S}) where {T,S} Q = promote_type(T,S) ok = (isa(x, Array) || isa(x, SharedArray)) && (isa(y, Array) || isa(y, SharedArray)) - if ok && isbitstype(Q) && use_shared_array[] && Distributed.check_same_host([workers()..., 1]) - return SharedArray{Q}(length(x)+length(y), pids=Distributed.procs()) + if ok && isbitstype(Q) && use_shared_array[] && check_same_host([workers()..., 1]) + return SharedArray{Q}(length(x)+length(y), pids=procs()) else return similar(x, Q, length(x)+length(y)) end diff --git a/src/sch/Sch.jl b/src/sch/Sch.jl index 73bb07bf..5c330841 100644 --- a/src/sch/Sch.jl +++ b/src/sch/Sch.jl @@ -1,6 +1,12 @@ module Sch -import Distributed: Future, ProcessExitedException, RemoteChannel, RemoteException, myid, remote_do, remotecall_fetch, remotecall_wait, workers +import Preferences: @load_preference +if @load_preference("distributed-package") == "DistributedNext" + import DistributedNext: Future, ProcessExitedException, RemoteChannel, RemoteException, myid, remote_do, remotecall_fetch, remotecall_wait, workers +else + import Distributed: Future, ProcessExitedException, RemoteChannel, RemoteException, myid, remote_do, remotecall_fetch, remotecall_wait, workers +end + import MemPool import MemPool: DRef, StorageResource import MemPool: poolset, storage_capacity, storage_utilized diff --git a/test/Project.toml b/test/Project.toml index 64e46aa8..19e64882 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,7 +2,6 @@ ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" -Dagger = "d58978e5-989f-55fb-8d15-ea34adc7bf54" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" diff --git a/test/preferences.jl b/test/preferences.jl new file mode 100644 index 00000000..649ba0f9 --- /dev/null +++ b/test/preferences.jl @@ -0,0 +1,21 @@ +import Preferences: load_preference, set_preferences! + +@testset "Preferences" begin + cmd = `$(Base.julia_cmd()) --startup-file=no --project -E 'using Dagger; parentmodule(Dagger.myid)'` + + try + # Disabling the precompilation workload shaves off over half the time + # this test takes. + set_preferences!(Dagger, "precompile_workload" => false; force=true) + + cd(dirname(Base.active_project())) do + Dagger.set_distributed_package!("Distributed") + @test readchomp(cmd) == "Distributed" + + Dagger.set_distributed_package!("DistributedNext") + @test readchomp(cmd) == "DistributedNext" + end + finally + set_preferences!(Dagger, "precompile_workload" => true; force=true) + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 04871f6b..cfdab817 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -24,6 +24,7 @@ tests = [ ("Disk Caching", "diskcaching.jl"), ("File IO", "file-io.jl"), ("External Languages - Python", "extlang/python.jl"), + ("Preferences", "preferences.jl") #("Fault Tolerance", "fault-tolerance.jl"), ] all_test_names = map(test -> replace(last(test), ".jl"=>""), tests)