From 635f294258b27dd5cb384cc4d6a0e2a88d23787b Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Wed, 4 Dec 2024 16:29:27 -0500 Subject: [PATCH] Split tests into groups + SafeTestsets (#30) --- Project.toml | 4 +- templates/project/Project.toml | 6 ++- templates/test/test/Project.toml | 5 -- .../test/test/{ => basics}/test_basics.jl | 2 - templates/test/test/runtests.jl | 53 ++++++++++++++++--- templates/test/test/test_aqua.jl | 2 - templates/test/test/test_examples.jl | 15 ------ test/Project.toml | 6 --- test/{ => basics}/test_basics.jl | 2 - test/runtests.jl | 53 ++++++++++++++++--- test/test_aqua.jl | 2 - test/test_examples.jl | 15 ------ 12 files changed, 98 insertions(+), 67 deletions(-) delete mode 100644 templates/test/test/Project.toml rename templates/test/test/{ => basics}/test_basics.jl (78%) delete mode 100644 templates/test/test/test_examples.jl delete mode 100644 test/Project.toml rename test/{ => basics}/test_basics.jl (98%) delete mode 100644 test/test_examples.jl diff --git a/Project.toml b/Project.toml index 30b5fef..a4c2df8 100644 --- a/Project.toml +++ b/Project.toml @@ -20,13 +20,15 @@ Git_jll = "2.46.2" LibGit2 = "1.10" PkgSkeleton = "1.3.1" Preferences = "1.4.3" +SafeTestsets = "0.1" Suppressor = "0.2.8" Test = "1.10" julia = "1.10" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Test"] +test = ["Aqua", "Test", "SafeTestsets"] diff --git a/templates/project/Project.toml b/templates/project/Project.toml index 5028069..4bd9dbe 100644 --- a/templates/project/Project.toml +++ b/templates/project/Project.toml @@ -5,12 +5,16 @@ version = "0.1.0" [compat] Aqua = "0.8.9" +SafeTestsets = "0.1" +Suppressor = "0.2" Test = "1.10" julia = "1.10" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" +Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" [targets] -test = ["Aqua", "Test"] +test = ["Aqua", "Test", "Suppressor", "SafeTestsets"] diff --git a/templates/test/test/Project.toml b/templates/test/test/Project.toml deleted file mode 100644 index ca8650a..0000000 --- a/templates/test/test/Project.toml +++ /dev/null @@ -1,5 +0,0 @@ -[deps] -{PKGNAME} = "{UUID}" -Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" -Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/templates/test/test/test_basics.jl b/templates/test/test/basics/test_basics.jl similarity index 78% rename from templates/test/test/test_basics.jl rename to templates/test/test/basics/test_basics.jl index 2722ccb..9754f09 100644 --- a/templates/test/test/test_basics.jl +++ b/templates/test/test/basics/test_basics.jl @@ -1,8 +1,6 @@ -@eval module $(gensym()) using {PKGNAME}: {PKGNAME} using Test: @test, @testset @testset "{PKGNAME}" begin # Tests go here. end -end diff --git a/templates/test/test/runtests.jl b/templates/test/test/runtests.jl index a15a1fd..bb62115 100644 --- a/templates/test/test/runtests.jl +++ b/templates/test/test/runtests.jl @@ -1,12 +1,49 @@ -@eval module $(gensym()) -using Test: @testset +using SafeTestsets: @safetestset +using Suppressor: @suppress -@testset "{PKGNAME}.jl" begin - filenames = filter(readdir(@__DIR__)) do f - startswith("test_")(f) && endswith(".jl")(f) +# check for filtered groups +# either via `--group=ALL` or through ENV["GROUP"] +const pat = r"(?:--group=)(\w+)" +arg_id = findfirst(contains(pat), ARGS) +const GROUP = uppercase( + if isnothing(arg_id) + get(ENV, "GROUP", "ALL") + else + only(match(pat, ARGS[arg_id]).captures) + end, +) + +function istestfile(filename) + return isfile(filename) && + endswith(filename, ".jl") && + startswith(basename(filename), "test") +end + +@time begin + # tests in groups based on folder structure + for testgroup in filter(isdir, readdir(@__DIR__)) + if GROUP == "ALL" || GROUP == uppercase(testgroup) + for file in filter(istestfile, readdir(joinpath(@__DIR__, testgroup); join=true)) + @eval @safetestset $file begin + include($file) + end + end + end end - @testset "Test $filename" for filename in filenames - include(filename) + + # single files in top folder + for file in filter(istestfile, readdir(@__DIR__)) + (file == basename(@__FILE__)) && continue + @eval @safetestset $file begin + include($file) + end + end + + # test examples + examplepath = joinpath(@__DIR__, "..", "examples") + for file in filter(endswith(".jl"), readdir(examplepath; join=true)) + @suppress @eval @safetestset $file begin + include($file) + end end -end end diff --git a/templates/test/test/test_aqua.jl b/templates/test/test/test_aqua.jl index d3d64af..c4a576f 100644 --- a/templates/test/test/test_aqua.jl +++ b/templates/test/test/test_aqua.jl @@ -1,4 +1,3 @@ -@eval module $(gensym()) using {PKGNAME}: {PKGNAME} using Aqua: Aqua using Test: @testset @@ -6,4 +5,3 @@ using Test: @testset @testset "Code quality (Aqua.jl)" begin Aqua.test_all({PKGNAME}) end -end diff --git a/templates/test/test/test_examples.jl b/templates/test/test/test_examples.jl deleted file mode 100644 index c347012..0000000 --- a/templates/test/test/test_examples.jl +++ /dev/null @@ -1,15 +0,0 @@ -@eval module $(gensym()) -using {PKGNAME}: {PKGNAME} -using Suppressor: @suppress -using Test: @testset - -@testset "{PKGNAME}.jl examples" begin - examples_path = joinpath(pkgdir({PKGNAME}), "examples") - filenames = filter(readdir(examples_path; join=true)) do f - endswith(".jl")(f) - end - @testset "Test $filename" for filename in filenames - @suppress include(filename) - end -end -end diff --git a/test/Project.toml b/test/Project.toml deleted file mode 100644 index 5db5691..0000000 --- a/test/Project.toml +++ /dev/null @@ -1,6 +0,0 @@ -[deps] -Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" -Git_jll = "f8c6e375-362e-5223-8a59-34ff63f689eb" -ITensorPkgSkeleton = "3d388ab1-018a-49f4-ae50-18094d5f71ea" -Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/test_basics.jl b/test/basics/test_basics.jl similarity index 98% rename from test/test_basics.jl rename to test/basics/test_basics.jl index 9e6f46c..036b7b2 100644 --- a/test/test_basics.jl +++ b/test/basics/test_basics.jl @@ -1,4 +1,3 @@ -@eval module $(gensym()) using ITensorPkgSkeleton: ITensorPkgSkeleton using Test: @test, @testset @@ -43,4 +42,3 @@ using Test: @test, @testset end end end -end diff --git a/test/runtests.jl b/test/runtests.jl index 8c6aae1..bb62115 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,12 +1,49 @@ -@eval module $(gensym()) -using Test: @testset +using SafeTestsets: @safetestset +using Suppressor: @suppress -@testset "ITensorPkgSkeleton.jl" begin - filenames = filter(readdir(@__DIR__)) do f - startswith("test_")(f) && endswith(".jl")(f) +# check for filtered groups +# either via `--group=ALL` or through ENV["GROUP"] +const pat = r"(?:--group=)(\w+)" +arg_id = findfirst(contains(pat), ARGS) +const GROUP = uppercase( + if isnothing(arg_id) + get(ENV, "GROUP", "ALL") + else + only(match(pat, ARGS[arg_id]).captures) + end, +) + +function istestfile(filename) + return isfile(filename) && + endswith(filename, ".jl") && + startswith(basename(filename), "test") +end + +@time begin + # tests in groups based on folder structure + for testgroup in filter(isdir, readdir(@__DIR__)) + if GROUP == "ALL" || GROUP == uppercase(testgroup) + for file in filter(istestfile, readdir(joinpath(@__DIR__, testgroup); join=true)) + @eval @safetestset $file begin + include($file) + end + end + end end - @testset "Test $filename" for filename in filenames - include(filename) + + # single files in top folder + for file in filter(istestfile, readdir(@__DIR__)) + (file == basename(@__FILE__)) && continue + @eval @safetestset $file begin + include($file) + end + end + + # test examples + examplepath = joinpath(@__DIR__, "..", "examples") + for file in filter(endswith(".jl"), readdir(examplepath; join=true)) + @suppress @eval @safetestset $file begin + include($file) + end end -end end diff --git a/test/test_aqua.jl b/test/test_aqua.jl index b1bda03..1bfd3a0 100644 --- a/test/test_aqua.jl +++ b/test/test_aqua.jl @@ -1,4 +1,3 @@ -@eval module $(gensym()) using ITensorPkgSkeleton: ITensorPkgSkeleton using Aqua: Aqua using Test: @testset @@ -6,4 +5,3 @@ using Test: @testset @testset "Code quality (Aqua.jl)" begin Aqua.test_all(ITensorPkgSkeleton) end -end diff --git a/test/test_examples.jl b/test/test_examples.jl deleted file mode 100644 index a983e25..0000000 --- a/test/test_examples.jl +++ /dev/null @@ -1,15 +0,0 @@ -@eval module $(gensym()) -using ITensorPkgSkeleton: ITensorPkgSkeleton -using Suppressor: @suppress -using Test: @testset - -@testset "ITensorPkgSkeleton.jl examples" begin - examples_path = joinpath(pkgdir(ITensorPkgSkeleton), "examples") - filenames = filter(readdir(examples_path; join=true)) do f - endswith(".jl")(f) - end - @testset "Test $filename" for filename in filenames - @suppress include(filename) - end -end -end