diff --git a/.gitignore b/.gitignore index af42c787ff3..90e067be2e1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,9 +10,9 @@ Manifest.toml \.DS_Store -WGLMakie/test/recorded_reference_images/ -GLMakie/test/recorded_reference_images/ -CairoMakie/test/recorded_reference_images/ +WGLMakie/test/reference_images/ +GLMakie/test/reference_images/ +CairoMakie/test/reference_images/ RPRMakie/test/recorded/ CairoMakie/src/display.svg diff --git a/CairoMakie/test/runtests.jl b/CairoMakie/test/runtests.jl index d3dcc4c6c93..c5b7a69ccf3 100644 --- a/CairoMakie/test/runtests.jl +++ b/CairoMakie/test/runtests.jl @@ -192,7 +192,7 @@ functions = [:volume, :volume!, :uv_mesh] @testset "refimages" begin CairoMakie.activate!(type = "png", px_per_unit = 1) ReferenceTests.mark_broken_tests(excludes, functions=functions) - recorded_files, recording_dir = @include_reference_tests "refimages.jl" + recorded_files, recording_dir = @include_reference_tests CairoMakie "refimages.jl" missing_images, scores = ReferenceTests.record_comparison(recording_dir) - ReferenceTests.test_comparison(scores; threshold = 0.032) + ReferenceTests.test_comparison(scores; threshold = 0.001) end diff --git a/GLMakie/test/runtests.jl b/GLMakie/test/runtests.jl index 1ada23c2012..edbbcb7338b 100644 --- a/GLMakie/test/runtests.jl +++ b/GLMakie/test/runtests.jl @@ -27,21 +27,14 @@ end include("unit_tests.jl") @testset "Reference Tests" begin - n_missing_images = 0 @testset "refimages" begin ReferenceTests.mark_broken_tests() - recorded_files, recording_dir = @include_reference_tests "refimages.jl" + recorded_files, recording_dir = @include_reference_tests GLMakie "refimages.jl" joinpath(@__DIR__, "glmakie_refimages.jl") missing_images, scores = ReferenceTests.record_comparison(recording_dir) - n_missing_images += length(missing_images) - ReferenceTests.test_comparison(scores; threshold = 0.032) + n_missing_images = length(missing_images) + ReferenceTests.test_comparison(scores; threshold = 0.001) end - @testset "glmakie_refimages" begin - recorded_files, recording_dir = @include_reference_tests joinpath(@__DIR__, "glmakie_refimages.jl") - missing_images, scores = ReferenceTests.record_comparison(recording_dir) - n_missing_images += length(missing_images) - ReferenceTests.test_comparison(scores; threshold = 0.01) - end GLMakie.closeall() GC.gc(true) # make sure no finalizers act up! # pass on status for Github Actions diff --git a/ReferenceTests/.gitignore b/ReferenceTests/.gitignore index b2c00c6f8d3..c0af860522e 100644 --- a/ReferenceTests/.gitignore +++ b/ReferenceTests/.gitignore @@ -1,5 +1,3 @@ recorded/ -refimages/ -refimages.tar -glmakie_refimages/ -glmakie_refimages.tar +reference_images/ +reference_images.tar diff --git a/ReferenceTests/src/database.jl b/ReferenceTests/src/database.jl index 090a6ada7be..49c5bafdbf0 100644 --- a/ReferenceTests/src/database.jl +++ b/ReferenceTests/src/database.jl @@ -88,21 +88,25 @@ function mark_broken_tests(title_excludes = []; functions=[]) union!(SKIP_FUNCTIONS, functions) end -macro include_reference_tests(path) +macro include_reference_tests(backend::Symbol, path, paths...) toplevel_folder = dirname(string(__source__.file)) return esc(quote using ReferenceTests: @reference_test - name = splitext(basename($(path)))[1] - include_path = isdir($path) ? $path : joinpath(@__DIR__, "tests", $path) - recording_dir = joinpath($toplevel_folder, "recorded_reference_images", name) + include_paths = map([$path, $(paths...)]) do p + isdir(p) ? p : joinpath(@__DIR__, "tests", p) + end + recording_dir = joinpath($toplevel_folder, "reference_images") if isdir(recording_dir) rm(recording_dir; force=true, recursive=true) end - ReferenceTests.RECORDING_DIR[] = joinpath(recording_dir, "recorded") - mkpath(joinpath(recording_dir, "recorded")) - @testset "$name" begin + # prefix the recordings with the backend name so that each backend has its own versions + ReferenceTests.RECORDING_DIR[] = joinpath(recording_dir, "recorded", $(string(backend))) + mkpath(ReferenceTests.RECORDING_DIR[]) + @testset "Reference tests $($(string(backend)))" begin empty!(ReferenceTests.REGISTERED_TESTS) - include(include_path) + for include_path in include_paths + include(include_path) + end end recorded_files = collect(ReferenceTests.REGISTERED_TESTS) recording_dir = recording_dir diff --git a/ReferenceTests/src/image_download.jl b/ReferenceTests/src/image_download.jl index 8f3076fc100..e1e72cb9cf2 100644 --- a/ReferenceTests/src/image_download.jl +++ b/ReferenceTests/src/image_download.jl @@ -5,10 +5,10 @@ function last_major_version() return "v" * string(VersionNumber(version.major, version.minor)) end -function download_refimages(tag=last_major_version(); name="refimages") - url = "https://github.com/MakieOrg/Makie.jl/releases/download/$(tag)/$(name).tar" - images_tar = basedir("$(name).tar") - images = basedir(name) +function download_refimages(tag=last_major_version()) + url = "https://github.com/MakieOrg/Makie.jl/releases/download/$(tag)/reference_images.tar" + images_tar = basedir("reference_images.tar") + images = basedir("reference_images") if isfile(images_tar) if Bool(parse(Int, get(ENV, "REUSE_IMAGES_TAR", "0"))) @info "$images_tar already exists, skipping download as requested" diff --git a/ReferenceTests/src/runtests.jl b/ReferenceTests/src/runtests.jl index 13526823361..b626df33c6d 100644 --- a/ReferenceTests/src/runtests.jl +++ b/ReferenceTests/src/runtests.jl @@ -56,9 +56,9 @@ function get_all_relative_filepaths_recursively(dir) end end -function record_comparison(base_folder::String; record_folder_name="recorded", reference_name = basename(base_folder), tag=last_major_version()) +function record_comparison(base_folder::String; record_folder_name="recorded", tag=last_major_version()) record_folder = joinpath(base_folder, record_folder_name) - reference_folder = download_refimages(tag; name=reference_name) + reference_folder = download_refimages(tag) # we copy the reference images into the output folder, since we want to upload it all as an artifact, to know against what images we compared cp(reference_folder, joinpath(base_folder, "reference")) testimage_paths = get_all_relative_filepaths_recursively(record_folder) diff --git a/WGLMakie/test/runtests.jl b/WGLMakie/test/runtests.jl index 200d61142c4..30d9c203470 100644 --- a/WGLMakie/test/runtests.jl +++ b/WGLMakie/test/runtests.jl @@ -55,9 +55,9 @@ edisplay = Bonito.use_electron_display(devtools=true) @testset "refimages" begin WGLMakie.activate!() ReferenceTests.mark_broken_tests(excludes) - recorded_files, recording_dir = @include_reference_tests "refimages.jl" + recorded_files, recording_dir = @include_reference_tests WGLMakie "refimages.jl" missing_images, scores = ReferenceTests.record_comparison(recording_dir) - ReferenceTests.test_comparison(scores; threshold = 0.032) + ReferenceTests.test_comparison(scores; threshold = 0.001) end @testset "memory leaks" begin