use separate reference images per backend

.gitignore

@@ -10,9 +10,9 @@ Manifest.toml
 
 \.DS_Store
 
-WGLMakie/test/recorded_reference_images/
-GLMakie/test/recorded_reference_images/
-CairoMakie/test/recorded_reference_images/
+WGLMakie/test/reference_images/
+GLMakie/test/reference_images/
+CairoMakie/test/reference_images/
 RPRMakie/test/recorded/
 
 CairoMakie/src/display.svg

CairoMakie/test/runtests.jl

@@ -192,7 +192,7 @@ functions = [:volume, :volume!, :uv_mesh]
 @testset "refimages" begin
     CairoMakie.activate!(type = "png", px_per_unit = 1)
     ReferenceTests.mark_broken_tests(excludes, functions=functions)
-    recorded_files, recording_dir = @include_reference_tests "refimages.jl"
+    recorded_files, recording_dir = @include_reference_tests CairoMakie "refimages.jl"
     missing_images, scores = ReferenceTests.record_comparison(recording_dir)
-    ReferenceTests.test_comparison(scores; threshold = 0.032)
+    ReferenceTests.test_comparison(scores; threshold = 0.001)
 end

GLMakie/test/runtests.jl

@@ -27,21 +27,14 @@ end
 include("unit_tests.jl")
 
 @testset "Reference Tests" begin
-    n_missing_images = 0
     @testset "refimages" begin
         ReferenceTests.mark_broken_tests()
-        recorded_files, recording_dir = @include_reference_tests "refimages.jl"
+        recorded_files, recording_dir = @include_reference_tests GLMakie "refimages.jl" joinpath(@__DIR__, "glmakie_refimages.jl")
         missing_images, scores = ReferenceTests.record_comparison(recording_dir)
-        n_missing_images += length(missing_images)
-        ReferenceTests.test_comparison(scores; threshold = 0.032)
+        n_missing_images = length(missing_images)
+        ReferenceTests.test_comparison(scores; threshold = 0.001)
     end
 
-    @testset "glmakie_refimages" begin
-        recorded_files, recording_dir = @include_reference_tests joinpath(@__DIR__, "glmakie_refimages.jl")
-        missing_images, scores = ReferenceTests.record_comparison(recording_dir)
-        n_missing_images += length(missing_images)
-        ReferenceTests.test_comparison(scores; threshold = 0.01)
-    end
     GLMakie.closeall()
     GC.gc(true) # make sure no finalizers act up!
     # pass on status for Github Actions

ReferenceTests/.gitignore

@@ -1,5 +1,3 @@
 recorded/
-refimages/
-refimages.tar
-glmakie_refimages/
-glmakie_refimages.tar
+reference_images/
+reference_images.tar

ReferenceTests/src/database.jl

@@ -88,21 +88,25 @@ function mark_broken_tests(title_excludes = []; functions=[])
     union!(SKIP_FUNCTIONS, functions)
 end
 
-macro include_reference_tests(path)
+macro include_reference_tests(backend::Symbol, path, paths...)
     toplevel_folder = dirname(string(__source__.file))
     return esc(quote
         using ReferenceTests: @reference_test
-        name = splitext(basename($(path)))[1]
-        include_path = isdir($path) ? $path : joinpath(@__DIR__, "tests", $path)
-        recording_dir = joinpath($toplevel_folder, "recorded_reference_images", name)
+        include_paths = map([$path, $(paths...)]) do p
+            isdir(p) ? p : joinpath(@__DIR__, "tests", p)
+        end
+        recording_dir = joinpath($toplevel_folder, "reference_images")
         if isdir(recording_dir)
             rm(recording_dir; force=true, recursive=true)
         end
-        ReferenceTests.RECORDING_DIR[] = joinpath(recording_dir, "recorded")
-        mkpath(joinpath(recording_dir, "recorded"))
-        @testset "$name" begin
+        # prefix the recordings with the backend name so that each backend has its own versions
+        ReferenceTests.RECORDING_DIR[] = joinpath(recording_dir, "recorded", $(string(backend)))
+        mkpath(ReferenceTests.RECORDING_DIR[])
+        @testset "Reference tests $($(string(backend)))" begin
             empty!(ReferenceTests.REGISTERED_TESTS)
-            include(include_path)
+            for include_path in include_paths
+                include(include_path)
+            end
         end
         recorded_files = collect(ReferenceTests.REGISTERED_TESTS)
         recording_dir = recording_dir

ReferenceTests/src/image_download.jl

@@ -5,10 +5,10 @@ function last_major_version()
     return "v" * string(VersionNumber(version.major, version.minor))
 end
 
-function download_refimages(tag=last_major_version(); name="refimages")
-    url = "https://github.com/MakieOrg/Makie.jl/releases/download/$(tag)/$(name).tar"
-    images_tar = basedir("$(name).tar")
-    images = basedir(name)
+function download_refimages(tag=last_major_version())
+    url = "https://github.com/MakieOrg/Makie.jl/releases/download/$(tag)/reference_images.tar"
+    images_tar = basedir("reference_images.tar")
+    images = basedir("reference_images")
     if isfile(images_tar)
         if Bool(parse(Int, get(ENV, "REUSE_IMAGES_TAR", "0")))
             @info "$images_tar already exists, skipping download as requested"

ReferenceTests/src/runtests.jl

@@ -56,9 +56,9 @@ function get_all_relative_filepaths_recursively(dir)
     end
 end
 
-function record_comparison(base_folder::String; record_folder_name="recorded", reference_name = basename(base_folder), tag=last_major_version())
+function record_comparison(base_folder::String; record_folder_name="recorded", tag=last_major_version())
     record_folder = joinpath(base_folder, record_folder_name)
-    reference_folder = download_refimages(tag; name=reference_name)
+    reference_folder = download_refimages(tag)
     # we copy the reference images into the output folder, since we want to upload it all as an artifact, to know against what images we compared
     cp(reference_folder, joinpath(base_folder, "reference"))
     testimage_paths = get_all_relative_filepaths_recursively(record_folder)

WGLMakie/test/runtests.jl

@@ -55,9 +55,9 @@ edisplay = Bonito.use_electron_display(devtools=true)
 @testset "refimages" begin
     WGLMakie.activate!()
     ReferenceTests.mark_broken_tests(excludes)
-    recorded_files, recording_dir = @include_reference_tests "refimages.jl"
+    recorded_files, recording_dir = @include_reference_tests WGLMakie "refimages.jl"
     missing_images, scores = ReferenceTests.record_comparison(recording_dir)
-    ReferenceTests.test_comparison(scores; threshold = 0.032)
+    ReferenceTests.test_comparison(scores; threshold = 0.001)
 end
 
 @testset "memory leaks" begin