diff --git a/.github/workflows/reference_tests.yml b/.github/workflows/reference_tests.yml index 47a28f69985..86cd448f184 100644 --- a/.github/workflows/reference_tests.yml +++ b/.github/workflows/reference_tests.yml @@ -50,11 +50,10 @@ jobs: julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("CairoMakie", coverage=true)' && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV - name: Upload test Artifacts - if: matrix.version == '1' uses: actions/upload-artifact@v4 with: - name: ReferenceImages_CairoMakie - path: ./CairoMakie/test/recorded_reference_images/ + name: ReferenceImages_CairoMakie_${{ matrix.version }} + path: ./CairoMakie/test/reference_images/ - name: Fail after artifacts if tests failed if: ${{ env.TESTS_SUCCESSFUL != 'true' }} run: exit 1 @@ -100,27 +99,11 @@ jobs: DISPLAY=:0 xvfb-run -s '-screen 0 1024x768x24' julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("GLMakie", coverage=true)' && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV - name: Upload test Artifacts - if: matrix.version == '1' uses: actions/upload-artifact@v4 with: - name: ReferenceImages_GLMakie + name: ReferenceImages_GLMakie_${{ matrix.version }} path: | - ./GLMakie/test/recorded_reference_images/ - - name: Save number of missing refimages to file - if: matrix.version == '1' - env: - N_MISSING: ${{ steps.referencetests.outputs.n_missing_refimages }} - COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - run: | - mkdir -p ./n_missing - echo $N_MISSING > ./n_missing/n_missing_refimages - echo $COMMIT_SHA >> ./n_missing/n_missing_refimages - - name: Upload artifact with number of missing refimages - if: matrix.version == '1' - uses: actions/upload-artifact@v4 - with: - name: n_missing_refimages - path: n_missing/ + ./GLMakie/test/reference_images/ - name: Fail after artifacts if tests failed if: ${{ env.TESTS_SUCCESSFUL != 'true' }} run: exit 1 @@ -163,11 +146,10 @@ jobs: DISPLAY=:0 xvfb-run -s '-screen 0 1024x768x24' julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("WGLMakie", coverage=true)' && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV - name: Upload test Artifacts - if: matrix.version == '1' uses: actions/upload-artifact@v4 with: - name: ReferenceImages_WGLMakie - path: ./WGLMakie/test/recorded_reference_images/ + name: ReferenceImages_WGLMakie_${{ matrix.version }} + path: ./WGLMakie/test/reference_images/ - name: Fail after artifacts if tests failed if: ${{ env.TESTS_SUCCESSFUL != 'true' }} run: exit 1 @@ -175,3 +157,65 @@ jobs: - uses: codecov/codecov-action@v3 with: file: lcov.info + + consolidation: + name: Merge artifacts + runs-on: ubuntu-20.04 + if: ${{ always() }} # run even if any of the three backend test jobs failed + needs: [cairomakie, glmakie, wglmakie] + steps: + - uses: actions/download-artifact@v4 + with: + name: ReferenceImages_WGLMakie_1 + path: ./ReferenceImages/WGLMakie + - uses: actions/download-artifact@v4 + with: + name: ReferenceImages_CairoMakie_1 + path: ./ReferenceImages/CairoMakie + - uses: actions/download-artifact@v4 + with: + name: ReferenceImages_GLMakie_1 + path: ./ReferenceImages/GLMakie + - name: Consolidate reference image folders + run: | + baseDir="./ReferenceImages" + + # Create new top-level directory for combined files + mkdir -p "./ReferenceImagesCombined" + + # Copy the reference folder from GLMakie, it's the same for all backends + cp -r "${baseDir}/GLMakie/reference/." "./ReferenceImagesCombined/reference/" + + # Initialize empty files for concatenation + > "./ReferenceImagesCombined/scores.tsv" + > "./ReferenceImagesCombined/new_files.txt" + + # Loop through the directories and concatenate the files, and copy recorded folders + for dir in WGLMakie CairoMakie GLMakie; do + # Concatenate scores.tsv and new_files.txt + cat "${baseDir}/${dir}/scores.tsv" >> "./ReferenceImagesCombined/scores.tsv" + cat "${baseDir}/${dir}/new_files.txt" >> "./ReferenceImagesCombined/new_files.txt" + + # Copy recorded folder + mkdir -p "./ReferenceImagesCombined/recorded/${dir}/" + cp -r "${baseDir}/${dir}/recorded/${dir}/." "./ReferenceImagesCombined/recorded/${dir}/" + done + + echo "Files and folders have been successfully combined into ReferenceImagesCombined." + - uses: actions/upload-artifact@v4 + with: + name: ReferenceImages + path: ./ReferenceImagesCombined/ + - name: Save number of missing refimages to file + env: + COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p ./n_missing + N_MISSING=$(wc -l < ./ReferenceImagesCombined/new_files.txt) + echo $N_MISSING > ./n_missing/n_missing_refimages + echo $COMMIT_SHA >> ./n_missing/n_missing_refimages + - name: Upload artifact with number of missing refimages + uses: actions/upload-artifact@v4 + with: + name: n_missing_refimages + path: n_missing/ diff --git a/.gitignore b/.gitignore index af42c787ff3..90e067be2e1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,9 +10,9 @@ Manifest.toml \.DS_Store -WGLMakie/test/recorded_reference_images/ -GLMakie/test/recorded_reference_images/ -CairoMakie/test/recorded_reference_images/ +WGLMakie/test/reference_images/ +GLMakie/test/reference_images/ +CairoMakie/test/reference_images/ RPRMakie/test/recorded/ CairoMakie/src/display.svg diff --git a/CairoMakie/test/runtests.jl b/CairoMakie/test/runtests.jl index d3dcc4c6c93..afac97bc2d5 100644 --- a/CairoMakie/test/runtests.jl +++ b/CairoMakie/test/runtests.jl @@ -192,7 +192,7 @@ functions = [:volume, :volume!, :uv_mesh] @testset "refimages" begin CairoMakie.activate!(type = "png", px_per_unit = 1) ReferenceTests.mark_broken_tests(excludes, functions=functions) - recorded_files, recording_dir = @include_reference_tests "refimages.jl" + recorded_files, recording_dir = @include_reference_tests CairoMakie "refimages.jl" missing_images, scores = ReferenceTests.record_comparison(recording_dir) - ReferenceTests.test_comparison(scores; threshold = 0.032) + ReferenceTests.test_comparison(scores; threshold = 0.05) end diff --git a/GLMakie/test/runtests.jl b/GLMakie/test/runtests.jl index 1ada23c2012..98e30bc505c 100644 --- a/GLMakie/test/runtests.jl +++ b/GLMakie/test/runtests.jl @@ -27,23 +27,13 @@ end include("unit_tests.jl") @testset "Reference Tests" begin - n_missing_images = 0 @testset "refimages" begin ReferenceTests.mark_broken_tests() - recorded_files, recording_dir = @include_reference_tests "refimages.jl" + recorded_files, recording_dir = @include_reference_tests GLMakie "refimages.jl" joinpath(@__DIR__, "glmakie_refimages.jl") missing_images, scores = ReferenceTests.record_comparison(recording_dir) - n_missing_images += length(missing_images) - ReferenceTests.test_comparison(scores; threshold = 0.032) + ReferenceTests.test_comparison(scores; threshold = 0.05) end - @testset "glmakie_refimages" begin - recorded_files, recording_dir = @include_reference_tests joinpath(@__DIR__, "glmakie_refimages.jl") - missing_images, scores = ReferenceTests.record_comparison(recording_dir) - n_missing_images += length(missing_images) - ReferenceTests.test_comparison(scores; threshold = 0.01) - end GLMakie.closeall() GC.gc(true) # make sure no finalizers act up! - # pass on status for Github Actions - println("::set-output name=n_missing_refimages::$n_missing_images") end diff --git a/ReferenceTests/.gitignore b/ReferenceTests/.gitignore index b2c00c6f8d3..c0af860522e 100644 --- a/ReferenceTests/.gitignore +++ b/ReferenceTests/.gitignore @@ -1,5 +1,3 @@ recorded/ -refimages/ -refimages.tar -glmakie_refimages/ -glmakie_refimages.tar +reference_images/ +reference_images.tar diff --git a/ReferenceTests/src/database.jl b/ReferenceTests/src/database.jl index 090a6ada7be..49c5bafdbf0 100644 --- a/ReferenceTests/src/database.jl +++ b/ReferenceTests/src/database.jl @@ -88,21 +88,25 @@ function mark_broken_tests(title_excludes = []; functions=[]) union!(SKIP_FUNCTIONS, functions) end -macro include_reference_tests(path) +macro include_reference_tests(backend::Symbol, path, paths...) toplevel_folder = dirname(string(__source__.file)) return esc(quote using ReferenceTests: @reference_test - name = splitext(basename($(path)))[1] - include_path = isdir($path) ? $path : joinpath(@__DIR__, "tests", $path) - recording_dir = joinpath($toplevel_folder, "recorded_reference_images", name) + include_paths = map([$path, $(paths...)]) do p + isdir(p) ? p : joinpath(@__DIR__, "tests", p) + end + recording_dir = joinpath($toplevel_folder, "reference_images") if isdir(recording_dir) rm(recording_dir; force=true, recursive=true) end - ReferenceTests.RECORDING_DIR[] = joinpath(recording_dir, "recorded") - mkpath(joinpath(recording_dir, "recorded")) - @testset "$name" begin + # prefix the recordings with the backend name so that each backend has its own versions + ReferenceTests.RECORDING_DIR[] = joinpath(recording_dir, "recorded", $(string(backend))) + mkpath(ReferenceTests.RECORDING_DIR[]) + @testset "Reference tests $($(string(backend)))" begin empty!(ReferenceTests.REGISTERED_TESTS) - include(include_path) + for include_path in include_paths + include(include_path) + end end recorded_files = collect(ReferenceTests.REGISTERED_TESTS) recording_dir = recording_dir diff --git a/ReferenceTests/src/image_download.jl b/ReferenceTests/src/image_download.jl index 8f3076fc100..e1e72cb9cf2 100644 --- a/ReferenceTests/src/image_download.jl +++ b/ReferenceTests/src/image_download.jl @@ -5,10 +5,10 @@ function last_major_version() return "v" * string(VersionNumber(version.major, version.minor)) end -function download_refimages(tag=last_major_version(); name="refimages") - url = "https://github.com/MakieOrg/Makie.jl/releases/download/$(tag)/$(name).tar" - images_tar = basedir("$(name).tar") - images = basedir(name) +function download_refimages(tag=last_major_version()) + url = "https://github.com/MakieOrg/Makie.jl/releases/download/$(tag)/reference_images.tar" + images_tar = basedir("reference_images.tar") + images = basedir("reference_images") if isfile(images_tar) if Bool(parse(Int, get(ENV, "REUSE_IMAGES_TAR", "0"))) @info "$images_tar already exists, skipping download as requested" diff --git a/ReferenceTests/src/runtests.jl b/ReferenceTests/src/runtests.jl index 13526823361..cb0d0672322 100644 --- a/ReferenceTests/src/runtests.jl +++ b/ReferenceTests/src/runtests.jl @@ -2,7 +2,10 @@ function get_frames(a, b) return (get_frames(a), get_frames(b)) end -function get_frames(video) +rgbf_convert(x::AbstractMatrix{<:RGB}) = convert(Matrix{RGBf}, x) +rgbf_convert(x::AbstractMatrix{<:RGBA}) = convert(Matrix{RGBAf}, x) + +function get_frames(video::AbstractString) mktempdir() do folder afolder = joinpath(folder, "a") mkpath(afolder) @@ -20,23 +23,45 @@ function get_frames(video) end end -function compare_media(a::Matrix, b::Matrix; sigma=[1,1]) - Images.test_approx_eq_sigma_eps(a, b, sigma, Inf) +function compare_images(a::AbstractMatrix{<:Union{RGB,RGBA}}, b::AbstractMatrix{<:Union{RGB,RGBA}}) + + a = rgbf_convert(a) + b = rgbf_convert(b) + + if size(a) != size(b) + @warn "images don't have the same size, difference will be Inf" + return Inf + end + + approx_tile_size_px = 30 + + range_dim1 = round.(Int, range(0, size(a, 1), length = ceil(Int, size(a, 1) / approx_tile_size_px))) + range_dim2 = round.(Int, range(0, size(a, 2), length = ceil(Int, size(a, 2) / approx_tile_size_px))) + + boundary_iter(boundaries) = zip(boundaries[1:end-1] .+ 1, boundaries[2:end]) + + _norm(rgb1::RGBf, rgb2::RGBf) = sqrt(sum(((rgb1.r - rgb2.r)^2, (rgb1.g - rgb2.g)^2, (rgb1.b - rgb2.b)^2))) + _norm(rgba1::RGBAf, rgba2::RGBAf) = sqrt(sum(((rgba1.r - rgba2.r)^2, (rgba1.g - rgba2.g)^2, (rgba1.b - rgba2.b)^2, (rgba1.alpha - rgba2.alpha)^2))) + + # compute the difference score as the maximum of the mean squared differences over the color + # values of tiles over the image. using tiles is a simple way to increase the local sensitivity + # without directly going to pixel-based comparison + # it also makes the scores more comparable between reference images of different sizes, because the same + # local differences would be normed to different mean scores if the images have different numbers of pixels + return maximum(Iterators.product(boundary_iter(range_dim1), boundary_iter(range_dim2))) do ((mi1, ma1), (mi2, ma2)) + @views mean(_norm.(a[mi1:ma1, mi2:ma2], b[mi1:ma1, mi2:ma2])) + end end -function compare_media(a, b; sigma=[1,1]) - file, ext = splitext(a) +function compare_media(a::AbstractString, b::AbstractString) + _, ext = splitext(a) if ext in (".png", ".jpg", ".jpeg", ".JPEG", ".JPG") imga = load(a) imgb = load(b) - if size(imga) != size(imgb) - @warn "images don't have the same size, difference will be Inf" - return Inf - end - conv(x) = convert(Matrix{RGBf}, x) - return compare_media(conv(imga), conv(imgb), sigma=sigma) + return compare_images(imga, imgb) elseif ext in (".mp4", ".gif") - aframes, bframes = get_frames(a, b) + aframes = get_frames(a) + bframes = get_frames(b) # Frames can differ in length, which usually shouldn't be the case but can happen # when the implementation of record changes, or when the example changes its number of frames # In that case, we just return inf + warn @@ -44,7 +69,7 @@ function compare_media(a, b; sigma=[1,1]) @warn "not the same number of frames in video, difference will be Inf" return Inf end - return mean(compare_media.(aframes, bframes; sigma=sigma)) + return maximum(compare_images.(aframes, bframes)) else error("Unknown media extension: $ext") end @@ -56,11 +81,16 @@ function get_all_relative_filepaths_recursively(dir) end end -function record_comparison(base_folder::String; record_folder_name="recorded", reference_name = basename(base_folder), tag=last_major_version()) +function record_comparison(base_folder::String; record_folder_name="recorded", tag=last_major_version()) record_folder = joinpath(base_folder, record_folder_name) - reference_folder = download_refimages(tag; name=reference_name) + @info "Downloading reference images" + reference_folder = download_refimages(tag) # we copy the reference images into the output folder, since we want to upload it all as an artifact, to know against what images we compared - cp(reference_folder, joinpath(base_folder, "reference")) + local_reference_copy_dir = joinpath(base_folder, "reference") + if isdir(local_reference_copy_dir) + rm(local_reference_copy_dir, recursive = true) + end + cp(reference_folder, local_reference_copy_dir) testimage_paths = get_all_relative_filepaths_recursively(record_folder) missing_refimages, scores = compare(testimage_paths, reference_folder, record_folder) diff --git a/ReferenceTests/src/tests/examples2d.jl b/ReferenceTests/src/tests/examples2d.jl index 735be7bb507..5cdd906ccdb 100644 --- a/ReferenceTests/src/tests/examples2d.jl +++ b/ReferenceTests/src/tests/examples2d.jl @@ -1171,50 +1171,54 @@ end fig end -@reference_test "Triplot of a constrained triangulation with holes and a custom bounding box" begin - curve_1 = [[ - (0.0, 0.0), (4.0, 0.0), (8.0, 0.0), (12.0, 0.0), (12.0, 4.0), - (12.0, 8.0), (14.0, 10.0), (16.0, 12.0), (16.0, 16.0), - (14.0, 18.0), (12.0, 20.0), (12.0, 24.0), (12.0, 28.0), - (8.0, 28.0), (4.0, 28.0), (0.0, 28.0), (-2.0, 26.0), (0.0, 22.0), - (0.0, 18.0), (0.0, 10.0), (0.0, 8.0), (0.0, 4.0), (-4.0, 4.0), - (-4.0, 0.0), (0.0, 0.0), - ]] - curve_2 = [[ - (4.0, 26.0), (8.0, 26.0), (10.0, 26.0), (10.0, 24.0), - (10.0, 22.0), (10.0, 20.0), (8.0, 20.0), (6.0, 20.0), - (4.0, 20.0), (4.0, 22.0), (4.0, 24.0), (4.0, 26.0) - ]] - curve_3 = [[(4.0, 16.0), (12.0, 16.0), (12.0, 14.0), (4.0, 14.0), (4.0, 16.0)]] - curve_4 = [[(4.0, 8.0), (10.0, 8.0), (8.0, 6.0), (6.0, 6.0), (4.0, 8.0)]] - curves = [curve_1, curve_2, curve_3, curve_4] - points = [ - (2.0, 26.0), (2.0, 24.0), (6.0, 24.0), (6.0, 22.0), (8.0, 24.0), (8.0, 22.0), - (2.0, 22.0), (0.0, 26.0), (10.0, 18.0), (8.0, 18.0), (4.0, 18.0), (2.0, 16.0), - (2.0, 12.0), (6.0, 12.0), (2.0, 8.0), (2.0, 4.0), (4.0, 2.0), - (-2.0, 2.0), (4.0, 6.0), (10.0, 2.0), (10.0, 6.0), (8.0, 10.0), (4.0, 10.0), - (10.0, 12.0), (12.0, 12.0), (14.0, 26.0), (16.0, 24.0), (18.0, 28.0), - (16.0, 20.0), (18.0, 12.0), (16.0, 8.0), (14.0, 4.0), (14.0, -2.0), - (6.0, -2.0), (2.0, -4.0), (-4.0, -2.0), (-2.0, 8.0), (-2.0, 16.0), - (-4.0, 22.0), (-4.0, 26.0), (-2.0, 28.0), (6.0, 15.0), (7.0, 15.0), - (8.0, 15.0), (9.0, 15.0), (10.0, 15.0), (6.2, 7.8), - (5.6, 7.8), (5.6, 7.6), (5.6, 7.4), (6.2, 7.4), (6.0, 7.6), - (7.0, 7.8), (7.0, 7.4)] - boundary_nodes, points = convert_boundary_points_to_indices(curves; existing_points=points) - tri = triangulate(points; boundary_nodes=boundary_nodes, rng = RNG.STABLE_RNG) - refine!(tri, max_area = 1e-3get_total_area(tri), rng = RNG.STABLE_RNG) - fig, ax, sc = triplot(tri, - show_points=true, - show_constrained_edges=true, - constrained_edge_linewidth=2, - strokewidth=0.2, - markersize=15, - point_color=:blue, - show_ghost_edges=true, # not as good because the outer boundary is not convex, but just testing - marker='x', - bounding_box = (-5,20,-5,35)) # also testing the conversion to Float64 for bbox here - fig -end +# TODO: as noted in https://github.com/MakieOrg/Makie.jl/pull/3520#issuecomment-1873382060 +# this test has some issues with random number generation across Julia 1.6 and 1, for now +# it's disabled until someone has time to look into it + +# @reference_test "Triplot of a constrained triangulation with holes and a custom bounding box" begin +# curve_1 = [[ +# (0.0, 0.0), (4.0, 0.0), (8.0, 0.0), (12.0, 0.0), (12.0, 4.0), +# (12.0, 8.0), (14.0, 10.0), (16.0, 12.0), (16.0, 16.0), +# (14.0, 18.0), (12.0, 20.0), (12.0, 24.0), (12.0, 28.0), +# (8.0, 28.0), (4.0, 28.0), (0.0, 28.0), (-2.0, 26.0), (0.0, 22.0), +# (0.0, 18.0), (0.0, 10.0), (0.0, 8.0), (0.0, 4.0), (-4.0, 4.0), +# (-4.0, 0.0), (0.0, 0.0), +# ]] +# curve_2 = [[ +# (4.0, 26.0), (8.0, 26.0), (10.0, 26.0), (10.0, 24.0), +# (10.0, 22.0), (10.0, 20.0), (8.0, 20.0), (6.0, 20.0), +# (4.0, 20.0), (4.0, 22.0), (4.0, 24.0), (4.0, 26.0) +# ]] +# curve_3 = [[(4.0, 16.0), (12.0, 16.0), (12.0, 14.0), (4.0, 14.0), (4.0, 16.0)]] +# curve_4 = [[(4.0, 8.0), (10.0, 8.0), (8.0, 6.0), (6.0, 6.0), (4.0, 8.0)]] +# curves = [curve_1, curve_2, curve_3, curve_4] +# points = [ +# (2.0, 26.0), (2.0, 24.0), (6.0, 24.0), (6.0, 22.0), (8.0, 24.0), (8.0, 22.0), +# (2.0, 22.0), (0.0, 26.0), (10.0, 18.0), (8.0, 18.0), (4.0, 18.0), (2.0, 16.0), +# (2.0, 12.0), (6.0, 12.0), (2.0, 8.0), (2.0, 4.0), (4.0, 2.0), +# (-2.0, 2.0), (4.0, 6.0), (10.0, 2.0), (10.0, 6.0), (8.0, 10.0), (4.0, 10.0), +# (10.0, 12.0), (12.0, 12.0), (14.0, 26.0), (16.0, 24.0), (18.0, 28.0), +# (16.0, 20.0), (18.0, 12.0), (16.0, 8.0), (14.0, 4.0), (14.0, -2.0), +# (6.0, -2.0), (2.0, -4.0), (-4.0, -2.0), (-2.0, 8.0), (-2.0, 16.0), +# (-4.0, 22.0), (-4.0, 26.0), (-2.0, 28.0), (6.0, 15.0), (7.0, 15.0), +# (8.0, 15.0), (9.0, 15.0), (10.0, 15.0), (6.2, 7.8), +# (5.6, 7.8), (5.6, 7.6), (5.6, 7.4), (6.2, 7.4), (6.0, 7.6), +# (7.0, 7.8), (7.0, 7.4)] +# boundary_nodes, points = convert_boundary_points_to_indices(curves; existing_points=points) +# tri = triangulate(points; boundary_nodes=boundary_nodes, rng = RNG.STABLE_RNG) +# refine!(tri, max_area = 1e-3get_total_area(tri), rng = RNG.STABLE_RNG) +# fig, ax, sc = triplot(tri, +# show_points=true, +# show_constrained_edges=true, +# constrained_edge_linewidth=2, +# strokewidth=0.2, +# markersize=15, +# point_color=:blue, +# show_ghost_edges=true, # not as good because the outer boundary is not convex, but just testing +# marker='x', +# bounding_box = (-5,20,-5,35)) # also testing the conversion to Float64 for bbox here +# fig +# end @reference_test "Triplot with nonlinear transformation" begin f = Figure() diff --git a/ReferenceTests/src/tests/updating.jl b/ReferenceTests/src/tests/updating.jl index d19c97647fd..3c362fbea7f 100644 --- a/ReferenceTests/src/tests/updating.jl +++ b/ReferenceTests/src/tests/updating.jl @@ -74,8 +74,8 @@ function compare_videos(reference, vpath, dir) @test n == length(reference) @test all(1:n) do i - v = ReferenceTests.compare_media(reference[i], to_compare[i]) - return v < 0.02 + v = ReferenceTests.compare_images(reference[i], to_compare[i]) + return v < 0.2 end end diff --git a/ReferenceTests/src/visual-regression.jl b/ReferenceTests/src/visual-regression.jl deleted file mode 100644 index a0a362e94df..00000000000 --- a/ReferenceTests/src/visual-regression.jl +++ /dev/null @@ -1,107 +0,0 @@ -""" -`m = maxfinite(A)` calculates the maximum value in `A`, ignoring any values that are not finite (Inf or NaN). -""" -function maxfinite(A::AbstractArray{T}) where T - ret = sentinel_max(T) - for a in A - ret = maxfinite_scalar(a, ret) - end - ret -end - -function maxfinite(f, A::AbstractArray) - ret = sentinel_max(typeof(f(first(A)))) - for a in A - ret = maxfinite_scalar(f(a), ret) - end - ret -end - -""" -`m = maxabsfinite(A)` calculates the maximum absolute value in `A`, ignoring any values that are not finite (Inf or NaN). -""" -function maxabsfinite(A::AbstractArray{T}) where T - ret = sentinel_min(typeof(abs(A[1]))) - for a in A - ret = maxfinite_scalar(abs(a), ret) - end - ret -end - -minfinite_scalar(a::T, b::T) where {T} = isfinite(a) ? (b < a ? b : a) : b -maxfinite_scalar(a::T, b::T) where {T} = isfinite(a) ? (b > a ? b : a) : b -minfinite_scalar(a::T, b::T) where {T <: Union{Integer,FixedPoint}} = b < a ? b : a -maxfinite_scalar(a::T, b::T) where {T <: Union{Integer,FixedPoint}} = b > a ? b : a -minfinite_scalar(a, b) = minfinite_scalar(promote(a, b)...) -maxfinite_scalar(a, b) = maxfinite_scalar(promote(a, b)...) - -function minfinite_scalar(c1::C, c2::C) where C <: AbstractRGB - C(minfinite_scalar(c1.r, c2.r), - minfinite_scalar(c1.g, c2.g), - minfinite_scalar(c1.b, c2.b)) -end -function maxfinite_scalar(c1::C, c2::C) where C <: AbstractRGB - C(maxfinite_scalar(c1.r, c2.r), - maxfinite_scalar(c1.g, c2.g), - maxfinite_scalar(c1.b, c2.b)) -end - -sentinel_min(::Type{T}) where {T <: Union{Integer,FixedPoint}} = typemax(T) -sentinel_max(::Type{T}) where {T <: Union{Integer,FixedPoint}} = typemin(T) -sentinel_min(::Type{T}) where {T <: AbstractFloat} = convert(T, NaN) -sentinel_max(::Type{T}) where {T <: AbstractFloat} = convert(T, NaN) -sentinel_min(::Type{C}) where {C <: AbstractRGB} = _sentinel_min(C, eltype(C)) -_sentinel_min(::Type{C},::Type{T}) where {C <: AbstractRGB,T} = (s = sentinel_min(T); C(s, s, s)) -sentinel_max(::Type{C}) where {C <: AbstractRGB} = _sentinel_max(C, eltype(C)) -_sentinel_max(::Type{C},::Type{T}) where {C <: AbstractRGB,T} = (s = sentinel_max(T); C(s, s, s)) - -difftype(::Type{T}) where {T <: Integer} = Int -difftype(::Type{T}) where {T <: Real} = Float32 -difftype(::Type{Float64}) = Float64 -difftype(::Type{CV}) where {CV <: Colorant} = difftype(CV, eltype(CV)) -difftype(::Type{CV}, ::Type{T}) where {CV <: RGBA,T <: Real} = RGBA{Float32} -difftype(::Type{CV}, ::Type{Float64}) where {CV <: RGBA} = RGBA{Float64} -difftype(::Type{CV}, ::Type{Float64}) where {CV <: Gray} = Gray{Float64} -difftype(::Type{CV}, ::Type{T}) where {CV <: BGRA,T <: Real} = BGRA{Float32} -difftype(::Type{CV}, ::Type{Float64}) where {CV <: BGRA} = BGRA{Float64} -difftype(::Type{CV}, ::Type{T}) where {CV <: AbstractRGB,T <: Real} = RGB{Float32} -difftype(::Type{CV}, ::Type{Float64}) where {CV <: AbstractRGB} = RGB{Float64} - -accum(::Type{T}) where {T <: Integer} = Int -accum(::Type{Float32}) = Float32 -accum(::Type{T}) where {T <: Real} = Float64 -accum(::Type{C}) where {C <: Colorant} = base_colorant_type(C){accum(eltype(C))} -function sumdiff(f, A::AbstractArray, B::AbstractArray) - axes(A) == axes(B) || throw(DimensionMismatch("A and B must have the same axes")) - T = promote_type(difftype(eltype(A)), difftype(eltype(B))) - s = zero(accum(eltype(T))) - for (a, b) in zip(A, B) - x = convert(T, a) - convert(T, b) - s += f(x) - end - s -end - -"`s = ssd(A, B)` computes the sum-of-squared differences over arrays/images A and B" -ssd(A::AbstractArray, B::AbstractArray) = sumdiff(abs2, A, B) - -"`s = sad(A, B)` computes the sum-of-absolute differences over arrays/images A and B" -sad(A::AbstractArray, B::AbstractArray) = sumdiff(abs, A, B) - - -function approx_difference( - A::AbstractArray, B::AbstractArray, - sigma::AbstractVector{T}=ones(ndims(A)), - eps::AbstractFloat=1e-2 - ) where T <: Real - - if length(sigma) != ndims(A) - error("Invalid sigma in test_approx_eq_sigma_eps. Should be ndims(A)-length vector of the number of pixels to blur. Got: $sigma") - end - kern = KernelFactors.IIRGaussian(sigma) - Af = imfilter(A, kern, NA()) - Bf = imfilter(B, kern, NA()) - diffscale = max(maxabsfinite(Af), maxabsfinite(Bf)) - d = sad(Af, Bf) - return d / (length(Af) * diffscale) -end diff --git a/ReferenceUpdater/README.md b/ReferenceUpdater/README.md index a051d6af7bc..b97e69a6d63 100644 --- a/ReferenceUpdater/README.md +++ b/ReferenceUpdater/README.md @@ -15,7 +15,7 @@ E.g., to see the results after running `]test WGLMakie` you can run: ```julia using ReferenceUpdater -ReferenceUpdater.serve_update_page_from_dir("Makie/WGLMakie/test/recorded_reference_images/refimages") +ReferenceUpdater.serve_update_page_from_dir("Makie/WGLMakie/test/reference_images/") ``` `ReferenceUpdater` is an unregistered package which is located in the Makie monorepo. To add it, run `Pkg.dev(joinpath(dirname(dirname(pathof(Makie))), "ReferenceUpdater"))`, assuming that you've `dev`'ed Makie. You should be given a choice of different backend workflow runs. diff --git a/ReferenceUpdater/src/image_download.jl b/ReferenceUpdater/src/image_download.jl index 0c0025033de..3dbe20d9365 100644 --- a/ReferenceUpdater/src/image_download.jl +++ b/ReferenceUpdater/src/image_download.jl @@ -11,9 +11,9 @@ function last_major_version() return "v" * string(VersionNumber(version.major, version.minor)) end -function upload_reference_images(path=basedir("recorded"), tag=last_major_version(); name="refimages") +function upload_reference_images(path=basedir("recorded"), tag=last_major_version()) mktempdir() do dir - tarfile = joinpath(dir, "$(name).tar") + tarfile = joinpath(dir, "reference_images.tar") Tar.create(path, tarfile) upload_release("MakieOrg", "Makie.jl", ENV["GITHUB_TOKEN"], tag, tarfile) end diff --git a/ReferenceUpdater/src/local_server.jl b/ReferenceUpdater/src/local_server.jl index 39f9833e423..b76ba90eb2f 100644 --- a/ReferenceUpdater/src/local_server.jl +++ b/ReferenceUpdater/src/local_server.jl @@ -4,9 +4,7 @@ function serve_update_page_from_dir(folder) folder = realpath(folder) @assert isdir(folder) "$folder is not a valid directory." - refimages_name = last(splitpath(realpath(folder))) - @info "Refimage set name is $refimages_name" - + router = HTTP.Router() function receive_update(req) @@ -32,7 +30,7 @@ function serve_update_page_from_dir(folder) @info "Uploading updated reference images under tag \"$tag\"" try - upload_reference_images(tempdir, tag; name = refimages_name) + upload_reference_images(tempdir, tag) @info "Upload successful. You can ctrl+c out now." HTTP.Response(200, "Upload successful") catch e @@ -100,16 +98,8 @@ function serve_update_page(; commit = nothing, pr = nothing) checkruns = filter(checksinfo["check_runs"]) do checkrun name = checkrun["name"] id = checkrun["id"] - right_combination = any(["GLMakie", "CairoMakie", "WGLMakie"]) do package - # We need to match the name quite specifically, since we need to keep this synchronized to the CI script anyways. - name == "$package Julia 1" - end - if right_combination - if name in unique_artifacts - return false - else - push!(unique_artifacts, name) - end + + if name == "Merge artifacts" job = JSON3.read(authget("https://api.github.com/repos/MakieOrg/Makie.jl/actions/jobs/$(id)").body) run = JSON3.read(authget(job["run_url"]).body) if run["status"] != "completed" @@ -123,17 +113,13 @@ function serve_update_page(; commit = nothing, pr = nothing) end end if isempty(checkruns) - error("No check runs fit the criteria, check if something about names or versions might have changed.") + error("\"Merge artifacts\" run is not available.") end - - menu = REPL.TerminalMenus.RadioMenu([x["name"] for x in checkruns]) - choice = REPL.TerminalMenus.request("Choose a workflow run:", menu) - - if choice == -1 - error("Cancelled") + if length(checkruns) > 1 + error("Found multiple checkruns for \"Merge artifacts\", this is unexpected.") end - check = checkruns[choice] - chosen_backend = match(r"(\w+) Julia 1", check["name"])[1] + + check = only(checkruns) job = JSON3.read(authget("https://api.github.com/repos/MakieOrg/Makie.jl/actions/jobs/$(check["id"])").body) run = JSON3.read(authget(job["run_url"]).body) @@ -141,8 +127,8 @@ function serve_update_page(; commit = nothing, pr = nothing) artifacts = JSON3.read(authget(run["artifacts_url"]).body)["artifacts"] for a in artifacts - if a["name"] == "ReferenceImages_$chosen_backend" - @info "Choosing artifact $(a["name"])" + if a["name"] == "ReferenceImages" + @info "Choosing artifact \"$(a["name"])\"" download_url = a["archive_download_url"] if !haskey(URL_CACHE, download_url) @info "Downloading artifact from $download_url" @@ -156,28 +142,13 @@ function serve_update_page(; commit = nothing, pr = nothing) @info "$download_url cached at $tmpdir" end - folders = readdir(tmpdir) - if length(folders) == 0 - error("No folder in zip") - elseif length(folders) == 1 - folder = only(folders) - else - menu = REPL.TerminalMenus.RadioMenu(folders, pagesize=4) - choice = REPL.TerminalMenus.request("Choose a reference image set:", menu) - - if choice == -1 - error("Cancelled") - end - folder = folders[choice] - end - - @info "Serving update page for folder $folder." - serve_update_page_from_dir(joinpath(tmpdir, folder)) + @info "Serving update page from folder $tmpdir." + serve_update_page_from_dir(tmpdir) return end end error(""" - Reference image artifacts found for commit $headsha and job id $(check["id"]). + No \"ReferenceImages\" artifact found for commit $headsha and job id $(check["id"]). This could be because the job's workflow run ($(job["run_url"])) has not completed, yet. Artifacts are only available for complete runs. """) diff --git a/ReferenceUpdater/src/reference_images.html b/ReferenceUpdater/src/reference_images.html index 363ea80349c..b339f313983 100644 --- a/ReferenceUpdater/src/reference_images.html +++ b/ReferenceUpdater/src/reference_images.html @@ -62,9 +62,9 @@

Images with references

` if (path.endsWith(".png")){ - div.innerHTML += `` + div.innerHTML += `` } else if (path.endsWith(".mp4")){ - div.innerHTML += `