Add tick event (#3948)

* initial Makie setup

* update CairoMakie

* track delta time and adjust states

* implement tick events in GLMakie

* implement ticks in WGLMakie

* track frame_delta_time in record

* update some comments

* cleanup CairoMakie

* cleanup WGLMakie

* add refimg test for record

* add frame count, time since start, skip event ticks

* update test

* update docs

* update changelog

* add more tests

* try more time

* make record ticks match set framerate

* add safeguard

* try stabilize normals of cat

* try fix tests

* try fix tests

* drop docs ref to enum to avoid docs failure

* add more documentation

* add tick safeguards to record()

* improve WGLMakie record suggestion & general formating

* move tick filtering to VideoStream

* update docs

* move tick generating functor to Makie

* run WGLMakie ticks on an independent clock

* fix missing namespace

* use BudgetedTimer for WGLMakie

---------

Co-authored-by: Simon <sdanisch@protonmail.com>

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [Unreleased]
 
+- Added `events.tick` to allow linking actions like animations to the renderloop. [#3948](https://github.com/MakieOrg/Makie.jl/pull/3948)
 - Added the `uv_transform` attribute for meshscatter, mesh, surface and image [#1406](https://github.com/MakieOrg/Makie.jl/pull/1406).
 - Added the ability to use textures with `meshscatter` in WGLMakie [#1406](https://github.com/MakieOrg/Makie.jl/pull/1406).
 - Don't remove underlying VideoStream file when doing save() [#3883](https://github.com/MakieOrg/Makie.jl/pull/3883).

CairoMakie/test/runtests.jl

@@ -249,3 +249,47 @@ end
 
     @test_throws ArgumentError save(filename, Figure(), pdf_version="foo")
 end
+
+@testset "Tick Events" begin
+    f, a, p = scatter(rand(10));
+    @test events(f).tick[] == Makie.Tick()
+
+    filename = "$(tempname()).png"
+    try
+        save(filename, f)
+        tick = events(f).tick[]
+        @test tick.state == Makie.OneTimeRenderTick
+        @test tick.count == 0
+        @test tick.time == 0.0
+        @test tick.delta_time == 0.0
+    finally
+        rm(filename)
+    end
+
+    filename = "$(tempname()).mp4"
+    try
+        tick_record = Makie.Tick[]
+        record(_ -> push!(tick_record, events(f).tick[]), f, filename, 1:10, framerate = 30)
+        dt = 1.0 / 30.0
+
+        for (i, tick) in enumerate(tick_record)
+            @test tick.state == Makie.OneTimeRenderTick
+            @test tick.count == i-1
+            @test tick.time ≈ dt * (i-1)
+            @test tick.delta_time ≈ dt
+        end
+    finally
+        rm(filename)
+    end
+
+    # test destruction of tick overwrite
+    f, a, p = scatter(rand(10));
+    let
+        io = VideoStream(f)
+        @test events(f).tick[] == Makie.Tick(Makie.OneTimeRenderTick, 0, 0.0, 1.0 / io.options.framerate)
+        nothing
+    end
+    tick = Makie.Tick(Makie.UnknownTickState, 1, 1.0, 1.0)
+    events(f).tick[] = tick
+    @test events(f).tick[] == tick
+end

GLMakie/src/display.jl

@@ -10,7 +10,7 @@ function Base.display(screen::Screen, scene::Scene; connect=true)
     else
         @assert screen.root_scene === scene "internal error. Scene already displayed by screen but not as root scene"
     end
-    pollevents(screen)
+    pollevents(screen, Makie.BackendTick)
     return screen
 end
 

GLMakie/src/drawing_primitives.jl

@@ -24,7 +24,7 @@ function handle_lights(attr::Dict, screen::Screen, lights::Vector{Makie.Abstract
     attr[:light_colors]     = Observable(sizehint!(RGBf[], MAX_LIGHTS))
     attr[:light_parameters] = Observable(sizehint!(Float32[], MAX_PARAMS))
 
-    on(screen.render_tick, priority = typemin(Int)) do _
+    on(screen.render_tick, priority = -1000) do _
         # derive number of lights from available lights. Both MAX_LIGHTS and
         # MAX_PARAMS are considered for this.
         n_lights = 0
@@ -231,7 +231,7 @@ const EXCLUDE_KEYS = Set([:transformation, :tickranges, :ticklabels, :raw, :SSAO
 
 function cached_robj!(robj_func, screen, scene, plot::AbstractPlot)
     # poll inside functions to make wait on compile less prominent
-    pollevents(screen)
+    pollevents(screen, Makie.BackendTick)
     robj = get!(screen.cache, objectid(plot)) do
 
         filtered = filter(plot.attributes) do (k, v)
@@ -319,13 +319,13 @@ function Base.insert!(screen::Screen, scene::Scene, @nospecialize(x::Plot))
     ShaderAbstractions.switch_context!(screen.glscreen)
     add_scene!(screen, scene)
     # poll inside functions to make wait on compile less prominent
-    pollevents(screen)
+    pollevents(screen, Makie.BackendTick)
     if isempty(x.plots) # if no plots inserted, this truly is an atomic
         draw_atomic(screen, scene, x)
     else
         foreach(x.plots) do x
             # poll inside functions to make wait on compile less prominent
-            pollevents(screen)
+            pollevents(screen, Makie.BackendTick)
             insert!(screen, scene, x)
         end
     end

GLMakie/src/events.jl

@@ -186,7 +186,7 @@ struct MousePositionUpdater
     hasfocus::Observable{Bool}
 end
 
-function (p::MousePositionUpdater)(::Nothing)
+function (p::MousePositionUpdater)(::Makie.TickState)
     !p.hasfocus[] && return
     nw = to_native(p.screen)
     x, y = GLFW.GetCursorPos(nw)
@@ -295,3 +295,15 @@ end
 function Makie.disconnect!(window::GLFW.Window, ::typeof(entered_window))
     GLFW.SetCursorEnterCallback(window, nothing)
 end
+
+function Makie.frame_tick(scene::Scene, screen::Screen)
+    # Separating screen ticks from event ticks allows us to sanitize:
+    # Internal on-tick event updates happen first (mouseposition), 
+    # consuming in event.tick listeners doesn't affect backend ticks,
+    # more control/consistent order
+    on(Makie.TickCallback(scene), scene, screen.render_tick, priority = typemin(Int))
+end
+function Makie.disconnect!(screen::Screen, ::typeof(Makie.frame_tick))
+    connections = filter(x -> x[2] isa Makie.TickCallback, screen.render_tick.listeners)
+    foreach(x -> off(screen.render_tick, x[2]), connections)
+end

GLMakie/src/screen.jl

@@ -176,7 +176,7 @@ mutable struct Screen{GLWindow} <: MakieScreen
     cache::Dict{UInt64, RenderObject}
     cache2plot::Dict{UInt32, AbstractPlot}
     framecache::Matrix{RGB{N0f8}}
-    render_tick::Observable{Nothing} # listeners must not Consume(true)
+    render_tick::Observable{Makie.TickState} # listeners must not Consume(true)
     window_open::Observable{Bool}
     scalefactor::Observable{Float32}
 
@@ -210,7 +210,7 @@ mutable struct Screen{GLWindow} <: MakieScreen
             config, stop_renderloop, rendertask, BudgetedTimer(1.0 / 30.0),
             Observable(0f0), screen2scene,
             screens, renderlist, postprocessors, cache, cache2plot,
-            Matrix{RGB{N0f8}}(undef, s), Observable(nothing),
+            Matrix{RGB{N0f8}}(undef, s), Observable(Makie.UnknownTickState),
             Observable(true), Observable(0f0), nothing, reuse, true, false
         )
         push!(ALL_SCREENS, screen) # track all created screens
@@ -478,10 +478,10 @@ function Screen(scene::Scene, config::ScreenConfig, ::Makie.ImageStorageFormat;
     return screen
 end
 
-function pollevents(screen::Screen)
+function pollevents(screen::Screen, frame_state::Makie.TickState)
     ShaderAbstractions.switch_context!(screen.glscreen)
     GLFW.PollEvents()
-    notify(screen.render_tick)
+    screen.render_tick[] = frame_state
     return
 end
 
@@ -770,7 +770,7 @@ function Makie.colorbuffer(screen::Screen, format::Makie.ImageStorageFormat = Ma
     ctex = screen.framebuffer.buffers[:color]
     # polling may change window size, when its bigger than monitor!
     # we still need to poll though, to get all the newest events!
-    pollevents(screen)
+    pollevents(screen, Makie.BackendTick)
     # keep current buffer size to allows larger-than-window renders
     render_frame(screen, resize_buffers=false) # let it render
     if screen.config.visible
@@ -903,7 +903,7 @@ function set_framerate!(screen::Screen, fps=30)
 end
 
 function refreshwindowcb(screen, window)
-    screen.render_tick[] = nothing
+    screen.render_tick[] = Makie.BackendTick
     render_frame(screen)
     GLFW.SwapBuffers(window)
     return
@@ -929,14 +929,13 @@ end
 scalechangeobs(screen) = scalefactor -> scalechangeobs(screen, scalefactor)
 
 
-# TODO add render_tick event to scene events
 function vsynced_renderloop(screen)
     while isopen(screen) && !screen.stop_renderloop
         if screen.config.pause_renderloop
-            pollevents(screen); sleep(0.1)
+            pollevents(screen, Makie.PausedRenderTick); sleep(0.1)
             continue
         end
-        pollevents(screen) # GLFW poll
+        pollevents(screen, Makie.RegularRenderTick) # GLFW poll
         render_frame(screen)
         yield()
         GC.safepoint()
@@ -947,10 +946,10 @@ end
 function fps_renderloop(screen::Screen)
     reset!(screen.timer, 1.0 / screen.config.framerate)
     while isopen(screen) && !screen.stop_renderloop
-        pollevents(screen)
-
-        if !screen.config.pause_renderloop
-            pollevents(screen) # GLFW poll
+        if screen.config.pause_renderloop
+            pollevents(screen, Makie.PausedRenderTick)
+        else
+            pollevents(screen, Makie.RegularRenderTick)
             render_frame(screen)
             GLFW.SwapBuffers(to_native(screen))
         end
@@ -973,14 +972,18 @@ end
 # const time_record = sizehint!(Float64[], 100_000)
 
 function on_demand_renderloop(screen::Screen)
+    tick_state = Makie.UnknownTickState
     # last_time = time_ns()
     reset!(screen.timer, 1.0 / screen.config.framerate)
     while isopen(screen) && !screen.stop_renderloop
-        pollevents(screen) # GLFW poll
+        pollevents(screen, tick_state) # GLFW poll
 
         if !screen.config.pause_renderloop && requires_update(screen)
+            tick_state = Makie.RegularRenderTick
             render_frame(screen)
             GLFW.SwapBuffers(to_native(screen))
+        else
+            tick_state = ifelse(screen.config.pause_renderloop, Makie.PausedRenderTick, Makie.SkippedRenderTick)
         end
 
         GC.safepoint()

GLMakie/test/runtests.jl

@@ -37,3 +37,87 @@ include("unit_tests.jl")
     GLMakie.closeall()
     GC.gc(true) # make sure no finalizers act up!
 end
+
+@testset "Tick Events" begin
+    function check_tick(tick, state, count)
+        @test tick.state == state
+        @test tick.count == count
+        @test tick.time > 1e-9
+        @test tick.delta_time > 1e-9
+    end
+
+    f, a, p = scatter(rand(10));
+    @test events(f).tick[] == Makie.Tick()
+
+    filename = "$(tempname()).png"
+    try
+        save(filename, f)
+        tick = events(f).tick[]
+        @test tick.state == Makie.OneTimeRenderTick
+        @test tick.count == 0
+        @test tick.time == 0.0
+        @test tick.delta_time == 0.0
+    finally
+        rm(filename)
+    end
+
+    filename = "$(tempname()).mp4"
+    try
+        tick_record = Makie.Tick[]
+        record(_ -> push!(tick_record, events(f).tick[]), f, filename, 1:10, framerate = 30)
+        dt = 1.0 / 30.0
+
+        for (i, tick) in enumerate(tick_record)
+            @test tick.state == Makie.OneTimeRenderTick
+            @test tick.count == i-1
+            @test tick.time ≈ dt * (i-1)
+            @test tick.delta_time ≈ dt
+        end
+    finally
+        rm(filename)
+    end
+
+    # test destruction of tick overwrite
+    f, a, p = scatter(rand(10));
+    let
+        io = VideoStream(f)
+        @test events(f).tick[] == Makie.Tick(Makie.OneTimeRenderTick, 0, 0.0, 1.0 / io.options.framerate)
+        nothing
+    end
+    tick = Makie.Tick(Makie.UnknownTickState, 1, 1.0, 1.0)
+    events(f).tick[] = tick
+    @test events(f).tick[] == tick
+
+
+    f, a, p = scatter(rand(10));
+    tick_record = Makie.Tick[]
+    on(t -> push!(tick_record, t), events(f).tick)
+    screen = GLMakie.Screen(render_on_demand = true, framerate = 30.0, pause_rendering = false, visible = false)
+    display(screen, f.scene)
+    sleep(0.15)
+    GLMakie.pause_renderloop!(screen)
+    sleep(0.1)
+    GLMakie.closeall()
+
+        # Why does it start with a skipped tick?
+    i = 1
+    while tick_record[i].state == Makie.SkippedRenderTick
+        check_tick(tick_record[1], Makie.SkippedRenderTick, i)
+        i += 1
+    end
+
+    check_tick(tick_record[i], Makie.RegularRenderTick, i)
+    i += 1
+
+    while tick_record[i].state == Makie.SkippedRenderTick
+            check_tick(tick_record[i], Makie.SkippedRenderTick, i)
+            i += 1
+        end
+
+    while (i <= length(tick_record)) && (tick_record[i].state == Makie.PausedRenderTick)
+        check_tick(tick_record[i], Makie.PausedRenderTick, i)
+        i += 1
+    end
+
+    @test i == length(tick_record)+1
+end

ReferenceTests/src/tests/examples3d.jl

@@ -424,12 +424,13 @@ end
 
 @reference_test "Normals of a Cat" begin
     x = loadasset("cat.obj")
-    mesh(x, color=:black)
+    f, a, p = mesh(x, color=:black)
     pos = map(decompose(Point3f, x), GeometryBasics.normals(x)) do p, n
         p => p .+ Point(normalize(n) .* 0.05f0)
     end
     linesegments!(pos, color=:blue)
-    current_figure()
+    Makie.update_state_before_display!(f)
+    f
 end
 
 @reference_test "Sphere Mesh" begin

ReferenceTests/src/tests/updating.jl

@@ -173,3 +173,22 @@ end
     ax.title = "identity"
     Makie.step!(st)
 end
+
+
+@reference_test "event ticks in record" begin
+    # Checks whether record calculates and triggers event.tick by drawing a 
+    # Point at y = 1 for each frame where it does. The animation is irrelevant
+    # here, so we can just check the final image.
+    # The first point maybe at 0 depending on when the backend sets up it's
+    # reference time
+    ps = Observable(Point2f[])
+    f, a, p = scatter(ps)
+    xlims!(a, 0, 61)
+    ylims!(a, -0.1, 1.1)
+    Record(f, 1:60, framerate = 30) do i
+        push!(ps.val,  Point2f(i, f.scene.events.tick[].delta_time > 1e-6))
+        notify(ps)
+        f.scene.events.tick[] = Makie.Tick(Makie.UnknownTickState, 0, 0.0, 0.0)
+    end
+    f
+end

WGLMakie/src/display.jl

@@ -53,8 +53,16 @@ mutable struct Screen <: Makie.MakieScreen
     displayed_scenes::Set{String}
     config::ScreenConfig
     canvas::Union{Nothing,Bonito.HTMLElement}
+    tick_clock::Makie.BudgetedTimer
     function Screen(scene::Union{Nothing,Scene}, config::ScreenConfig)
-        return new(Channel{Bool}(1), nothing, scene, Set{String}(), config, nothing)
+        timer = Makie.BudgetedTimer(1.0 / 30.0)
+        screen = new(Channel{Bool}(1), nothing, scene, Set{String}(), config, nothing, timer)
+
+        finalizer(screen) do screen
+            close(screen.tick_clock)
+        end
+
+        return screen
     end
 end