Add uv_transform attribute to mesh, meshscatter, surface and image (#…

…1406)

* replay changes from master (add per-instance uv_scale)

* define uv_transform instead of just scale

* fix shader

* add uv_transform attribute

* add UVTransform and conversions

* merge uv_scale into uv_transform

* disable in WGLMakie

* extend usage to mesh and surface

* export UVTransform

* update changelog

* prototype Mat transforms

* fix order of matrix dimensions

* use only one buffer

* update other shaders

* update defaults

* update image

* switch to master defaults

* fix patterns

* add more named transforms

* apply suggestions + cleanup

* add symbol -> plot defaults

* add tests for conversions

* fix tests

* drop support for rotations

* add refimg

* allow rotation in uv_transform() function only

* document uv_transform() function

* update attribute docs

* update changelog

* fix rotation direction of surface

* update test

* add uv_transforms to CairoMakie

* fix tests

* restore comment

* actually use convert_arguments on uv_transform

* get mesh, surface, image working in WGLMakie

* fix heatmap

* add textures to meshscatter with static uv_transform

* implement per-instance uv_transform

* add test for per-element uv_transform in meshscatter

* allow operation chaining in uv_transform

* fix chaining & add test

* add docs example

* update changelog

* sample color in vertex shader for particles

---------

Co-authored-by: Simon <sdanisch@protonmail.com>

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [Unreleased]
 
+- Added the `uv_transform` attribute for meshscatter, mesh, surface and image [#1406](https://github.com/MakieOrg/Makie.jl/pull/1406).
+- Added the ability to use textures with `meshscatter` in WGLMakie [#1406](https://github.com/MakieOrg/Makie.jl/pull/1406).
 - Don't remove underlying VideoStream file when doing save() [#3883](https://github.com/MakieOrg/Makie.jl/pull/3883).
 - Fix label/legend for plotlist [#4079](https://github.com/MakieOrg/Makie.jl/pull/4079).
 - Fix wrong order for colors in RPRMakie [#4098](https://github.com/MakieOrg/Makie.jl/pull/4098).

CairoMakie/src/cairo-extension.jl

@@ -10,6 +10,16 @@ function get_font_matrix(ctx)
     return matrix
 end
 
+function pattern_set_matrix(ctx, matrix)
+    ccall((:cairo_pattern_set_matrix, Cairo.libcairo), Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}), ctx.ptr, Ref(matrix))
+end
+
+function pattern_get_matrix(ctx)
+    matrix = Cairo.CairoMatrix()
+    ccall((:cairo_pattern_get_matrix, Cairo.libcairo), Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}), ctx.ptr, Ref(matrix))
+    return matrix
+end
+
 function cairo_font_face_destroy(font_face)
     ccall(
         (:cairo_font_face_destroy, Cairo.libcairo),

CairoMakie/src/primitives.jl

@@ -821,17 +821,7 @@ function draw_atomic(scene::Scene, screen::Screen{RT}, @nospecialize(primitive::
     t = Makie.transform_func(primitive)
     identity_transform = (t === identity || t isa Tuple && all(x-> x === identity, t)) && (abs(model[1, 2]) < 1e-15)
     regular_grid = xs isa AbstractRange && ys isa AbstractRange
-    xy_aligned = let
-        # Only allow scaling and translation
-        pv = scene.camera.projectionview[]
-        M = Mat4f(
-            pv[1, 1], 0.0,      0.0,      0.0,
-            0.0,      pv[2, 2], 0.0,      0.0,
-            0.0,      0.0,      pv[3, 3], 0.0,
-            pv[1, 4], pv[2, 4], pv[3, 4], 1.0
-        )
-        pv ≈ M
-    end
+    xy_aligned = Makie.is_translation_scale_matrix(scene.camera.projectionview[])
 
     if interpolate
         if !regular_grid
@@ -850,6 +840,21 @@ function draw_atomic(scene::Scene, screen::Screen{RT}, @nospecialize(primitive::
     xymax = project_position(primitive, space, Point2(last.(imsize)), model)
     w, h = xymax .- xy
 
+    uv_transform = if primitive isa Image
+        val = to_value(get(primitive, :uv_transform, I))
+        T = Makie.convert_attribute(val, Makie.key"uv_transform"(), Makie.key"image"())
+        # Cairo uses pixel units so we need to transform those to a 0..1 range, 
+        # then apply uv_transform, then scale them back to pixel units.
+        # Cairo also doesn't have the yflip we have in OpenGL, so we need to
+        # invert y.
+        T3 = Mat3f(T[1], T[2], 0, T[3], T[4], 0, T[5], T[6], 1)
+        T3 = Makie.uv_transform(Vec2f(size(image))) * T3 * 
+            Makie.uv_transform(Vec2f(0, 1), 1f0 ./ Vec2f(size(image, 1), -size(image, 2)))
+        T3[Vec(1, 2), Vec(1,2,3)]
+    else
+        Mat{2, 3, Float32}(1,0,0,1,0,0)
+    end
+
     can_use_fast_path = !(is_vector && !interpolate) && regular_grid && identity_transform &&
         (interpolate || xy_aligned)
     use_fast_path = can_use_fast_path && !disable_fast_path
@@ -874,8 +879,10 @@ function draw_atomic(scene::Scene, screen::Screen{RT}, @nospecialize(primitive::
         end
         filt = interpolate ? Cairo.FILTER_BILINEAR : Cairo.FILTER_NEAREST
         Cairo.pattern_set_filter(p, filt)
+        pattern_set_matrix(p, Cairo.CairoMatrix(uv_transform...))
         Cairo.fill(ctx)
         Cairo.restore(ctx)
+        pattern_set_matrix(p, Cairo.CairoMatrix(1, 0, 0, 1, 0, 0))
     else
         # find projected image corners
         # this already takes care of flipping the image to correct cairo orientation
@@ -940,7 +947,8 @@ function draw_atomic(scene::Scene, screen::Screen, @nospecialize(primitive::Maki
         if !haskey(primitive, :faceculling)
             primitive[:faceculling] = Observable(-10)
         end
-        draw_mesh3D(scene, screen, primitive, mesh)
+        uv_transform = Makie.convert_attribute(primitive[:uv_transform][], Makie.key"uv_transform"(), Makie.key"mesh"())
+        draw_mesh3D(scene, screen, primitive, mesh; uv_transform = uv_transform)
     end
     return nothing
 end
@@ -952,6 +960,11 @@ function draw_mesh2D(scene, screen, @nospecialize(plot), @nospecialize(mesh))
     vs = project_position(scene, transform_func, space, decompose(Point, mesh), model)
     fs = decompose(GLTriangleFace, mesh)::Vector{GLTriangleFace}
     uv = decompose_uv(mesh)::Union{Nothing, Vector{Vec2f}}
+    # Note: This assume the function is only called from mesh plots
+    uv_transform = Makie.convert_attribute(plot[:uv_transform][], Makie.key"uv_transform"(), Makie.key"mesh"())
+    if uv isa Vector{Vec2f} && to_value(uv_transform) !== nothing
+        uv = map(uv -> uv_transform * to_ndim(Vec3f, uv, 1), uv)
+    end
     color = hasproperty(mesh, :color) ? to_color(mesh.color) : plot.calculated_colors[]
     cols = per_face_colors(color, nothing, fs, nothing, uv)
     return draw_mesh2D(screen, cols, vs, fs)
@@ -1000,7 +1013,10 @@ end
 nan2zero(x) = !isnan(x) * x
 
 
-function draw_mesh3D(scene, screen, attributes, mesh; pos = Vec4f(0), scale = 1f0, rotation = Mat4f(I))
+function draw_mesh3D(
+        scene, screen, attributes, mesh; pos = Vec4f(0), scale = 1f0, rotation = Mat4f(I),
+        uv_transform = Mat{2, 3, Float32}(1,0,0,1,0,0)
+    )
     @get_attribute(attributes, (shading, diffuse, specular, shininess, faceculling))
 
     matcap = to_value(get(attributes, :matcap, nothing))
@@ -1009,9 +1025,12 @@ function draw_mesh3D(scene, screen, attributes, mesh; pos = Vec4f(0), scale = 1f
     meshnormals = decompose_normals(mesh)::Vector{Vec3f} # note: can be made NaN-aware.
     meshuvs = texturecoordinates(mesh)::Union{Nothing, Vector{Vec2f}}
 
+    if meshuvs isa Vector{Vec2f} && to_value(uv_transform) !== nothing
+        meshuvs = map(uv -> uv_transform * to_ndim(Vec3f, uv, 1), meshuvs)
+    end
+
     # Priorize colors of the mesh if present
     color = hasproperty(mesh, :color) ? mesh.color : to_value(attributes.calculated_colors)
-
     per_face_col = per_face_colors(color, matcap, meshfaces, meshnormals, meshuvs)
 
     model = attributes.model[]::Mat4d
@@ -1197,7 +1216,8 @@ function draw_atomic(scene::Scene, screen::Screen, @nospecialize(primitive::Maki
     if !haskey(primitive, :faceculling)
         primitive[:faceculling] = Observable(-10)
     end
-    draw_mesh3D(scene, screen, primitive, mesh)
+    uv_transform = Makie.convert_attribute(primitive[:uv_transform][], Makie.key"uv_transform"(), Makie.key"surface"())
+    draw_mesh3D(scene, screen, primitive, mesh; uv_transform = uv_transform)
     primitive[:color] = old
     return nothing
 end
@@ -1235,22 +1255,20 @@ function draw_atomic(scene::Scene, screen::Screen, @nospecialize(primitive::Maki
 
     submesh[:model] = model
     scales = primitive[:markersize][]
+    uv_transform = Makie.convert_attribute(primitive[:uv_transform][], Makie.key"uv_transform"(), Makie.key"meshscatter"())
     for i in zorder
         p = pos[i]
         if color isa AbstractVector
             submesh[:calculated_colors] = color[i]
         end
         scale = markersize isa Vector ? markersize[i] : markersize
-        _rotation = if rotation isa Vector
-            Makie.rotationmatrix4(to_rotation(rotation[i]))
-        else
-            Makie.rotationmatrix4(to_rotation(rotation))
-        end
+        _rotation = Makie.rotationmatrix4(to_rotation(Makie.sv_getindex(rotation, i)))
+        _uv_transform = Makie.sv_getindex(uv_transform, i)
 
         draw_mesh3D(
             scene, screen, submesh, marker, pos = p,
             scale = scale isa Real ? Vec3f(scale) : to_ndim(Vec3f, scale, 1f0),
-            rotation = _rotation
+            rotation = _rotation, uv_transform = _uv_transform
         )
     end
 

CairoMakie/src/utils.jl

@@ -261,11 +261,12 @@ function per_face_colors(_color, matcap, faces, normals, uv)
         # let next level extend and fill with CairoPattern
         return color
     elseif color isa AbstractMatrix{<: Colorant} && !isnothing(uv)
-        wsize = reverse(size(color))
+        wsize = size(color)
         wh = wsize .- 1
+        # nearest
         cvec = map(uv) do uv
             x, y = clamp.(round.(Int, Tuple(uv) .* wh) .+ 1, 1, wsize)
-            return color[end - (y - 1), x]
+            return color[x, y]
         end
         # TODO This is wrong and doesn't actually interpolate
         # Inside the triangle sampling the color image

GLMakie/assets/shader/mesh.frag

@@ -12,6 +12,7 @@ in vec3 o_view_normal;
 in vec4 o_color;
 in vec2 o_uv;
 flat in uvec2 o_id;
+flat in int o_InstanceID;
 
 {{matcap_type}} matcap;
 {{image_type}} image;
@@ -79,18 +80,28 @@ vec4 get_color(sampler1D color, vec2 uv, vec2 color_norm, sampler1D color_map, s
 }
 
 uniform bool fetch_pixel;
-uniform vec2 uv_scale;
+{{uv_transform_type}} uv_transform;
+vec2 apply_uv_transform(Nothing t1, int i, vec2 uv){ return uv; }
+vec2 apply_uv_transform(mat3x2 transform, int i, vec2 uv){ return transform * vec3(uv, 1); }
+vec2 apply_uv_transform(samplerBuffer transforms, int index, vec2 uv){
+    // can't have matrices in a texture so we have 3x vec2 instead
+    mat3x2 transform;
+    transform[0] = texelFetch(transforms, 3 * index + 0).xy;
+    transform[1] = texelFetch(transforms, 3 * index + 1).xy;
+    transform[2] = texelFetch(transforms, 3 * index + 2).xy;
+    return transform * vec3(uv, 1);
+}
 
 vec4 get_pattern_color(sampler1D color) {
     int size = textureSize(color, 0);
-    vec2 pos = gl_FragCoord.xy * uv_scale;
+    vec2 pos = apply_uv_transform(uv_transform, o_InstanceID, gl_FragCoord.xy);
     int idx = int(mod(pos.x, size));
     return texelFetch(color, idx, 0);
 }
 
 vec4 get_pattern_color(sampler2D color){
     ivec2 size = textureSize(color, 0);
-    vec2 pos = gl_FragCoord.xy * uv_scale;
+    vec2 pos = apply_uv_transform(uv_transform, o_InstanceID, gl_FragCoord.xy);
     return texelFetch(color, ivec2(mod(pos.x, size.x), mod(pos.y, size.y)), 0);
 }
 

GLMakie/assets/shader/mesh.vert

@@ -20,8 +20,9 @@ void render(vec4 position_world, vec3 normal, mat4 view, mat4 projection);
 vec4 get_color_from_cmap(float value, sampler1D color_map, vec2 colorrange);
 
 uniform uint objectid;
+
 flat out uvec2 o_id;
-uniform vec2 uv_scale;
+flat out int o_InstanceID;
 out vec2 o_uv;
 out vec4 o_color;
 
@@ -31,6 +32,10 @@ vec3 to_3d(vec3 v){return v;}
 vec2 to_2d(float v){return vec2(v, 0);}
 vec2 to_2d(vec2 v){return v;}
 
+{{uv_transform_type}} uv_transform;
+vec2 apply_uv_transform(Nothing t1, vec2 uv){ return uv; }
+vec2 apply_uv_transform(mat3x2 transform, vec2 uv){ return transform * vec3(uv, 1); }
+
 vec4 to_color(vec3 c, Nothing color_map, Nothing color_norm){
     return vec4(c, 1);
 }
@@ -59,8 +64,9 @@ void main()
 {
     o_id = uvec2(objectid, gl_VertexID+1);
     vec2 tex_uv = to_2d(texturecoordinates);
-    o_uv = vec2(1.0 - tex_uv.y, tex_uv.x) * uv_scale;
+    o_uv = apply_uv_transform(uv_transform, tex_uv);
     o_color = to_color(vertex_color, color_map, color_norm);
+    o_InstanceID = 0;
     vec3 v = to_3d(vertices);
     render(model * vec4(v, 1), normals, view, projection);
 }

GLMakie/assets/shader/particles.vert

@@ -33,6 +33,7 @@ uniform uint objectid;
 uniform int len;
 
 flat out uvec2 o_id;
+flat out int o_InstanceID;
 out vec4 o_color;
 out vec2 o_uv;
 
@@ -92,8 +93,22 @@ vec4 get_particle_color(sampler2D color, Nothing intensity, Nothing color_map, N
 
 void render(vec4 position_world, vec3 normal, mat4 view, mat4 projection);
 
-vec2 get_uv(Nothing x){return vec2(0.0);}
-vec2 get_uv(vec2 x){return vec2(1.0 - x.y, x.x);}
+{{uv_transform_type}} uv_transform;
+vec2 apply_uv_transform(Nothing t1, int i, vec2 uv){ return uv; }
+vec2 apply_uv_transform(mat3x2 transform, int i, vec2 uv){ return transform * vec3(uv, 1); }
+vec2 apply_uv_transform(samplerBuffer transforms, int index, vec2 uv){
+    // can't have matrices in a texture so we have 3x vec2 instead
+    mat3x2 transform;
+    transform[0] = texelFetch(transforms, 3 * index + 0).xy;
+    transform[1] = texelFetch(transforms, 3 * index + 1).xy;
+    transform[2] = texelFetch(transforms, 3 * index + 2).xy;
+    return transform * vec3(uv, 1);
+}
+
+vec2 get_uv(int index, Nothing uv){ return vec2(0.0); }
+vec2 get_uv(int index, vec2 uv){
+    return apply_uv_transform(uv_transform, index, uv);
+}
 
 void main(){
     int index = gl_InstanceID;
@@ -105,7 +120,8 @@ void main(){
     {{position_calc}}
     o_color = get_particle_color(color, intensity, color_map, color_norm, index, len);
     o_color = o_color * to_color(vertex_color);
-    o_uv = get_uv(texturecoordinates);
+    o_uv = get_uv(index, texturecoordinates);
+    o_InstanceID = index;
     rotate(rotation, index, V, N);
     render(model * vec4(pos + V, 1), N, view, projection);
 }

GLMakie/assets/shader/surface.vert

@@ -30,7 +30,6 @@ uniform vec4 nan_color;
 vec4 color_lookup(float intensity, sampler1D color, vec2 norm);
 
 uniform vec3 scale;
-
 uniform mat4 view, model, projection;
 
 // See util.vert for implementations
@@ -41,6 +40,9 @@ vec2 linear_index(ivec2 dims, int index);
 vec2 linear_index(ivec2 dims, int index, vec2 offset);
 vec4 linear_texture(sampler2D tex, int index, vec2 offset);
 
+{{uv_transform_type}} uv_transform;
+vec2 apply_uv_transform(Nothing t1, vec2 uv){ return uv; }
+vec2 apply_uv_transform(mat3x2 transform, vec2 uv){ return transform * vec3(uv, 1); }
 
 // Normal generation
 
@@ -147,8 +149,8 @@ vec3 getnormal(Nothing pos, sampler1D xs, sampler1D ys, sampler2D zs, ivec2 uv){
 }
 
 uniform uint objectid;
-uniform vec2 uv_scale;
 flat out uvec2 o_id;
+flat out int o_InstanceID; // dummy for compat with meshscatter in mesh.frag
 out vec4 o_color;
 out vec2 o_uv;
 
@@ -162,7 +164,9 @@ void main()
     {{position_calc}}
 
     o_id = uvec2(objectid, index1D+1);
-    o_uv = index01 * uv_scale;
+    o_InstanceID = 0;
+    // match up with mesh
+    o_uv = apply_uv_transform(uv_transform, vec2(index01.x, 1 - index01.y));
     vec3 normalvec = {{normal_calc}};
 
     o_color = vec4(0.0);

GLMakie/src/GLAbstraction/GLUniforms.jl

@@ -30,7 +30,7 @@ function uniformfunc(typ::DataType, dims::Tuple{Int})
 end
 function uniformfunc(typ::DataType, dims::Tuple{Int, Int})
     M, N = dims
-    Symbol(string("glUniformMatrix", M == N ? "$M" : "$(M)x$(N)", opengl_postfix(typ)))
+    Symbol(string("glUniformMatrix", M == N ? "$M" : "$(N)x$(M)", opengl_postfix(typ)))
 end
 
 gluniform(location::Integer, x::Nothing) = nothing
@@ -105,7 +105,7 @@ end
 
 function glsl_typename(t::Type{T}) where T <: Mat
     M, N = size(t)
-    string(opengl_prefix(eltype(t)), "mat", M==N ? M : string(M, "x", N))
+    string(opengl_prefix(eltype(t)), "mat", M==N ? M : string(N, "x", M))
 end
 toglsltype_string(t::Observable) = toglsltype_string(to_value(t))
 toglsltype_string(x::T) where {T<:Union{Real, Mat, StaticVector, Texture, Colorant, TextureBuffer, Nothing}} = "uniform $(glsl_typename(x))"

GLMakie/src/drawing_primitives.jl

@@ -647,9 +647,7 @@ function draw_atomic(screen::Screen, scene::Scene, plot::Image)
         gl_attributes[:vertices] = apply_transform_and_f32_conversion(scene, plot, position)
         rect = Rect2f(0, 0, 1, 1)
         gl_attributes[:faces] = decompose(GLTriangleFace, rect)
-        gl_attributes[:texturecoordinates] = map(decompose_uv(rect)) do uv
-            return 1.0f0 .- Vec2f(uv[2], uv[1])
-        end
+        gl_attributes[:texturecoordinates] = decompose_uv(rect)
         get!(gl_attributes, :shading, NoShading)
         _interp = to_value(pop!(gl_attributes, :interpolate, true))
         interp = _interp ? :linear : :nearest
@@ -677,6 +675,14 @@ function mesh_inner(screen::Screen, mesh, transfunc, gl_attributes, plot, space=
         img = lift(x -> el32convert(Makie.to_image(x)), plot, color)
         gl_attributes[:image] = ShaderAbstractions.Sampler(img, x_repeat=:repeat, minfilter=:nearest)
         get!(gl_attributes, :fetch_pixel, true)
+        # different default with Patterns (no swapping and flipping of axes)
+        gl_attributes[:uv_transform] = map(plot, plot.attributes[:uv_transform]) do uv_transform
+            if uv_transform === Makie.automatic
+                return Mat{2,3,Float32}(1,0,0,1,0,0)
+            else
+                return convert_attribute(uv_transform, key"uv_transform"())
+            end
+        end
     elseif to_value(color) isa AbstractMatrix{<:Colorant}
         gl_attributes[:image] = Texture(lift(el32convert, plot, color), minfilter = interp)
         delete!(gl_attributes, :color_map)

GLMakie/src/glshaders/mesh.jl

@@ -50,7 +50,7 @@ function draw_mesh(screen, data::Dict)
         color_norm = nothing
         fetch_pixel = false
         texturecoordinates = Vec2f(0) => GLBuffer
-        uv_scale = Vec2f(1)
+        uv_transform = Vec4f(1, 1, 0, 0)
         transparency = false
         interpolate_in_fragment_shader = true
         shader = GLVisualizeShader(