Fix playground pointer position (#1292)

The BabylonNative playground app pick points are inaccurate on Win32, UWP, macOS and iOS. This change fixes the issues and also adds mouse-move tracking on macOS. Fixes #1291. Tested on playground https://playground.babylonjs.com/#VXUJZA. Tested for regressions per suggestions from @PolygonalSun on: - Default playground for FreeCamera. - https://playground.babylonjs.com/#SRZRWV#839 for ArcRotateCamera. - https://playground.babylonjs.com/#H5PZ4Q for picking. - https://playground.babylonjs.com/#SG9ZZB for gizmos.
BabylonJS · Sep 22, 2023 · bb5bfec · bb5bfec
1 parent c5a544b
commit bb5bfec
Show file tree

Hide file tree

Showing 6 changed files with 122 additions and 88 deletions.
diff --git a/Apps/Playground/UWP/App.cpp b/Apps/Playground/UWP/App.cpp
@@ -258,8 +258,8 @@ void App::OnPointerMoved(CoreWindow^, PointerEventArgs^ args)
         const auto deviceType = args->CurrentPoint->PointerDevice->PointerDeviceType;
         const auto deviceSlot = args->CurrentPoint->PointerId;
         const auto updateKind = args->CurrentPoint->Properties->PointerUpdateKind;
-        const auto x = static_cast<int>(position.X);
-        const auto y = static_cast<int>(position.Y);
+        const auto x = static_cast<int>(position.X * m_displayScale);
+        const auto y = static_cast<int>(position.Y * m_displayScale);
 
         if (deviceType == Windows::Devices::Input::PointerDeviceType::Mouse)
         {
@@ -285,8 +285,8 @@ void App::OnPointerPressed(CoreWindow^, PointerEventArgs^ args)
         const auto deviceType = args->CurrentPoint->PointerDevice->PointerDeviceType;
         const auto deviceSlot = args->CurrentPoint->PointerId;
         const auto updateKind = args->CurrentPoint->Properties->PointerUpdateKind;
-        const auto x = static_cast<int>(position.X);
-        const auto y = static_cast<int>(position.Y);
+        const auto x = static_cast<int>(position.X * m_displayScale);
+        const auto y = static_cast<int>(position.Y * m_displayScale);
 
         if (deviceType == Windows::Devices::Input::PointerDeviceType::Mouse)
         {
@@ -307,8 +307,8 @@ void App::OnPointerReleased(CoreWindow^, PointerEventArgs^ args)
         const auto deviceType = args->CurrentPoint->PointerDevice->PointerDeviceType;
         const auto deviceSlot = args->CurrentPoint->PointerId;
         const auto updateKind = args->CurrentPoint->Properties->PointerUpdateKind;
-        const auto x = static_cast<int>(position.X);
-        const auto y = static_cast<int>(position.Y);
+        const auto x = static_cast<int>(position.X * m_displayScale);
+        const auto y = static_cast<int>(position.Y * m_displayScale);
 
         if (deviceType == Windows::Devices::Input::PointerDeviceType::Mouse)
         {

diff --git a/Apps/Playground/Win32/App.cpp b/Apps/Playground/Win32/App.cpp
@@ -427,11 +427,12 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
             {
                 POINTER_INFO info;
                 auto pointerId = GET_POINTERID_WPARAM(wParam);
+                POINT origin{0, 0};
 
-                if (GetPointerInfo(pointerId, &info))
+                if (GetPointerInfo(pointerId, &info) && ClientToScreen(hWnd, &origin))
                 {
-                    auto x = GET_X_LPARAM(lParam);
-                    auto y = GET_Y_LPARAM(lParam);
+                    auto x = GET_X_LPARAM(lParam) - origin.x;
+                    auto y = GET_Y_LPARAM(lParam) - origin.y;
 
                     if (info.pointerType == PT_MOUSE)
                     {
@@ -449,13 +450,14 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
         {
             if (nativeInput != nullptr)
             {
-                POINTER_INFO info;
                 auto pointerId = GET_POINTERID_WPARAM(wParam);
+                POINTER_INFO info;
+                POINT origin{0, 0};
 
-                if (GetPointerInfo(pointerId, &info))
+                if (GetPointerInfo(pointerId, &info) && ClientToScreen(hWnd, &origin))
                 {
-                    auto x = GET_X_LPARAM(lParam);
-                    auto y = GET_Y_LPARAM(lParam);
+                    auto x = GET_X_LPARAM(lParam) - origin.x;
+                    auto y = GET_Y_LPARAM(lParam) - origin.y;
 
                     if (info.pointerType == PT_MOUSE)
                     {
@@ -474,13 +476,14 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
         {
             if (nativeInput != nullptr)
             {
-                POINTER_INFO info;
                 auto pointerId = GET_POINTERID_WPARAM(wParam);
+                POINTER_INFO info;
+                POINT origin{0, 0};
 
-                if (GetPointerInfo(pointerId, &info))
+                if (GetPointerInfo(pointerId, &info) && ClientToScreen(hWnd, &origin))
                 {
-                    auto x = GET_X_LPARAM(lParam);
-                    auto y = GET_Y_LPARAM(lParam);
+                    auto x = GET_X_LPARAM(lParam) - origin.x;
+                    auto y = GET_Y_LPARAM(lParam) - origin.y;
 
                     if (info.pointerType == PT_MOUSE)
                     {

diff --git a/Apps/Playground/iOS/LibNativeBridge.h b/Apps/Playground/iOS/LibNativeBridge.h
@@ -9,7 +9,7 @@
 - (instancetype)init;
 - (void)dealloc;
 
-- (void)init:(MTKView*)inView width:(int)inWidth height:(int)inHeight xrView:(void*)xrView;
+- (void)init:(MTKView*)inView screenScale:(float)inScreenScale width:(int)inWidth height:(int)inHeight xrView:(void*)xrView;
 - (void)resize:(int)inWidth height:(int)inHeight;
 - (void)render;
 - (void)setTouchDown:(int)pointerId x:(int)inX y:(int)inY;

diff --git a/Apps/Playground/iOS/LibNativeBridge.mm b/Apps/Playground/iOS/LibNativeBridge.mm
@@ -22,6 +22,7 @@
 std::optional<Babylon::Plugins::NativeXr> nativeXr{};
 Babylon::Plugins::NativeInput* nativeInput{};
 bool isXrActive{};
+float screenScale{1.0f};
 
 @implementation LibNativeBridge
 
@@ -47,8 +48,9 @@ - (void)dealloc
     device.reset();
 }
 
-- (void)init:(MTKView*)view width:(int)inWidth height:(int)inHeight xrView:(void*)xrView
+- (void)init:(MTKView*)view screenScale:(float)inScreenScale width:(int)inWidth height:(int)inHeight xrView:(void*)xrView
 {
+    screenScale = inScreenScale;
     float width = inWidth;
     float height = inHeight;
 
@@ -130,21 +132,21 @@ - (void)render
 - (void)setTouchDown:(int)pointerId x:(int)inX y:(int)inY
 {
     if (nativeInput != nullptr) {
-        nativeInput->TouchDown(pointerId, inX, inY);
+        nativeInput->TouchDown(pointerId, inX * screenScale, inY * screenScale);
     }
 }
 
 - (void)setTouchMove:(int)pointerId x:(int)inX y:(int)inY
 {
     if (nativeInput != nullptr) {
-        nativeInput->TouchMove(pointerId, inX, inY);
+        nativeInput->TouchMove(pointerId, inX * screenScale, inY * screenScale);
     }
 }
 
 - (void)setTouchUp:(int)pointerId x:(int)inX y:(int)inY
 {
     if (nativeInput != nullptr) {
-        nativeInput->TouchUp(pointerId, inX, inY);
+        nativeInput->TouchUp(pointerId, inX * screenScale, inY * screenScale);
     }
 }
 

diff --git a/Apps/Playground/iOS/ViewController.swift b/Apps/Playground/iOS/ViewController.swift
@@ -110,7 +110,7 @@ class ViewController: UIViewController, MTKViewDelegate {
             view.addConstraints(NSLayoutConstraint.constraints(withVisualFormat: "|[xrView]|", options: [], metrics: nil, views: ["xrView" : xrView]))
             view.addConstraints(NSLayoutConstraint.constraints(withVisualFormat: "V:|[xrView]|", options: [], metrics: nil, views: ["xrView" : xrView]))
 
-            appDelegate!._bridge!.init(mtkView, width:Int32(width * scale), height:Int32(height * scale), xrView:Unmanaged.passUnretained(xrView).toOpaque())
+            appDelegate!._bridge!.init(mtkView, screenScale:Float(UIScreen.main.scale), width:Int32(width * scale), height:Int32(height * scale), xrView:Unmanaged.passUnretained(xrView).toOpaque())
         }
     }
 

diff --git a/Apps/Playground/macOS/ViewController.mm b/Apps/Playground/macOS/ViewController.mm
@@ -57,6 +57,16 @@ @implementation ViewController
 
 - (void)viewDidLoad {
     [super viewDidLoad];
+
+    // Required for mouseMoved events.
+    NSTrackingArea* trackingArea = [
+        [NSTrackingArea alloc]
+        initWithRect:NSZeroRect
+        options:NSTrackingActiveAlways | NSTrackingInVisibleRect | NSTrackingMouseMoved
+        owner:self
+        userInfo:nil
+        ];
+    [[self view] addTrackingArea:trackingArea];
 }
 
 - (void)uninitialize {
@@ -176,85 +186,104 @@ - (CGFloat)getScreenHeight {
     return [self view].frame.size.height;
 }
 
+- (void)mouseMoved:(NSEvent *) theEvent {
+    if (nativeInput)
+    {
+        NSPoint eventLocation = [theEvent locationInWindow];
+        auto invertedY = [self getScreenHeight] - eventLocation.y;
+        CGFloat screenScale = [[NSScreen mainScreen] backingScaleFactor];
+        nativeInput->MouseMove(eventLocation.x * screenScale, invertedY * screenScale);
+    }
+}
+
 - (void)mouseDown:(NSEvent *) theEvent {
-     if (nativeInput)
-     {
-         NSPoint eventLocation = [theEvent locationInWindow];
-         auto invertedY = [self getScreenHeight] - eventLocation.y;
-         nativeInput->MouseDown(Babylon::Plugins::NativeInput::LEFT_MOUSE_BUTTON_ID, eventLocation.x, invertedY);
-     }
+    if (nativeInput)
+    {
+        NSPoint eventLocation = [theEvent locationInWindow];
+        auto invertedY = [self getScreenHeight] - eventLocation.y;
+        CGFloat screenScale = [[NSScreen mainScreen] backingScaleFactor];
+        nativeInput->MouseDown(Babylon::Plugins::NativeInput::LEFT_MOUSE_BUTTON_ID, eventLocation.x * screenScale, invertedY * screenScale);
+    }
  }
 
- - (void)mouseDragged:(NSEvent *)theEvent {
-     if (nativeInput)
-     {
-         NSPoint eventLocation = [theEvent locationInWindow];
-         auto invertedY = [self getScreenHeight] - eventLocation.y;
-         nativeInput->MouseMove(eventLocation.x, invertedY);
-     }
- }
+- (void)mouseDragged:(NSEvent *)theEvent {
+    if (nativeInput)
+    {
+        NSPoint eventLocation = [theEvent locationInWindow];
+        auto invertedY = [self getScreenHeight] - eventLocation.y;
+        CGFloat screenScale = [[NSScreen mainScreen] backingScaleFactor];
+        nativeInput->MouseMove(eventLocation.x * screenScale, invertedY * screenScale);
+    }
+}
 
- - (void)mouseUp:(NSEvent *) theEvent {
-     if (nativeInput)
-     {
-         NSPoint eventLocation = [theEvent locationInWindow];
-         auto invertedY = [self getScreenHeight] - eventLocation.y;
-         nativeInput->MouseUp(Babylon::Plugins::NativeInput::LEFT_MOUSE_BUTTON_ID, eventLocation.x, invertedY);
-     }
+- (void)mouseUp:(NSEvent *) theEvent {
+    if (nativeInput)
+    {
+        NSPoint eventLocation = [theEvent locationInWindow];
+        auto invertedY = [self getScreenHeight] - eventLocation.y;
+        CGFloat screenScale = [[NSScreen mainScreen] backingScaleFactor];
+        nativeInput->MouseUp(Babylon::Plugins::NativeInput::LEFT_MOUSE_BUTTON_ID, eventLocation.x * screenScale, invertedY * screenScale);
+    }
 }
 
 - (void)otherMouseDown:(NSEvent *) theEvent {
-     if (nativeInput)
-     {
-         NSPoint eventLocation = [theEvent locationInWindow];
-         auto invertedY = [self getScreenHeight] - eventLocation.y;
-         nativeInput->MouseDown(Babylon::Plugins::NativeInput::MIDDLE_MOUSE_BUTTON_ID, eventLocation.x, invertedY);
-     }
- }
+    if (nativeInput)
+    {
+        NSPoint eventLocation = [theEvent locationInWindow];
+        auto invertedY = [self getScreenHeight] - eventLocation.y;
+        CGFloat screenScale = [[NSScreen mainScreen] backingScaleFactor];
+        nativeInput->MouseDown(Babylon::Plugins::NativeInput::MIDDLE_MOUSE_BUTTON_ID, eventLocation.x * screenScale, invertedY * screenScale);
+    }
+}
 
- - (void)otherMouseDragged:(NSEvent *)theEvent {
-     if (nativeInput)
-     {
-         NSPoint eventLocation = [theEvent locationInWindow];
-         auto invertedY = [self getScreenHeight] - eventLocation.y;
-         nativeInput->MouseMove(eventLocation.x, invertedY);
-     }
- }
+- (void)otherMouseDragged:(NSEvent *)theEvent {
+    if (nativeInput)
+    {
+        NSPoint eventLocation = [theEvent locationInWindow];
+        auto invertedY = [self getScreenHeight] - eventLocation.y;
+        CGFloat screenScale = [[NSScreen mainScreen] backingScaleFactor];
+        nativeInput->MouseMove(eventLocation.x * screenScale, invertedY * screenScale);
+    }
+}
 
- - (void)otherMouseUp:(NSEvent *) theEvent {
-     if (nativeInput)
-     {
-         NSPoint eventLocation = [theEvent locationInWindow];
-         auto invertedY = [self getScreenHeight] - eventLocation.y;
-         nativeInput->MouseUp(Babylon::Plugins::NativeInput::MIDDLE_MOUSE_BUTTON_ID, eventLocation.x, invertedY);
-     }
+- (void)otherMouseUp:(NSEvent *) theEvent {
+    if (nativeInput)
+    {
+        NSPoint eventLocation = [theEvent locationInWindow];
+        auto invertedY = [self getScreenHeight] - eventLocation.y;
+        CGFloat screenScale = [[NSScreen mainScreen] backingScaleFactor];
+        nativeInput->MouseUp(Babylon::Plugins::NativeInput::MIDDLE_MOUSE_BUTTON_ID, eventLocation.x * screenScale, invertedY * screenScale);
+    }
 }
 
 - (void)rightMouseDown:(NSEvent *) theEvent {
-     if (nativeInput)
-     {
-         NSPoint eventLocation = [theEvent locationInWindow];
-         auto invertedY = [self getScreenHeight] - eventLocation.y;
-         nativeInput->MouseDown(Babylon::Plugins::NativeInput::RIGHT_MOUSE_BUTTON_ID, eventLocation.x, invertedY);
-     }
- }
+    if (nativeInput)
+    {
+        NSPoint eventLocation = [theEvent locationInWindow];
+        auto invertedY = [self getScreenHeight] - eventLocation.y;
+        CGFloat screenScale = [[NSScreen mainScreen] backingScaleFactor];
+        nativeInput->MouseDown(Babylon::Plugins::NativeInput::RIGHT_MOUSE_BUTTON_ID, eventLocation.x * screenScale, invertedY * screenScale);
+    }
+}
 
- - (void)rightMouseDragged:(NSEvent *)theEvent {
-     if (nativeInput)
-     {
-         NSPoint eventLocation = [theEvent locationInWindow];
-         auto invertedY = [self getScreenHeight] - eventLocation.y;
-         nativeInput->MouseMove(eventLocation.x, invertedY);
-     }
- }
+- (void)rightMouseDragged:(NSEvent *)theEvent {
+    if (nativeInput)
+    {
+        NSPoint eventLocation = [theEvent locationInWindow];
+        auto invertedY = [self getScreenHeight] - eventLocation.y;
+        CGFloat screenScale = [[NSScreen mainScreen] backingScaleFactor];
+        nativeInput->MouseMove(eventLocation.x * screenScale, invertedY * screenScale);
+    }
+}
 
- - (void)rightMouseUp:(NSEvent *) theEvent {
-     if (nativeInput)
-     {
-         NSPoint eventLocation = [theEvent locationInWindow];
-         auto invertedY = [self getScreenHeight] - eventLocation.y;
-         nativeInput->MouseUp(Babylon::Plugins::NativeInput::RIGHT_MOUSE_BUTTON_ID, eventLocation.x, invertedY);
-     }
+- (void)rightMouseUp:(NSEvent *) theEvent {
+    if (nativeInput)
+    {
+        NSPoint eventLocation = [theEvent locationInWindow];
+        auto invertedY = [self getScreenHeight] - eventLocation.y;
+        CGFloat screenScale = [[NSScreen mainScreen] backingScaleFactor];
+        nativeInput->MouseUp(Babylon::Plugins::NativeInput::RIGHT_MOUSE_BUTTON_ID, eventLocation.x * screenScale, invertedY * screenScale);
+    }
 }
 
 - (void)scrollWheel:(NSEvent *) theEvent {