ImageCapture native polyfill (#1283)

This change provides an initial implementation of a polyfill for
[`ImageCapture`](https://developer.mozilla.org/en-US/docs/Web/API/ImageCapture)
to enable high res photo capture. This builds on top of the existing
`MediaCapture` polyfill, and follows existing patterns. Some notes on
this:
- MediaStream and MediaStreamTrack are polyfilled by the same class on
the C++ side, which means that class (`MediaStream`) is what gets passed
into the `ImageCapture` constructor. For this PR, I am considering
`ImageCapture` to be part of the same polyfill. It is not independent,
and has internal knowledge that the object passed into the
`ImageCapture` constructor is actually a `MediaStream`. The
`MediaStream` now exposes the underlying (platform specific)
`CameraDevice`.
- The `CameraDevice` remains the primary camera abstraction across
platforms. As such, it is the thing that exposes high res photo
capabilities, settings, and the ability to actually take a high res
photo. Since capabilities are normally track specific, it just returns
capabilities for the opened camera session. This should be fine because
we only support opening a single camera stream at a time, and
`ImageCapture` isn't constructed until you have a track (which is the
result of opening a camera stream).

This PR includes all the common glue code (e.g. napi) and platform
abstractions, plus an initial iOS implementation (probably needs a bit
more work, but the basics are in place).

Apps/Playground/Scripts/experience.js

@@ -16,6 +16,7 @@ var meshDetection = false;
 var text = false;
 var hololens = false;
 var cameraTexture = false;
+var imageCapture = false;
 var imageTracking = false;
 const readPixels = false;
 
@@ -88,14 +89,37 @@ CreateBoxAsync(scene).then(function () {
         var mat = new BABYLON.StandardMaterial("mat", scene);
         mat.diffuseColor = BABYLON.Color3.Black();
 
-        var tex = BABYLON.VideoTexture.CreateFromWebCam(scene, function(videoTexture) {
-            const videoSize = videoTexture.getSize();
-            mat.emissiveTexture = videoTexture;
-            plane.material = mat;
-            plane.scaling.x = 5;
-            plane.scaling.y = 5 * (videoSize.height / videoSize.width);
-            console.log("Video texture size: " + videoSize);
-        }, { maxWidth: 1280, maxHeight: 720, facingMode: 'environment'});
+        const constraints = { maxWidth: 1280, maxHeight: 720, facingMode: 'environment'};
+         navigator.mediaDevices.getUserMedia({ video: constraints }).then((stream) => {
+            BABYLON.VideoTexture.CreateFromStreamAsync(scene, stream, constraints).then((videoTexture) => {
+                const videoSize = videoTexture.getSize();
+                mat.emissiveTexture = videoTexture;
+                plane.material = mat;
+                plane.scaling.x = 5;
+                plane.scaling.y = 5 * (videoSize.height / videoSize.width);
+                console.log("Video texture size (NEW): " + videoSize);
+            });
+
+            if (imageCapture) {
+                new Promise(resolve => setTimeout(resolve, 1000)).then(() => {
+                    const imageCapture = new ImageCapture(stream.getVideoTracks()[0]);
+                    console.log(`Capabilities: ${JSON.stringify(imageCapture.getPhotoCapabilities(), null, 2)}`);
+                    console.log(`Settings: ${JSON.stringify(imageCapture.getPhotoSettings(), null, 2)}`);
+                    imageCapture.takePhoto().then((blob) => {
+                        console.log(`takePhoto finished with a blob of size ${blob.size} and type '${blob.type}'`);
+                        blob.arrayBuffer().then((buffer) => {
+                            const imageData = new Uint8Array(buffer);
+                            console.log(`Retrieved photo ArrayBuffer of size ${imageData.byteLength}`);
+                            console.log(`JPEG header bytes should be 0xff, 0xd8, 0xff.`);
+                            console.log(`Header bytes are 0x${imageData[0].toString(16)}, 0x${imageData[1].toString(16)}, 0x${imageData[2].toString(16)}`);
+
+                            const imageTexture = new BABYLON.Texture("data:fromblob", scene, true, undefined, undefined, undefined, undefined, imageData);
+                            mat.emissiveTexture = imageTexture;
+                        });
+                    });
+                });
+            }
+        });
     }
 
     if (readPixels) {

Plugins/NativeCamera/CMakeLists.txt

@@ -10,6 +10,8 @@ set(SOURCES
     "Source/Capability.cpp"
     "Source/Constraint.h"
     "Source/Constraint.cpp"
+    "Source/ImageCapture.h"
+    "Source/ImageCapture.cpp"
     "Source/MediaDevices.h"
     "Source/MediaDevices.cpp"
     "Source/MediaStream.h"

Plugins/NativeCamera/Source/Android/CameraDevice.cpp

@@ -74,6 +74,8 @@ namespace Babylon::Plugins
 
         std::vector<CameraTrack> supportedResolutions{};
         std::vector<std::unique_ptr<Capability>> capabilities{};
+        std::optional<Plugins::PhotoCapabilities> photoCapabilities{};
+        std::optional<Plugins::PhotoSettings> defaultPhotoSettings{};
         std::string cameraID{};
         int32_t sensorRotation{};
         bool facingUser{};
@@ -505,6 +507,11 @@ namespace Babylon::Plugins
                    : CameraDimensions{m_impl->cameraDimensions.height, m_impl->cameraDimensions.width};
     }
 
+    CameraDevice::TakePhotoTask CameraDevice::TakePhotoAsync(PhotoSettings /*photoSettings*/)
+    {
+        throw std::runtime_error{"TakePhoto not implemented for this platform."};
+    }
+
     void CameraDevice::Close()
     {
         if (m_impl->textureSession == nullptr)