diff --git a/Mediapipe.Net.Examples.FaceMesh/Mediapipe.Net.Examples.FaceMesh.csproj b/Mediapipe.Net.Examples.FaceMesh/Mediapipe.Net.Examples.FaceMesh.csproj index 4531f62d..7081fda1 100644 --- a/Mediapipe.Net.Examples.FaceMesh/Mediapipe.Net.Examples.FaceMesh.csproj +++ b/Mediapipe.Net.Examples.FaceMesh/Mediapipe.Net.Examples.FaceMesh.csproj @@ -11,7 +11,7 @@ - + diff --git a/Mediapipe.Net.Examples.FaceMesh/Options.cs b/Mediapipe.Net.Examples.FaceMesh/Options.cs index 0ee56205..1dbe41bb 100644 --- a/Mediapipe.Net.Examples.FaceMesh/Options.cs +++ b/Mediapipe.Net.Examples.FaceMesh/Options.cs @@ -10,5 +10,17 @@ public class Options { [Option('c', "camera", Default = 0, HelpText = "The index of the camera to use")] public int CameraIndex { get; set; } + + [Option('f', "input-format", Default = null, HelpText = "The format of the camera input")] + public string? InputFormat { get; set; } + + [Option('r', "fps", Default = null, HelpText = "The framerate of the camera input")] + public int? Framerate { get; set; } + + [Option('w', "width", Default = null, HelpText = "The width of the camera input")] + public int? Width { get; set; } + + [Option('h', "height", Default = null, HelpText = "The height of the camera input")] + public int? Height { get; set; } } } diff --git a/Mediapipe.Net.Examples.FaceMesh/Program.cs b/Mediapipe.Net.Examples.FaceMesh/Program.cs index 211c8f92..28fe04b2 100644 --- a/Mediapipe.Net.Examples.FaceMesh/Program.cs +++ b/Mediapipe.Net.Examples.FaceMesh/Program.cs @@ -5,11 +5,13 @@ using System; using System.Collections.Generic; using CommandLine; +using FFmpeg.AutoGen; using Mediapipe.Net.Calculators; using Mediapipe.Net.External; using Mediapipe.Net.Framework.Format; using Mediapipe.Net.Framework.Protobuf; using SeeShark; +using SeeShark.Device; using SeeShark.FFmpeg; namespace Mediapipe.Net.Examples.FaceMesh @@ -24,6 +26,15 @@ public static void Main(string[] args) { // Get and parse command line arguments Options parsed = Parser.Default.ParseArguments(args).Value; + + (int, int)? videoSize = null; + if (parsed.Width != null && parsed.Height != null) + videoSize = ((int)parsed.Width, (int)parsed.Height); + else if (parsed.Width != null && parsed.Height == null) + Console.Error.WriteLine("Specifying width requires to specify height"); + else if (parsed.Width == null && parsed.Height != null) + Console.Error.WriteLine("Specifying height requires to specify width"); + FFmpegManager.SetupFFmpeg("/usr/lib"); Glog.Initialize("stuff"); @@ -32,7 +43,17 @@ public static void Main(string[] args) { try { - camera = manager.GetCamera(parsed.CameraIndex); + camera = manager.GetDevice(parsed.CameraIndex, + new VideoInputOptions + { + InputFormat = parsed.InputFormat, + Framerate = parsed.Framerate == null ? null : new AVRational + { + num = (int)parsed.Framerate, + den = 1, + }, + VideoSize = videoSize, + }); Console.WriteLine($"Using camera {camera.Info}"); } catch (Exception) @@ -41,41 +62,30 @@ public static void Main(string[] args) return; } } - camera.OnFrame += onFrame; calculator = new FaceMeshCpuCalculator(); calculator.OnResult += handleLandmarks; calculator.Run(); - camera.StartCapture(); Console.CancelKeyPress += (sender, eventArgs) => exit(); - Console.ReadLine(); - } - - private static void handleLandmarks(object? sender, List landmarks) - { - Console.WriteLine($"Got a list of {landmarks[0].Landmark.Count} landmarks at frame {calculator?.CurrentFrame}"); - } - - private static unsafe void onFrame(object? sender, FrameEventArgs e) - { - if (calculator == null) - return; - - var frame = e.Frame; - converter ??= new FrameConverter(frame, PixelFormat.Rgba); + while (true) + { + var frame = camera.GetFrame(); - // Don't use a frame if it's not new - if (e.Status != DecodeStatus.NewFrame) - return; + converter ??= new FrameConverter(frame, PixelFormat.Rgba); - Frame cFrame = converter.Convert(frame); + Frame cFrame = converter.Convert(frame); - ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba, + using ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba, cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData); - using ImageFrame img = calculator.Send(imgframe); - imgframe.Dispose(); + using ImageFrame img = calculator.Send(imgframe); + } + } + + private static void handleLandmarks(object? sender, List landmarks) + { + Console.WriteLine($"Got a list of {landmarks[0].Landmark.Count} landmarks at frame {calculator?.CurrentFrame}"); } // Dispose everything on exit diff --git a/Mediapipe.Net.Examples.FaceMesh/mediapipe b/Mediapipe.Net.Examples.FaceMesh/mediapipe new file mode 120000 index 00000000..b55890d4 --- /dev/null +++ b/Mediapipe.Net.Examples.FaceMesh/mediapipe @@ -0,0 +1 @@ +../mediapipe \ No newline at end of file diff --git a/Mediapipe.Net.Examples.FaceMeshGpu/Mediapipe.Net.Examples.FaceMeshGpu.csproj b/Mediapipe.Net.Examples.FaceMeshGpu/Mediapipe.Net.Examples.FaceMeshGpu.csproj index fb899b2c..1657ea06 100644 --- a/Mediapipe.Net.Examples.FaceMeshGpu/Mediapipe.Net.Examples.FaceMeshGpu.csproj +++ b/Mediapipe.Net.Examples.FaceMeshGpu/Mediapipe.Net.Examples.FaceMeshGpu.csproj @@ -11,7 +11,7 @@ - + diff --git a/Mediapipe.Net.Examples.FaceMeshGpu/Options.cs b/Mediapipe.Net.Examples.FaceMeshGpu/Options.cs index bc9df7e5..5ba66f7e 100644 --- a/Mediapipe.Net.Examples.FaceMeshGpu/Options.cs +++ b/Mediapipe.Net.Examples.FaceMeshGpu/Options.cs @@ -10,5 +10,17 @@ public class Options { [Option('c', "camera", Default = 0, HelpText = "The index of the camera to use")] public int CameraIndex { get; set; } + + [Option('f', "input-format", Default = null, HelpText = "The format of the camera input")] + public string? InputFormat { get; set; } + + [Option('r', "fps", Default = null, HelpText = "The framerate of the camera input")] + public int? Framerate { get; set; } + + [Option('w', "width", Default = null, HelpText = "The width of the camera input")] + public int? Width { get; set; } + + [Option('h', "height", Default = null, HelpText = "The height of the camera input")] + public int? Height { get; set; } } } diff --git a/Mediapipe.Net.Examples.FaceMeshGpu/Program.cs b/Mediapipe.Net.Examples.FaceMeshGpu/Program.cs index ab11a69f..e9f7e14a 100644 --- a/Mediapipe.Net.Examples.FaceMeshGpu/Program.cs +++ b/Mediapipe.Net.Examples.FaceMeshGpu/Program.cs @@ -6,11 +6,13 @@ using System.Collections.Generic; using System.Runtime.Versioning; using CommandLine; +using FFmpeg.AutoGen; using Mediapipe.Net.Calculators; using Mediapipe.Net.External; using Mediapipe.Net.Framework.Format; using Mediapipe.Net.Framework.Protobuf; using SeeShark; +using SeeShark.Device; using SeeShark.FFmpeg; namespace Mediapipe.Net.Examples.FaceMeshGpu @@ -26,6 +28,15 @@ public static void Main(string[] args) { // Get and parse command line arguments Options parsed = Parser.Default.ParseArguments(args).Value; + + (int, int)? videoSize = null; + if (parsed.Width != null && parsed.Height != null) + videoSize = ((int)parsed.Width, (int)parsed.Height); + else if (parsed.Width != null && parsed.Height == null) + Console.Error.WriteLine("Specifying width requires to specify height"); + else if (parsed.Width == null && parsed.Height != null) + Console.Error.WriteLine("Specifying height requires to specify width"); + FFmpegManager.SetupFFmpeg("/usr/lib"); Glog.Initialize("stuff"); @@ -34,7 +45,17 @@ public static void Main(string[] args) { try { - camera = manager.GetCamera(parsed.CameraIndex); + camera = manager.GetDevice(parsed.CameraIndex, + new VideoInputOptions + { + InputFormat = parsed.InputFormat, + Framerate = parsed.Framerate == null ? null : new AVRational + { + num = (int)parsed.Framerate, + den = 1, + }, + VideoSize = videoSize, + }); Console.WriteLine($"Using camera {camera.Info}"); } catch (Exception) @@ -43,41 +64,31 @@ public static void Main(string[] args) return; } } - camera.OnFrame += onFrame; calculator = new FaceMeshGpuCalculator(); calculator.OnResult += handleLandmarks; calculator.Run(); - camera.StartCapture(); Console.CancelKeyPress += (sender, eventArgs) => exit(); - Console.ReadLine(); - } - - private static void handleLandmarks(object? sender, List landmarks) - { - Console.WriteLine($"Got a list of {landmarks[0].Landmark.Count} landmarks at frame {calculator?.CurrentFrame}"); - } - - private static unsafe void onFrame(object? sender, FrameEventArgs e) - { - if (calculator == null) - return; - - var frame = e.Frame; - converter ??= new FrameConverter(frame, PixelFormat.Rgba); + while (true) + { + var frame = camera.GetFrame(); - // Don't use a frame if it's not new - if (e.Status != DecodeStatus.NewFrame) - return; + converter ??= new FrameConverter(frame, PixelFormat.Rgba); - Frame cFrame = converter.Convert(frame); + Frame cFrame = converter.Convert(frame); - ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba, + ImageFrame imgframe = new ImageFrame(ImageFormat.Srgba, cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData); - using ImageFrame img = calculator.Send(imgframe); - imgframe.Dispose(); + using ImageFrame img = calculator.Send(imgframe); + imgframe.Dispose(); + } + } + + private static void handleLandmarks(object? sender, List landmarks) + { + Console.WriteLine($"Got a list of {landmarks[0].Landmark.Count} landmarks at frame {calculator?.CurrentFrame}"); } // Dispose everything on exit diff --git a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Mediapipe.Net.Examples.OsuFrameworkVisualTests.csproj b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Mediapipe.Net.Examples.OsuFrameworkVisualTests.csproj index 2d50ed4d..f73c9377 100644 --- a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Mediapipe.Net.Examples.OsuFrameworkVisualTests.csproj +++ b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Mediapipe.Net.Examples.OsuFrameworkVisualTests.csproj @@ -10,15 +10,16 @@ + - - - - + + + + - + diff --git a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/MediapipeDrawable.cs b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/MediapipeDrawable.cs index 0c635fa9..34bc2037 100644 --- a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/MediapipeDrawable.cs +++ b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/MediapipeDrawable.cs @@ -5,37 +5,32 @@ using System; using Mediapipe.Net.Calculators; using Mediapipe.Net.Framework.Format; +using osu.Framework.Allocation; using osu.Framework.Graphics; using osu.Framework.Graphics.Containers; using osu.Framework.Graphics.Sprites; using osu.Framework.Graphics.Textures; using SeeShark; -using SeeShark.FFmpeg; +using SeeShark.Decode; +using SeeShark.Device; using SixLabors.ImageSharp.PixelFormats; +using Image = SixLabors.ImageSharp.Image; namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests { public class MediapipeDrawable : CompositeDrawable { - public readonly Camera Camera; + private Camera? camera; private FrameConverter? converter; - private readonly FaceMeshCpuCalculator calculator; + private FaceMeshCpuCalculator? calculator; private readonly Sprite sprite; private Texture? texture; - public MediapipeDrawable(int cameraIndex = 0) + public MediapipeDrawable() { - var manager = new CameraManager(); - Camera = manager.GetCamera(cameraIndex); - manager.Dispose(); - - Camera.OnFrame += onFrame; - calculator = new FaceMeshCpuCalculator(); - Masking = true; CornerRadius = 10; - AddInternal(sprite = new Sprite { Anchor = Anchor.Centre, @@ -46,53 +41,38 @@ public MediapipeDrawable(int cameraIndex = 0) }); } - public void Start() +#pragma warning disable IDE0051 + [BackgroundDependencyLoader] + private void load(Camera camera, FrameConverter converter, FaceMeshCpuCalculator calculator) { - calculator.Run(); - Camera.StartCapture(); + this.camera = camera; + this.converter = converter; + this.calculator = calculator; } +#pragma warning restore IDE0051 - private unsafe void onFrame(object? sender, FrameEventArgs e) + protected override unsafe void Update() { - var frame = e.Frame; - if (converter == null) - converter = new FrameConverter(frame, PixelFormat.Rgba); + base.Update(); + if (camera == null || converter == null || calculator == null) + return; - // Don't use a frame if it's not new - if (e.Status != DecodeStatus.NewFrame) + if (camera.TryGetFrame(out Frame frame) != DecodeStatus.NewFrame) return; Frame cFrame = converter.Convert(frame); - - ImageFrame imgFrame; - fixed (byte* rawDataPtr = cFrame.RawData) - { - imgFrame = new ImageFrame(ImageFormat.Srgba, cFrame.Width, cFrame.Height, cFrame.WidthStep, - rawDataPtr); - } - + ImageFrame imgFrame = new ImageFrame(ImageFormat.Srgba, + cFrame.Width, cFrame.Height, cFrame.WidthStep, cFrame.RawData); using ImageFrame outImgFrame = calculator.Send(imgFrame); imgFrame.Dispose(); var span = new ReadOnlySpan(outImgFrame.MutablePixelData, outImgFrame.Height * outImgFrame.WidthStep); - var pixelData = SixLabors.ImageSharp.Image.LoadPixelData(span, cFrame.Width, cFrame.Height); + var pixelData = Image.LoadPixelData(span, cFrame.Width, cFrame.Height); texture ??= new Texture(cFrame.Width, cFrame.Height); texture.SetData(new TextureUpload(pixelData)); sprite.FillAspectRatio = (float)cFrame.Width / cFrame.Height; sprite.Texture = texture; } - - public new void Dispose() - { - if (IsDisposed) - return; - - Camera.StopCapture(); - Camera.Dispose(); - converter?.Dispose(); - calculator.Dispose(); - base.Dispose(); - } } } diff --git a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Options.cs b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Options.cs new file mode 100644 index 00000000..68f7bbfe --- /dev/null +++ b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Options.cs @@ -0,0 +1,26 @@ +// Copyright (c) homuler and The Vignette Authors +// This file is part of MediaPipe.NET. +// MediaPipe.NET is licensed under the MIT License. See LICENSE for details. + +using CommandLine; + +namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests +{ + public class Options + { + [Option('c', "camera", Default = 0, HelpText = "The index of the camera to use")] + public int CameraIndex { get; set; } + + [Option('f', "input-format", Default = null, HelpText = "The format of the camera input")] + public string? InputFormat { get; set; } + + [Option('r', "fps", Default = null, HelpText = "The framerate of the camera input")] + public int? Framerate { get; set; } + + [Option('w', "width", Default = null, HelpText = "The width of the camera input")] + public int? Width { get; set; } + + [Option('h', "height", Default = null, HelpText = "The height of the camera input")] + public int? Height { get; set; } + } +} diff --git a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/OsuFrameworkVisualTestsGameBase.cs b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/OsuFrameworkVisualTestsGameBase.cs index 56f91a37..7e167cb2 100644 --- a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/OsuFrameworkVisualTestsGameBase.cs +++ b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/OsuFrameworkVisualTestsGameBase.cs @@ -2,17 +2,26 @@ // This file is part of MediaPipe.NET. // MediaPipe.NET is licensed under the MIT License. See LICENSE for details. +using System; +using CommandLine; +using FFmpeg.AutoGen; +using Mediapipe.Net.Calculators; using osu.Framework; using osu.Framework.Allocation; using osu.Framework.Graphics; using osu.Framework.Graphics.Containers; using osuTK; +using SeeShark; +using SeeShark.Device; namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests { public class OsuFrameworkVisualTestsGameBase : Game { - private MediapipeDrawable? mediapipeDrawable; + private Camera? camera; + private FrameConverter? converter; + private FaceMeshCpuCalculator? calculator; + protected override Container Content { get; } private DependencyContainer? dependencies; @@ -31,20 +40,47 @@ protected OsuFrameworkVisualTestsGameBase() [BackgroundDependencyLoader] private void load() { - mediapipeDrawable = new MediapipeDrawable - { - Anchor = Anchor.Centre, - Origin = Anchor.Centre, - Size = new Vector2(1280, 720), - FillMode = FillMode.Fit - }; - dependencies?.Cache(mediapipeDrawable); + string[] args = Environment.GetCommandLineArgs(); + Options parsed = Parser.Default.ParseArguments(args).Value; + + (int, int)? videoSize = null; + if (parsed.Width != null && parsed.Height != null) + videoSize = ((int)parsed.Width, (int)parsed.Height); + else if (parsed.Width != null && parsed.Height == null) + Console.Error.WriteLine("Specifying width requires to specify height"); + else if (parsed.Width == null && parsed.Height != null) + Console.Error.WriteLine("Specifying height requires to specify width"); + + var manager = new CameraManager(); + camera = manager.GetDevice(parsed.CameraIndex, + new VideoInputOptions + { + InputFormat = parsed.InputFormat, + Framerate = parsed.Framerate == null ? null : new AVRational + { + num = (int)parsed.Framerate, + den = 1, + }, + VideoSize = videoSize, + }); + dependencies?.Cache(camera); + manager.Dispose(); + + var dummyFrame = camera.GetFrame(); + converter = new FrameConverter(dummyFrame, PixelFormat.Rgba); + dependencies?.Cache(converter); + + calculator = new FaceMeshCpuCalculator(); + calculator.Run(); + dependencies?.Cache(calculator); } #pragma warning restore IDE0051 protected override bool OnExiting() { - mediapipeDrawable?.Dispose(); + calculator?.Dispose(); + converter?.Dispose(); + camera?.Dispose(); return base.OnExiting(); } } diff --git a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Program.cs b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Program.cs index 84f67ebc..ac128da2 100644 --- a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Program.cs +++ b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Program.cs @@ -13,7 +13,7 @@ public static class Program public static void Main() { FFmpegManager.SetupFFmpeg("/usr/lib"); - using GameHost host = Host.GetSuitableHost("visual-tests"); + using GameHost host = Host.GetSuitableDesktopHost("visual-tests"); using var game = new OsuFrameworkVisualTestsTestBrowser(); host.Run(game); } diff --git a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Visual/TestSceneFaceMeshCpu.cs b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Visual/TestSceneFaceMeshCpu.cs index b1b3b733..77bf447d 100644 --- a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Visual/TestSceneFaceMeshCpu.cs +++ b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/Visual/TestSceneFaceMeshCpu.cs @@ -7,6 +7,7 @@ using osu.Framework.Graphics; using osu.Framework.Graphics.Containers; using osu.Framework.Graphics.Shapes; +using osuTK; namespace Mediapipe.Net.Examples.OsuFrameworkVisualTests.Visual { @@ -14,7 +15,7 @@ public class TestSceneFaceMeshCpu : OsuFrameworkVisualTestsTestScene { #pragma warning disable IDE0051 [BackgroundDependencyLoader] - private void load(MediapipeDrawable mediapipeDrawable) + private void load() { Add(new Container { @@ -26,10 +27,15 @@ private void load(MediapipeDrawable mediapipeDrawable) RelativeSizeAxes = Axes.Both, Colour = Color4Extensions.FromHex(@"272727"), }, - mediapipeDrawable, + new MediapipeDrawable + { + Anchor = Anchor.Centre, + Origin = Anchor.Centre, + Size = new Vector2(1280, 720), + FillMode = FillMode.Fit, + }, }, }); - mediapipeDrawable.Start(); } #pragma warning restore IDE0051 } diff --git a/Mediapipe.Net.Examples.OsuFrameworkVisualTests/mediapipe b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/mediapipe new file mode 120000 index 00000000..b55890d4 --- /dev/null +++ b/Mediapipe.Net.Examples.OsuFrameworkVisualTests/mediapipe @@ -0,0 +1 @@ +../mediapipe \ No newline at end of file diff --git a/Mediapipe.Net/External/SerializedProtoVector.cs b/Mediapipe.Net/External/SerializedProtoVector.cs index daa9f6b0..51d21735 100644 --- a/Mediapipe.Net/External/SerializedProtoVector.cs +++ b/Mediapipe.Net/External/SerializedProtoVector.cs @@ -17,7 +17,12 @@ internal unsafe struct SerializedProtoVector // TODO: This is looking just as sus as SerializedProto.Dispose(). // Should be investigated in the same way. - public void Dispose() => UnsafeNativeMethods.mp_api_SerializedProtoArray__delete(Data); + public void Dispose() + { + for (int i = 0; i < Size; i++) + Data[i].Dispose(); + UnsafeNativeMethods.mp_api_SerializedProtoArray__delete(Data); + } public List Deserialize(MessageParser parser) where T : IMessage { diff --git a/Mediapipe.Net/Framework/CalculatorGraph.cs b/Mediapipe.Net/Framework/CalculatorGraph.cs index bf4c0f8b..0d00ebad 100644 --- a/Mediapipe.Net/Framework/CalculatorGraph.cs +++ b/Mediapipe.Net/Framework/CalculatorGraph.cs @@ -89,6 +89,7 @@ public Status ObserveOutputStream(string streamName, PacketCall { var packet = (TPacket?)Activator.CreateInstance(typeof(TPacket), (IntPtr)packetPtr, false); status = packetCallback(packet); + packet?.Dispose(); } catch (Exception e) { diff --git a/Mediapipe.Net/Mediapipe.Net.csproj b/Mediapipe.Net/Mediapipe.Net.csproj index 65d9f7a2..469f028b 100644 --- a/Mediapipe.Net/Mediapipe.Net.csproj +++ b/Mediapipe.Net/Mediapipe.Net.csproj @@ -8,7 +8,7 @@ - +